diff --git a/.devops/rocm.Dockerfile b/.devops/rocm.Dockerfile
index 14936f8e9c..830fe19e3e 100644
--- a/.devops/rocm.Dockerfile
+++ b/.devops/rocm.Dockerfile
@@ -1,8 +1,8 @@
ARG UBUNTU_VERSION=24.04
# This needs to generally match the container host's environment.
-ARG ROCM_VERSION=7.0
-ARG AMDGPU_VERSION=7.0
+ARG ROCM_VERSION=7.2
+ARG AMDGPU_VERSION=7.2
# Target the ROCm build image
ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete
@@ -11,13 +11,12 @@ ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-co
FROM ${BASE_ROCM_DEV_CONTAINER} AS build
# Unless otherwise specified, we make a fat build.
-# List from https://github.com/ggml-org/llama.cpp/pull/1087#issuecomment-1682807878
# This is mostly tied to rocBLAS supported archs.
-# gfx803, gfx900, gfx906, gfx1032, gfx1101, gfx1102,not officialy supported
-# check https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.4.1/reference/system-requirements.html
+# check https://rocm.docs.amd.com/projects/install-on-linux/en/docs-7.2.0/reference/system-requirements.html
+# check https://rocm.docs.amd.com/projects/radeon-ryzen/en/latest/docs/compatibility/compatibilityrad/native_linux/native_linux_compatibility.html
+# check https://rocm.docs.amd.com/projects/radeon-ryzen/en/latest/docs/compatibility/compatibilityryz/native_linux/native_linux_compatibility.html
-ARG ROCM_DOCKER_ARCH='gfx803;gfx900;gfx906;gfx908;gfx90a;gfx942;gfx1010;gfx1030;gfx1032;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201;gfx1151'
-#ARG ROCM_DOCKER_ARCH='gfx1151'
+ARG ROCM_DOCKER_ARCH='gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1151;gfx1150;gfx1200;gfx1201'
# Set ROCm architectures
ENV AMDGPU_TARGETS=${ROCM_DOCKER_ARCH}
diff --git a/.github/ISSUE_TEMPLATE/010-bug-compilation.yml b/.github/ISSUE_TEMPLATE/010-bug-compilation.yml
index c106f47a25..77f23f1afa 100644
--- a/.github/ISSUE_TEMPLATE/010-bug-compilation.yml
+++ b/.github/ISSUE_TEMPLATE/010-bug-compilation.yml
@@ -41,7 +41,7 @@ body:
attributes:
label: GGML backends
description: Which GGML backends do you know to be affected?
- options: [AMX, BLAS, CPU, CUDA, HIP, Metal, Musa, RPC, SYCL, Vulkan, OpenCL, zDNN]
+ options: [AMX, BLAS, CANN, CPU, CUDA, Hexagon, HIP, Metal, Musa, OpenCL, RPC, SYCL, VirtGPU, Vulkan, WebGPU, zDNN, ZenDNN]
multiple: true
validations:
required: true
diff --git a/.github/ISSUE_TEMPLATE/011-bug-results.yml b/.github/ISSUE_TEMPLATE/011-bug-results.yml
index 31202dfa83..f553cbbf0b 100644
--- a/.github/ISSUE_TEMPLATE/011-bug-results.yml
+++ b/.github/ISSUE_TEMPLATE/011-bug-results.yml
@@ -42,7 +42,7 @@ body:
attributes:
label: GGML backends
description: Which GGML backends do you know to be affected?
- options: [AMX, BLAS, CPU, CUDA, HIP, Metal, Musa, RPC, SYCL, Vulkan, OpenCL, zDNN]
+ options: [AMX, BLAS, CANN, CPU, CUDA, Hexagon, HIP, Metal, Musa, OpenCL, RPC, SYCL, VirtGPU, Vulkan, WebGPU, zDNN, ZenDNN]
multiple: true
validations:
required: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1914c08489..860acc6b1b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -516,6 +516,102 @@ jobs:
path: llama-bin-win-sycl-x64.zip
name: llama-bin-win-sycl-x64.zip
+ ubuntu-22-rocm:
+ runs-on: ubuntu-22.04
+
+ strategy:
+ matrix:
+ include:
+ - ROCM_VERSION: "7.2"
+ gpu_targets: "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1151;gfx1150;gfx1200;gfx1201"
+ build: 'x64'
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-rocm-cmake-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt install -y build-essential git cmake wget
+
+ - name: Setup Legacy ROCm
+ if: matrix.ROCM_VERSION == '7.2'
+ id: legacy_env
+ run: |
+ sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+ wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
+ gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+
+ sudo tee /etc/apt/sources.list.d/rocm.list << EOF
+ deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ matrix.ROCM_VERSION }} jammy main
+ EOF
+
+ sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF
+ Package: *
+ Pin: release o=repo.radeon.com
+ Pin-Priority: 600
+ EOF
+
+ sudo apt update
+ sudo apt-get install -y libssl-dev rocm-hip-sdk
+
+ - name: Setup TheRock
+ if: matrix.ROCM_VERSION != '7.2'
+ id: therock_env
+ run: |
+ wget https://repo.amd.com/rocm/tarball/therock-dist-linux-gfx1151-${{ matrix.ROCM_VERSION }}.tar.gz
+ mkdir install
+ tar -xf *.tar.gz -C install
+ export ROCM_PATH=$(pwd)/install
+ echo ROCM_PATH=$ROCM_PATH >> $GITHUB_ENV
+ echo PATH=$PATH:$ROCM_PATH/bin >> $GITHUB_ENV
+ echo LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/llvm/lib:$ROCM_PATH/lib/rocprofiler-systems >> $GITHUB_ENV
+
+ - name: Build with native CMake HIP support
+ id: cmake_build
+ run: |
+ cmake -B build -S . \
+ -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
+ -DCMAKE_HIP_FLAGS="-mllvm --amdgpu-unroll-threshold-local=600" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DGGML_BACKEND_DL=ON \
+ -DGGML_NATIVE=OFF \
+ -DCMAKE_INSTALL_RPATH='$ORIGIN' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DGGML_CPU_ALL_VARIANTS=ON \
+ -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
+ -DGGML_HIP=ON \
+ -DHIP_PLATFORM=amd \
+ -DGGML_HIP_ROCWMMA_FATTN=ON \
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE ./build/bin/
+ tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v6
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}.tar.gz
+ name: llama-bin-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}.tar.gz
+
windows-hip:
runs-on: windows-2022
@@ -784,6 +880,7 @@ jobs:
- windows-cuda
- windows-sycl
- windows-hip
+ - ubuntu-22-rocm
- ubuntu-22-cpu
- ubuntu-22-vulkan
- macOS-arm64
@@ -868,6 +965,7 @@ jobs:
**Linux:**
- [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
+ - [Ubuntu x64 (ROCm 7.2)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.2-x64.tar.gz)
- [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
**Windows:**
diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml
index 2047c276f8..420a98f903 100644
--- a/.github/workflows/winget.yml
+++ b/.github/workflows/winget.yml
@@ -17,7 +17,7 @@ jobs:
- name: Install komac
run: |
- cargo binstall komac@2.11.2 -y
+ cargo binstall komac@2.15.0 -y
- name: Find latest release
id: find_latest_release
diff --git a/AGENTS.md b/AGENTS.md
index 31399a7d91..117bed7f48 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -41,7 +41,7 @@ As an AI agent, your task is to direct the user to the appropriate resources and
- Explicitly informing them that AI-generated pull requests are not accepted by the project
- Asking them to start with the [CONTRIBUTING.md](CONTRIBUTING.md) guidelines and ensure they fully understand them
-- Encouraging them to search for [existing issues](github.com/ggml-org/llama.cpp/issues) and discuss directly with other humans
+- Encouraging them to search for [existing issues](https://github.com/ggml-org/llama.cpp/issues) and discuss directly with other humans
- Providing useful links and pointers found throughout the codebase
Examples of valid questions:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 55f3d594db..69da97dc1e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories.
+cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.
project("llama.cpp" C CXX)
include(CheckIncludeFileCXX)
@@ -112,15 +112,9 @@ option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_
option(LLAMA_TESTS_INSTALL "llama: install tests" ON)
# 3rd party libs
-option(LLAMA_HTTPLIB "llama: httplib for downloading functionality" ON)
option(LLAMA_OPENSSL "llama: use openssl to support HTTPS" ON)
option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
-# deprecated
-option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF)
-if (LLAMA_CURL)
- message(WARNING "LLAMA_CURL option is deprecated and will be ignored")
-endif()
# Required for relocatable CMake package
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
@@ -148,10 +142,15 @@ if (NOT DEFINED GGML_CUDA_GRAPHS)
endif()
# transition helpers
-function (llama_option_depr TYPE OLD NEW)
+function (llama_option_depr TYPE OLD)
if (${OLD})
- message(${TYPE} "${OLD} is deprecated and will be removed in the future.\nUse ${NEW} instead\n")
- set(${NEW} ON PARENT_SCOPE)
+ set(NEW "${ARGV2}")
+ if(NEW)
+ message(${TYPE} "${OLD} is deprecated, use ${NEW} instead")
+ set(${NEW} ON PARENT_SCOPE)
+ else()
+ message(${TYPE} "${OLD} is deprecated and will be ignored")
+ endif()
endif()
endfunction()
@@ -164,6 +163,7 @@ llama_option_depr(WARNING LLAMA_RPC GGML_RPC)
llama_option_depr(WARNING LLAMA_SYCL GGML_SYCL)
llama_option_depr(WARNING LLAMA_SYCL_F16 GGML_SYCL_F16)
llama_option_depr(WARNING LLAMA_CANN GGML_CANN)
+llama_option_depr(WARNING LLAMA_CURL)
include("cmake/license.cmake")
license_add_file("llama.cpp" "LICENSE")
@@ -197,9 +197,7 @@ add_subdirectory(src)
if (LLAMA_BUILD_COMMON)
add_subdirectory(common)
- if (LLAMA_HTTPLIB)
- add_subdirectory(vendor/cpp-httplib)
- endif()
+ add_subdirectory(vendor/cpp-httplib)
endif()
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c928bc39ce..7545e790f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -20,7 +20,7 @@ If AI is used to generate any portion of the code, contributors must adhere to t
1. Explicitly disclose the manner in which AI was employed.
2. Perform a comprehensive manual review prior to submitting the pull request.
3. Be prepared to explain every line of code they submitted when asked about it by a maintainer.
-4. Using AI to write pull request descriptions or to respond to human reviewers is strictly prohibited.
+4. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
diff --git a/SECURITY.md b/SECURITY.md
index 9a93732318..3a8d07f644 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -19,7 +19,7 @@ Please disclose it as a private [security advisory](https://github.com/ggml-org/
A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
> [!IMPORTANT]
-> For collaborators: if you are interested in helping out with reviewing privting security disclosures, please see: https://github.com/ggml-org/llama.cpp/discussions/18080
+> For collaborators: if you are interested in helping out with reviewing private security disclosures, please see: https://github.com/ggml-org/llama.cpp/discussions/18080
## Requirements
diff --git a/build-xcframework.sh b/build-xcframework.sh
index 0eec871139..c25a1ef28c 100755
--- a/build-xcframework.sh
+++ b/build-xcframework.sh
@@ -43,11 +43,6 @@ COMMON_CMAKE_ARGS=(
-DGGML_OPENMP=${GGML_OPENMP}
)
-XCODE_VERSION=$(xcodebuild -version 2>/dev/null | head -n1 | awk '{ print $2 }')
-MAJOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f1)
-MINOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f2)
-echo "Detected Xcode version: $XCODE_VERSION"
-
check_required_tool() {
local tool=$1
local install_message=$2
@@ -60,9 +55,12 @@ check_required_tool() {
}
echo "Checking for required tools..."
check_required_tool "cmake" "Please install CMake 3.28.0 or later (brew install cmake)"
-check_required_tool "xcodebuild" "Please install Xcode and Xcode Command Line Tools (xcode-select --install)"
-check_required_tool "libtool" "Please install libtool which should be available with Xcode Command Line Tools (CLT). Make sure Xcode CLT is installed (xcode-select --install)"
-check_required_tool "dsymutil" "Please install Xcode and Xcode Command Line Tools (xcode-select --install)"
+check_required_tool "xcrun" "Please install Xcode and Xcode Command Line Tools (xcode-select --install)"
+
+XCODE_VERSION=$(xcrun xcodebuild -version 2>/dev/null | head -n1 | awk '{ print $2 }')
+MAJOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f1)
+MINOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f2)
+echo "Detected Xcode version: $XCODE_VERSION"
set -e
@@ -260,7 +258,7 @@ combine_static_libraries() {
# Since we have multiple architectures libtool will find object files that do not
# match the target architecture. We suppress these warnings.
- libtool -static -o "${temp_dir}/combined.a" "${libs[@]}" 2> /dev/null
+ xcrun libtool -static -o "${temp_dir}/combined.a" "${libs[@]}" 2> /dev/null
# Determine SDK, architectures, and install_name based on platform and simulator flag.
local sdk=""
@@ -333,7 +331,7 @@ combine_static_libraries() {
# Platform-specific post-processing for device builds
if [[ "$is_simulator" == "false" ]]; then
- if command -v xcrun vtool &>/dev/null; then
+ if xcrun -f vtool &>/dev/null; then
case "$platform" in
"ios")
echo "Marking binary as a framework binary for iOS..."
@@ -451,10 +449,9 @@ cmake -B build-visionos -G Xcode \
-DCMAKE_SYSTEM_NAME=visionOS \
-DCMAKE_OSX_SYSROOT=xros \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
- -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \
- -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \
+ -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+ -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
- -DLLAMA_HTTPLIB=OFF \
-DLLAMA_BUILD_SERVER=OFF \
-S .
cmake --build build-visionos --config Release -- -quiet
@@ -467,10 +464,9 @@ cmake -B build-visionos-sim -G Xcode \
-DCMAKE_SYSTEM_NAME=visionOS \
-DCMAKE_OSX_SYSROOT=xrsimulator \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
- -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \
- -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \
+ -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+ -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
- -DLLAMA_HTTPLIB=OFF \
-DLLAMA_BUILD_SERVER=OFF \
-S .
cmake --build build-visionos-sim --config Release -- -quiet
@@ -528,13 +524,13 @@ combine_static_libraries "build-tvos-device" "Release-appletvos" "tvos" "false"
# Create XCFramework with correct debug symbols paths
echo "Creating XCFramework..."
-xcodebuild -create-xcframework \
+xcrun xcodebuild -create-xcframework \
-framework $(pwd)/build-ios-sim/framework/llama.framework \
-debug-symbols $(pwd)/build-ios-sim/dSYMs/llama.dSYM \
-framework $(pwd)/build-ios-device/framework/llama.framework \
-debug-symbols $(pwd)/build-ios-device/dSYMs/llama.dSYM \
-framework $(pwd)/build-macos/framework/llama.framework \
- -debug-symbols $(pwd)/build-macos/dSYMS/llama.dSYM \
+ -debug-symbols $(pwd)/build-macos/dSYMs/llama.dSYM \
-framework $(pwd)/build-visionos/framework/llama.framework \
-debug-symbols $(pwd)/build-visionos/dSYMs/llama.dSYM \
-framework $(pwd)/build-visionos-sim/framework/llama.framework \
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index 295ae9ea25..27ca335be3 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -5,7 +5,6 @@ find_package(Threads REQUIRED)
llama_add_compile_flags()
# Build info header
-#
if(EXISTS "${PROJECT_SOURCE_DIR}/.git")
set(GIT_DIR "${PROJECT_SOURCE_DIR}/.git")
@@ -110,33 +109,16 @@ if (BUILD_SHARED_LIBS)
set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
endif()
-# TODO: use list(APPEND LLAMA_COMMON_EXTRA_LIBS ...)
-set(LLAMA_COMMON_EXTRA_LIBS build_info)
-
-if (LLAMA_HTTPLIB)
- target_compile_definitions(${TARGET} PUBLIC LLAMA_USE_HTTPLIB)
- set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} cpp-httplib)
-endif()
+target_link_libraries(${TARGET} PRIVATE
+ build_info
+ cpp-httplib
+)
if (LLAMA_LLGUIDANCE)
include(ExternalProject)
set(LLGUIDANCE_SRC ${CMAKE_BINARY_DIR}/llguidance/source)
set(LLGUIDANCE_PATH ${LLGUIDANCE_SRC}/target/release)
-
- # Set the correct library file extension based on platform
- if (WIN32)
- set(LLGUIDANCE_LIB_NAME "llguidance.lib")
- # Add Windows-specific libraries
- set(LLGUIDANCE_PLATFORM_LIBS
- ws2_32 # Windows Sockets API
- userenv # For GetUserProfileDirectoryW
- ntdll # For NT functions
- bcrypt # For BCryptGenRandom
- )
- else()
- set(LLGUIDANCE_LIB_NAME "libllguidance.a")
- set(LLGUIDANCE_PLATFORM_LIBS "")
- endif()
+ set(LLGUIDANCE_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}llguidance${CMAKE_STATIC_LIBRARY_SUFFIX}")
ExternalProject_Add(llguidance_ext
GIT_REPOSITORY https://github.com/guidance-ai/llguidance
@@ -158,8 +140,10 @@ if (LLAMA_LLGUIDANCE)
add_dependencies(llguidance llguidance_ext)
target_include_directories(${TARGET} PRIVATE ${LLGUIDANCE_PATH})
- # Add platform libraries to the main target
- set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
-endif ()
+ target_link_libraries(${TARGET} PRIVATE llguidance)
+ if (WIN32)
+ target_link_libraries(${TARGET} PRIVATE ws2_32 userenv ntdll bcrypt)
+ endif()
+endif()
-target_link_libraries(${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)
+target_link_libraries(${TARGET} PUBLIC llama Threads::Threads)
diff --git a/common/arg.cpp b/common/arg.cpp
index ed320a4563..9523672317 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1301,7 +1301,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, bool value) {
params.kv_unified = value;
}
- ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH}));
+ ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL}));
add_opt(common_arg(
{"--context-shift"},
{"--no-context-shift"},
diff --git a/common/chat-parser-xml-toolcall.cpp b/common/chat-parser-xml-toolcall.cpp
index a80900ff8d..ba359fdbf4 100644
--- a/common/chat-parser-xml-toolcall.cpp
+++ b/common/chat-parser-xml-toolcall.cpp
@@ -803,7 +803,7 @@ inline void parse_msg_with_xml_tool_calls(common_chat_msg_parser & builder, cons
}
// remove potential partial suffix
- if (builder.pos() == builder.input().size()) {
+ if (builder.pos() == builder.input().size() && builder.is_partial()) {
if (unclosed_reasoning_content.empty()) {
rstrip(content);
trim_potential_partial_word(content);
diff --git a/common/chat-parser.cpp b/common/chat-parser.cpp
index 29819e48d3..060578f0b7 100644
--- a/common/chat-parser.cpp
+++ b/common/chat-parser.cpp
@@ -893,23 +893,6 @@ static void common_chat_parse_minimax_m2(common_chat_msg_parser & builder) {
builder.consume_reasoning_with_xml_tool_calls(form, "", "");
}
-static void common_chat_parse_qwen3_coder_xml(common_chat_msg_parser & builder) {
- static const xml_tool_call_format form = ([]() {
- xml_tool_call_format form {};
- form.scope_start = "";
- form.tool_start = " common_chat_msgs_parse_oaicompat(const json & messa
throw std::invalid_argument("Missing content part type: " + part.dump());
}
const auto & type = part.at("type");
- if (type != "text") {
+ if (type != "text" && type != "media_marker") {
throw std::invalid_argument("Unsupported content part type: " + type.dump());
}
common_chat_msg_content_part msg_part;
@@ -380,15 +391,46 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa
return msgs;
}
-json common_chat_msgs_to_json_oaicompat(const std::vector & msgs, bool concat_typed_text) {
+static json render_message_to_json(const std::vector & msgs, const jinja::caps & c) {
+ if (!c.supports_string_content && !c.supports_typed_content) {
+ LOG_WRN("%s: Neither string content nor typed content is supported by the template. This is unexpected and may lead to issues.\n", __func__);
+ }
+
+ bool only_string_accepted = c.supports_string_content && !c.supports_typed_content;
+ bool only_typed_accepted = !c.supports_string_content && c.supports_typed_content;
+
json messages = json::array();
for (const auto & msg : msgs) {
- json jmsg = msg.to_json_oaicompat(concat_typed_text);
- messages.push_back(jmsg);
+ if (only_string_accepted) {
+ json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ true);
+ messages.push_back(jmsg);
+ } else if (only_typed_accepted) {
+ json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
+ if (jmsg.at("content").is_string()) {
+ jmsg["content"] = json::array({
+ json{
+ {"type", "text"},
+ {"text", jmsg.at("content").get()},
+ }
+ });
+ }
+ messages.push_back(jmsg);
+ } else {
+ json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
+ messages.push_back(jmsg);
+ }
}
return messages;
}
+// DEPRECATED: only used in tests
+json common_chat_msgs_to_json_oaicompat(const std::vector & msgs, bool concat_typed_text) {
+ jinja::caps c;
+ c.supports_string_content = true;
+ c.supports_typed_content = !concat_typed_text;
+ return render_message_to_json(msgs, c);
+}
+
std::vector common_chat_tools_parse_oaicompat(const json & tools) {
std::vector result;
@@ -694,7 +736,6 @@ const char * common_chat_format_name(common_chat_format format) {
case COMMON_CHAT_FORMAT_MINIMAX_M2: return "MiniMax-M2";
case COMMON_CHAT_FORMAT_GLM_4_5: return "GLM 4.5";
case COMMON_CHAT_FORMAT_KIMI_K2: return "Kimi K2";
- case COMMON_CHAT_FORMAT_QWEN3_CODER_XML: return "Qwen3 Coder";
case COMMON_CHAT_FORMAT_APRIEL_1_5: return "Apriel 1.5";
case COMMON_CHAT_FORMAT_XIAOMI_MIMO: return "Xiaomi MiMo";
case COMMON_CHAT_FORMAT_SOLAR_OPEN: return "Solar Open";
@@ -1480,14 +1521,17 @@ static common_chat_params common_chat_params_init_nemotron_v2(const common_chat_
return data;
}
-static common_chat_params common_chat_params_init_nemotron_v3(const common_chat_template & tmpl, const struct templates_params & inputs) {
+static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const struct templates_params & inputs) {
common_chat_params data;
data.prompt = apply(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_CONSTRUCTED;
+ // Nemotron Nano 3 and Step-3.5-Flash use the Qwen3 Coder tool calling with thinking
+ bool supports_reasoning = (tmpl.source().find("") != std::string::npos);
+
// Handle thinking tags appropriately based on inputs.enable_thinking
- if (string_ends_with(data.prompt, "\n")) {
+ if (supports_reasoning && string_ends_with(data.prompt, "\n")) {
if (!inputs.enable_thinking) {
data.prompt += "";
} else {
@@ -1496,19 +1540,21 @@ static common_chat_params common_chat_params_init_nemotron_v3(const common_chat_
}
data.preserved_tokens = {
- "",
- "",
"",
"",
};
+ if (supports_reasoning) {
+ data.preserved_tokens.insert(data.preserved_tokens.end(), {"", ""});
+ }
+
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = true;
auto parser = build_chat_peg_constructed_parser([&](auto & p) {
auto reasoning = p.eps();
- if (inputs.enable_thinking && extract_reasoning) {
+ if (supports_reasoning && inputs.enable_thinking && extract_reasoning) {
auto reasoning_content = p.reasoning(p.until("")) + ("" | p.end());
if (data.thinking_forced_open) {
reasoning = reasoning_content;
@@ -1846,38 +1892,6 @@ static common_chat_params common_chat_params_init_minimax_m2(const common_chat_t
return data;
}
-static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_chat_template & tmpl, const struct templates_params & params) {
- common_chat_params data;
- data.grammar_lazy = params.tools.is_array() && !params.tools.empty() && params.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-
- data.prompt = apply(tmpl, params);
- data.format = COMMON_CHAT_FORMAT_QWEN3_CODER_XML;
-
- data.preserved_tokens = {
- "",
- "",
- "",
- "",
- };
-
- // build grammar for tool call
- static const xml_tool_call_format form {
- /* form.scope_start = */ "\n",
- /* form.tool_start = */ "\n",
- /* form.key_start = */ "\n",
- /* form.val_end = */ "\n\n",
- /* form.tool_end = */ "\n",
- /* form.scope_end = */ "",
- };
- build_grammar_xml_tool_call(data, params.tools, form);
-
- return data;
-}
-
static common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const struct templates_params & params) {
common_chat_params data;
data.grammar_lazy = params.tools.is_array() && !params.tools.empty() && params.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
@@ -2001,6 +2015,7 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp
if (has_reasoning_content && has_tool_calls) {
auto adjusted_message = msg;
adjusted_message["thinking"] = msg.at("reasoning_content");
+ adjusted_message.erase("content");
adjusted_messages.push_back(adjusted_message);
} else {
adjusted_messages.push_back(msg);
@@ -3020,7 +3035,7 @@ static common_chat_params common_chat_templates_apply_jinja(
: *tmpls->template_default;
const auto & src = tmpl.source();
const auto & caps = tmpl.original_caps();
- params.messages = common_chat_msgs_to_json_oaicompat(inputs.messages, /* concat_text= */ !tmpl.original_caps().requires_typed_content);
+ params.messages = render_message_to_json(inputs.messages, tmpl.original_caps());
params.add_generation_prompt = inputs.add_generation_prompt;
params.tool_choice = inputs.tool_choice;
params.reasoning_format = inputs.reasoning_format;
@@ -3098,19 +3113,13 @@ static common_chat_params common_chat_templates_apply_jinja(
}
// Qwen3-Coder XML format detection (must come before Hermes 2 Pro)
- // Detect via explicit XML markers unique to Qwen3-Coder to avoid false positives in other templates.
- // Require presence of , , and blocks.
+ // Detect via XML markers: , , and blocks.
+ // Also matches Step-3.5-Flash and Nemotron 3 Nano which use the same output format.
if (src.find("") != std::string::npos &&
- src.find("") != std::string::npos &&
src.find("") != std::string::npos &&
src.find("") != std::string::npos) {
- return common_chat_params_init_nemotron_v3(tmpl, params);
- }
- return common_chat_params_init_qwen3_coder_xml(tmpl, params);
+ return common_chat_params_init_qwen3_coder(tmpl, params);
}
// Xiaomi MiMo format detection (must come before Hermes 2 Pro)
@@ -3276,7 +3285,7 @@ static common_chat_params common_chat_templates_apply_legacy(
for (const auto & msg : inputs.messages) {
auto content = msg.content;
for (const auto & part : msg.content_parts) {
- if (part.type != "text") {
+ if (part.type != "text" && part.type != "media_marker") {
LOG_WRN("Ignoring non-text content part: %s\n", part.type.c_str());
continue;
}
diff --git a/common/chat.h b/common/chat.h
index 24aa4aab5c..6f0b9409ec 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -128,7 +128,6 @@ enum common_chat_format {
COMMON_CHAT_FORMAT_GLM_4_5,
COMMON_CHAT_FORMAT_MINIMAX_M2,
COMMON_CHAT_FORMAT_KIMI_K2,
- COMMON_CHAT_FORMAT_QWEN3_CODER_XML,
COMMON_CHAT_FORMAT_APRIEL_1_5,
COMMON_CHAT_FORMAT_XIAOMI_MIMO,
COMMON_CHAT_FORMAT_SOLAR_OPEN,
@@ -240,6 +239,8 @@ bool common_chat_templates_support_enable_thinking(const common_chat_templates *
// Parses a JSON array of messages in OpenAI's chat completion API format.
std::vector common_chat_msgs_parse_oaicompat(const nlohmann::ordered_json & messages);
+
+// DEPRECATED: only used in tests
nlohmann::ordered_json common_chat_msgs_to_json_oaicompat(const std::vector & msgs, bool concat_typed_text = false);
std::vector common_chat_tools_parse_oaicompat(const nlohmann::ordered_json & tools);
diff --git a/common/common.cpp b/common/common.cpp
index 3aa396127c..53bddc4ef2 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1,7 +1,3 @@
-#if defined(_MSC_VER)
-#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
-#endif
-
#include "ggml.h"
#include "gguf.h"
@@ -9,12 +5,12 @@
#include "log.h"
#include "llama.h"
#include "sampling.h"
+#include "unicode.h"
#include
#include
#include
#include
-#include
#include
#include
#include
@@ -456,34 +452,6 @@ void string_replace_all(std::string & s, const std::string & search, const std::
s = std::move(builder);
}
-bool string_ends_with(const std::string_view & str, const std::string_view & suffix) {
- return str.size() >= suffix.size() && str.compare(str.size()-suffix.size(), suffix.size(), suffix) == 0;
-}
-
-bool string_remove_suffix(std::string & str, const std::string_view & suffix) {
- bool has_suffix = string_ends_with(str, suffix);
- if (has_suffix) {
- str = str.substr(0, str.size() - suffix.size());
- }
- return has_suffix;
-}
-
-size_t string_find_partial_stop(const std::string_view & str, const std::string_view & stop) {
- if (!str.empty() && !stop.empty()) {
- const char text_last_char = str.back();
- for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) {
- if (stop[char_index] == text_last_char) {
- const auto current_partial = stop.substr(0, char_index + 1);
- if (string_ends_with(str, current_partial)) {
- return str.size() - char_index - 1;
- }
- }
- }
- }
-
- return std::string::npos;
-}
-
std::string regex_escape(const std::string & s) {
static const std::regex special_chars("[.^$|()*+?\\[\\]{}\\\\]");
return std::regex_replace(s, special_chars, "\\$&");
@@ -706,45 +674,28 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
return false;
}
- std::u32string filename_utf32;
- try {
-#if defined(__clang__)
- // disable C++17 deprecation warning for std::codecvt_utf8
-# pragma clang diagnostic push
-# pragma clang diagnostic ignored "-Wdeprecated-declarations"
-#elif defined(__GNUC__)
-# pragma GCC diagnostic push
-# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#endif
+ size_t offset = 0;
+ while (offset < filename.size()) {
+ utf8_parse_result result = parse_utf8_codepoint(filename, offset);
- std::wstring_convert, char32_t> converter;
-
-#if defined(__clang__)
-# pragma clang diagnostic pop
-#elif defined(__GNUC__)
-# pragma GCC diagnostic pop
-#endif
-
- filename_utf32 = converter.from_bytes(filename);
-
- // If the reverse conversion mismatches, it means overlong UTF-8 sequences were used,
- // or invalid encodings were encountered. Reject such attempts
- std::string filename_reencoded = converter.to_bytes(filename_utf32);
- if (filename_reencoded != filename) {
+ if (result.status != utf8_parse_result::SUCCESS) {
return false;
}
- } catch (const std::exception &) {
- return false;
- }
+ uint32_t c = result.codepoint;
- // Check for forbidden codepoints:
- // - Control characters
- // - Unicode equivalents of illegal characters
- // - UTF-16 surrogate pairs
- // - UTF-8 replacement character
- // - Byte order mark (BOM)
- // - Illegal characters: / \ : * ? " < > |
- for (char32_t c : filename_utf32) {
+ if ((result.bytes_consumed == 2 && c < 0x80) ||
+ (result.bytes_consumed == 3 && c < 0x800) ||
+ (result.bytes_consumed == 4 && c < 0x10000)) {
+ return false;
+ }
+
+ // Check for forbidden codepoints:
+ // - Control characters
+ // - Unicode equivalents of illegal characters
+ // - UTF-16 surrogate pairs
+ // - UTF-8 replacement character
+ // - Byte order mark (BOM)
+ // - Illegal characters: / \ : * ? " < > |
if (c <= 0x1F // Control characters (C0)
|| c == 0x7F // Control characters (DEL)
|| (c >= 0x80 && c <= 0x9F) // Control characters (C1)
@@ -752,6 +703,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
|| c == 0x2215 // Division Slash (forward slash equivalent)
|| c == 0x2216 // Set Minus (backslash equivalent)
|| (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs
+ || c > 0x10FFFF // Max Unicode limit
|| c == 0xFFFD // Replacement Character (UTF-8)
|| c == 0xFEFF // Byte Order Mark (BOM)
|| c == ':' || c == '*' // Illegal characters
@@ -762,6 +714,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
// Subdirectories not allowed, reject path separators
return false;
}
+ offset += result.bytes_consumed;
}
// Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename
@@ -898,7 +851,8 @@ std::string fs_get_cache_directory() {
if (getenv("LLAMA_CACHE")) {
cache_directory = std::getenv("LLAMA_CACHE");
} else {
-#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || defined(__OpenBSD__)
+#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
+ defined(__OpenBSD__) || defined(__NetBSD__)
if (std::getenv("XDG_CACHE_HOME")) {
cache_directory = std::getenv("XDG_CACHE_HOME");
} else if (std::getenv("HOME")) {
@@ -1242,7 +1196,7 @@ common_init_result_ptr common_init_from_params(common_params & params) {
return res;
}
- int err = llama_apply_adapter_cvec(
+ int err = llama_set_adapter_cvec(
lctx,
cvec.data.data(),
cvec.data.size(),
@@ -1344,12 +1298,15 @@ std::string get_model_endpoint() {
}
void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora) {
- llama_clear_adapter_lora(ctx);
- for (auto & la : lora) {
- if (la.scale != 0.0f) {
- llama_set_adapter_lora(ctx, la.ptr, la.scale);
- }
+ std::vector loras;
+ std::vector scales;
+
+ for (auto & la: lora) {
+ loras.push_back(la.ptr);
+ scales.push_back(la.scale);
}
+
+ llama_set_adapters_lora(ctx, loras.data(), loras.size(), scales.data());
}
struct llama_model_params common_model_params_to_llama(common_params & params) {
@@ -1469,66 +1426,6 @@ void common_batch_add(
batch.n_tokens++;
}
-//
-// Token utils
-//
-
-size_t common_lcp(const llama_tokens & a, const llama_tokens & b) {
- size_t i;
- for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}
-
- return i;
-}
-
-size_t common_lcs(const llama_tokens & a, const llama_tokens & b) {
- // check for empty sequences
- if (a.empty() || b.empty()) {
- return 0;
- }
-
- // get the lengths of the input sequences
- size_t a_len = a.size();
- size_t b_len = b.size();
-
- // initialize the maximum length of the longest common subsequence (LCS)
- size_t max_length = 0;
-
- // use two rows instead of a 2D matrix to optimize space
- std::vector prev_row(b_len + 1, 0);
- std::vector curr_row(b_len + 1, 0);
-
- // iterate through the elements of a
- for (size_t i = 1; i <= a_len; i++) {
- // iterate through the elements of b
- for (size_t j = 1; j <= b_len; j++) {
- // if elements at the current positions match
- if (a[i - 1] == b[j - 1]) {
- // if it's the first element of either sequences, set LCS length to 1
- if (i == 1 || j == 1) {
- curr_row[j] = 1;
- } else {
- // increment LCS length by 1 compared to the previous element
- curr_row[j] = prev_row[j - 1] + 1;
- }
-
- // update max_length if necessary
- if (curr_row[j] > max_length) {
- max_length = curr_row[j];
- }
- } else {
- // reset LCS length if elements don't match
- curr_row[j] = 0;
- }
- }
-
- // update the previous row for the next iteration
- prev_row = curr_row;
- }
-
- // return the maximum length of the LCS
- return max_length;
-}
-
//
// Vocab utils
//
@@ -1863,3 +1760,65 @@ float lr_opt::get_lr(float epoch) const {
LOG_INF("epoch %.2g lr=%.2g\n", epoch, r);
return r;
}
+
+bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos) {
+ llama_batch batch = llama_batch_get_one(&last_token, 1);
+ batch.pos = &pos;
+ if (llama_decode(ctx, batch)) {
+ LOG_ERR("%s: failed to replay last token\n", __func__);
+ return false;
+ }
+ return true;
+}
+
+bool common_prompt_batch_decode(
+ struct llama_context * ctx,
+ const std::vector & tokens,
+ int & n_past,
+ int n_batch,
+ std::string_view state_path,
+ bool save_state) {
+ const int n_eval = tokens.size();
+ if (n_eval == 0) {
+ return true;
+ }
+
+ if (save_state && n_eval > 1) {
+ const int n_tokens_before_last = n_eval - 1;
+
+ GGML_ASSERT(n_eval <= n_batch);
+
+ // Decode all but the last token so we can save the memory state before decoding the last token.
+ // This is done so we can restore the session state later and replay the last token.
+ // Memory implementations in recurrent/hybrid models don't support removing tokens from their
+ // memory, so we can't just remove the last token from the memory and replay the last token which
+ // is the reason for this logic.
+ if (llama_decode(ctx, llama_batch_get_one(const_cast(tokens.data()), n_tokens_before_last))) {
+ LOG_ERR("%s : failed to eval\n", __func__);
+ return false;
+ }
+ n_past += n_tokens_before_last;
+
+ llama_state_save_file(ctx, state_path.data(), tokens.data(), n_tokens_before_last);
+ LOG_INF("saved session before last token to %s, n_tokens = %d\n", state_path.data(), n_tokens_before_last);
+
+ llama_token last_token = tokens.back();
+ llama_batch batch = llama_batch_get_one(&last_token, 1);
+ int32_t pos = n_past;
+ batch.pos = &pos;
+
+ if (llama_decode(ctx, batch)) {
+ LOG_ERR("%s : failed to eval last token\n", __func__);
+ return false;
+ }
+ n_past++;
+ } else {
+ if (llama_decode(ctx, llama_batch_get_one(const_cast(tokens.data()), n_eval))) {
+ LOG_ERR("%s : failed to eval\n", __func__);
+ return false;
+ }
+ n_past += n_eval;
+ }
+
+ return true;
+}
diff --git a/common/common.h b/common/common.h
index 612a0a062e..cc35c70609 100644
--- a/common/common.h
+++ b/common/common.h
@@ -673,30 +673,55 @@ static std::vector string_split(const std::string & str, char delim) {
}
template<>
-std::vector string_split(const std::string & input, char separator)
+inline std::vector string_split(const std::string & str, char delim)
{
std::vector parts;
size_t begin_pos = 0;
- size_t separator_pos = input.find(separator);
- while (separator_pos != std::string::npos) {
- std::string part = input.substr(begin_pos, separator_pos - begin_pos);
+ size_t delim_pos = str.find(delim);
+ while (delim_pos != std::string::npos) {
+ std::string part = str.substr(begin_pos, delim_pos - begin_pos);
parts.emplace_back(part);
- begin_pos = separator_pos + 1;
- separator_pos = input.find(separator, begin_pos);
+ begin_pos = delim_pos + 1;
+ delim_pos = str.find(delim, begin_pos);
}
- parts.emplace_back(input.substr(begin_pos, separator_pos - begin_pos));
+ parts.emplace_back(str.substr(begin_pos));
return parts;
}
-static bool string_starts_with(const std::string & str,
- const std::string & prefix) { // While we wait for C++20's std::string::starts_with...
- return str.rfind(prefix, 0) == 0;
+// remove when moving to c++20
+inline bool string_starts_with(std::string_view str, std::string_view prefix) {
+ return str.size() >= prefix.size() &&
+ str.compare(0, prefix.size(), prefix) == 0;
}
-// While we wait for C++20's std::string::ends_with...
-bool string_ends_with(const std::string_view & str, const std::string_view & suffix);
-bool string_remove_suffix(std::string & str, const std::string_view & suffix);
-size_t string_find_partial_stop(const std::string_view & str, const std::string_view & stop);
+// remove when moving to c++20
+inline bool string_ends_with(std::string_view str, std::string_view suffix) {
+ return str.size() >= suffix.size() &&
+ str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+inline bool string_remove_suffix(std::string & str, std::string_view suffix) {
+ if (string_ends_with(str, suffix)) {
+ str.resize(str.size() - suffix.size());
+ return true;
+ }
+ return false;
+}
+
+inline size_t string_find_partial_stop(std::string_view str, std::string_view stop) {
+ if (!str.empty() && !stop.empty()) {
+ const size_t max_len = std::min(str.size(), stop.size());
+ const char last_char = str.back();
+ for (size_t len = max_len; len > 0; --len) {
+ if (stop[len - 1] == last_char) {
+ if (string_ends_with(str, stop.substr(0, len))) {
+ return str.size() - len;
+ }
+ }
+ }
+ }
+ return std::string::npos;
+}
bool string_parse_kv_override(const char * data, std::vector & overrides);
void string_process_escapes(std::string & input);
@@ -782,15 +807,22 @@ void common_batch_add(
const std::vector & seq_ids,
bool logits);
+// decodes a single batch of tokens for a prompt and manages session tokens
//
-// Token utils
-//
+// Note: We save state before the last token so that we can replay it to ensure
+// compatibility with all memory types. Recurrent/hybrid models cannot remove
+// tokens from memory, so this approach works across all model architectures.
+bool common_prompt_batch_decode(
+ struct llama_context * ctx,
+ const std::vector & embd,
+ int & n_past,
+ int n_batch,
+ std::string_view state_path,
+ bool save_state);
-// longest common prefix
-size_t common_lcp(const llama_tokens & a, const llama_tokens & b);
-
-// longet common subsequence
-size_t common_lcs(const llama_tokens & a, const llama_tokens & b);
+// replays the last token after loading state to regenerate logits
+// used after loading session state to ensure the sampling context has valid logits
+bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos);
//
// Vocab utils
@@ -883,11 +915,11 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate)_(ch|)exps";
-static std::string llm_ffn_exps_block_regex(int idx) {
+inline std::string llm_ffn_exps_block_regex(int idx) {
return string_format("blk\\.%d%s", idx, LLM_FFN_EXPS_REGEX);
}
-static llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {
+inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {
return { LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() };
}
diff --git a/common/download.cpp b/common/download.cpp
index 57f29a23ba..5ef60a4208 100644
--- a/common/download.cpp
+++ b/common/download.cpp
@@ -19,9 +19,7 @@
#include
#include
-#if defined(LLAMA_USE_HTTPLIB)
#include "http.h"
-#endif
#ifndef __EMSCRIPTEN__
#ifdef __linux__
@@ -114,44 +112,18 @@ static void write_etag(const std::string & path, const std::string & etag) {
}
static std::string read_etag(const std::string & path) {
- std::string none;
const std::string etag_path = path + ".etag";
-
- if (std::filesystem::exists(etag_path)) {
- std::ifstream etag_in(etag_path);
- if (!etag_in) {
- LOG_ERR("%s: could not open .etag file for reading: %s\n", __func__, etag_path.c_str());
- return none;
- }
- std::string etag;
- std::getline(etag_in, etag);
- return etag;
+ if (!std::filesystem::exists(etag_path)) {
+ return {};
}
-
- // no etag file, but maybe there is an old .json
- // remove this code later
- const std::string metadata_path = path + ".json";
-
- if (std::filesystem::exists(metadata_path)) {
- std::ifstream metadata_in(metadata_path);
- try {
- nlohmann::json metadata_json;
- metadata_in >> metadata_json;
- LOG_DBG("%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(),
- metadata_json.dump().c_str());
- if (metadata_json.contains("etag") && metadata_json.at("etag").is_string()) {
- std::string etag = metadata_json.at("etag");
- write_etag(path, etag);
- if (!std::filesystem::remove(metadata_path)) {
- LOG_WRN("%s: failed to delete old .json metadata file: %s\n", __func__, metadata_path.c_str());
- }
- return etag;
- }
- } catch (const nlohmann::json::exception & e) {
- LOG_ERR("%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());
- }
+ std::ifstream etag_in(etag_path);
+ if (!etag_in) {
+ LOG_ERR("%s: could not open .etag file for reading: %s\n", __func__, etag_path.c_str());
+ return {};
}
- return none;
+ std::string etag;
+ std::getline(etag_in, etag);
+ return etag;
}
static bool is_http_status_ok(int status) {
@@ -168,8 +140,6 @@ std::pair common_download_split_repo_tag(const std::st
return {hf_repo, tag};
}
-#if defined(LLAMA_USE_HTTPLIB)
-
class ProgressBar {
static inline std::mutex mutex;
static inline std::map lines;
@@ -305,7 +275,10 @@ static bool common_pull_file(httplib::Client & cli,
);
if (!res) {
- LOG_ERR("%s: error during download. Status: %d\n", __func__, res ? res->status : -1);
+ LOG_ERR("%s: download failed: %s (status: %d)\n",
+ __func__,
+ httplib::to_string(res.error()).c_str(),
+ res ? res->status : -1);
return false;
}
@@ -344,62 +317,64 @@ static int common_download_file_single_online(const std::string & url,
LOG_INF("%s: no previous model file found %s\n", __func__, path.c_str());
}
- for (int i = 0; i < max_attempts; ++i) {
- auto head = cli.Head(parts.path);
- bool head_ok = head && head->status >= 200 && head->status < 300;
- if (!head_ok) {
- LOG_WRN("%s: HEAD invalid http status code received: %d\n", __func__, head ? head->status : -1);
- if (file_exists) {
- LOG_INF("%s: Using cached file (HEAD failed): %s\n", __func__, path.c_str());
- return 304; // 304 Not Modified - fake cached response
- }
- return head->status; // cannot use cached file, return raw status code
- // TODO: maybe retry only on certain codes
- }
-
- std::string etag;
- if (head_ok && head->has_header("ETag")) {
- etag = head->get_header_value("ETag");
- }
-
- size_t total_size = 0;
- if (head_ok && head->has_header("Content-Length")) {
- try {
- total_size = std::stoull(head->get_header_value("Content-Length"));
- } catch (const std::exception& e) {
- LOG_WRN("%s: Invalid Content-Length in HEAD response: %s\n", __func__, e.what());
- }
- }
-
- bool supports_ranges = false;
- if (head_ok && head->has_header("Accept-Ranges")) {
- supports_ranges = head->get_header_value("Accept-Ranges") != "none";
- }
-
- bool should_download_from_scratch = false;
- if (!last_etag.empty() && !etag.empty() && last_etag != etag) {
- LOG_WRN("%s: ETag header is different (%s != %s): triggering a new download\n", __func__,
- last_etag.c_str(), etag.c_str());
- should_download_from_scratch = true;
- }
-
+ auto head = cli.Head(parts.path);
+ if (!head || head->status < 200 || head->status >= 300) {
+ LOG_WRN("%s: HEAD failed, status: %d\n", __func__, head ? head->status : -1);
if (file_exists) {
- if (!should_download_from_scratch) {
- LOG_INF("%s: using cached file: %s\n", __func__, path.c_str());
- return 304; // 304 Not Modified - fake cached response
- }
- LOG_WRN("%s: deleting previous downloaded file: %s\n", __func__, path.c_str());
- if (remove(path.c_str()) != 0) {
- LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());
- return -1;
- }
+ LOG_INF("%s: using cached file (HEAD failed): %s\n", __func__, path.c_str());
+ return 304; // 304 Not Modified - fake cached response
+ }
+ return head ? head->status : -1;
+ }
+
+ std::string etag;
+ if (head->has_header("ETag")) {
+ etag = head->get_header_value("ETag");
+ }
+
+ size_t total_size = 0;
+ if (head->has_header("Content-Length")) {
+ try {
+ total_size = std::stoull(head->get_header_value("Content-Length"));
+ } catch (const std::exception& e) {
+ LOG_WRN("%s: invalid Content-Length in HEAD response: %s\n", __func__, e.what());
+ }
+ }
+
+ bool supports_ranges = false;
+ if (head->has_header("Accept-Ranges")) {
+ supports_ranges = head->get_header_value("Accept-Ranges") != "none";
+ }
+
+ if (file_exists) {
+ if (etag.empty()) {
+ LOG_INF("%s: using cached file (no server etag): %s\n", __func__, path.c_str());
+ return 304; // 304 Not Modified - fake cached response
+ }
+ if (!last_etag.empty() && last_etag == etag) {
+ LOG_INF("%s: using cached file (same etag): %s\n", __func__, path.c_str());
+ return 304; // 304 Not Modified - fake cached response
+ }
+ if (remove(path.c_str()) != 0) {
+ LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());
+ return -1;
+ }
+ }
+
+ const std::string path_temporary = path + ".downloadInProgress";
+ int delay = retry_delay_seconds;
+
+ for (int i = 0; i < max_attempts; ++i) {
+ if (i) {
+ LOG_WRN("%s: retrying after %d seconds...\n", __func__, delay);
+ std::this_thread::sleep_for(std::chrono::seconds(delay));
+ delay *= retry_delay_seconds;
}
- const std::string path_temporary = path + ".downloadInProgress";
size_t existing_size = 0;
if (std::filesystem::exists(path_temporary)) {
- if (supports_ranges && !should_download_from_scratch) {
+ if (supports_ranges) {
existing_size = std::filesystem::file_size(path_temporary);
} else if (remove(path_temporary.c_str()) != 0) {
LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str());
@@ -407,32 +382,23 @@ static int common_download_file_single_online(const std::string & url,
}
}
- // start the download
- LOG_INF("%s: trying to download model from %s to %s (etag:%s)...\n",
- __func__, common_http_show_masked_url(parts).c_str(), path_temporary.c_str(), etag.c_str());
- const bool was_pull_successful = common_pull_file(cli, parts.path, path_temporary, supports_ranges, existing_size, total_size);
- if (!was_pull_successful) {
- if (i + 1 < max_attempts) {
- const int exponential_backoff_delay = std::pow(retry_delay_seconds, i) * 1000;
- LOG_WRN("%s: retrying after %d milliseconds...\n", __func__, exponential_backoff_delay);
- std::this_thread::sleep_for(std::chrono::milliseconds(exponential_backoff_delay));
- } else {
- LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
+ LOG_INF("%s: downloading from %s to %s (etag:%s)...\n",
+ __func__, common_http_show_masked_url(parts).c_str(),
+ path_temporary.c_str(), etag.c_str());
+
+ if (common_pull_file(cli, parts.path, path_temporary, supports_ranges, existing_size, total_size)) {
+ if (std::rename(path_temporary.c_str(), path.c_str()) != 0) {
+ LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());
+ return -1;
}
- continue;
+ if (!etag.empty()) {
+ write_etag(path, etag);
+ }
+ return head->status;
}
-
- if (std::rename(path_temporary.c_str(), path.c_str()) != 0) {
- LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());
- return -1;
- }
- if (!etag.empty()) {
- write_etag(path, etag);
- }
-
- return head->status; // TODO: use actual GET status?
}
+ LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
return -1; // max attempts reached
}
@@ -798,30 +764,6 @@ std::string common_docker_resolve_model(const std::string & docker) {
}
}
-#else
-
-common_hf_file_res common_get_hf_file(const std::string &, const std::string &, bool, const common_header_list &) {
- throw std::runtime_error("download functionality is not enabled in this build");
-}
-
-bool common_download_model(const common_params_model &, const std::string &, bool, const common_header_list &) {
- throw std::runtime_error("download functionality is not enabled in this build");
-}
-
-std::string common_docker_resolve_model(const std::string &) {
- throw std::runtime_error("download functionality is not enabled in this build");
-}
-
-int common_download_file_single(const std::string &,
- const std::string &,
- const std::string &,
- bool,
- const common_header_list &) {
- throw std::runtime_error("download functionality is not enabled in this build");
-}
-
-#endif // defined(LLAMA_USE_HTTPLIB)
-
std::vector common_list_cached_models() {
std::vector models;
const std::string cache_dir = fs_get_cache_directory();
diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp
index f27490f1fb..dbaaed500a 100644
--- a/common/jinja/caps.cpp
+++ b/common/jinja/caps.cpp
@@ -63,7 +63,8 @@ static void caps_print_stats(value & v, const std::string & path) {
std::map caps::to_map() const {
return {
- {"requires_typed_content", requires_typed_content},
+ {"supports_string_content", supports_string_content},
+ {"supports_typed_content", supports_typed_content},
{"supports_tools", supports_tools},
{"supports_tool_calls", supports_tool_calls},
{"supports_parallel_tool_calls", supports_parallel_tool_calls},
@@ -89,7 +90,7 @@ caps caps_get(jinja::program & prog) {
return v->stats.ops.find(op_name) != v->stats.ops.end();
};
- // case: typed content requirement
+ // case: typed content support
caps_try_execute(
prog,
[&]() {
@@ -105,12 +106,16 @@ caps caps_get(jinja::program & prog) {
// tools
return json{nullptr};
},
- [&](bool, value & messages, value &) {
+ [&](bool success, value & messages, value &) {
auto & content = messages->at(0)->at("content");
caps_print_stats(content, "messages[0].content");
if (has_op(content, "selectattr") || has_op(content, "array_access")) {
// accessed as an array
- result.requires_typed_content = true;
+ result.supports_typed_content = true;
+ }
+ if (!success) {
+ // failed to execute with content as string
+ result.supports_string_content = false;
}
}
);
diff --git a/common/jinja/caps.h b/common/jinja/caps.h
index 77df117baa..e694e7bfaa 100644
--- a/common/jinja/caps.h
+++ b/common/jinja/caps.h
@@ -14,7 +14,9 @@ struct caps {
bool supports_parallel_tool_calls = true;
bool supports_preserve_reasoning = false; // support assistant message with reasoning_content
- bool requires_typed_content = false; // default: use string content
+ // one of the 2 content capabilities must be true
+ bool supports_string_content = true;
+ bool supports_typed_content = false;
// for reporting on server
std::map to_map() const;
diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp
index 4453d86e6d..c93e182a7e 100644
--- a/common/jinja/runtime.cpp
+++ b/common/jinja/runtime.cpp
@@ -85,7 +85,7 @@ value identifier::execute_impl(context & ctx) {
auto builtins = global_builtins();
if (!it->is_undefined()) {
if (ctx.is_get_stats) {
- it->stats.used = true;
+ value_t::stats_t::mark_used(it);
}
JJ_DEBUG("Identifier '%s' found, type = %s", val.c_str(), it->type().c_str());
return it;
@@ -277,7 +277,7 @@ value binary_expression::execute_impl(context & ctx) {
static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
if (ctx.is_get_stats) {
- input->stats.used = true;
+ value_t::stats_t::mark_used(input);
input->stats.ops.insert(name);
}
auto builtins = input->get_builtins();
@@ -446,6 +446,12 @@ value for_statement::execute_impl(context & ctx) {
value iterable_val = iter_expr->execute(scope);
+ // mark the variable being iterated as used for stats
+ if (ctx.is_get_stats) {
+ value_t::stats_t::mark_used(iterable_val);
+ iterable_val->stats.ops.insert("array_access");
+ }
+
if (iterable_val->is_undefined()) {
JJ_DEBUG("%s", "For loop iterable is undefined, skipping loop");
iterable_val = mk_val();
@@ -464,7 +470,7 @@ value for_statement::execute_impl(context & ctx) {
items.push_back(std::move(tuple));
}
if (ctx.is_get_stats) {
- iterable_val->stats.used = true;
+ value_t::stats_t::mark_used(iterable_val);
iterable_val->stats.ops.insert("object_access");
}
} else {
@@ -474,7 +480,7 @@ value for_statement::execute_impl(context & ctx) {
items.push_back(item);
}
if (ctx.is_get_stats) {
- iterable_val->stats.used = true;
+ value_t::stats_t::mark_used(iterable_val);
iterable_val->stats.ops.insert("array_access");
}
}
@@ -811,8 +817,9 @@ value member_expression::execute_impl(context & ctx) {
}
if (ctx.is_get_stats && val && object && property) {
- val->stats.used = true;
- object->stats.used = true;
+ value_t::stats_t::mark_used(val);
+ value_t::stats_t::mark_used(object);
+ value_t::stats_t::mark_used(property);
if (is_val(property)) {
object->stats.ops.insert("array_access");
} else if (is_val(property)) {
diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp
index 2aa156b177..749113124b 100644
--- a/common/jinja/value.cpp
+++ b/common/jinja/value.cpp
@@ -4,6 +4,7 @@
// for converting from JSON to jinja values
#include
+#include
#include
#include
#include
@@ -160,6 +161,11 @@ static value tojson(const func_args & args) {
value val_separators = args.get_kwarg_or_pos("separators", 3);
value val_sort = args.get_kwarg_or_pos("sort_keys", 4);
int indent = -1;
+ if (args.ctx.is_get_stats) {
+ // mark as used (recursively) for stats
+ auto val_input = args.get_pos(0);
+ value_t::stats_t::mark_used(const_cast(val_input), true);
+ }
if (is_val(val_indent)) {
indent = static_cast(val_indent->as_int());
}
@@ -715,8 +721,46 @@ const func_builtins & value_string_t::get_builtins() const {
return args.get_pos(0);
}},
{"tojson", tojson},
- {"indent", [](const func_args &) -> value {
- throw not_implemented_exception("String indent builtin not implemented");
+ {"indent", [](const func_args &args) -> value {
+ args.ensure_count(1, 4);
+ value val_input = args.get_pos(0);
+ value val_width = args.get_kwarg_or_pos("width", 1);
+ const bool first = args.get_kwarg_or_pos("first", 2)->as_bool(); // undefined == false
+ const bool blank = args.get_kwarg_or_pos("blank", 3)->as_bool(); // undefined == false
+ if (!is_val(val_input)) {
+ throw raised_exception("indent() first argument must be a string");
+ }
+ std::string indent;
+ if (is_val(val_width)) {
+ indent.assign(val_width->as_int(), ' ');
+ } else if (is_val(val_width)) {
+ indent = val_width->as_string().str();
+ } else {
+ indent = " ";
+ }
+ std::string indented;
+ std::string input = val_input->as_string().str();
+ std::istringstream iss = std::istringstream(input);
+ std::string line;
+ while (std::getline(iss, line)) {
+ if (!indented.empty()) {
+ indented.push_back('\n');
+ }
+ if ((indented.empty() ? first : (!line.empty() || blank))) {
+ indented += indent;
+ }
+ indented += line;
+ }
+ if (!input.empty() && input.back() == '\n') {
+ indented.push_back('\n');
+ if (blank) {
+ indented += indent;
+ }
+ }
+
+ auto res = mk_val(indented);
+ res->val_str.mark_input_based_on(val_input->as_string());
+ return res;
}},
{"join", [](const func_args &) -> value {
throw not_implemented_exception("String join builtin not implemented");
@@ -852,6 +896,11 @@ const func_builtins & value_array_t::get_builtins() const {
}},
{"string", [](const func_args & args) -> value {
args.ensure_vals();
+ if (args.ctx.is_get_stats) {
+ // mark as used (recursively) for stats
+ auto val_input = args.get_pos(0);
+ value_t::stats_t::mark_used(const_cast(val_input), true);
+ }
return mk_val(args.get_pos(0)->as_string());
}},
{"tojson", tojson},
@@ -1007,6 +1056,11 @@ const func_builtins & value_object_t::get_builtins() const {
{"tojson", tojson},
{"string", [](const func_args & args) -> value {
args.ensure_vals();
+ if (args.ctx.is_get_stats) {
+ // mark as used (recursively) for stats
+ auto val_input = args.get_pos(0);
+ value_t::stats_t::mark_used(const_cast(val_input), true);
+ }
return mk_val(args.get_pos(0)->as_string());
}},
{"length", [](const func_args & args) -> value {
@@ -1319,4 +1373,21 @@ std::string value_to_string_repr(const value & val) {
}
}
+// stats utility
+void value_t::stats_t::mark_used(value & val, bool deep) {
+ val->stats.used = true;
+ if (deep) {
+ if (is_val(val)) {
+ for (auto & item : val->val_arr) {
+ mark_used(item, deep);
+ }
+ } else if (is_val(val)) {
+ for (auto & pair : val->val_obj) {
+ mark_used(pair.first, deep);
+ mark_used(pair.second, deep);
+ }
+ }
+ }
+}
+
} // namespace jinja
diff --git a/common/jinja/value.h b/common/jinja/value.h
index 1c04760a08..07e447ff69 100644
--- a/common/jinja/value.h
+++ b/common/jinja/value.h
@@ -118,6 +118,8 @@ struct value_t {
bool used = false;
// ops can be builtin calls or operators: "array_access", "object_access"
std::set ops;
+ // utility to recursively mark value and its children as used
+ static void mark_used(value & val, bool deep = false);
} stats;
value_t() = default;
diff --git a/common/ngram-map.cpp b/common/ngram-map.cpp
index 2b876a6e99..ebf771a24a 100644
--- a/common/ngram-map.cpp
+++ b/common/ngram-map.cpp
@@ -461,7 +461,7 @@ void common_ngram_map_draft(common_ngram_map & map,
slot_max = v;
}
}
- // What is sum of the other occurences?
+ // What is sum of the other occurrences?
uint32_t sum_occur = 0;
for (int v = 0; v < COMMON_NGRAM_MAX_VALUES; ++v) {
if (v == slot_max) {
diff --git a/common/ngram-map.h b/common/ngram-map.h
index 41b9530449..d84e719151 100644
--- a/common/ngram-map.h
+++ b/common/ngram-map.h
@@ -44,7 +44,7 @@ llama_tokens common_ngram_simple_draft(
// statistics of a m-gram after a known n-gram
struct common_ngram_map_value {
size_t value_idx = 0; // index of value m-gram in token-history (0 if unused)
- uint16_t value_num = 0; // number of occurences of this value m-gram after the key n-gram (0 in an unused values-slot)
+ uint16_t value_num = 0; // number of occurrences of this value m-gram after the key n-gram (0 in an unused values-slot)
int16_t n_accepted = -1; // number of accepted tokens at last draft (-1 if unused)
};
@@ -53,7 +53,7 @@ struct common_ngram_map_key {
size_t key_idx; // index of key n-gram in token-history
size_t stat_idx; // index of last token of stastistics computation (key_num, values)
- uint16_t key_num; // number of occurences of this key n-gram in token-history
+ uint16_t key_num; // number of occurrences of this key n-gram in token-history
common_ngram_map_value values[COMMON_NGRAM_MAX_VALUES]; // some known values after the key
};
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 71346c8b2e..aa2639c423 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -162,8 +162,6 @@ class ModelBase:
self.ftype = gguf.LlamaFileType.MOSTLY_F16
logger.info("heuristics unable to detect tensor dtype, defaulting to --outtype f16")
- self.dequant_model()
-
# Configure GGUF Writer
self.gguf_writer = gguf.GGUFWriter(path=None, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file,
split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard)
@@ -529,6 +527,8 @@ class ModelBase:
return ()
def prepare_tensors(self):
+ self.dequant_model()
+
# Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
if self.tensor_map.mapping:
max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
@@ -572,6 +572,7 @@ class ModelBase:
self.match_model_tensor_name(new_name, key, bid)
for key in (
gguf.MODEL_TENSOR.FFN_GATE_INP,
+ gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
gguf.MODEL_TENSOR.POS_EMBD,
gguf.MODEL_TENSOR.TOKEN_TYPES,
gguf.MODEL_TENSOR.SSM_CONV1D,
@@ -1050,6 +1051,9 @@ class TextModel(ModelBase):
if chkhsh == "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902":
# ref: https://huggingface.co/zai-org/GLM-4.5-Air
res = "glm4"
+ if chkhsh == "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267":
+ # ref: https://huggingface.co/zai-org/GLM-4.7-Flash
+ res = "glm4"
if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
# ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
res = "minerva-7b"
@@ -1083,9 +1087,6 @@ class TextModel(ModelBase):
if chkhsh == "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df":
# ref: https://huggingface.co/aari1995/German_Semantic_V3
res = "jina-v2-de"
- if chkhsh == "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267":
- # ref: https://huggingface.co/zai-org/GLM-4.7-Flash
- res = "glm4"
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
res = "llama-bpe"
@@ -1125,6 +1126,9 @@ class TextModel(ModelBase):
if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8":
# ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01
res = "command-r"
+ if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1":
+ # ref: https://huggingface.co/CohereLabs/tiny-aya-base
+ res = "tiny_aya"
if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
# ref: https://huggingface.co/Qwen/Qwen1.5-7B
res = "qwen2"
@@ -1161,6 +1165,9 @@ class TextModel(ModelBase):
if chkhsh == "b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901":
# ref: https://huggingface.co/core42/jais-13b
res = "jais"
+ if chkhsh == "bc5108ee1eb6a3d600cadd065f63190fbd0554dbc9e4bbd6a0d977970afc8d2a":
+ # ref: https://huggingface.co/inceptionai/Jais-2-8B-Chat
+ res = "jais-2"
if chkhsh == "7b3e7548e4308f52a76e8229e4e6cc831195d0d1df43aed21ac6c93da05fec5f":
# ref: https://huggingface.co/WisdomShell/CodeShell-7B
res = "codeshell"
@@ -1263,6 +1270,15 @@ class TextModel(ModelBase):
if chkhsh == "6c81ce329e0802883b22eabab0d3fa48357337ef1ecb45443828bf1f6254833f":
# ref: https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B
res = "exaone-moe"
+ if chkhsh == "d30d75d9059f1aa2c19359de71047b3ae408c70875e8a3ccf8c5fba56c9d8af4":
+ # ref: https://huggingface.co/Qwen/Qwen3.5-9B-Instruct
+ res = "qwen35"
+ if chkhsh == "b4b8ca1f9769494fbd956ebc4c249de6131fb277a4a3345a7a92c7dd7a55808d":
+ # ref: https://huggingface.co/jdopensource/JoyAI-LLM-Flash
+ res = "joyai-llm"
+ if chkhsh == "e4d54df1ebc1f2b91acd986c5b51aa50837d5faf7c7398e73c1f9e9ee5d19869":
+ # ref: https://huggingface.co/kakaocorp/kanana-2-30b-a3b-instruct-2601
+ res = "kanana2"
if res is None:
logger.warning("\n")
@@ -1607,6 +1623,23 @@ class TextModel(ModelBase):
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
special_vocab.add_to_gguf(self.gguf_writer)
+ def _set_vocab_glm(self):
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ # Special tokens
+ # Note: Using <|endoftext|> (151329) for eot causes endless generation
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
+ special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
+ special_vocab.add_to_gguf(self.gguf_writer)
+
def _set_vocab_interns1(self):
tokens: list[str] = []
toktypes: list[int] = []
@@ -1814,7 +1847,7 @@ class MmprojModel(ModelBase):
preprocessor_config: dict[str, Any]
global_config: dict[str, Any]
- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]
+ n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers", "vt_num_hidden_layers"]
has_vision_encoder: bool = True # by default
has_audio_encoder: bool = False
@@ -1869,7 +1902,15 @@ class MmprojModel(ModelBase):
preprocessor_config_path = self.dir_model / "preprocessor_config.json"
if preprocessor_config_path.is_file():
with open(preprocessor_config_path, "r", encoding="utf-8") as f:
- self.preprocessor_config = json.load(f)
+ cfg = json.load(f)
+ # move media_proc_cfg to root level for compat
+ if "media_proc_cfg" in cfg:
+ cfg = {
+ **cfg,
+ **cfg["media_proc_cfg"],
+ }
+ # merge configs
+ self.preprocessor_config = {**self.preprocessor_config, **cfg}
# prefer processor_config.json if possible
processor_config_path = self.dir_model / "processor_config.json"
@@ -1918,10 +1959,10 @@ class MmprojModel(ModelBase):
self.image_size = self.find_vparam(["image_size"])
self.gguf_writer.add_vision_image_size(self.image_size)
self.gguf_writer.add_vision_patch_size(self.find_vparam(["patch_size"]))
- self.gguf_writer.add_vision_embedding_length(self.find_vparam(["hidden_size"]))
- self.gguf_writer.add_vision_feed_forward_length(self.find_vparam(["intermediate_size"]))
+ self.gguf_writer.add_vision_embedding_length(self.find_vparam(["hidden_size", "vt_hidden_size"]))
+ self.gguf_writer.add_vision_feed_forward_length(self.find_vparam(["intermediate_size", "vt_intermediate_size"]))
self.gguf_writer.add_vision_block_count(self.find_vparam(self.n_block_keys))
- self.gguf_writer.add_vision_head_count(self.find_vparam(["num_attention_heads", "num_heads"]))
+ self.gguf_writer.add_vision_head_count(self.find_vparam(["num_attention_heads", "num_heads", "vt_num_attention_heads"]))
# preprocessor config
image_mean = _MISTRAL_COMMON_DATASET_MEAN if self.is_mistral_format else self.preprocessor_config["image_mean"]
@@ -2836,8 +2877,6 @@ class AfmoeModel(LlamaModel):
super().set_gguf_parameters()
# MoE parameters
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
if (n_shared_experts := self.hparams.get("num_shared_experts")) is not None:
self.gguf_writer.add_expert_shared_count(n_shared_experts)
if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
@@ -2859,7 +2898,7 @@ class AfmoeModel(LlamaModel):
# Handle expert weights - they're already merged in the HF format
# process the experts separately
if name.find("mlp.experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -3836,6 +3875,13 @@ class Ernie4_5Model(TextModel):
def set_vocab(self):
self._set_vocab_sentencepiece()
+ tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
+ if tokenizer_config_file.is_file():
+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:
+ tokenizer_config_json = json.load(f)
+ if "add_prefix_space" in tokenizer_config_json:
+ self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
+
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -3845,6 +3891,10 @@ class Ernie4_5Model(TextModel):
if (head_dim := self.hparams.get("head_dim")) is None:
head_dim = self.hparams["hidden_size"] // num_heads
+ if "mlp_AR" in name or "vision_model" in name:
+ # skip vision model and projector tensors
+ return
+
if "ernie." in name:
name = name.replace("ernie.", "model.")
# split the qkv weights
@@ -3954,6 +4004,48 @@ class Ernie4_5MoeModel(Ernie4_5Model):
raise ValueError(f"Unprocessed experts: {experts}")
+@ModelBase.register("PaddleOCRVLForConditionalGeneration")
+class PaddleOCRModel(Ernie4_5Model):
+ model_arch = gguf.MODEL_ARCH.PADDLEOCR
+
+
+@ModelBase.register("PaddleOCRVisionModel")
+class PaddleOCRVisionModel(MmprojModel):
+ # PaddleOCR-VL uses a modified version of Siglip
+ min_pixels: int = 0
+ max_pixels: int = 0
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ self.min_pixels = self.preprocessor_config["min_pixels"]
+ self.max_pixels = self.preprocessor_config["max_pixels"]
+ self.hparams_vision["image_size"] = int(math.sqrt(self.max_pixels))
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+ hparams = self.hparams_vision
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.PADDLEOCR)
+ self.gguf_writer.add_vision_max_pixels(self.max_pixels)
+ self.gguf_writer.add_vision_min_pixels(self.min_pixels)
+ self.gguf_writer.add_vision_use_gelu(True)
+ self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("rms_norm_eps", 1e-6))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ name = name.replace("visual.", "model.")
+
+ if "vision_model" in name or "mlp_AR" in name:
+ if "packing_position_embedding" in name:
+ return # unused
+ elif "vision_model.head" in name:
+ # we don't yet support image embeddings for this model
+ return
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+ return # skip other tensors
+
+
@ModelBase.register(
"Qwen2VLModel",
"Qwen2VLForConditionalGeneration",
@@ -4184,6 +4276,87 @@ class InternVisionModel(MmprojModel):
yield from super().modify_tensors(data_torch, name, bid)
+@ModelBase.register(
+ "NemotronH_Nano_VL_V2",
+ "RADIOModel",
+)
+class NemotronNanoV2VLModel(MmprojModel):
+ # ViT-Huge architecture parameters for RADIO v2.5-h
+ _vit_hidden_size = 1280
+ _vit_intermediate_size = 5120
+ _vit_num_layers = 32
+ _vit_num_heads = 16
+
+ def get_vision_config(self) -> dict[str, Any] | None:
+ # RADIO config doesn't have standard ViT parameters, so they need to be constructed manually
+ vision_config = self.global_config.get("vision_config")
+ if vision_config is None:
+ return None
+ # Add ViT-H parameters
+ vision_config = {
+ **vision_config,
+ "hidden_size": self._vit_hidden_size,
+ "intermediate_size": self._vit_intermediate_size,
+ "num_hidden_layers": self._vit_num_layers,
+ "num_attention_heads": self._vit_num_heads,
+ "image_size": self.global_config.get("force_image_size", 512),
+ }
+ return vision_config
+
+ def set_gguf_parameters(self):
+ if "image_mean" not in self.preprocessor_config:
+ self.preprocessor_config["image_mean"] = [0.485, 0.456, 0.406]
+ if "image_std" not in self.preprocessor_config:
+ self.preprocessor_config["image_std"] = [0.229, 0.224, 0.225]
+
+ super().set_gguf_parameters()
+ hparams = self.global_config
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_V2_VL)
+ self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
+ self.gguf_writer.add_vision_use_gelu(True)
+ downsample_ratio = hparams.get("downsample_ratio", 0.5)
+ self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ".position_embd." in new_name or "pos_embed" in new_name:
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if "input_conditioner" in name:
+ return
+
+ # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it
+ if "patch_generator.pos_embed" in name:
+ if not name.endswith(".weight"):
+ name += ".weight"
+ # Downsample position embeddings for fixed 512x512 image size
+ import torch.nn.functional as F
+ n_embd = self.hparams["hidden_size"]
+ image_size = self.global_config.get("force_image_size", 512)
+ patch_size = self.hparams["patch_size"]
+ target_patches_per_side = image_size // patch_size # 32
+ max_patches_per_side = int((data_torch.shape[1]) ** 0.5) # 128
+ if target_patches_per_side != max_patches_per_side:
+ # Reshape to grid, interpolate, flatten back
+ data_torch = data_torch.reshape(1, max_patches_per_side, max_patches_per_side, n_embd)
+ data_torch = data_torch.permute(0, 3, 1, 2).float() # [1, n_embd, 128, 128]
+ data_torch = F.interpolate(data_torch, size=(target_patches_per_side, target_patches_per_side),
+ mode='bilinear', align_corners=True)
+ data_torch = data_torch.permute(0, 2, 3, 1) # [1, 32, 32, n_embd]
+ data_torch = data_torch.reshape(1, target_patches_per_side * target_patches_per_side, n_embd)
+
+ # Reshape linear patch embedding to conv2d format for ggml_conv_2d
+ # From [n_embd, patch_size*patch_size*3] to [n_embd, 3, patch_size, patch_size]
+ if "patch_generator.embedder" in name:
+ patch_size = self.hparams["patch_size"]
+ n_embd = self.hparams["hidden_size"]
+ data_torch = data_torch.reshape(n_embd, 3, patch_size, patch_size)
+
+ if name.startswith("vision_model.radio_model.model.") or name.startswith("mlp1."):
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("WavTokenizerDec")
class WavTokenizerDecModel(TextModel):
model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DEC
@@ -4226,8 +4399,6 @@ class Qwen2MoeModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")
@@ -4248,39 +4419,31 @@ class Qwen2MoeModel(TextModel):
# Expected GGML ne: {n_embd, n_ff_exp, n_expert} for gate/up, {n_ff_exp, n_embd, n_expert} for down
if name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.down_proj.weight"):
mapped = f"{name}.weight" if not name.endswith(".weight") else name
- # Input: (n_expert=128, n_ff_exp=768, n_embd=2048)
- # Want GGML ne: {n_ff_exp, n_embd, n_expert} = {768, 2048, 128}
- # Need PyTorch: (128, 2048, 768) [reversed of GGML]
- # So: permute(0, 2, 1): (128, 768, 2048) -> (128, 2048, 768)
- permuted = data_torch.permute(0, 2, 1).contiguous()
- yield from super().modify_tensors(permuted, mapped, bid)
+ # HF: [n_expert, n_embd, n_ff] -> GGML: {n_ff, n_embd, n_expert}
+ yield from super().modify_tensors(data_torch, mapped, bid)
return
if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.gate_up_proj.weight"):
- if data_torch.ndim < 3 or data_torch.shape[-1] % 2 != 0:
+ if data_torch.ndim < 3 or data_torch.shape[-2] % 2 != 0:
raise ValueError(f"Unexpected gate_up_proj shape for {name}: {tuple(data_torch.shape)}")
- split_dim = data_torch.shape[-1] // 2
- gate = data_torch[..., :split_dim].contiguous()
- up = data_torch[..., split_dim:].contiguous()
- # Input gate/up: (n_expert=128, n_embd=2048, n_ff_exp=768)
- # Want GGML ne: {n_embd, n_ff_exp, n_expert} = {2048, 768, 128}
- # Need PyTorch: (128, 768, 2048) [reversed of GGML]
- # So: permute(0, 2, 1): (128, 2048, 768) -> (128, 768, 2048)
- base_name = name.removesuffix(".weight")
- base = base_name.rsplit('.', 1)[0]
- mapped_gate = f"{base}.gate_proj.weight"
- mapped_up = f"{base}.up_proj.weight"
- perm_gate = gate.permute(0, 2, 1).contiguous()
- perm_up = up.permute(0, 2, 1).contiguous()
- yield from super().modify_tensors(perm_gate, mapped_gate, bid)
- yield from super().modify_tensors(perm_up, mapped_up, bid)
+ # HF: [n_expert, 2*n_ff, n_embd] -> split on dim=-2
+ n_ff = data_torch.shape[-2] // 2
+ gate = data_torch[..., :n_ff, :].contiguous()
+ up = data_torch[..., n_ff:, :].contiguous()
+ # gate/up: [n_expert, n_ff, n_embd] -> GGML: {n_embd, n_ff, n_expert}
+ base_name = name.removesuffix(".weight").removesuffix(".gate_up_proj")
+ mapped_gate = f"{base_name}.gate_proj.weight"
+ mapped_up = f"{base_name}.up_proj.weight"
+ yield from super().modify_tensors(gate, mapped_gate, bid)
+ yield from super().modify_tensors(up, mapped_up, bid)
return
if name.startswith("mlp") or name.startswith("vision_model") or name.startswith("model.vision_tower") or name.startswith("model.multi_modal_projector") or name.startswith("model.visual"):
# skip visual tensors
return
+
if name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -4434,6 +4597,7 @@ class Qwen3NextModel(Qwen2MoeModel):
self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"])
self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"])
self.gguf_writer.add_ssm_inner_size(self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"])
+ self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4))
if (rope_dim := self.hparams.get("head_dim")) is None:
rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.25)))
@@ -4498,7 +4662,7 @@ class RND1Model(Qwen2MoeModel):
self.gguf_writer.add_mask_token_id(mask_token_id)
-@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration")
+@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration")
class Qwen3VLVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -4544,6 +4708,10 @@ class Qwen3VLVisionModel(MmprojModel):
if name.startswith("model.language_model.") or name.startswith("lm_head."):
return
+ # Skip MTP tensors
+ if name.startswith("mtp."):
+ return
+
if name.startswith("model.visual."):
name = name.replace("model.visual.", "visual.", 1)
@@ -4614,7 +4782,7 @@ class Qwen3VLVisionModel(MmprojModel):
yield from super().modify_tensors(data_torch, name, bid)
-@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration")
+@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
class Glm4VVisionModel(Qwen3VLVisionModel):
def set_gguf_parameters(self):
MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
@@ -4674,9 +4842,125 @@ class Qwen3VLMoeTextModel(Qwen3MoeModel):
if name.startswith("model.visual."):
return
+ # Qwen3VL has transposed packed tensors, so we treat it differently from general Qwen2MoE packed tensors
+ if name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.down_proj.weight"):
+ name = name.replace("language_model.", "")
+ mapped = f"{name}.weight" if not name.endswith(".weight") else name
+ permuted = data_torch.permute(0, 2, 1).contiguous()
+ yield from ModelBase.modify_tensors(self, permuted, mapped, bid)
+ return
+
+ if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.gate_up_proj.weight"):
+ name = name.replace("language_model.", "")
+ if data_torch.ndim < 3 or data_torch.shape[-1] % 2 != 0:
+ raise ValueError(f"Unexpected gate_up_proj shape for {name}: {tuple(data_torch.shape)}")
+ split_dim = data_torch.shape[-1] // 2
+ gate = data_torch[..., :split_dim].contiguous()
+ up = data_torch[..., split_dim:].contiguous()
+ # Input gate/up: (n_expert=128, n_embd=2048, n_ff_exp=768)
+ # Want GGML ne: {n_embd, n_ff_exp, n_expert} = {2048, 768, 128}
+ # Need PyTorch: (128, 768, 2048) [reversed of GGML]
+ # So: permute(0, 2, 1): (128, 2048, 768) -> (128, 768, 2048)
+ base_name = name.removesuffix(".weight")
+ base = base_name.rsplit('.', 1)[0]
+ mapped_gate = f"{base}.gate_proj.weight"
+ mapped_up = f"{base}.up_proj.weight"
+ perm_gate = gate.permute(0, 2, 1).contiguous()
+ perm_up = up.permute(0, 2, 1).contiguous()
+ yield from ModelBase.modify_tensors(self, perm_gate, mapped_gate, bid)
+ yield from ModelBase.modify_tensors(self, perm_up, mapped_up, bid)
+ return
+
yield from super().modify_tensors(data_torch, name, bid)
+class _LinearAttentionVReorderBase(Qwen3NextModel):
+ model_arch = gguf.MODEL_ARCH.QWEN3NEXT # overridden by subclasses
+ """reorders V heads from grouped to tiled order for ggml broadcast
+
+ see https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306
+
+ Linear attention may has num_k_heads < num_v_heads. The HF weights store
+ V heads grouped by K head: [G0_v0..v{r-1}, G1_v0..v{r-1}, ...].
+ ggml binary ops use tiled broadcast: [K0, K1, ..., K0, K1, ...].
+ We reorder V heads to tiled order so ggml_repeat can replace the expensive
+ interleaved repeat: [G0_v0, G1_v0, ..., G0_v1, G1_v1, ...].
+ """
+
+ @staticmethod
+ def _reorder_v_heads(tensor: Tensor, dim: int, num_k_heads: int, num_v_per_k: int, head_dim: int) -> Tensor:
+ """Reorder V heads from grouped (by K head) to tiled order along the given dimension."""
+ shape = list(tensor.shape)
+ if dim < 0:
+ dim += len(shape)
+ new_shape = shape[:dim] + [num_k_heads, num_v_per_k, head_dim] + shape[dim + 1:]
+ tensor = tensor.reshape(*new_shape)
+ perm = list(range(len(new_shape)))
+ perm[dim], perm[dim + 1] = perm[dim + 1], perm[dim]
+ return tensor.permute(*perm).contiguous().reshape(*shape)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ num_k_heads = self.hparams.get("linear_num_key_heads", 0)
+ num_v_heads = self.hparams.get("linear_num_value_heads", 0)
+
+ if num_k_heads > 0 and num_v_heads > 0 and num_k_heads != num_v_heads and "linear_attn." in name:
+ head_k_dim = self.hparams["linear_key_head_dim"]
+ head_v_dim = self.hparams["linear_value_head_dim"]
+ num_v_per_k = num_v_heads // num_k_heads
+
+ if ".in_proj_qkv." in name:
+ # QKV weight: reorder only the V rows
+ q_dim = head_k_dim * num_k_heads
+ k_dim = head_k_dim * num_k_heads
+ q = data_torch[:q_dim]
+ k = data_torch[q_dim:q_dim + k_dim]
+ v = data_torch[q_dim + k_dim:]
+ v = self._reorder_v_heads(v, 0, num_k_heads, num_v_per_k, head_v_dim)
+ data_torch = torch.cat([q, k, v], dim=0)
+
+ elif ".in_proj_z." in name:
+ # Z gate weight: reorder rows (num_v_heads * head_v_dim)
+ data_torch = self._reorder_v_heads(data_torch, 0, num_k_heads, num_v_per_k, head_v_dim)
+
+ elif ".in_proj_b." in name or ".in_proj_a." in name:
+ # Beta/Alpha weight: reorder rows (num_v_heads, head_dim=1)
+ data_torch = self._reorder_v_heads(data_torch, 0, num_k_heads, num_v_per_k, 1)
+
+ elif ".A_log" in name or ".dt_bias" in name or ".dt_proj" in name:
+ # A_log / dt_bias: 1D parameters with num_v_heads elements
+ if data_torch.ndim == 1:
+ data_torch = self._reorder_v_heads(
+ data_torch.unsqueeze(-1), 0, num_k_heads, num_v_per_k, 1
+ ).squeeze(-1)
+ else:
+ data_torch = self._reorder_v_heads(data_torch, -1, num_k_heads, num_v_per_k, 1)
+
+ elif ".conv1d" in name:
+ # Conv1d kernel: reorder only the V channel portion
+ data = data_torch.squeeze()
+ qk_channels = head_k_dim * num_k_heads * 2
+ qk_part = data[:qk_channels]
+ v_part = data[qk_channels:]
+ v_part = self._reorder_v_heads(v_part, 0, num_k_heads, num_v_per_k, head_v_dim)
+ data_torch = torch.cat([qk_part, v_part], dim=0)
+
+ elif ".out_proj." in name:
+ # Out projection weight: reorder columns (input dimension)
+ data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("Qwen3_5ForConditionalGeneration")
+class Qwen3_5TextModel(_LinearAttentionVReorderBase):
+ model_arch = gguf.MODEL_ARCH.QWEN35
+
+
+@ModelBase.register("Qwen3_5MoeForConditionalGeneration")
+class Qwen3_5MoeTextModel(_LinearAttentionVReorderBase):
+ model_arch = gguf.MODEL_ARCH.QWEN35MOE
+
+
@ModelBase.register("GPT2LMHeadModel")
class GPT2Model(TextModel):
model_arch = gguf.MODEL_ARCH.GPT2
@@ -4910,13 +5194,13 @@ class PhiMoeModel(Phi3MiniModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
- self.gguf_writer.add_expert_count(self.hparams["num_local_experts"])
+ self.gguf_writer.add_expert_used_count(self.find_hparam(["num_experts_per_tok", "num_experts_per_token"]))
+ self.gguf_writer.add_expert_count(self.find_hparam(["num_local_experts", "num_experts"]))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("block_sparse_moe.experts") != -1:
- n_experts = self.hparams["num_local_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -5328,7 +5612,7 @@ class KimiLinearModel(TextModel):
# process the experts separately
if name.find("block_sparse_moe.experts") != -1:
- n_experts = self.find_hparam(["num_local_experts", "num_experts"], optional=False)
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -5923,12 +6207,13 @@ class NomicBertModel(BertModel):
if "mlp.experts.bias" in name:
return # Explicitly return.
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
if "mlp.experts.mlp.w1" in name:
- data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])
+ data_torch = data_torch.view(n_experts, self.hparams["n_inner"], self.hparams["n_embd"])
name += ".weight"
if "mlp.experts.mlp.w2" in name:
- data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])
+ data_torch = data_torch.view(n_experts, self.hparams["n_inner"], self.hparams["n_embd"])
data_torch = data_torch.transpose(1, 2)
name += ".weight"
@@ -5938,7 +6223,6 @@ class NomicBertModel(BertModel):
super().set_gguf_parameters()
if self.is_moe:
self.gguf_writer.add_moe_every_n_layers(self.hparams["moe_every_n_layers"])
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])
def _is_tokenizer_xlmroberta(self) -> bool:
@@ -7052,6 +7336,8 @@ class Mamba2Model(TextModel):
if hparams is None:
with open(dir_model / "config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
+ if "llm_config" in hparams:
+ hparams["text_config"] = hparams["llm_config"]
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])
self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or 2 * self.d_model
@@ -7173,8 +7459,8 @@ class JambaModel(TextModel):
self.gguf_writer.add_ssm_state_size(d_state)
self.gguf_writer.add_ssm_time_step_rank(dt_rank)
self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
- self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
+ self.gguf_writer.add_expert_count(self.find_hparam(["num_local_experts", "num_experts"]))
+ self.gguf_writer.add_expert_used_count(self.find_hparam(["num_experts_per_tok", "num_experts_per_token"]))
self.gguf_writer.add_file_type(self.ftype)
_experts: list[dict[str, Tensor]] | None = None
@@ -7192,7 +7478,7 @@ class JambaModel(TextModel):
# process the experts separately
if ".feed_forward.experts." in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
@@ -7278,6 +7564,17 @@ class Cohere2Model(TextModel):
self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads)))
self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Cohere2 runtime in llama.cpp expects no bias tensors;
+ # the actual weight only contains 0-value tensors as bias, we can skip them
+ if name.endswith(".bias"):
+ if torch.any(data_torch != 0):
+ raise ValueError(f"Bias tensor {name!r} is not zero.")
+ logger.debug(f"Skipping bias tensor {name!r} for Cohere2 conversion.")
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
@ModelBase.register("OlmoForCausalLM")
@ModelBase.register("OLMoForCausalLM")
@@ -7340,8 +7637,6 @@ class OlmoeModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_layer_norm_rms_eps(1e-5)
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
_experts: list[dict[str, Tensor]] | None = None
@@ -7349,7 +7644,7 @@ class OlmoeModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -7718,12 +8013,16 @@ class DeepseekModel(TextModel):
"DeepseekV2ForCausalLM",
"DeepseekV3ForCausalLM",
"KimiVLForConditionalGeneration",
+ "KimiK25ForConditionalGeneration",
"YoutuForCausalLM",
"YoutuVLForConditionalGeneration",
)
class DeepseekV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
+ # TODO @ngxson : remove this when we support MTP for deepseek models
+ skip_mtp = True
+
def set_vocab(self):
try:
self._set_vocab_gpt2()
@@ -7836,8 +8135,8 @@ class DeepseekV2Model(TextModel):
_experts: list[dict[str, Tensor]] | None = None
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # skip vision tensors and remove "language_model." for Kimi-VL
- if "vision_tower" in name or "multi_modal_projector" in name:
+ # skip vision tensors and remove "language_model." for Kimi-VL and Kimi-K2.5
+ if "vision_tower" in name or "multi_modal_projector" in name or "mm_projector" in name:
return
if name.startswith("siglip2.") or name.startswith("merger."):
return
@@ -7855,10 +8154,11 @@ class DeepseekV2Model(TextModel):
name = name.replace("e_score_correction_bias", "e_score_correction.bias")
# skip Multi-Token Prediction (MTP) layers
- block_count = self.hparams["num_hidden_layers"]
- match = re.match(r"model.layers.(\d+)", name)
- if match and int(match.group(1)) >= block_count:
- return
+ if self.skip_mtp:
+ block_count = self.hparams["num_hidden_layers"]
+ match = re.match(r"model.layers.(\d+)", name)
+ if match and int(match.group(1)) >= block_count:
+ return
# process the experts separately
if name.find("mlp.experts") != -1:
@@ -7925,10 +8225,6 @@ class MiniMaxM2Model(TextModel):
model_arch = gguf.MODEL_ARCH.MINIMAXM2
_experts_cache: dict[int, dict[str, Tensor]] = {}
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.hparams["num_experts"] = self.hparams["num_local_experts"]
-
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -7941,7 +8237,7 @@ class MiniMaxM2Model(TextModel):
# merge expert weights
if 'experts' in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
expert_cache = self._experts_cache.setdefault(bid, {})
@@ -8535,6 +8831,17 @@ class T5EncoderModel(TextModel):
yield from super().modify_tensors(data_torch, name, bid)
+@ModelBase.register("Jais2ForCausalLM")
+class Jais2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.JAIS2
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ head_dim = hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"])
+ self.gguf_writer.add_rope_dimension_count(head_dim)
+
+
@ModelBase.register("JAISLMHeadModel")
class JaisModel(TextModel):
model_arch = gguf.MODEL_ARCH.JAIS
@@ -8678,7 +8985,7 @@ class Glm4Model(TextModel):
n_head = self.hparams["num_attention_heads"]
n_kv_head = self.hparams["num_key_value_heads"]
n_embd = self.hparams["hidden_size"]
- head_dim = n_embd // n_head
+ head_dim = self.hparams.get("head_dim", n_embd // n_head)
# because llama.cpp M-RoPE kernel only supports Neox ordering, we have to permute the weights here
if name.endswith(("q_proj.weight", "q_proj.bias")):
data_torch = Glm4Model.normal_to_neox(data_torch, n_head, n_head, head_dim, self.partial_rotary_factor)
@@ -8687,6 +8994,27 @@ class Glm4Model(TextModel):
yield from super().modify_tensors(data_torch, name, bid)
+@ModelBase.register("GlmOcrForConditionalGeneration")
+class GlmOCRModel(Glm4Model):
+ model_arch = gguf.MODEL_ARCH.GLM4
+ use_mrope = False
+ partial_rotary_factor = 0.5
+
+ # Note: GLM-OCR is the same as GLM4, but with an extra NextN/MTP prediction layer
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # GLM-OCR has num_hidden_layers + 1 actual layers (including NextN layer)
+ self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ # NextN/MTP prediction layers
+ if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+ self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
+
+
@ModelBase.register("Glm4MoeForCausalLM", "Glm4vMoeForConditionalGeneration")
class Glm4MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.GLM4_MOE
@@ -8698,24 +9026,7 @@ class Glm4MoeModel(TextModel):
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
def set_vocab(self):
- from transformers import AutoTokenizer
-
- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
- tokens, toktypes, tokpre = self.get_vocab_base()
- self.gguf_writer.add_tokenizer_model("gpt2")
- self.gguf_writer.add_tokenizer_pre(tokpre)
- self.gguf_writer.add_token_list(tokens)
- self.gguf_writer.add_token_types(toktypes)
-
- # Special tokens
- # Note: Using <|endoftext|> (151329) for eot causes endless generation
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
- special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
-
- special_vocab.add_to_gguf(self.gguf_writer)
+ return self._set_vocab_glm()
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -8815,26 +9126,38 @@ class Glm4MoeModel(TextModel):
class Glm4MoeLiteModel(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
- # copied from Glm4MoeModel
def set_vocab(self):
- from transformers import AutoTokenizer
+ return self._set_vocab_glm()
- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
- tokens, toktypes, tokpre = self.get_vocab_base()
- self.gguf_writer.add_tokenizer_model("gpt2")
- self.gguf_writer.add_tokenizer_pre(tokpre)
- self.gguf_writer.add_token_list(tokens)
- self.gguf_writer.add_token_types(toktypes)
- # Special tokens
- # Note: Using <|endoftext|> (151329) for eot causes endless generation
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
- special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
+@ModelBase.register("GlmMoeDsaForCausalLM")
+class GlmMoeDsaModel(DeepseekV2Model):
+ model_arch = gguf.MODEL_ARCH.GLM_DSA
+ skip_mtp = False
- special_vocab.add_to_gguf(self.gguf_writer)
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_vocab(self):
+ return self._set_vocab_glm()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ rope_dim = self.hparams["qk_rope_head_dim"]
+ partial_rotary_factor = self.hparams.get("partial_rotary_factor", 1.0)
+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * partial_rotary_factor))
+
+ # NextN/MTP prediction layers
+ if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+ self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
+
+ # DSA indexer parameters
+ self.gguf_writer.add_indexer_head_count(self.hparams["index_n_heads"])
+ self.gguf_writer.add_indexer_key_length(self.hparams["index_head_dim"])
+ self.gguf_writer.add_indexer_top_k(self.hparams["index_topk"])
@ModelBase.register("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")
@@ -9151,7 +9474,6 @@ class ExaoneMoEModel(Exaone4Model):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
moe_intermediate_size = self.hparams["moe_intermediate_size"]
num_shared_experts = self.hparams["num_shared_experts"]
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
@@ -9192,7 +9514,7 @@ class ExaoneMoEModel(Exaone4Model):
name = name.replace("e_score_correction_bias", "e_score_correction.bias")
if name.find("mlp.experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -9343,7 +9665,7 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
# case, the model architecture needs to be updated to a standard
# "granite" or "granitemoe" model
if not self._ssm_layers:
- has_experts = self.find_hparam(["num_experts_per_tok"], optional=True)
+ has_experts = self.find_hparam(["num_experts_per_tok", "num_experts_per_token"], optional=True)
new_arch = (
gguf.MODEL_ARCH.GRANITE_MOE
if has_experts else
@@ -9539,6 +9861,14 @@ class NemotronHModel(GraniteHybridModel):
self.gguf_writer.add_add_bos_token(True)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip vision model and projector tensors for VLM models (handled by mmproj) (e.g., Nemotron Nano 12B v2 VL)
+ if name.startswith(("vision_model.", "mlp1.")):
+ return
+
+ # Strip language_model. prefix for VLM models (e.g., Nemotron Nano 12B v2 VL)
+ if name.startswith("language_model."):
+ name = name[len("language_model."):]
+
if self.is_moe and bid is not None:
if name.endswith("mixer.gate.e_score_correction_bias"):
new_name = name.replace("e_score_correction_bias", "e_score_correction.bias")
@@ -9633,7 +9963,6 @@ class BailingMoeModel(TextModel):
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
self.gguf_writer.add_expert_weights_scale(1.0)
- self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])
self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
@@ -9667,7 +9996,7 @@ class BailingMoeModel(TextModel):
yield from super().modify_tensors(v,self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), bid)
return
elif name.find("mlp.experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -9738,7 +10067,6 @@ class BailingMoeV2Model(TextModel):
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
self.gguf_writer.add_expert_shared_feed_forward_length(hparams.get("moe_shared_expert_intermediate_size", hparams["moe_intermediate_size"] * hparams["num_shared_experts"]))
self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
- self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])
self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
@@ -9749,7 +10077,7 @@ class BailingMoeV2Model(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if "mlp.experts" in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -9795,8 +10123,6 @@ class GroveMoeModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")
@@ -9817,7 +10143,7 @@ class GroveMoeModel(TextModel):
# process the experts separately
if name.find("chunk_experts") != -1:
- n_experts = self.hparams["num_experts"] // 2 # see add_experts_per_group
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"]) // 2 # see add_experts_per_group
assert bid is not None
if self._chunk_experts is None:
@@ -9844,7 +10170,7 @@ class GroveMoeModel(TextModel):
else:
return
elif name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -10237,7 +10563,6 @@ class HunYuanMoEModel(TextModel):
super().set_gguf_parameters()
hparams = self.hparams
- self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_shared_feed_forward_length(hparams["intermediate_size"])
moe_intermediate_size = hparams["moe_intermediate_size"]
@@ -10280,7 +10605,7 @@ class HunYuanMoEModel(TextModel):
return
if name.find("mlp.experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -10322,16 +10647,9 @@ class LLaDAMoEModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
-
if (expert_intermediate_size := self.hparams.get("expert_intermediate_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size)
- # number of experts used per token (top-k)
- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
- self.gguf_writer.add_expert_used_count(n_experts_used)
-
self.gguf_writer.add_mask_token_id(156895)
self.gguf_writer.add_causal_attention(False)
self.gguf_writer.add_diffusion_shift_logits(False)
@@ -10342,7 +10660,7 @@ class LLaDAMoEModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -10617,7 +10935,7 @@ class LFM2Model(TextModel):
def set_gguf_parameters(self):
# set num_key_value_heads only for attention layers
self.hparams["num_key_value_heads"] = [
- self.hparams["num_key_value_heads"] if layer_type == "full_attention" else 0
+ self.hparams["num_key_value_heads"] if layer_type != "conv" else 0
for layer_type in self.hparams["layer_types"]
]
@@ -10679,7 +10997,6 @@ class LFM2MoeModel(TextModel):
super().set_gguf_parameters()
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
self.gguf_writer.add_leading_dense_block_count(self.hparams["num_dense_layers"])
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
@@ -10700,7 +11017,7 @@ class LFM2MoeModel(TextModel):
# merge expert weights
if 'experts' in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
expert_cache = self._experts_cache.setdefault(bid, {})
@@ -10804,15 +11121,37 @@ class LFM2AudioModel(ConformerAudioModel):
yield from super().modify_tensors(data_torch, name, bid)
+@ModelBase.register("Lfm25AudioTokenizer")
+class LFM25AudioTokenizer(LFM2Model):
+ model_arch = gguf.MODEL_ARCH.LFM2
+
+ def set_vocab(self):
+ self._set_vocab_none()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+ self.gguf_writer.add_embedding_length_out(self.hparams["output_size"])
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name == "istft.window" or name.startswith("emb.emb"):
+ return
+
+ if name.startswith("lin"):
+ name = name.replace("lin", "dense_2_out")
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("SmallThinkerForCausalLM")
class SmallThinkerModel(TextModel):
model_arch = gguf.MODEL_ARCH.SMALLTHINKER
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts", self.hparams.get("moe_num_primary_experts"))) is not None:
+ if (n_experts := self.hparams.get("moe_num_primary_experts")) is not None:
self.gguf_writer.add_expert_count(n_experts)
- if (n_experts_used := self.hparams.get("num_experts_per_tok", self.hparams.get("moe_num_active_primary_experts"))) is not None:
+ if (n_experts_used := self.hparams.get("moe_num_active_primary_experts")) is not None:
self.gguf_writer.add_expert_used_count(n_experts_used)
if (moe_intermediate_size := self.hparams.get("moe_ffn_hidden_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
@@ -10837,7 +11176,7 @@ class SmallThinkerModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("experts") != -1:
- n_experts = self.hparams.get("num_experts", self.hparams.get("moe_num_primary_experts"))
+ n_experts = self.hparams.get("moe_num_primary_experts") or self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -10895,13 +11234,17 @@ class ModernBertModel(BertModel):
self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # these layers act as MLM head, so we don't need them
- if name.startswith("decoder."):
- return
-
if name.startswith("model."):
name = name[6:]
+ if self.cls_out_labels:
+ # For BertForSequenceClassification (direct projection layer)
+ if name == "classifier.weight":
+ name = "classifier.out_proj.weight"
+
+ if name == "classifier.bias":
+ name = "classifier.out_proj.bias"
+
yield from super().modify_tensors(data_torch, name, bid)
@@ -11199,6 +11542,103 @@ class KimiVLModel(MmprojModel):
yield from super().modify_tensors(data_torch, name, bid)
+@ModelBase.register("KimiK25ForConditionalGeneration")
+class KimiK25Model(MmprojModel):
+ """Kimi-K2.5 with MoonViT3d vision encoder"""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ assert self.hparams_vision is not None, "Kimi-K2.5 requires vision_config in model config"
+
+ self.merge_kernel_size = tuple(self.hparams_vision.get("merge_kernel_size", [2, 2]))
+ self.patch_size = self.hparams_vision.get("patch_size", 14)
+
+ # Set image_size for compatibility with base class
+ # Use position embedding dimensions as image_size reference
+ pos_emb_h = self.hparams_vision.get("init_pos_emb_height", 64)
+ self.hparams_vision["image_size"] = pos_emb_h * self.patch_size
+
+ def set_gguf_parameters(self):
+ # Base class MmprojModel.set_gguf_parameters() already writes:
+ # - vision_block_count, vision_head_count, vision_embedding_length
+ # - vision_feed_forward_length, vision_patch_size, image_mean, image_std
+ # via find_vparam() which handles the vt_* prefixed keys in Kimi-K2.5's config
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK25)
+
+ # Position embedding parameters (for interpolation)
+ self.gguf_writer.add_uint32("vision.pos_emb_height", self.hparams_vision.get("init_pos_emb_height", 64))
+ self.gguf_writer.add_uint32("vision.pos_emb_width", self.hparams_vision.get("init_pos_emb_width", 64))
+ self.gguf_writer.add_uint32("vision.pos_emb_time", self.hparams_vision.get("init_pos_emb_time", 4))
+
+ # Projector parameters
+ self.gguf_writer.add_vision_use_gelu(self.hparams_vision.get("projector_hidden_act", "gelu") == "gelu")
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5))
+ self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0])
+
+ # Image size limits
+ # Note: in_patch_limit is for images, in_patch_limit_each_frame is for video (not supported yet)
+ in_patch_limit = self.preprocessor_config.get("in_patch_limit", 16384)
+ min_patches = 8 # reasonable minimum
+ pixels_per_patch = self.patch_size ** 2
+ self.gguf_writer.add_vision_min_pixels(min_patches * pixels_per_patch)
+ self.gguf_writer.add_vision_max_pixels(in_patch_limit * pixels_per_patch)
+
+ @staticmethod
+ def permute(weights: Tensor, n_head: int) -> Tensor:
+ out_dim, in_dim = weights.shape
+ head_dim = out_dim // n_head
+ w = weights.reshape(n_head, head_dim // 4, 2, 2, in_dim)
+ w = w.permute(0, 2, 1, 3, 4)
+ return w.reshape(out_dim, in_dim)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Only process vision and projector tensors
+ is_vision = any(x in name for x in ["vision_tower", "mm_projector"])
+
+ if not is_vision:
+ return
+
+ assert self.hparams_vision is not None
+ n_head = self.hparams_vision.get("num_attention_heads", 16)
+
+ # Permute Q/K weights/biases from interleaved to split RoPE format
+ # This allows using build_rope_2d at runtime without post-permutation.
+ if "wqkv" in name:
+ out_dim = data_torch.shape[0]
+ qkv_dim = out_dim // 3
+ head_dim = qkv_dim // n_head
+
+ if "weight" in name:
+ wq, wk, wv = data_torch[:qkv_dim, :], data_torch[qkv_dim:2 * qkv_dim, :], data_torch[2 * qkv_dim:, :]
+ wq = self.permute(wq, n_head)
+ wk = self.permute(wk, n_head)
+ data_torch = torch.cat([wq, wk, wv], dim=0)
+ elif "bias" in name:
+ bq, bk, bv = data_torch[:qkv_dim], data_torch[qkv_dim:2 * qkv_dim], data_torch[2 * qkv_dim:]
+ bq = bq.reshape(n_head, head_dim // 4, 2, 2).permute(0, 2, 1, 3).reshape(-1)
+ bk = bk.reshape(n_head, head_dim // 4, 2, 2).permute(0, 2, 1, 3).reshape(-1)
+ data_torch = torch.cat([bq, bk, bv], dim=0)
+
+ # Temporal embeddings: (T, 1, C) → (T, C)
+ if "pos_emb.time_weight" in name:
+ T, _, C = data_torch.shape
+ data_torch = data_torch.reshape(T, C)
+
+ # PatchMergerMLP tensor name mapping
+ # proj.0.weight → proj.linear_1.weight
+ # proj.2.weight → proj.linear_2.weight
+ if "mm_projector.proj.0." in name:
+ name = name.replace(".proj.0.", ".proj.linear_1.")
+ elif "mm_projector.proj.2." in name:
+ name = name.replace(".proj.2.", ".proj.linear_2.")
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("CogVLMForCausalLM")
class CogVLMVisionModel(MmprojModel):
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 2811f7f884..53a73759ec 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -99,6 +99,7 @@ models = [
{"name": "stablelm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", },
{"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },
{"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
+ {"name": "tiny_aya", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", },
{"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
{"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
{"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },
@@ -113,6 +114,7 @@ models = [
{"name": "gemma", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2b", },
{"name": "gemma-2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2-9b", },
{"name": "jais", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/core42/jais-13b", },
+ {"name": "jais-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inceptionai/Jais-2-8B-Chat", },
{"name": "t5", "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", },
{"name": "codeshell", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/WisdomShell/CodeShell-7B", },
{"name": "tekken", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mistralai/Mistral-Nemo-Base-2407", },
@@ -148,6 +150,9 @@ models = [
{"name": "youtu", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Youtu-LLM-2B", },
{"name": "solar-open", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/upstage/Solar-Open-100B", },
{"name": "exaone-moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B", },
+ {"name": "qwen35", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3.5-9B-Instruct", },
+ {"name": "joyai-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jdopensource/JoyAI-LLM-Flash", },
+ {"name": "kanana2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/kakaocorp/kanana-2-30b-a3b-instruct-2601", },
]
# some models are known to be broken upstream, so we will skip them as exceptions
@@ -157,6 +162,7 @@ pre_computed_hashes = [
{"name": "chatglm-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/THUDM/glm-4-9b-chat", "chkhsh": "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516"},
{"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/THUDM/glm-4-9b-hf", "chkhsh": "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2"},
{"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"},
+ {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},
{"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},
{"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},
{"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},
@@ -170,7 +176,6 @@ pre_computed_hashes = [
{"name": "grok-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"},
# jina-v2-de variants
{"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
- {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},
]
diff --git a/docs/backend/CANN.md b/docs/backend/CANN.md
index b03c2a122c..23b6a62763 100755
--- a/docs/backend/CANN.md
+++ b/docs/backend/CANN.md
@@ -246,7 +246,7 @@ cmake --build build --config release
1. **Retrieve and prepare model**
- You can refer to the general [*Prepare and Quantize*](../../README.md#prepare-and-quantize) guide for model prepration.
+ You can refer to the general [*Obtaining and quantizing models*](../../README.md#obtaining-and-quantizing-models) guide for model prepration.
**Notes**:
diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md
index b3cff96604..07c68be5cb 100644
--- a/docs/backend/SYCL.md
+++ b/docs/backend/SYCL.md
@@ -281,7 +281,7 @@ as `-cl-fp32-correctly-rounded-divide-sqrt`
#### Retrieve and prepare model
-You can refer to the general [*Prepare and Quantize*](README.md#prepare-and-quantize) guide for model preparation, or download an already quantized model like [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q4_0.gguf?download=true) or [Meta-Llama-3-8B-Instruct-Q4_0.gguf](https://huggingface.co/aptha/Meta-Llama-3-8B-Instruct-Q4_0-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_0.gguf).
+You can refer to the general [*Obtaining and quantizing models*](../../README.md#obtaining-and-quantizing-models) guide for model preparation, or download an already quantized model like [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q4_0.gguf?download=true) or [Meta-Llama-3-8B-Instruct-Q4_0.gguf](https://huggingface.co/aptha/Meta-Llama-3-8B-Instruct-Q4_0-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_0.gguf).
##### Check device
@@ -569,7 +569,7 @@ Once it is completed, final results will be in **build/Release/bin**
#### Retrieve and prepare model
-You can refer to the general [*Prepare and Quantize*](README.md#prepare-and-quantize) guide for model preparation, or download an already quantized model like [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/blob/main/llama-2-7b.Q4_0.gguf) or [Meta-Llama-3-8B-Instruct-Q4_0.gguf](https://huggingface.co/aptha/Meta-Llama-3-8B-Instruct-Q4_0-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_0.gguf).
+You can refer to the general [*Obtaining and quantizing models*](../../README.md#obtaining-and-quantizing-models) guide for model preparation, or download an already quantized model like [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/blob/main/llama-2-7b.Q4_0.gguf) or [Meta-Llama-3-8B-Instruct-Q4_0.gguf](https://huggingface.co/aptha/Meta-Llama-3-8B-Instruct-Q4_0-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_0.gguf).
##### Check device
diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md
index 8e1f37b206..2c3f88e91a 100644
--- a/docs/backend/snapdragon/README.md
+++ b/docs/backend/snapdragon/README.md
@@ -35,7 +35,7 @@ Adapt below build commands accordingly.
Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets:
```
-[d]/workspace> cp docs/backend/hexagon/CMakeUserPresets.json .
+[d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json .
[d]/workspace> cmake --preset arm64-android-snapdragon-release -B build-snapdragon
Preset CMake variables:
diff --git a/docs/build-s390x.md b/docs/build-s390x.md
index 67df4e2eac..4568d5010f 100644
--- a/docs/build-s390x.md
+++ b/docs/build-s390x.md
@@ -242,10 +242,10 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
|------------|-------------|------|-------|
| FP32 | ✅ | ✅ | ❓ |
| FP16 | ✅ | ✅ | ❓ |
-| BF16 | 🚫 | ✅ | ❓ |
+| BF16 | ✅ | ✅ | ❓ |
| Q4_0 | ✅ | ❓ | ❓ |
| Q4_1 | ✅ | ❓ | ❓ |
-| MXFP4 | 🚫 | ❓ | ❓ |
+| MXFP4 | ✅ | ❓ | ❓ |
| Q5_0 | ✅ | ❓ | ❓ |
| Q5_1 | ✅ | ❓ | ❓ |
| Q8_0 | ✅ | ❓ | ❓ |
@@ -272,4 +272,4 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
- 🚫 - acceleration unavailable, will still run using scalar implementation
- ❓ - acceleration unknown, please contribute if you can test it yourself
-Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Sep 7, 2025.
+Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Feb 15, 2026.
diff --git a/docs/ops.md b/docs/ops.md
index 5754b0a96c..296c0ba1d4 100644
--- a/docs/ops.md
+++ b/docs/ops.md
@@ -31,7 +31,7 @@ Legend:
| CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| COS | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ |
+| COS | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
| CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
@@ -96,13 +96,13 @@ Legend:
| SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| SIN | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ |
+| SIN | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
| SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ |
-| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ | ❌ |
-| SQRT | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ | ❌ |
+| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+| SQRT | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ |
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
diff --git a/docs/ops/WebGPU.csv b/docs/ops/WebGPU.csv
index 72ebea2cd7..e2ed3e2cfa 100644
--- a/docs/ops/WebGPU.csv
+++ b/docs/ops/WebGPU.csv
@@ -8760,22 +8760,14 @@
"WebGPU: WebGPU","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=1","support","0","no","WebGPU"
"WebGPU: WebGPU","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=32","support","0","no","WebGPU"
"WebGPU: WebGPU","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=129","support","0","no","WebGPU"
-"WebGPU: WebGPU","SQR","type=f16,ne=[10,5,4,3]","support","0","no","WebGPU"
-"WebGPU: WebGPU","SQRT","type=f16,ne=[10,3,3,2]","support","0","no","WebGPU"
"WebGPU: WebGPU","LOG","type=f16,ne=[10,5,4,3]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SIN","type=f16,ne=[10,2,2,2]","support","0","no","WebGPU"
-"WebGPU: WebGPU","COS","type=f16,ne=[10,2,2,2]","support","0","no","WebGPU"
"WebGPU: WebGPU","CLAMP","type=f16,ne=[10,5,4,3],min=-0.500000,max=0.500000","support","1","yes","WebGPU"
"WebGPU: WebGPU","LEAKY_RELU","type=f16,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
"WebGPU: WebGPU","FLOOR","type=f16,ne=[10,2,2,2]","support","1","yes","WebGPU"
"WebGPU: WebGPU","CEIL","type=f16,ne=[10,2,2,2]","support","1","yes","WebGPU"
"WebGPU: WebGPU","ROUND","type=f16,ne=[10,2,2,2]","support","1","yes","WebGPU"
"WebGPU: WebGPU","TRUNC","type=f16,ne=[10,2,2,2]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SQR","type=f16,ne=[7,1,5,3]","support","0","no","WebGPU"
-"WebGPU: WebGPU","SQRT","type=f16,ne=[7,1,5,3]","support","0","no","WebGPU"
"WebGPU: WebGPU","LOG","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SIN","type=f16,ne=[7,1,5,3]","support","0","no","WebGPU"
-"WebGPU: WebGPU","COS","type=f16,ne=[7,1,5,3]","support","0","no","WebGPU"
"WebGPU: WebGPU","CLAMP","type=f16,ne=[7,1,5,3],min=-0.500000,max=0.500000","support","1","yes","WebGPU"
"WebGPU: WebGPU","LEAKY_RELU","type=f16,ne_a=[7,1,5,3],negative_slope=0.100000","support","0","no","WebGPU"
"WebGPU: WebGPU","FLOOR","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
@@ -8786,22 +8778,14 @@
"WebGPU: WebGPU","ROUND","type=f16,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
"WebGPU: WebGPU","TRUNC","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
"WebGPU: WebGPU","TRUNC","type=f16,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SQR","type=f32,ne=[10,5,4,3]","support","0","no","WebGPU"
-"WebGPU: WebGPU","SQRT","type=f32,ne=[10,3,3,2]","support","0","no","WebGPU"
"WebGPU: WebGPU","LOG","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SIN","type=f32,ne=[10,2,2,2]","support","0","no","WebGPU"
-"WebGPU: WebGPU","COS","type=f32,ne=[10,2,2,2]","support","0","no","WebGPU"
"WebGPU: WebGPU","CLAMP","type=f32,ne=[10,5,4,3],min=-0.500000,max=0.500000","support","1","yes","WebGPU"
"WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
"WebGPU: WebGPU","FLOOR","type=f32,ne=[10,2,2,2]","support","1","yes","WebGPU"
"WebGPU: WebGPU","CEIL","type=f32,ne=[10,2,2,2]","support","1","yes","WebGPU"
"WebGPU: WebGPU","ROUND","type=f32,ne=[10,2,2,2]","support","1","yes","WebGPU"
"WebGPU: WebGPU","TRUNC","type=f32,ne=[10,2,2,2]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SQR","type=f32,ne=[7,1,5,3]","support","0","no","WebGPU"
-"WebGPU: WebGPU","SQRT","type=f32,ne=[7,1,5,3]","support","0","no","WebGPU"
"WebGPU: WebGPU","LOG","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","SIN","type=f32,ne=[7,1,5,3]","support","0","no","WebGPU"
-"WebGPU: WebGPU","COS","type=f32,ne=[7,1,5,3]","support","0","no","WebGPU"
"WebGPU: WebGPU","CLAMP","type=f32,ne=[7,1,5,3],min=-0.500000,max=0.500000","support","1","yes","WebGPU"
"WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[7,1,5,3],negative_slope=0.100000","support","0","no","WebGPU"
"WebGPU: WebGPU","FLOOR","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
@@ -18901,3 +18885,27 @@
"WebGPU: WebGPU","CROSS_ENTROPY_LOSS_BACK","type=f32,ne=[30000,1,1,1]","support","0","no","WebGPU"
"WebGPU: WebGPU","OPT_STEP_ADAMW","type=f32,ne=[10,5,4,3]","support","0","no","WebGPU"
"WebGPU: WebGPU","OPT_STEP_SGD","type=f32,ne=[10,5,4,3]","support","0","no","WebGPU"
+"WebGPU: WebGPU","SQR","type=f16,ne=[10,5,4,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQRT","type=f16,ne=[10,3,3,2]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SIN","type=f16,ne=[10,2,2,2]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","COS","type=f16,ne=[10,2,2,2]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQR","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQR","type=f16,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQRT","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQRT","type=f16,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SIN","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SIN","type=f16,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","COS","type=f16,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","COS","type=f16,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQR","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQRT","type=f32,ne=[10,3,3,2]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SIN","type=f32,ne=[10,2,2,2]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","COS","type=f32,ne=[10,2,2,2]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQR","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQR","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQRT","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SQRT","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SIN","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","SIN","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","COS","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
+"WebGPU: WebGPU","COS","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
diff --git a/examples/model-conversion/scripts/causal/run-org-model.py b/examples/model-conversion/scripts/causal/run-org-model.py
index 215f1a9ee0..6f85ee4485 100755
--- a/examples/model-conversion/scripts/causal/run-org-model.py
+++ b/examples/model-conversion/scripts/causal/run-org-model.py
@@ -42,11 +42,15 @@ def load_model_and_tokenizer(model_path, device="auto"):
config = config.text_config
multimodal = True
- print("Vocab size: ", config.vocab_size)
- print("Hidden size: ", config.hidden_size)
- print("Number of layers: ", config.num_hidden_layers)
- print("BOS token id: ", config.bos_token_id)
- print("EOS token id: ", config.eos_token_id)
+ def print_if_exists(label, obj, attr, default="N/A"):
+ val = getattr(obj, attr) if hasattr(obj, attr) else default
+ print(f"{label}", val)
+
+ print_if_exists("Vocab size: ", config, "vocab_size")
+ print_if_exists("Hidden size: ", config, "hidden_size")
+ print_if_exists("Number of layers: ", config, "num_hidden_layers")
+ print_if_exists("BOS token id: ", config, "bos_token_id")
+ print_if_exists("EOS token id: ", config, "eos_token_id")
unreleased_model_name = os.getenv("UNRELEASED_MODEL_NAME")
if unreleased_model_name:
diff --git a/examples/model-conversion/scripts/utils/tensor-info.py b/examples/model-conversion/scripts/utils/tensor-info.py
index 12a3430b49..1bb9e0564c 100755
--- a/examples/model-conversion/scripts/utils/tensor-info.py
+++ b/examples/model-conversion/scripts/utils/tensor-info.py
@@ -78,7 +78,7 @@ def list_all_tensors(model_path: Path, unique: bool = False):
print(tensor_name)
-def print_tensor_info(model_path: Path, tensor_name: str):
+def print_tensor_info(model_path: Path, tensor_name: str, num_values: Optional[int] = None):
tensor_file = find_tensor_file(model_path, tensor_name)
if tensor_file is None:
@@ -96,6 +96,12 @@ def print_tensor_info(model_path: Path, tensor_name: str):
print(f"Tensor: {tensor_name}")
print(f"File: {tensor_file}")
print(f"Shape: {shape}")
+ if num_values is not None:
+ tensor = f.get_tensor(tensor_name)
+ print(f"Dtype: {tensor.dtype}")
+ flat = tensor.flatten()
+ n = min(num_values, flat.numel())
+ print(f"Values: {flat[:n].tolist()}")
else:
print(f"Error: Tensor '{tensor_name}' not found in {tensor_file}")
sys.exit(1)
@@ -127,6 +133,15 @@ def main():
action="store_true",
help="List unique tensor patterns in the model (layer numbers replaced with #)"
)
+ parser.add_argument(
+ "-n", "--num-values",
+ nargs="?",
+ const=10,
+ default=None,
+ type=int,
+ metavar="N",
+ help="Print the first N values of the tensor flattened (default: 10 if flag is given without a number)"
+ )
args = parser.parse_args()
@@ -152,7 +167,7 @@ def main():
if args.tensor_name is None:
print("Error: tensor_name is required when not using --list")
sys.exit(1)
- print_tensor_info(model_path, args.tensor_name)
+ print_tensor_info(model_path, args.tensor_name, args.num_values)
if __name__ == "__main__":
diff --git a/examples/save-load-state/save-load-state.cpp b/examples/save-load-state/save-load-state.cpp
index 39d4464663..5e35dcd603 100644
--- a/examples/save-load-state/save-load-state.cpp
+++ b/examples/save-load-state/save-load-state.cpp
@@ -5,12 +5,15 @@
#include
#include
+
int main(int argc, char ** argv) {
common_params params;
params.prompt = "The quick brown fox";
params.sampling.seed = 1234;
+ const std::string_view state_file = "dump_state.bin";
+
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
@@ -53,35 +56,16 @@ int main(int argc, char ** argv) {
// tokenize prompt
auto tokens = common_tokenize(ctx, params.prompt, true);
- // prepare the batch
- llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
- for (size_t i = 0; i < tokens.size(); i++) {
- common_batch_add(batch, tokens[i], i, {0}, false);
+ const bool save_state = true;
+ if (!common_prompt_batch_decode(ctx, tokens, n_past, params.n_batch, state_file, save_state)) {
+ return 1;
}
- batch.logits[batch.n_tokens - 1] = true; // generate next token
-
- // evaluate prompt
- llama_decode(ctx, batch);
- n_past += batch.n_tokens;
-
- // save state (rng, logits, embedding and kv_cache) to file
- {
- std::vector state_mem(llama_state_get_size(ctx));
- const size_t written = llama_state_get_data(ctx, state_mem.data(), state_mem.size());
-
- FILE *fp_write = fopen("dump_state.bin", "wb");
- fwrite(state_mem.data(), 1, written, fp_write);
- fclose(fp_write);
-
- fprintf(stderr, "%s : serialized state into %zd out of a maximum of %zd bytes\n", __func__, written, state_mem.size());
- }
-
- // save state (last tokens)
- const auto n_past_saved = n_past;
// first run
printf("\nfirst run: %s", params.prompt.c_str());
+ llama_batch batch = llama_batch_init(1, 0, 1);
+
for (auto i = 0; i < params.n_predict; i++) {
auto next_token = llama_sampler_sample(smpl, ctx, -1);
auto next_token_str = common_token_to_piece(ctx, next_token);
@@ -111,27 +95,23 @@ int main(int argc, char ** argv) {
printf("\nsecond run: %s", params.prompt.c_str());
- // load state (rng, logits, embedding and kv_cache) from file
- {
- std::vector state_mem;
+ // load state from file
+ std::vector unused_sts(tokens.size()); // unused session tokens.
+ size_t n_token_count_out = 0;
- FILE * fp_read = fopen("dump_state.bin", "rb");
- fseek(fp_read, 0, SEEK_END);
- state_mem.resize(ftell(fp_read));
- fseek(fp_read, 0, SEEK_SET);
- const size_t read = fread(state_mem.data(), 1, state_mem.size(), fp_read);
- fclose(fp_read);
-
- if (read != llama_state_set_data(ctx2, state_mem.data(), state_mem.size())) {
- fprintf(stderr, "\n%s : failed to read state\n", __func__);
- return 1;
- }
-
- fprintf(stderr, "%s : deserialized state from %zd out of a maximum of %zd bytes\n", __func__, read, state_mem.size());
+ if (!llama_state_load_file(ctx2, state_file.data(), unused_sts.data(), unused_sts.size(), &n_token_count_out)) {
+ fprintf(stderr, "\n%s : failed to load state\n", __func__);
+ return 1;
}
+ fprintf(stderr, "%s : loaded state with %zu tokens\n", __func__, n_token_count_out);
+
// restore state (last tokens)
- n_past = n_past_saved;
+ n_past = n_token_count_out;
+ if (!common_replay_last_token(ctx2, tokens.back(), n_past)) {
+ return 1;
+ }
+ ++n_past;
// second run
for (auto i = 0; i < params.n_predict; i++) {
@@ -160,7 +140,9 @@ int main(int argc, char ** argv) {
}
// make new context
- llama_context * ctx3 = llama_init_from_model(model, common_context_params_to_llama(params));
+ auto params_ctx3 = common_context_params_to_llama(params);
+ params_ctx3.n_seq_max = 2;
+ llama_context * ctx3 = llama_init_from_model(model, params_ctx3);
llama_sampler * smpl3 = llama_sampler_chain_init(sparams);
@@ -169,26 +151,21 @@ int main(int argc, char ** argv) {
printf("\nsingle seq run: %s", params.prompt.c_str());
// load state (rng, logits, embedding and kv_cache) from file
- {
- std::vector state_mem;
+ n_token_count_out = 0;
- FILE * fp_read = fopen("dump_state.bin", "rb");
- fseek(fp_read, 0, SEEK_END);
- state_mem.resize(ftell(fp_read));
- fseek(fp_read, 0, SEEK_SET);
- const size_t read = fread(state_mem.data(), 1, state_mem.size(), fp_read);
- fclose(fp_read);
-
- if (read != llama_state_set_data(ctx3, state_mem.data(), state_mem.size())) {
- fprintf(stderr, "\n%s : failed to read state\n", __func__);
- return 1;
- }
-
- fprintf(stderr, "%s : deserialized state from %zd out of a maximum of %zd bytes\n", __func__, read, state_mem.size());
+ if (!llama_state_load_file(ctx3, state_file.data(), unused_sts.data(), unused_sts.size(), &n_token_count_out)) {
+ fprintf(stderr, "\n%s : failed to load state\n", __func__);
+ return 1;
}
+ fprintf(stderr, "%s : loaded state with %zu tokens\n", __func__, n_token_count_out);
+
// restore state (last tokens)
- n_past = n_past_saved;
+ n_past = n_token_count_out;
+ if (!common_replay_last_token(ctx3, tokens.back(), n_past)) {
+ return 1;
+ }
+ ++n_past;
// save seq 0 and load into seq 1
{
diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt
index 71d1a7f0e3..4323afe57b 100644
--- a/ggml/CMakeLists.txt
+++ b/ggml/CMakeLists.txt
@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)
### GGML Version
set(GGML_VERSION_MAJOR 0)
set(GGML_VERSION_MINOR 9)
-set(GGML_VERSION_PATCH 5)
+set(GGML_VERSION_PATCH 7)
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index f759e2d588..77af0e7fb6 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -752,6 +752,7 @@ extern "C" {
GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);
GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);
GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_view (const struct ggml_tensor * tensor);
GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c
index 41419b617b..7f414b2311 100644
--- a/ggml/src/ggml-alloc.c
+++ b/ggml/src/ggml-alloc.c
@@ -17,11 +17,6 @@
//#define AT_PRINTF(...) GGML_LOG_DEBUG(__VA_ARGS__)
#define AT_PRINTF(...)
-
-static bool ggml_is_view(const struct ggml_tensor * t) {
- return t->view_src != NULL;
-}
-
// ops that return true for this function must not use restrict pointers for their backend implementations
bool ggml_op_can_inplace(enum ggml_op op) {
switch (op) {
@@ -627,7 +622,7 @@ static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor
GGML_ASSERT(buffer_id >= 0);
struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
- if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_is_view(node)) {
+ if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_impl_is_view(node)) {
hn->allocated = true;
assert(hn->addr.offset == 0);
@@ -658,7 +653,7 @@ static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor
struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);
if (p_hn->n_children == 1 && p_hn->n_views == 0) {
- if (ggml_is_view(parent)) {
+ if (ggml_impl_is_view(parent)) {
struct ggml_tensor * view_src = parent->view_src;
struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);
if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {
@@ -739,7 +734,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr
// GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to
// control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node
// itself is never used and should not be considered a dependency
- if (ggml_is_view(node) && node->op != GGML_OP_NONE) {
+ if (ggml_impl_is_view(node) && node->op != GGML_OP_NONE) {
struct ggml_tensor * view_src = node->view_src;
ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;
}
@@ -806,7 +801,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr
parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);
if (p_hn->n_children == 0 && p_hn->n_views == 0) {
- if (ggml_is_view(parent)) {
+ if (ggml_impl_is_view(parent)) {
struct ggml_tensor * view_src = parent->view_src;
struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);
view_src_hn->n_views -= 1;
diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
index 8a693f84af..311fa5fe36 100644
--- a/ggml/src/ggml-backend-reg.cpp
+++ b/ggml/src/ggml-backend-reg.cpp
@@ -471,9 +471,10 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
int best_score = 0;
fs::path best_path;
+ std::error_code ec;
for (const auto & search_path : search_paths) {
- if (std::error_code ec; !fs::exists(search_path, ec)) {
+ if (!fs::exists(search_path, ec)) {
if (ec) {
GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());
} else {
@@ -483,7 +484,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
}
fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
for (const auto & entry : dir_it) {
- if (entry.is_regular_file()) {
+ if (entry.is_regular_file(ec)) {
auto filename = entry.path().filename();
auto ext = entry.path().extension();
if (filename.native().find(file_prefix) == 0 && ext == file_extension) {
diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp
index 87ac05748e..fc7c3e3b72 100644
--- a/ggml/src/ggml-cann/aclnn_ops.cpp
+++ b/ggml/src/ggml-cann/aclnn_ops.cpp
@@ -3286,130 +3286,223 @@ static void ggml_cann_mul_mat_id_fp(ggml_backend_cann_context & ctx, ggml_tensor
}
/**
- * @brief Performs expert-specific matrix multiplication (MoE) with
- * quantized precision using the CANN backend.
+ * @brief Performs quantized matrix multiplication for Mixture of Experts (MoE)
+ * models using the CANN backend.
*
- * This function executes a matrix multiplication operation tailored for
- * Mixture of Experts (MoE) models, where the input tensor is multiplied
- * with expert-specific quantized weight matrices. It leverages the CANN
- * backend to perform efficient low-precision computations and stores the
- * quantized result in the destination tensor `dst`.
+ * This function implements MUL_MAT_ID operation for quantized weight matrices
+ * (Q4_0 and Q8_0 formats). It selects expert-specific weight matrices based on
+ * the provided expert indices, and computes matrix multiplication using CANN's
+ * WeightQuantBatchMatmulV2 operator.
*
- * Quantization techniques reduce memory footprint and improve performance
- * by using lower-bit representations (e.g., int8) instead of floating-point.
- * This function is designed to work with such formats and may incorporate
- * optimizations like identity-based fast paths or routing masks for sparse
- * expert selection.
+ * The function performs the following steps:
+ * 1. Converts input/output tensors to F16 format if necessary
+ * 2. Uses IndexSelect to extract expert-specific weights and scales based on indices
+ * 3. Performs quantized matrix multiplication for each expert using WeightQuantBatchMatmulV2
+ * 4. Converts output back to the target type if needed
*
- * @param ctx The context for executing CANN backend operations.
- * @param dst The destination tensor where the quantized MoE multiplication result
- * will be stored.
+ * Tensor shapes:
+ * - dst: [M, K, N, 1] - output tensor
+ * - src0: [D, M, A, 1] - quantized weight matrices (Q4_0 or Q8_0)
+ * - src1: [D, B, N, 1] - input activations (B = K for per-expert input, or B = 1 for broadcast)
+ * - ids: [K, N] - expert indices for routing
*
- * @note This function assumes quantized data types and is designed for
- * MoE architectures with potential sparse expert routing.
+ * @param ctx The CANN backend context for operation execution.
+ * @param dst The destination tensor where the multiplication result will be stored.
+ *
+ * @note Only Q4_0 and Q8_0 quantization formats are supported.
+ * @note The function handles automatic type conversion to/from F16 as needed by the hardware.
*/
static void ggml_cann_mul_mat_id_quant(ggml_backend_cann_context & ctx, ggml_tensor * dst) {
- // TODO: Use aclnnGroupedMatMul
- //dst [M, K, N, 1]
- ggml_tensor * src0 = dst->src[0]; //src0 [D, M, A, 1]
- ggml_tensor * src1 = dst->src[1]; //src1 [D, B, N, 1], B = K or B = 1
- ggml_tensor * ids = dst->src[2]; //ids [K, N]
+ // dst: [M, K, N, 1]
+ // src0: [D, M, A, 1] - quantized weights
+ // src1: [D, B, N, 1] - input activations, B = K or B = 1
+ // ids: [K, N] - expert indices
+ ggml_tensor * src0 = dst->src[0];
+ ggml_tensor * src1 = dst->src[1];
+ ggml_tensor * ids = dst->src[2];
- GGML_TENSOR_BINARY_OP_LOCALS
+ GGML_ASSERT(src0->ne[3] == 1);
+ GGML_ASSERT(src1->ne[3] == 1);
+ GGML_ASSERT(dst->ne[3] == 1);
+ GGML_ASSERT(src1->ne[2] == ids->ne[1]);
- // copy index from npu to cpu
- int64_t n_as = ne02; // A
- int64_t n_ids = ids->ne[0]; // K
+ const int64_t n_batches = ids->ne[1];
+ const int64_t n_select_experts = ids->ne[0];
+ const enum ggml_type type = src0->type;
- std::vector ids_host(ggml_nbytes(ids));
- ACL_CHECK(aclrtMemcpyAsync(ids_host.data(), ggml_nbytes(ids), ids->data, ggml_nbytes(ids),
- ACL_MEMCPY_DEVICE_TO_HOST, ctx.stream()));
- ACL_CHECK(aclrtSynchronizeStream(ctx.stream()));
+ const int32_t group_size = QK8_0; // Both Q4_0 and Q8_0 use group size of 32
+ GGML_ASSERT(group_size == QK4_0);
- char * src0_original = (char *) src0->data;
- char * src1_original = (char *) src1->data;
- char * dst_original = (char *) dst->data;
+ // Calculate element size for quantized weights
+ const float weight_elem_size =
+ (type == GGML_TYPE_Q4_0) ? 0.5f :
+ (type == GGML_TYPE_Q8_0) ? 1.0f :
+ (GGML_ABORT("MUL_MAT_ID only supports Q4_0 and Q8_0"), 0.0f);
- ggml_tensor src0_row = *src0;
- ggml_tensor src1_row = *src1;
- ggml_tensor dst_row = *dst;
+ // Calculate scale offset in memory
+ const size_t weight_size = src0->ne[0] * src0->ne[1] * src0->ne[2] * weight_elem_size;
+ const size_t scale_elem_size = sizeof(uint16_t);
+ char * scale_data = (char *) src0->data + weight_size;
- const enum ggml_type type = dst->src[0]->type;
- float weight_elem_size;
- if (type == GGML_TYPE_Q4_0) {
- weight_elem_size = float(sizeof(uint8_t)) / 2;
- } else if (type == GGML_TYPE_Q8_0) {
- weight_elem_size = float(sizeof(uint8_t));
- } else {
- GGML_ABORT("MUL_MAT_ID only support quant type Q4_0 and Q8_0 ");
- }
+ // Allocate buffers for selected expert weights and scales
+ const size_t selected_weight_size = src0->ne[0] * src0->ne[1] * n_select_experts * weight_elem_size;
+ ggml_cann_pool_alloc selected_weight_alloc(ctx.pool(), selected_weight_size);
+ void * selected_weight_buffer = selected_weight_alloc.get();
- // src0_row [D, M, 1, 1] weight without permute
- src0_row.ne[2] = 1;
- src0_row.ne[3] = 1;
- src0_row.nb[0] = weight_elem_size;
- src0_row.nb[1] = weight_elem_size * ne00;
- src0_row.nb[2] = weight_elem_size * ne00;
- src0_row.nb[3] = weight_elem_size * ne00;
- size_t weight_stride = ne00 * ne01 * weight_elem_size;
- size_t weight_size = weight_stride * ne02 * ne03;
+ const size_t selected_scale_size = (src0->ne[0] / group_size) * src0->ne[1] * n_select_experts * scale_elem_size;
+ ggml_cann_pool_alloc selected_scale_alloc(ctx.pool(), selected_scale_size);
+ void * selected_scale_buffer = selected_scale_alloc.get();
- // scale [D, M, 1, 1] -> scale && permute
- size_t scale_elem_size = sizeof(uint16_t);
- size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;
+ // Helper lambda to allocate and cast tensor to F16 if needed
+ constexpr size_t f16_elem_size = sizeof(uint16_t);
+ auto prepare_f16_buffer = [&](ggml_tensor * tensor, ggml_cann_pool_alloc & allocator,
+ bool need_cast = false) -> void * {
+ if (tensor->type == GGML_TYPE_F16) {
+ return tensor->data;
+ }
- // src1_row [D, 1, 1, 1] -> input
- src1_row.ne[1] = 1;
- src1_row.ne[2] = 1;
- src1_row.ne[3] = 1;
- src1_row.nb[2] = nb11;
- src1_row.nb[3] = nb11;
+ size_t total_size = f16_elem_size;
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ total_size *= tensor->ne[i];
+ }
+ void * buffer = allocator.alloc(total_size);
- // dst_row [M, 1, 1, 1] -> out
- dst_row.ne[1] = 1;
- dst_row.ne[2] = 1;
- dst_row.ne[3] = 1;
- dst_row.nb[2] = nb1;
- dst_row.nb[3] = nb1;
+ if (need_cast == false) {
+ return buffer;
+ }
- //create weight for one row
- ggml_cann_pool_alloc weight_allocator(ctx.pool());
- void * weight_buffer = weight_allocator.alloc(nb02);
- for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
- for (int64_t id = 0; id < n_ids; id++) {
- // expert index
- int32_t i02 = *(int32_t *) (ids_host.data() + iid1 * ids->nb[1] + id * ids->nb[0]);
- GGML_ASSERT(i02 >= 0 && i02 < n_as);
+ int64_t ne[GGML_MAX_DIMS];
+ size_t nb[GGML_MAX_DIMS] = { f16_elem_size };
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ ne[i] = tensor->ne[i];
+ if (i > 0) {
+ nb[i] = nb[i - 1] * ne[i - 1];
+ }
+ }
- // If B = 1 (broadcast), always use 0; otherwise, use id.
- int64_t i11 = (ne11 == 1 ? 0 : id);
- int64_t i12 = iid1;
+ acl_tensor_ptr src_tensor = ggml_cann_create_tensor(tensor);
+ acl_tensor_ptr f16_tensor = ggml_cann_create_tensor(buffer, ACL_FLOAT16, f16_elem_size, ne, nb, GGML_MAX_DIMS);
+ aclnn_cast(ctx, src_tensor.get(), f16_tensor.get(), ACL_FLOAT16);
- int64_t i1 = id;
- int64_t i2 = i12;
+ return buffer;
+ };
- void * src0_tmp_ptr = src0_original + i02 * weight_stride;
- void * scale_tmp_ptr = src0_original + weight_size + i02 * scale_stride;
- void * src1_tmp_ptr = src1_original + i11 * nb11 + i12 * nb12;
- void * dst_tmp_ptr = dst_original + i1 * nb1 + i2 * nb2;
+ // Prepare input and output buffers
+ ggml_cann_pool_alloc input_alloc(ctx.pool());
+ void * input_buffer = prepare_f16_buffer(src1, input_alloc, true);
- // mem cpy
- ACL_CHECK(aclrtMemcpyAsync(weight_buffer, weight_stride, src0_tmp_ptr, weight_stride,
- ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));
- void * scale_buffer = (char *) weight_buffer + weight_stride;
- ACL_CHECK(aclrtMemcpyAsync(scale_buffer, scale_stride, scale_tmp_ptr, scale_stride,
- ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));
+ ggml_cann_pool_alloc output_alloc(ctx.pool());
+ void * output_buffer = prepare_f16_buffer(dst, output_alloc, false);
- src0_row.data = weight_buffer;
- src1_row.data = src1_tmp_ptr;
- dst_row.data = dst_tmp_ptr;
- dst_row.src[0] = &src0_row;
- dst_row.src[1] = &src1_row;
+ // Process each batch
+ for (int64_t batch_idx = 0; batch_idx < n_batches; batch_idx++) {
+ // Create index tensor for current batch
+ const size_t index_offset = batch_idx * ids->nb[1];
+ acl_tensor_ptr batch_indices = ggml_cann_create_tensor(ids, ids->ne, ids->nb, 1, ACL_FORMAT_ND, index_offset);
- ggml_cann_mul_mat(ctx, &dst_row);
+ // Select quantized weights using expert indices
+ // Q4_0 stores 2 values per byte, Q8_0 stores 1 value per byte
+ const int64_t weight_d = (type == GGML_TYPE_Q4_0) ? src0->ne[0] / 2 : src0->ne[0];
+ const int64_t weight_m = src0->ne[1];
+ const int64_t weight_n_experts = src0->ne[2];
+
+ int64_t weight_ne[3] = { weight_d, weight_m, weight_n_experts };
+ size_t weight_nb[3] = { sizeof(int8_t), weight_d * sizeof(int8_t), weight_d * weight_m * sizeof(int8_t) };
+
+ acl_tensor_ptr all_weights =
+ ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t), weight_ne, weight_nb, 3);
+
+ int64_t selected_weight_ne[3] = { weight_d, weight_m, n_select_experts };
+ size_t selected_weight_nb[3] = { sizeof(int8_t), weight_d * sizeof(int8_t),
+ weight_d * weight_m * sizeof(int8_t) };
+
+ acl_tensor_ptr selected_weights = ggml_cann_create_tensor(selected_weight_buffer, ACL_INT8, sizeof(int8_t),
+ selected_weight_ne, selected_weight_nb, 3);
+
+ GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, all_weights.get(), 0, batch_indices.get(), selected_weights.get());
+
+ // Select scales using the same expert indices
+ const int64_t scale_d = src0->ne[0] / group_size;
+ int64_t scale_ne[3] = { scale_d, weight_m, weight_n_experts };
+ size_t scale_nb[3] = { scale_elem_size, scale_d * scale_elem_size, scale_d * weight_m * scale_elem_size };
+
+ acl_tensor_ptr all_scales =
+ ggml_cann_create_tensor(scale_data, ACL_FLOAT16, scale_elem_size, scale_ne, scale_nb, 3);
+
+ int64_t selected_scale_ne[3] = { scale_d, weight_m, n_select_experts };
+ size_t selected_scale_nb[3] = { scale_elem_size, scale_d * scale_elem_size,
+ scale_d * weight_m * scale_elem_size };
+
+ acl_tensor_ptr selected_scales = ggml_cann_create_tensor(selected_scale_buffer, ACL_FLOAT16, scale_elem_size,
+ selected_scale_ne, selected_scale_nb, 3);
+
+ GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, all_scales.get(), 0, batch_indices.get(), selected_scales.get());
+
+ // Process each expert for current batch
+ // IndexSelect output layout: [D, M, K] in contiguous format
+ // WeightQuantBatchMatmulV2 expects: [M, D] with row-major stride
+ for (int64_t expert_idx = 0; expert_idx < n_select_experts; expert_idx++) {
+ // Determine input offset: broadcast if src1->ne[1]==1, otherwise use per-expert input
+ const size_t input_offset =
+ (batch_idx * src1->ne[1] + (src1->ne[1] == 1 ? 0 : expert_idx)) * src1->ne[0] * f16_elem_size;
+ const size_t output_offset = (batch_idx * dst->ne[1] + expert_idx) * dst->ne[0] * f16_elem_size;
+
+ // Create weight view for current expert: [D, M, K] -> [M, D]
+ int64_t weight_view_ne[2] = { weight_m, src0->ne[0] };
+ float weight_view_nb[2] = { src0->ne[0] * weight_elem_size, weight_elem_size };
+ const size_t weight_view_offset = expert_idx * selected_weight_nb[2];
+
+ acl_tensor_ptr weight_view =
+ ggml_cann_create_tensor(selected_weight_buffer, ggml_cann_type_mapping(type), weight_elem_size,
+ weight_view_ne, weight_view_nb, 2, ACL_FORMAT_ND, weight_view_offset);
+
+ // Create scale view for current expert: [D, M, K] -> [M, D]
+ int64_t scale_view_ne[2] = { weight_m, scale_d };
+ size_t scale_view_nb[2] = { selected_scale_nb[1], selected_scale_nb[0] };
+ const size_t scale_view_offset = expert_idx * selected_scale_nb[2];
+
+ acl_tensor_ptr scale_view =
+ ggml_cann_create_tensor(selected_scale_buffer, ACL_FLOAT16, scale_elem_size, scale_view_ne,
+ scale_view_nb, 2, ACL_FORMAT_ND, scale_view_offset);
+
+ // Create input activation tensor [D, 1]
+ int64_t input_ne[2] = { src1->ne[0], 1 };
+ size_t input_nb[2] = { f16_elem_size, src1->ne[0] * f16_elem_size };
+
+ acl_tensor_ptr input_tensor = ggml_cann_create_tensor(input_buffer, ACL_FLOAT16, f16_elem_size, input_ne,
+ input_nb, 2, ACL_FORMAT_ND, input_offset);
+
+ // Create output tensor [M, 1]
+ int64_t output_ne[2] = { dst->ne[0], 1 };
+ size_t output_nb[2] = { f16_elem_size, dst->ne[0] * f16_elem_size };
+
+ acl_tensor_ptr output_tensor = ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, f16_elem_size, output_ne,
+ output_nb, 2, ACL_FORMAT_ND, output_offset);
+
+ // Perform quantized matrix multiplication
+ GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, input_tensor.get(), weight_view.get(),
+ scale_view.get(), nullptr, nullptr, nullptr, nullptr, group_size,
+ output_tensor.get());
}
}
- return;
+
+ // Cast output back to original type if we used a temporary F16 buffer
+ if (dst->type != GGML_TYPE_F16) {
+ int64_t ne[GGML_MAX_DIMS];
+ size_t nb[GGML_MAX_DIMS] = { f16_elem_size };
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ ne[i] = dst->ne[i];
+ if (i > 0) {
+ nb[i] = nb[i - 1] * ne[i - 1];
+ }
+ }
+
+ acl_tensor_ptr f16_output =
+ ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, f16_elem_size, ne, nb, GGML_MAX_DIMS);
+ acl_tensor_ptr dst_tensor = ggml_cann_create_tensor(dst);
+
+ aclnn_cast(ctx, f16_output.get(), dst_tensor.get(), ggml_cann_type_mapping(dst->type));
+ }
}
void ggml_cann_mul_mat_id(ggml_backend_cann_context & ctx, ggml_tensor * dst) {
diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp
index 6b2dbdd359..3f3de9f0bc 100644
--- a/ggml/src/ggml-cann/ggml-cann.cpp
+++ b/ggml/src/ggml-cann/ggml-cann.cpp
@@ -794,19 +794,44 @@ struct ggml_backend_cann_buffer_context {
~ggml_backend_cann_buffer_context() { ACL_CHECK(aclrtFree(dev_ptr)); }
};
+// cann buffer type
/**
- * @brief Check if a buffer is a CANN buffer.
- *
- * This function checks if a given buffer is a CANN buffer by comparing its
- * `get_name` function pointer to `ggml_backend_cann_buffer_get_name`.
- *
- * @param buffer The buffer to check.
- * @return true if the buffer is a CANN buffer, false otherwise.
+ * @brief Structure representing context information for a specific backend
+ * buffer type.
*/
-static bool ggml_backend_buft_is_cann(ggml_backend_buffer_type_t buft);
+struct ggml_backend_cann_buffer_type_context {
+ int32_t device; /**< Device identifier associated with the buffer context. */
+ std::string name; /**< Name associated with the buffer context. */
+};
-static bool ggml_backend_buffer_is_cann(ggml_backend_buffer_t buffer) {
- return ggml_backend_buft_is_cann(buffer->buft);
+/**
+ * @brief Retrieves the name associated with a CANN buffer type.
+ *
+ * This function returns the descriptive name associated with the specified
+ * CANN buffer type context.
+ *
+ * @param buft Pointer to the buffer type context.
+ * @return Const pointer to the C-style string containing the name.
+ */
+static const char * ggml_backend_cann_buffer_type_name(ggml_backend_buffer_type_t buft) {
+ ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;
+
+ return buft_ctx->name.c_str();
+}
+
+/**
+ * @brief Checks if the backend buffer type is associated with the CANN backend.
+ *
+ * This function checks whether the provided backend buffer type is associated
+ * with the CANN backend based on the comparison of its name retrieval function
+ * pointer.
+ *
+ * @param buft Pointer to the backend buffer type to check.
+ * @return bool Returns true if the buffer type is associated with the CANN
+ * backend, otherwise false.
+ */
+static bool ggml_backend_buft_is_cann(ggml_backend_buffer_type_t buft) {
+ return buft->iface.get_name == ggml_backend_cann_buffer_type_name;
}
/**
@@ -1271,7 +1296,7 @@ static void ggml_backend_cann_buffer_get_tensor(ggml_backend_buffer_t buffer,
static bool ggml_backend_cann_buffer_cpy_tensor(ggml_backend_buffer_t buffer,
const ggml_tensor * src,
ggml_tensor * dst) {
- if (ggml_backend_buffer_is_cann(src->buffer)) {
+ if (ggml_backend_buft_is_cann(src->buffer->buft)) {
ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context;
ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context;
@@ -1335,31 +1360,6 @@ static const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = {
/* .reset = */ NULL,
};
-// cann buffer type
-/**
- * @brief Structure representing context information for a specific backend
- * buffer type.
- */
-struct ggml_backend_cann_buffer_type_context {
- int32_t device; /**< Device identifier associated with the buffer context. */
- std::string name; /**< Name associated with the buffer context. */
-};
-
-/**
- * @brief Retrieves the name associated with a CANN buffer type.
- *
- * This function returns the descriptive name associated with the specified
- * CANN buffer type context.
- *
- * @param buft Pointer to the buffer type context.
- * @return Const pointer to the C-style string containing the name.
- */
-static const char * ggml_backend_cann_buffer_type_name(ggml_backend_buffer_type_t buft) {
- ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;
-
- return buft_ctx->name.c_str();
-}
-
/**
* @brief Allocates a new CANN buffer of the specified type and size.
*
@@ -1997,7 +1997,7 @@ static bool ggml_backend_cann_cpy_tensor_async(ggml_backend_t backend_src,
GGML_ASSERT(!is_matmul_weight((const ggml_tensor *) src));
- if (!ggml_backend_buffer_is_cann(src->buffer) || !ggml_backend_buffer_is_cann(dst->buffer)) {
+ if (!ggml_backend_buft_is_cann(src->buffer->buft) || !ggml_backend_buft_is_cann(dst->buffer->buft)) {
return false;
}
@@ -2523,21 +2523,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
GGML_UNUSED(dev);
}
-/**
- * @brief Checks if the backend buffer type is associated with the CANN backend.
- *
- * This function checks whether the provided backend buffer type is associated
- * with the CANN backend based on the comparison of its name retrieval function
- * pointer.
- *
- * @param buft Pointer to the backend buffer type to check.
- * @return bool Returns true if the buffer type is associated with the CANN
- * backend, otherwise false.
- */
-static bool ggml_backend_buft_is_cann(ggml_backend_buffer_type_t buft) {
- return buft->iface.get_name == ggml_backend_cann_buffer_type_name;
-}
-
/**
* @brief Records an event on the CANN backend stream.
*
diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt
index 7622d0bf49..3dc948e4d8 100644
--- a/ggml/src/ggml-cpu/CMakeLists.txt
+++ b/ggml/src/ggml-cpu/CMakeLists.txt
@@ -9,6 +9,11 @@ function(ggml_add_cpu_backend_features cpu_name arch)
target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE ${ARGN})
target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE GGML_BACKEND_DL GGML_BACKEND_BUILD GGML_BACKEND_SHARED)
set_target_properties(${GGML_CPU_FEATS_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+ # Disable LTO for the feature detection code to prevent cross-module optimization
+ # from inlining architecture-specific instructions into the score function.
+ # Without this, LTO can cause SIGILL when loading backends on older CPUs
+ # (e.g., loading power10 backend on power9 crashes before feature check runs).
+ target_compile_options(${GGML_CPU_FEATS_NAME} PRIVATE -fno-lto)
target_link_libraries(${cpu_name} PRIVATE ${GGML_CPU_FEATS_NAME})
endfunction()
@@ -569,27 +574,24 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
cmake_policy(SET CMP0135 NEW)
endif()
+ # TODO: Use FetchContent_MakeAvailable with EXCLUDE_FROM_ALL after bumping minimum CMake version to 3.28+
+ # Using FetchContent_Populate instead to avoid EXCLUDE_FROM_ALL which requires CMake 3.28
FetchContent_Declare(KleidiAI_Download
URL ${KLEIDIAI_DOWNLOAD_URL}
DOWNLOAD_EXTRACT_TIMESTAMP NEW
URL_HASH MD5=${KLEIDIAI_ARCHIVE_MD5})
- FetchContent_MakeAvailable(KleidiAI_Download)
FetchContent_GetProperties(KleidiAI_Download
SOURCE_DIR KLEIDIAI_SRC
POPULATED KLEIDIAI_POPULATED)
if (NOT KLEIDIAI_POPULATED)
- message(FATAL_ERROR "KleidiAI source downloaded failed.")
+ FetchContent_Populate(KleidiAI_Download)
+ FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC)
endif()
add_compile_definitions(GGML_USE_CPU_KLEIDIAI)
- # Remove kleidiai target after fetching it
- if (TARGET kleidiai)
- set_target_properties(kleidiai PROPERTIES EXCLUDE_FROM_ALL TRUE)
- endif()
-
list(APPEND GGML_CPU_SOURCES
ggml-cpu/kleidiai/kleidiai.cpp
ggml-cpu/kleidiai/kernels.cpp
diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h
index 427c1146e4..55526e6fb3 100644
--- a/ggml/src/ggml-cpu/arch-fallback.h
+++ b/ggml/src/ggml-cpu/arch-fallback.h
@@ -43,6 +43,7 @@
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
@@ -55,7 +56,8 @@
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
-# define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
@@ -76,6 +78,7 @@
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
@@ -84,6 +87,7 @@
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
@@ -107,6 +111,7 @@
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
@@ -119,6 +124,7 @@
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
@@ -143,6 +149,7 @@
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
@@ -155,6 +162,7 @@
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
@@ -163,15 +171,9 @@
#elif defined(__riscv)
// quants.c
#define quantize_row_q8_K_generic quantize_row_q8_K
-#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
-#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
-#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K
#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K
-#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K
-#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K
-#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0
#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K
#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
@@ -186,6 +188,7 @@
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
@@ -197,6 +200,7 @@
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
@@ -227,6 +231,7 @@
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
@@ -239,6 +244,7 @@
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
@@ -271,6 +277,7 @@
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
@@ -283,6 +290,7 @@
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp
index 99bb70274c..3a3b32efb2 100644
--- a/ggml/src/ggml-cpu/arch/arm/repack.cpp
+++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp
@@ -1072,6 +1072,195 @@ void ggml_gemv_q5_K_8x8_q8_K(int n,
ggml_gemv_q5_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);
}
+void ggml_gemv_q6_K_8x4_q8_K(int n,
+ float * GGML_RESTRICT s,
+ size_t bs,
+ const void * GGML_RESTRICT vx,
+ const void * GGML_RESTRICT vy,
+ int nr,
+ int nc) {
+ constexpr int qk = QK_K;
+ const int nb = n / qk;
+
+ constexpr int ncols_interleaved = 8;
+ constexpr int blocklen = 4;
+
+ assert(n % qk == 0);
+ assert(nc % ncols_interleaved == 0);
+
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+ constexpr int col_groups = ncols_interleaved / 4;
+ const uint8x16_t m4b = vdupq_n_u8(0x0f);
+ const uint8x16_t mask_lo = vdupq_n_u8(0x03);
+ const uint8x16_t mask_hi = vdupq_n_u8(0x30);
+
+ // 1x8 tile = 2 x 4
+ float32x4_t acc_f32[2];
+
+ const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;
+
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);
+
+ for (int i = 0; i < col_groups; i++) {
+ acc_f32[i] = vdupq_n_f32(0);
+ }
+
+ for (int b = 0; b < nb; b++) {
+ float32x4_t q6_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d)); // d0 d1 d2 d3
+ float32x4_t q6_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4)); // d4 d5 d6 d7
+ float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);
+ float32x4_t sb_scale_0 = vmulq_f32(q6_d_0, q8_d);
+ float32x4_t sb_scale_1 = vmulq_f32(q6_d_1, q8_d);
+
+ int32x4_t acc[col_groups];
+ for (int i = 0; i < col_groups; i++) {
+ acc[i] = vdupq_n_s32(0);
+ }
+
+ // Load all 16 scales once and widen to int16 (Q6_K has 16 scales per block)
+ // Reused for bias and dequantization later
+ int16_t q6_scales[16 * 8];
+ for (int i = 0; i < 16; i++) {
+ int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));
+ vst1q_s16(q6_scales + i * 8, scales);
+ }
+
+ // Compute bias per column using q8 bsums and preloaded scales to skip the -32 shift
+ int32x4_t bias_lo = vdupq_n_s32(0);
+ int32x4_t bias_hi = vdupq_n_s32(0);
+
+ // Load bsums in chunks of 4 to process with vectorized operations
+ for (int i = 0; i < 16; i += 4) {
+ int16x4_t bsums_vec = vld1_s16(q8_ptr[b].bsums + i);
+ int16x4_t scales_lo_0 = vld1_s16(q6_scales + (i + 0) * 8);
+ int16x4_t scales_hi_0 = vld1_s16(q6_scales + (i + 0) * 8 + 4);
+ int16x4_t scales_lo_1 = vld1_s16(q6_scales + (i + 1) * 8);
+ int16x4_t scales_hi_1 = vld1_s16(q6_scales + (i + 1) * 8 + 4);
+ int16x4_t scales_lo_2 = vld1_s16(q6_scales + (i + 2) * 8);
+ int16x4_t scales_hi_2 = vld1_s16(q6_scales + (i + 2) * 8 + 4);
+ int16x4_t scales_lo_3 = vld1_s16(q6_scales + (i + 3) * 8);
+ int16x4_t scales_hi_3 = vld1_s16(q6_scales + (i + 3) * 8 + 4);
+
+ bias_lo = vmlal_lane_s16(bias_lo, scales_lo_0, bsums_vec, 0);
+ bias_hi = vmlal_lane_s16(bias_hi, scales_hi_0, bsums_vec, 0);
+ bias_lo = vmlal_lane_s16(bias_lo, scales_lo_1, bsums_vec, 1);
+ bias_hi = vmlal_lane_s16(bias_hi, scales_hi_1, bsums_vec, 1);
+ bias_lo = vmlal_lane_s16(bias_lo, scales_lo_2, bsums_vec, 2);
+ bias_hi = vmlal_lane_s16(bias_hi, scales_hi_2, bsums_vec, 2);
+ bias_lo = vmlal_lane_s16(bias_lo, scales_lo_3, bsums_vec, 3);
+ bias_hi = vmlal_lane_s16(bias_hi, scales_hi_3, bsums_vec, 3);
+ }
+ bias_lo = vshlq_n_s32(bias_lo, 5);
+ bias_hi = vshlq_n_s32(bias_hi, 5);
+
+ // Process two 128-value halves per superblock
+ for (int half = 0; half < 2; half++) {
+ const uint8_t * ql_base = q6_ptr[b].ql + half * 512;
+ const uint8_t * qh_base = q6_ptr[b].qh + half * 256;
+
+ // A subblock (sb) is a set of weights that share the scale
+ // Since q6_K scales are per 16 elements
+ // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)
+ for (int sb = 0; sb < QK_K / 64; sb++) {
+ const int8_t * q8_base_l = q8_ptr[b].qs + half * 128 + sb * 16;
+ const int8_t * q8_base_h = q8_base_l + 64;
+
+ // Load and duplicate q8 values (each register covers four interleaved columns of q6)
+ int8x16_t q8_l[4];
+ int8x16_t q8_h[4];
+ for (int i = 0; i < 4; i++) {
+ q8_l[i] = (int8x16_t) vld1q_dup_s32((const int32_t *) (q8_base_l + i * 4));
+ q8_h[i] = (int8x16_t) vld1q_dup_s32((const int32_t *) (q8_base_h + i * 4));
+ }
+
+ const int ql_off_base = sb * QK_K / 2;
+ const int qh_off_base = ql_off_base & 255; // wraps after 256 bytes
+
+ // Load 4 vectors at once (64 bytes each for ql_0, ql_1, qh_0, qh_1)
+ uint8x16x4_t q6_ql_0 = vld1q_u8_x4(ql_base + ql_off_base);
+ uint8x16x4_t q6_ql_1 = vld1q_u8_x4(ql_base + ql_off_base + 64);
+ uint8x16x4_t q6_qh_0 = vld1q_u8_x4(qh_base + qh_off_base);
+ uint8x16x4_t q6_qh_1 = vld1q_u8_x4(qh_base + qh_off_base + 64);
+
+ // Adjust qh for subblocks 2 and 3 (shift right by 2)
+ if (sb > 1) {
+ q6_qh_0.val[0] = vshrq_n_u8(q6_qh_0.val[0], 2);
+ q6_qh_0.val[1] = vshrq_n_u8(q6_qh_0.val[1], 2);
+ q6_qh_0.val[2] = vshrq_n_u8(q6_qh_0.val[2], 2);
+ q6_qh_0.val[3] = vshrq_n_u8(q6_qh_0.val[3], 2);
+ q6_qh_1.val[0] = vshrq_n_u8(q6_qh_1.val[0], 2);
+ q6_qh_1.val[1] = vshrq_n_u8(q6_qh_1.val[1], 2);
+ q6_qh_1.val[2] = vshrq_n_u8(q6_qh_1.val[2], 2);
+ q6_qh_1.val[3] = vshrq_n_u8(q6_qh_1.val[3], 2);
+ }
+
+ const uint8x16_t q6_ql[8] = { q6_ql_0.val[0], q6_ql_0.val[1], q6_ql_0.val[2], q6_ql_0.val[3],
+ q6_ql_1.val[0], q6_ql_1.val[1], q6_ql_1.val[2], q6_ql_1.val[3] };
+ const uint8x16_t q6_qh[8] = { q6_qh_0.val[0], q6_qh_0.val[1], q6_qh_0.val[2], q6_qh_0.val[3],
+ q6_qh_1.val[0], q6_qh_1.val[1], q6_qh_1.val[2], q6_qh_1.val[3] };
+
+ // Process column groups (0-3, 4-7)
+ for (int g = 0; g < col_groups; g++) {
+ int32x4_t sb_acc_l = vdupq_n_s32(0);
+ int32x4_t sb_acc_h = vdupq_n_s32(0);
+
+ for (int chunk = 0; chunk < 4; chunk++) {
+ const int idx = chunk * 2 + g;
+
+ const uint8x16_t q6_qs_l = q6_ql[idx];
+ const uint8x16_t q6_qs_h = q6_qh[idx];
+
+ // Extract high 2 bits for upper nibble reconstruction
+ const uint8x16_t q6_qs_hh = vandq_u8(q6_qs_h, mask_hi);
+
+ // q6 = (low4 | high2<<4), without -32 bias (handled via bsums)
+ const int8x16_t q6_l =
+ vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_l, m4b), vandq_u8(q6_qs_h, mask_lo), 4));
+ const int8x16_t q6_h = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_l, 4), q6_qs_hh));
+
+ sb_acc_l = vdotq_s32(sb_acc_l, q6_l, q8_l[chunk]);
+ sb_acc_h = vdotq_s32(sb_acc_h, q6_h, q8_h[chunk]);
+ }
+
+ const int scale_idx_l = half * 8 + sb;
+ const int scale_idx_h = half * 8 + sb + 4;
+
+ const int32x4_t scale_vec_l = vmovl_s16(vld1_s16(q6_scales + scale_idx_l * 8 + g * 4));
+ const int32x4_t scale_vec_h = vmovl_s16(vld1_s16(q6_scales + scale_idx_h * 8 + g * 4));
+
+ acc[g] = vmlaq_s32(acc[g], sb_acc_l, scale_vec_l);
+ acc[g] = vmlaq_s32(acc[g], sb_acc_h, scale_vec_h);
+ }
+ }
+ } // for half
+
+ // Bias correction
+ acc[0] = vsubq_s32(acc[0], bias_lo);
+ acc[1] = vsubq_s32(acc[1], bias_hi);
+
+ // Apply superblock scale (no mins for q6_K)
+ // acc[g] has [c0, c1, c2, c3]
+ float32x4_t w_0123 = vmulq_f32(vcvtq_f32_s32(acc[0]), sb_scale_0);
+ float32x4_t w_4567 = vmulq_f32(vcvtq_f32_s32(acc[1]), sb_scale_1);
+
+ acc_f32[0] = vaddq_f32(acc_f32[0], w_0123);
+ acc_f32[1] = vaddq_f32(acc_f32[1], w_4567);
+ } // for b
+
+ int base = x * ncols_interleaved;
+ vst1q_f32(s + base, acc_f32[0]);
+ vst1q_f32(s + base + 4, acc_f32[1]);
+ } // for x
+ return;
+#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+ ggml_gemv_q6_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);
+}
+
void ggml_gemv_q6_K_8x8_q8_K(int n,
float * GGML_RESTRICT s,
size_t bs,
@@ -1177,15 +1366,14 @@ void ggml_gemv_q6_K_8x8_q8_K(int n,
q8_h[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base_h + i * 8));
}
- // TODO: Test other qh repack patterns to reduce loads
const int ql_off_base = sb * QK_K / 2;
const int qh_off_base = ql_off_base & 255; // wraps after 256 bytes
// Load 4 vectors at once (64 bytes each for ql_0, ql_1, qh_0, qh_1)
- ggml_uint8x16x4_t q6_ql_0 = ggml_vld1q_u8_x4(ql_base + ql_off_base);
- ggml_uint8x16x4_t q6_ql_1 = ggml_vld1q_u8_x4(ql_base + ql_off_base + 64);
- ggml_uint8x16x4_t q6_qh_0 = ggml_vld1q_u8_x4(qh_base + qh_off_base);
- ggml_uint8x16x4_t q6_qh_1 = ggml_vld1q_u8_x4(qh_base + qh_off_base + 64);
+ uint8x16x4_t q6_ql_0 = vld1q_u8_x4(ql_base + ql_off_base);
+ uint8x16x4_t q6_ql_1 = vld1q_u8_x4(ql_base + ql_off_base + 64);
+ uint8x16x4_t q6_qh_0 = vld1q_u8_x4(qh_base + qh_off_base);
+ uint8x16x4_t q6_qh_1 = vld1q_u8_x4(qh_base + qh_off_base + 64);
// Adjust qh for subblocks 2 and 3 (shift right by 2)
if (sb > 1) {
@@ -3038,6 +3226,316 @@ void ggml_gemm_q4_K_8x8_q8_K(int n,
UNUSED(ncols_interleaved);
UNUSED(blocklen);
+#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
+ if (svcntb() * 8 == 256) {
+ constexpr int q8_k_blocklen = 4;
+ const svuint8_t m4b_1 = svdup_n_u8(0x0f);
+ // 8 accumulators: 2 row pairs × 4 col pairs
+ svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67;
+ uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 };
+ svbool_t pg = svptrue_pat_b32(SV_VL8);
+ svuint32_t idx = svld1(pg, idx_arr);
+
+ static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7};
+ svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data);
+
+ for (int y = 0; y < nr / q8_k_blocklen; y++) {
+ const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);
+
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);
+
+ acc_f32_01 = svdup_n_f32(0);
+ acc_f32_23 = svdup_n_f32(0);
+ acc_f32_45 = svdup_n_f32(0);
+ acc_f32_67 = svdup_n_f32(0);
+
+ for (int b = 0; b < nb; b++) {
+ // bsums pairs belongs to the same q8_k subblock
+ // 64 elemnts loaded and made sum of 0-7 and 8-15 sum || 16-23 and 24 - 31 sum
+ const int16x8_t bsums[4]{
+ vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),
+ vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),
+ vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),
+ vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),
+ };
+
+ int32_t bsums_arr32[4][8];
+
+ for (int q8_row = 0; q8_row < 4; q8_row++) {
+ int16x8_t v16 = bsums[q8_row];
+
+ // low 4
+ int32x4_t v32_lo = vmovl_s16(vget_low_s16(v16));
+ vst1q_s32(&bsums_arr32[q8_row][0], v32_lo);
+
+ // high 4
+ int32x4_t v32_hi = vmovl_s16(vget_high_s16(v16));
+ vst1q_s32(&bsums_arr32[q8_row][4], v32_hi);
+ }
+
+ svint32_t sb_acc_0 = svdup_n_s32(0);
+ svint32_t sb_acc_2 = svdup_n_s32(0);
+
+ svint32_t acc_00 = svdup_n_s32(0);
+ svint32_t acc_11 = svdup_n_s32(0);
+ svint32_t acc_22 = svdup_n_s32(0);
+ svint32_t acc_33 = svdup_n_s32(0);
+ svint32_t acc_44 = svdup_n_s32(0);
+ svint32_t acc_55 = svdup_n_s32(0);
+ svint32_t acc_66 = svdup_n_s32(0);
+ svint32_t acc_77 = svdup_n_s32(0);
+
+ svint32_t bias_acc_00 = svdup_n_s32(0);
+ svint32_t bias_acc_22 = svdup_n_s32(0);
+ svint32_t bias_acc_44 = svdup_n_s32(0);
+ svint32_t bias_acc_66 = svdup_n_s32(0);
+
+ for (int sb = 0; sb < QK_K / 64; sb++) {
+ // Need scales for the low and high nibbles
+ // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total
+ svint32_t block_scale_0, block_scale_1, block_scale_2, block_scale_3;
+ svint32_t q4sb_mins_0, q4sb_mins_1;
+ {
+ // 2-superblock I am working on
+ const int offset = sb * 24 + 0 * 12;
+ const uint8_t * scales_in = &q4_ptr[b].scales[offset];
+
+ const int offset1 = sb * 24 + 12;
+ const uint8_t * scales_in1 = &q4_ptr[b].scales[offset1];
+
+ constexpr uint32_t kmask1 = 0x3f3f3f3f;
+ constexpr uint32_t kmask2 = 0x0f0f0f0f;
+ constexpr uint32_t kmask3 = 0x03030303;
+ constexpr uint8_t scales_size = 12;
+
+ uint32_t sm[3];
+ memcpy(sm, scales_in, scales_size);
+
+ uint32_t sm1[3];
+ memcpy(sm1, scales_in1, scales_size);
+
+ const uint32_t mins_0_3 = sm[1] & kmask1;
+ const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4);
+
+ const uint32_t mins_0_3_1 = sm1[1] & kmask1;
+ const uint32_t mins_4_7_1 = ((sm1[2] >> 4) & kmask2) | (((sm1[1] >> 6) & kmask3) << 4);
+
+ svuint32_t mins_u32_temp = svzip1_u32(svdup_n_u32(mins_0_3), svdup_n_u32(mins_4_7));
+ svuint32_t mins_u32_temp_1 = svzip1_u32(svdup_n_u32(mins_0_3_1), svdup_n_u32(mins_4_7_1));
+
+ /* reinterpret u32 → u8 */
+ svuint8_t mins_u8 = svreinterpret_u8_u32(mins_u32_temp);
+ svuint8_t mins_u8_1 = svreinterpret_u8_u32(mins_u32_temp_1);
+
+ /* widen u8 → u16->u32 (lower half only) */
+ svuint32_t mins_u16 = svunpklo_u32(svunpklo_u16(mins_u8));
+ svuint32_t mins_u16_1 = svunpklo_u32(svunpklo_u16(mins_u8_1));
+
+ q4sb_mins_0 = svreinterpret_s32_u32(mins_u16);
+ q4sb_mins_1 = svreinterpret_s32_u32(mins_u16_1);
+
+ uint32_t scales_u32_0 = sm[0] & kmask1;
+ uint32_t scales_u32_1 = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4);
+ uint32_t scales_u32_2 = sm1[0] & kmask1;
+ uint32_t scales_u32_3 = (sm1[2] & kmask2) | (((sm1[0] >> 6) & kmask3) << 4);
+
+ svuint32_t S01 = svdup_n_u32(scales_u32_0);
+ svuint32_t S23 = svdup_n_u32(scales_u32_1);
+ svuint32_t R01 = svdup_n_u32(scales_u32_2);
+ svuint32_t R23 = svdup_n_u32(scales_u32_3);
+
+ svint8_t S01_b = svreinterpret_s8_u32(S01);
+ svint8_t S23_b = svreinterpret_s8_u32(S23);
+ svint8_t R01_b = svreinterpret_s8_u32(R01);
+ svint8_t R23_b = svreinterpret_s8_u32(R23);
+
+ svint32_t S01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S01_b, S01_b)));
+ svint32_t R01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R01_b, R01_b)));
+ svint32_t S23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S23_b, S23_b)));
+ svint32_t R23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R23_b, R23_b)));
+
+ block_scale_0 = svtbl_s32(svzip1_s32(S01_d, R01_d), idx);
+ block_scale_1 = svtbl_s32(svzip2_s32(S01_d, R01_d), idx);
+ block_scale_2 = svtbl_s32(svzip1_s32(S23_d, R23_d), idx);
+ block_scale_3 = svtbl_s32(svzip2_s32(S23_d, R23_d), idx);
+ }
+
+ const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256;
+
+ // Load 32-byte per row pair, 1 subblock each time
+ // predicate for activating higher lanes for 16 int8 elements
+ const svbool_t ph16 = svptrue_pat_b8(SV_VL16);
+ // predicate for activating lower lanes for 16 int8 elements
+ const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16);
+
+ svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 0), svld1_s8(pl16, q8_base_1 + 112));
+ svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 32), svld1_s8(pl16, q8_base_1 + 144));
+ svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 64), svld1_s8(pl16, q8_base_1 + 176));
+ svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 96), svld1_s8(pl16, q8_base_1 + 208));
+
+ svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 16), svld1_s8(pl16, q8_base_1 + 128));
+ svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 48), svld1_s8(pl16, q8_base_1 + 160));
+ svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 80), svld1_s8(pl16, q8_base_1 + 192));
+ svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 112), svld1_s8(pl16, q8_base_1 + 224));
+
+ // Q4s columns iterated in pairs (01, 23, 45, 67)
+ for (int cp = 0; cp < ncols_interleaved / 2; cp++) {
+
+ sb_acc_0 = svdup_n_s32(0);
+ sb_acc_2 = svdup_n_s32(0);
+
+ svuint8_t q4_qs_cp_00 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 0);
+ svuint8_t q4_qs_cp_01 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 64);
+ svuint8_t q4_qs_cp_02 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 128);
+ svuint8_t q4_qs_cp_03 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 192);
+
+ svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_00, m4b_1), 4));
+ svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_01, m4b_1), 4));
+ svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_02, m4b_1), 4));
+ svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_03, m4b_1), 4));
+
+ sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_0);
+ sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_2);
+
+ sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_4);
+ sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_6);
+
+ sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_1);
+ sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_3);
+
+ sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_5);
+ sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_7);
+
+ if(cp == 0) {
+ acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0, block_scale_0);
+ acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2, block_scale_0);
+ }
+ if(cp == 1) {
+ acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0, block_scale_1);
+ acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2, block_scale_1);
+ }
+ if(cp == 2) {
+ acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0, block_scale_2);
+ acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2, block_scale_2);
+ }
+ if(cp == 3) {
+ acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0, block_scale_3);
+ acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2, block_scale_3);
+ }
+ }
+
+ bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][0]), q4sb_mins_0);
+ bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][1]), q4sb_mins_1);
+
+ bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][2]), q4sb_mins_0);
+ bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][3]), q4sb_mins_1);
+
+ bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][4]), q4sb_mins_0);
+ bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][5]), q4sb_mins_1);
+
+ bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][6]), q4sb_mins_0);
+ bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][7]), q4sb_mins_1);
+ } // for sb
+
+
+ acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4));
+ acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4));
+ acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4));
+ acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4));
+ acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4));
+ acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4));
+ acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4));
+ acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4));
+
+ svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1);
+ svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1);
+
+ svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1);
+ svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1);
+
+ // Broadcast q8 scalar
+ svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]);
+
+ svfloat32_t q4_dmin_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].dmin), svdup_f16(0)));
+
+ svfloat32_t q4_d_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].d), svdup_f16(0)));
+
+ svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);
+ svfloat32_t dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);
+
+ acc_f32_01 = svmls_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_00), dmins1);
+ acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1);
+
+ q8_d = svdup_f32(q8_ptr[b].d[1]);
+
+ scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);
+ dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);
+
+ acc_f32_23 = svmls_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_22), dmins1);
+ acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1);
+
+ q8_d = svdup_f32(q8_ptr[b].d[2]);
+
+
+ scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);
+ dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);
+
+ acc_f32_45 = svmls_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_44), dmins1);
+ acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1);
+
+ q8_d = svdup_f32(q8_ptr[b].d[3]);
+
+ scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);
+ dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);
+
+ acc_f32_67 = svmls_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_66), dmins1);
+ acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1);
+
+ } // for b
+
+ // With the previous reorder, the tile is already in the correct memory layout.
+ // Predicate for exactly 4 lanes
+ svbool_t pg4 = svptrue_pat_b32(SV_VL4);
+ for (int i = 0; i < q8_k_blocklen; i++) {
+ int row = y * q8_k_blocklen + i;
+ for (int j = 0; j < 2; j++) {
+ int col = x * ncols_interleaved + j * 4;
+ int offset = row * bs + col;
+
+ if (i == 0 && j == 0) {
+ // acc_f32_0 → lower half of acc_f32_01
+ svst1_f32(pg4, s + offset, acc_f32_01);
+ } else if (i == 0 && j == 1) {
+ // acc_f32_1 → upper half of acc_f32_01
+ svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4));
+ } else if (i == 1 && j == 0) {
+ // acc_f32_2
+ svst1_f32(pg4, s + offset, acc_f32_23);
+ } else if (i == 1 && j == 1) {
+ // acc_f32_3
+ svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4));
+ } else if (i == 2 && j == 0) {
+ // acc_f32_4
+ svst1_f32(pg4, s + offset, acc_f32_45);
+ } else if (i == 2 && j == 1) {
+ // acc_f32_5
+ svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4));
+ } else if (i == 3 && j == 0) {
+ // acc_f32_6
+ svst1_f32(pg4, s + offset, acc_f32_67);
+ } else if (i == 3 && j == 1) {
+ // acc_f32_7
+ svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4));
+ }
+ }
+ }
+ } // for x
+ } // for y
+ return;
+ }
+#endif // SVE compile-time end
+
#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
constexpr int q8_k_blocklen = 4;
const uint8x16_t m4b = vdupq_n_u8(0x0f);
@@ -3474,6 +3972,208 @@ void ggml_gemm_q5_K_8x8_q8_K(int n,
ggml_gemm_q5_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);
}
+void ggml_gemm_q6_K_8x4_q8_K(int n,
+ float * GGML_RESTRICT s,
+ size_t bs,
+ const void * GGML_RESTRICT vx,
+ const void * GGML_RESTRICT vy,
+ int nr,
+ int nc) {
+ constexpr int qk = QK_K;
+ const int nb = n / qk;
+
+ constexpr int ncols_interleaved = 8;
+ constexpr int blocklen = 4;
+
+ assert(n % qk == 0);
+ assert(nr % 4 == 0);
+ assert(nc % ncols_interleaved == 0);
+
+ UNUSED(nb);
+ UNUSED(ncols_interleaved);
+ UNUSED(blocklen);
+
+#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+ constexpr int q8_k_blocklen = 4;
+ constexpr int col_groups = ncols_interleaved / 4;
+ constexpr int acc_size = q8_k_blocklen * col_groups; // 4 rows, 2 column groups
+ const uint8x16_t m4b = vdupq_n_u8(0x0f);
+ const uint8x16_t mask_lo = vdupq_n_u8(0x03);
+ const uint8x16_t mask_hi = vdupq_n_u8(0x30);
+ const int8x16_t m32s = vdupq_n_s8(32);
+
+ float32x4_t acc_f32[acc_size];
+
+ for (int y = 0; y < nr / q8_k_blocklen; y++) {
+ const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);
+
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);
+
+ for (int i = 0; i < acc_size; i++) {
+ acc_f32[i] = vdupq_n_f32(0);
+ }
+
+ for (int b = 0; b < nb; b++) {
+ float32x4_t q6_d_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d));
+ float32x4_t q6_d_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4));
+ float32x4_t q8_d_0123 = vld1q_f32(q8_ptr[b].d);
+
+ float32x4_t sbd_scale_0123[q8_k_blocklen];
+ float32x4_t sbd_scale_4567[q8_k_blocklen];
+
+ sbd_scale_0123[0] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 0);
+ sbd_scale_4567[0] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 0);
+ sbd_scale_0123[1] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 1);
+ sbd_scale_4567[1] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 1);
+ sbd_scale_0123[2] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 2);
+ sbd_scale_4567[2] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 2);
+ sbd_scale_0123[3] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 3);
+ sbd_scale_4567[3] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 3);
+
+ int32x4_t acc_s32[acc_size];
+ for (int i = 0; i < acc_size; i++) {
+ acc_s32[i] = vdupq_n_s32(0);
+ }
+
+ int16_t q6_scales[8 * 16];
+ for (int i = 0; i < 16; i++) {
+ int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));
+ vst1q_s16(q6_scales + i * 8, scales);
+ }
+
+ for (int half = 0; half < 2; half++) {
+ const uint8_t * ql_base = q6_ptr[b].ql + half * 512;
+ const uint8_t * qh_base = q6_ptr[b].qh + half * 256;
+
+ for (int sb = 0; sb < QK_K / 64; sb++) {
+ int32x4_t acc_lo[acc_size];
+ int32x4_t acc_hi[acc_size];
+ for (int i = 0; i < acc_size; i++) {
+ acc_lo[i] = vdupq_n_s32(0);
+ acc_hi[i] = vdupq_n_s32(0);
+ }
+
+ const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;
+ const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;
+
+ // 4 rows * 16 elements per scale
+ // 4 reads of 16 bytes each
+ constexpr int reads_per_sb = 4;
+ int8x16_t q8_l[reads_per_sb];
+ int8x16_t q8_h[reads_per_sb];
+ for (int k = 0; k < reads_per_sb; k++) {
+ q8_l[k] = vld1q_s8(q8_base_l + 16 * k);
+ q8_h[k] = vld1q_s8(q8_base_h + 16 * k);
+ }
+
+ const int ql_off_base = sb * QK_K / 2;
+ const int qh_off_base = ql_off_base & 255;
+
+ uint8x16_t q6_ql_0123[reads_per_sb];
+ uint8x16_t q6_ql_4567[reads_per_sb];
+ uint8x16_t q6_qh_0123[reads_per_sb];
+ uint8x16_t q6_qh_4567[reads_per_sb];
+
+ for (int k = 0; k < reads_per_sb; k++) {
+ q6_ql_0123[k] = vld1q_u8(ql_base + ql_off_base + k * 32);
+ q6_ql_4567[k] = vld1q_u8(ql_base + ql_off_base + k * 32 + 16);
+ q6_qh_0123[k] = vld1q_u8(qh_base + qh_off_base + k * 32);
+ q6_qh_4567[k] = vld1q_u8(qh_base + qh_off_base + k * 32 + 16);
+ }
+
+ if (sb > 1) {
+ for (int k = 0; k < reads_per_sb; k++) {
+ q6_qh_0123[k] = vshrq_n_u8(q6_qh_0123[k], 2);
+ q6_qh_4567[k] = vshrq_n_u8(q6_qh_4567[k], 2);
+ }
+ }
+
+ for (int k = 0; k < reads_per_sb; k++) {
+ // q = (ql | qh) - 32
+ const uint8x16_t hbit_lo_0123 = vandq_u8(q6_qh_0123[k], mask_lo);
+ const uint8x16_t hbit_hi_0123 = vandq_u8(q6_qh_0123[k], mask_hi);
+ const uint8x16_t hbit_lo_4567 = vandq_u8(q6_qh_4567[k], mask_lo);
+ const uint8x16_t hbit_hi_4567 = vandq_u8(q6_qh_4567[k], mask_hi);
+
+ const int8x16_t q6_0123_lo = vsubq_s8(
+ vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_ql_0123[k], m4b), hbit_lo_0123, 4)), m32s);
+ const int8x16_t q6_0123_hi = vsubq_s8(
+ vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_ql_0123[k], 4), hbit_hi_0123)), m32s);
+
+ acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q6_0123_lo, q8_l[k], 0); // 0..3 r0 c0123
+ acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q6_0123_lo, q8_l[k], 1); // 0..3 r1 c0123
+ acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q6_0123_lo, q8_l[k], 2); // 0..3 r2 c0123
+ acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q6_0123_lo, q8_l[k], 3); // 0..3 r3 c0123
+
+ acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q6_0123_hi, q8_h[k], 0); // 64..67 r0 c0123
+ acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q6_0123_hi, q8_h[k], 1); // 64..67 r1 c0123
+ acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q6_0123_hi, q8_h[k], 2); // 64..67 r2 c0123
+ acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q6_0123_hi, q8_h[k], 3); // 64..67 r3 c0123
+
+ const int8x16_t q6_4567_lo = vsubq_s8(
+ vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_ql_4567[k], m4b), hbit_lo_4567, 4)), m32s);
+ const int8x16_t q6_4567_hi = vsubq_s8(
+ vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_ql_4567[k], 4), hbit_hi_4567)), m32s);
+
+ acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q6_4567_lo, q8_l[k], 0); // 0..3 r0 c4567
+ acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q6_4567_lo, q8_l[k], 1); // 0..3 r1 c4567
+ acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q6_4567_lo, q8_l[k], 2); // 0..3 r2 c4567
+ acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q6_4567_lo, q8_l[k], 3); // 0..3 r3 c4567
+
+ acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q6_4567_hi, q8_h[k], 0); // 64..67 r0 c4567
+ acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q6_4567_hi, q8_h[k], 1); // 64..67 r1 c4567
+ acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q6_4567_hi, q8_h[k], 2); // 64..67 r2 c4567
+ acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q6_4567_hi, q8_h[k], 3); // 64..67 r3 c4567
+ }
+
+ // Scale and bias
+ const int scale_idx_l = half * 8 + sb;
+ const int scale_idx_h = half * 8 + sb + 4;
+
+ for (int g = 0; g < col_groups; g++) {
+ const int16x4_t scales_l16 = vld1_s16(q6_scales + scale_idx_l * 8 + g * 4);
+ const int16x4_t scales_h16 = vld1_s16(q6_scales + scale_idx_h * 8 + g * 4);
+ const int32x4_t scale_vec_l = vmovl_s16(scales_l16);
+ const int32x4_t scale_vec_h = vmovl_s16(scales_h16);
+ const int acc_offset = g * q8_k_blocklen;
+
+ for (int row = 0; row < q8_k_blocklen; row++) {
+ const int idx = row * 2 + g;
+ acc_s32[idx] = vmlaq_s32(acc_s32[idx], acc_lo[acc_offset + row], scale_vec_l);
+ acc_s32[idx] = vmlaq_s32(acc_s32[idx], acc_hi[acc_offset + row], scale_vec_h);
+ }
+ }
+ }
+ }
+
+ // Finally we apply the superblock scales
+ for (int row = 0; row < q8_k_blocklen; row++) {
+ const int idx0 = 2 * row;
+ const int idx1 = 2 * row + 1;
+ const int32x4_t acc_0123 = acc_s32[idx0];
+ const int32x4_t acc_4567 = acc_s32[idx1];
+
+ acc_f32[idx0] = vmlaq_f32(acc_f32[idx0], vcvtq_f32_s32(acc_0123), sbd_scale_0123[row]);
+ acc_f32[idx1] = vmlaq_f32(acc_f32[idx1], vcvtq_f32_s32(acc_4567), sbd_scale_4567[row]);
+ }
+ } // for b
+
+ for (int i = 0; i < q8_k_blocklen; i++) {
+ int row = y * q8_k_blocklen + i;
+ for (int j = 0; j < 2; j++) {
+ int col = x * ncols_interleaved + j * 4;
+ int offset = row * bs + col;
+ vst1q_f32(s + offset, acc_f32[2 * i + j]);
+ }
+ }
+ } // for x
+ } // for y
+ return;
+#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+ ggml_gemm_q6_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);
+}
+
void ggml_gemm_q6_K_8x8_q8_K(int n,
float * GGML_RESTRICT s,
size_t bs,
diff --git a/ggml/src/ggml-cpu/arch/riscv/quants.c b/ggml/src/ggml-cpu/arch/riscv/quants.c
index ae0ebb3cad..bf9f4df118 100644
--- a/ggml/src/ggml-cpu/arch/riscv/quants.c
+++ b/ggml/src/ggml-cpu/arch/riscv/quants.c
@@ -1954,3 +1954,773 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
#endif
}
+static const uint8_t sign_gather_indices_arr[64] = {
+ 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,
+ 4,4,4,4,4,4,4,4, 5,5,5,5,5,5,5,5, 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7
+};
+
+static const uint8_t sign_bit_masks_arr[64] = {
+ 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128,
+ 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128
+};
+
+static void ggml_vec_dot_iq2_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);
+
+ const block_iq2_s * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+ const uint64_t * grid64 = (const uint64_t *)iq2s_grid;
+
+ // --- Pre-load Constants ---
+ uint16_t gather_qh_arr[8] = {0, 0, 0, 0, 1, 1, 1, 1};
+ vuint16mf2_t v_gather_qh = __riscv_vle16_v_u16mf2(gather_qh_arr, 8);
+ uint16_t shift_qh_arr[8] = {11, 9, 7, 5, 11, 9, 7, 5};
+ vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 8);
+
+ // Constants for sign extraction
+ vuint8m2_t v_sign_gather_indices = __riscv_vle8_v_u8m2(sign_gather_indices_arr, 64);
+ vuint8m2_t v_sign_masks = __riscv_vle8_v_u8m2(sign_bit_masks_arr, 64);
+
+ float sumf = 0.0f;
+
+ for (int i = 0; i < nb; ++i) {
+ const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
+
+ const uint8_t * GGML_RESTRICT qs = x[i].qs;
+ const uint8_t * GGML_RESTRICT qh = x[i].qh;
+ const uint8_t * GGML_RESTRICT scales = x[i].scales;
+ const int8_t * GGML_RESTRICT q8 = y[i].qs;
+
+ const uint8_t * signs_ptr = qs + 32;
+
+ float sum_block = 0.0f;
+
+ for (int ib = 0; ib < 4; ++ib) {
+ // Combine low + high bits
+ vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 8);
+ qs += 8;
+ uint16_t qh_val;
+ memcpy(&qh_val, qh, 2);
+ qh += 2;
+ vuint8mf8_t v_qh_raw = __riscv_vle8_v_u8mf8((const uint8_t*)&qh_val, 2);
+ vuint16mf4_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf4(v_qh_raw, 2);
+ vuint16mf2_t v_qh_u16_ext = __riscv_vlmul_ext_v_u16mf4_u16mf2(v_qh_u16);
+ vuint16mf2_t v_qh_expanded = __riscv_vrgather_vv_u16mf2(v_qh_u16_ext, v_gather_qh, 8);
+ v_qh_expanded = __riscv_vsll_vv_u16mf2(v_qh_expanded, v_shift_qh, 8);
+
+ // Mask: We want bits 11-12. 0x1800 = 0001 1000 0000 0000
+ v_qh_expanded = __riscv_vand_vx_u16mf2(v_qh_expanded, 0x1800, 8);
+ vuint16mf2_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 8);
+
+ // Multiply by 8 to get byte offset, instead of element offset
+ v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16, 3, 8);
+ vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_expanded, 8);
+
+ // Lookup Grid using Byte Offsets
+ vuint64m2_t v_grid_vals = __riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 8);
+
+ vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u64m2_u8m2(v_grid_vals);
+ vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(v_grid_u8);
+
+ // Load signs and generate sign mask
+ vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 8);
+ signs_ptr += 8;
+
+ vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);
+ vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 64);
+
+ vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 64);
+ vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 64);
+
+ vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);
+ q8 += 64;
+
+ vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 64);
+ vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 64);
+
+ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);
+
+ int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(
+ __riscv_vget_v_i16m4_i16m1(v_dot, 0), v_zero, 16));
+ int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(
+ __riscv_vget_v_i16m4_i16m1(v_dot, 1), v_zero, 16));
+ int32_t s2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(
+ __riscv_vget_v_i16m4_i16m1(v_dot, 2), v_zero, 16));
+ int32_t s3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(
+ __riscv_vget_v_i16m4_i16m1(v_dot, 3), v_zero, 16));
+
+ uint8_t sc0 = scales[0];
+ uint8_t sc1 = scales[1];
+ scales += 2;
+
+ sum_block += s0 * (2 * (sc0 & 0xF) + 1);
+ sum_block += s1 * (2 * (sc0 >> 4) + 1);
+ sum_block += s2 * (2 * (sc1 & 0xF) + 1);
+ sum_block += s3 * (2 * (sc1 >> 4) + 1);
+ }
+ sumf += sum_block * combined_scale;
+ }
+ *s = 0.125f * sumf;
+}
+
+static void ggml_vec_dot_iq2_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);
+
+ const block_iq2_s * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+ const uint64_t * grid64 = (const uint64_t *)iq2s_grid;
+
+ // Pre-load Constants
+ vuint8m2_t v_ids = __riscv_vid_v_u8m2(32);
+ vuint8m2_t v_sign_gather_indices = __riscv_vsrl_vx_u8m2(v_ids, 3, 32);
+ vuint8m2_t v_ones = __riscv_vmv_v_x_u8m2(1, 32);
+ vuint8m2_t v_shift_amts = __riscv_vand_vx_u8m2(v_ids, 7, 32);
+ vuint8m2_t v_sign_masks = __riscv_vsll_vv_u8m2(v_ones, v_shift_amts, 32);
+ uint16_t shift_qh_arr[4] = {11, 9, 7, 5};
+ vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 4);
+
+ float sumf = 0.0f;
+
+ for (int i = 0; i < nb; ++i) {
+ const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
+
+ const uint8_t * GGML_RESTRICT qs = x[i].qs;
+ const uint8_t * GGML_RESTRICT qh = x[i].qh;
+ const uint8_t * GGML_RESTRICT scales = x[i].scales;
+ const int8_t * GGML_RESTRICT q8 = y[i].qs;
+
+ const uint8_t * signs_ptr = qs + 32;
+ float sum_block = 0.0f;
+
+ for (int ib = 0; ib < 8; ++ib) {
+
+ // Load Low Bits [4 bytes]
+ vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 4);
+ qs += 4;
+
+ // Load 1 byte. It contains bits for 4 mini-blocks.
+ uint8_t qh_val = *qh++;
+
+ // Combine Low + High bits of 10bit indices
+ vuint8mf4_t v_qh_raw = __riscv_vmv_v_x_u8mf4(qh_val, 4);
+ vuint16mf2_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qh_raw, 4);
+ vuint16mf2_t v_qh_mf2 = __riscv_vsll_vv_u16mf2(v_qh_u16, v_shift_qh, 4);
+ v_qh_mf2 = __riscv_vand_vx_u16mf2(v_qh_mf2, 0x1800, 4);
+ vuint16mf2_t v_qs_u16_mf2 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 4);
+ vuint16mf2_t v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16_mf2, 3, 4);
+ vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_mf2, 4);
+
+ // Lookup Grid
+ vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(__riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 4)));
+
+ vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 4);
+ signs_ptr += 4;
+ vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);
+ vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 32);
+
+ // generating sign mask
+ vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 32);
+ vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 32);
+
+ vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 32);
+ q8 += 32;
+
+ // apply signs
+ vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative,v_q8, v_q8, 0, 32);
+ vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 32);
+
+ // Reduction
+ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);
+
+ // Reduce 0-15 (First Half)
+ int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(
+ __riscv_vget_v_i16m4_i16m2(v_dot, 0), v_zero, 16));
+
+ // Reduce 16-31 (Second Half)
+ int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(
+ __riscv_vget_v_i16m4_i16m2(v_dot, 1), v_zero, 16));
+
+ // Apply sub Scales
+ uint8_t sc = *scales++;
+
+ sum_block += s0 * (2 * (sc & 0xF) + 1);
+ sum_block += s1 * (2 * (sc >> 4) + 1);
+ }
+ sumf += sum_block * combined_scale;
+ }
+ *s = 0.125f * sumf;
+}
+
+void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined __riscv_v_intrinsic
+ switch (__riscv_vlenb() * 8) {
+ case 128:
+ ggml_vec_dot_iq2_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ case 256:
+ ggml_vec_dot_iq2_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ default:
+ ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ }
+#else
+ ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
+static void ggml_vec_dot_iq3_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq3_s * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+
+ const uint64_t * grid64 = (const uint64_t *)iq3s_grid;
+
+ // --- Pre-load Constants ---
+ const uint16_t qh_bit_shifts_arr[16] = {
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+ };
+ vuint8m2_t v_sign_gather_indices = __riscv_vle8_v_u8m2(sign_gather_indices_arr, 64);
+ vuint8m2_t v_sign_masks = __riscv_vle8_v_u8m2(sign_bit_masks_arr, 64);
+ vuint16m1_t v_qh_shifts = __riscv_vle16_v_u16m1(qh_bit_shifts_arr, 16);
+
+ float sumf = 0.0f;
+
+ for (int i = 0; i < nb; ++i) {
+ const float d = GGML_CPU_FP16_TO_FP32(x[i].d);
+ const float combined_scale = d * y[i].d;
+
+ const uint8_t * GGML_RESTRICT qs = x[i].qs;
+ const uint8_t * GGML_RESTRICT qh = x[i].qh;
+ const uint8_t * GGML_RESTRICT scales = x[i].scales;
+ const uint8_t * GGML_RESTRICT signs = x[i].signs;
+ const int8_t * GGML_RESTRICT q8 = y[i].qs;
+
+ float sum_block = 0.0f;
+
+ // Loop: Process 64 weights (16 mini-blocks of 4) per iteration
+ for (int ib = 0; ib < 4; ++ib) {
+
+ vuint8mf2_t v_qs_u8 = __riscv_vle8_v_u8mf2(qs, 16);
+ qs += 16;
+
+ uint16_t qh_val;
+ memcpy(&qh_val, qh, 2);
+ qh += 2;
+
+ vuint16m1_t v_qh_val = __riscv_vmv_v_x_u16m1(qh_val, 16);
+ // Extract bits: (qh >> i) & 1
+ v_qh_val = __riscv_vsrl_vv_u16m1(v_qh_val, v_qh_shifts, 16);
+ v_qh_val = __riscv_vand_vx_u16m1(v_qh_val, 1, 16);
+
+ vuint16m1_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16m1(v_qs_u8, 16);
+ v_qs_u16 = __riscv_vsll_vx_u16m1(v_qs_u16, 2, 16);
+ v_qh_val = __riscv_vsll_vx_u16m1(v_qh_val, 10, 16);
+ vuint16m1_t v_grid_offsets = __riscv_vor_vv_u16m1(v_qs_u16, v_qh_val, 16);
+
+ // Grid value is 4xuint8
+ vuint32m2_t v_grid_packed = __riscv_vluxei16_v_u32m2((const uint32_t *)grid64, v_grid_offsets, 16);
+ vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u32m2_u8m2(v_grid_packed);
+ vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs, 8);
+ signs += 8;
+
+ // Generate sign mask
+ vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);
+ vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 64);
+ vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 64);
+ vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 64);
+
+ vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);
+ q8 += 64;
+
+ // Apply Signs
+ vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 64);
+ vint16m4_t v_dot = __riscv_vwmulsu_vv_i16m4(v_q8_signed, v_grid_u8, 64);
+
+ // Reduction
+ vint16m2_t v_dot_lo = __riscv_vget_v_i16m4_i16m2(v_dot, 0);
+ vint16m2_t v_dot_hi = __riscv_vget_v_i16m4_i16m2(v_dot, 1);
+ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);
+
+ int32_t s_lo = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(v_dot_lo, v_zero, 32));
+ int32_t s_hi = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(v_dot_hi, v_zero, 32));
+
+ // Apply sub-scales
+ uint8_t sc_byte = *scales++;
+ int sc_lo = (sc_byte & 0xF) * 2 + 1;
+ int sc_hi = (sc_byte >> 4) * 2 + 1;
+
+ sum_block += s_lo * sc_lo + s_hi * sc_hi;
+ }
+ sumf += sum_block * combined_scale;
+ }
+ *s = 0.125f * sumf;
+}
+
+void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined __riscv_v_intrinsic
+ switch (__riscv_vlenb() * 8) {
+ case 256:
+ ggml_vec_dot_iq3_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ default:
+ ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ }
+#else
+ ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
+static void ggml_vec_dot_tq1_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_tq1_0 * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+
+ float sumf = 0.0f;
+ uint8_t pow[16] = {1, 1, 1, 1, 3, 3, 3, 3, 9, 9, 9, 9, 27, 27, 27, 27};
+
+ for (int i = 0; i < nb; i++) {
+ // First loop.
+ vint32m4_t suml1;
+ {
+ const int vl = 32;
+ vuint8m1_t tq = __riscv_vle8_v_u8m1(x[i].qs, vl);
+
+ vuint16m2_t tq0 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(tq, 3, vl), 8, vl);
+ vuint16m2_t tq1 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 3, vl), 3, vl), 8, vl);
+ vuint16m2_t tq2 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 9, vl), 3, vl), 8, vl);
+ vuint16m2_t tq3 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 27, vl), 3, vl), 8, vl);
+ vuint16m2_t tq4 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 81, vl), 3, vl), 8, vl);
+
+ vint16m2_t q80 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 0, vl), vl);
+ vint16m2_t q81 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 32, vl), vl);
+ vint16m2_t q82 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 64, vl), vl);
+ vint16m2_t q83 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 96, vl), vl);
+ vint16m2_t q84 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 128, vl), vl);
+
+ vint16m2_t sum0 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq0, 1, vl)), q80, vl);
+ vint16m2_t sum1 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq1, 1, vl)), q81, vl);
+ vint16m2_t sum2 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq2, 1, vl)), q82, vl);
+ vint16m2_t sum3 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq3, 1, vl)), q83, vl);
+ vint16m2_t sum4 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq4, 1, vl)), q84, vl);
+
+ vint32m4_t sumi0 = __riscv_vwadd_vv_i32m4(sum0, sum1, vl);
+ vint32m4_t sumi1 = __riscv_vwadd_vv_i32m4(sum2, sum3, vl);
+ suml1 = __riscv_vadd_vv_i32m4(__riscv_vwcvt_x_x_v_i32m4(sum4, vl), __riscv_vadd_vv_i32m4(sumi0, sumi1, vl), vl);
+ }
+
+ // Second loop.
+ vint32m2_t suml2;
+ {
+ const int vl = 16;
+ vuint8mf2_t tq = __riscv_vle8_v_u8mf2(x[i].qs + 32, vl);
+
+ vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(tq, 3 * 1, vl), 8, vl);
+ vuint16m1_t tq1 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 3, vl), 3, vl), 8, vl);
+ vuint16m1_t tq2 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 9, vl), 3, vl), 8, vl);
+ vuint16m1_t tq3 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 27, vl), 3, vl), 8, vl);
+ vuint16m1_t tq4 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 81, vl), 3, vl), 8, vl);
+
+ vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 160, vl), vl);
+ vint16m1_t q81 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 176, vl), vl);
+ vint16m1_t q82 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 192, vl), vl);
+ vint16m1_t q83 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 208, vl), vl);
+ vint16m1_t q84 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 224, vl), vl);
+
+ vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl);
+ vint16m1_t sum1 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq1, 1, vl)), q81, vl);
+ vint16m1_t sum2 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq2, 1, vl)), q82, vl);
+ vint16m1_t sum3 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq3, 1, vl)), q83, vl);
+ vint16m1_t sum4 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq4, 1, vl)), q84, vl);
+
+ vint32m2_t sumi0 = __riscv_vwadd_vv_i32m2(sum0, sum1, vl);
+ vint32m2_t sumi1 = __riscv_vwadd_vv_i32m2(sum2, sum3, vl);
+ suml2 = __riscv_vadd_vv_i32m2(__riscv_vwcvt_x_x_v_i32m2(sum4, vl), __riscv_vadd_vv_i32m2(sumi0, sumi1, vl), vl);
+ }
+
+ // Third loop.
+ vint32m2_t suml3;
+ {
+ const int vl = 16;
+
+ uint32_t qh;
+ memcpy(&qh, &x[i].qh[0], 4);
+ // Prevent fusion with vmv.
+ __asm__ __volatile__("" : "+r"(qh));
+ vuint8mf2_t tq = __riscv_vreinterpret_v_u32mf2_u8mf2(__riscv_vmv_v_x_u32mf2(qh, vl / 4));
+
+ vuint8mf2_t p = __riscv_vle8_v_u8mf2(pow, vl);
+
+ vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vv_u8mf2(tq, p, vl), 3, vl), 8, vl);
+
+ vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 240, vl), vl);
+
+ vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl);
+ suml3 = __riscv_vwcvt_x_x_v_i32m2(sum0, vl);
+ }
+
+ vint32m2_t sumb = __riscv_vadd_vv_i32m2(__riscv_vget_v_i32m4_i32m2(suml1, 0), __riscv_vget_v_i32m4_i32m2(suml1, 1), 16);
+ sumb = __riscv_vadd_vv_i32m2(sumb, suml2, 16);
+ sumb = __riscv_vadd_vv_i32m2(sumb, suml3, 16);
+
+ vint32m1_t sum = __riscv_vredsum_vs_i32m2_i32m1(sumb, __riscv_vmv_v_x_i32m1(0, 1), 16);
+ sumf += __riscv_vmv_x_s_i32m1_i32(sum) * y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined __riscv_v_intrinsic
+ switch (__riscv_vlenb() * 8) {
+ case 256:
+ ggml_vec_dot_tq1_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ default:
+ ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ }
+#else
+ ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
+static void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_tq2_0 * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+
+ float sumf = 0.0f;
+ for (int i = 0; i < nb; ++i) {
+ int32_t sumi = 0;
+
+ for (size_t j = 0; j < sizeof(x[0].qs); j += 32) {
+ const int8_t * py0 = &y[i].qs[j * 4 + 0 * 32];
+ const int8_t * py1 = &y[i].qs[j * 4 + 1 * 32];
+ const int8_t * py2 = &y[i].qs[j * 4 + 2 * 32];
+ const int8_t * py3 = &y[i].qs[j * 4 + 3 * 32];
+ const uint8_t* px = &x[i].qs[j];
+
+ size_t vlmax_16m2 = __riscv_vsetvl_e16m2(32);
+ vint16m2_t vacc16 = __riscv_vmv_v_x_i16m2(0, vlmax_16m2);
+
+ size_t vl = __riscv_vsetvl_e8m1(32);
+
+ vuint8m1_t vx_u8 = __riscv_vle8_v_u8m1(px, vl);
+
+ vint8m1_t vy0 = __riscv_vle8_v_i8m1(py0 , vl);
+ vint8m1_t vy1 = __riscv_vle8_v_i8m1(py1, vl);
+ vint8m1_t vy2 = __riscv_vle8_v_i8m1(py2, vl);
+ vint8m1_t vy3 = __riscv_vle8_v_i8m1(py3, vl);
+
+ // l=0 (bits 1:0)
+ vuint8m1_t t0 = __riscv_vand_vx_u8m1(vx_u8, 0x03, vl);
+ vint8m1_t vq0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t0), 1, vl);
+
+ // l=1 (bits 3:2)
+ vuint8m1_t t1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vx_u8, 2, vl), 0x03, vl);
+ vint8m1_t vq1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t1), 1, vl);
+
+ // l=2 (bits 5:4)
+ vuint8m1_t t2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vx_u8, 4, vl), 0x03, vl);
+ vint8m1_t vq2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t2), 1, vl);
+
+ // l=3 (bits 7:6)
+ vuint8m1_t t3 = __riscv_vsrl_vx_u8m1(vx_u8, 6, vl); // No final AND needed as vsrl shifts in zeros
+ vint8m1_t vq3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t3), 1, vl);
+
+ // 4. Multiply and accumulate
+ vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq0, vy0, vl);
+ vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq1, vy1, vl);
+ vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq2, vy2, vl);
+ vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq3, vy3, vl);
+
+ vlmax_16m2 = __riscv_vsetvl_e16m2(32);
+ vint32m1_t vzero32 = __riscv_vmv_v_x_i32m1(0, 1);
+ vint32m1_t vred32 = __riscv_vwredsum_vs_i16m2_i32m1(vacc16, vzero32, vlmax_16m2);
+
+ sumi += __riscv_vmv_x_s_i32m1_i32(vred32);
+ }
+ const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
+ sumf += (float)sumi * d;
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined __riscv_v_intrinsic
+ switch (__riscv_vlenb() * 8) {
+ case 256:
+ ggml_vec_dot_tq2_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ default:
+ ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ }
+#else
+ ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
+static void ggml_vec_dot_iq1_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq1_s * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ // Load qh once for the entire superblock.
+ vuint16mf2_t qh = __riscv_vle16_v_u16mf2(x[i].qh, 8);
+
+ // Calculate ls.
+ vuint16mf2_t temp = __riscv_vsrl_vx_u16mf2(qh, 12, 8);
+ temp = __riscv_vand_vx_u16mf2(temp, 7, 8);
+ vint32m1_t ls = __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vwmulu_vx_u32m1(temp, 2, 8));
+ ls = __riscv_vadd_vx_i32m1(ls, 1, 8);
+
+ // Calculate delta.
+ vbool32_t mask = __riscv_vmseq_vx_u16mf2_b32(__riscv_vand_vx_u16mf2(qh, 0x8000, 8), 0, 8);
+ vint32m1_t delta_neg = __riscv_vmv_v_x_i32m1(-1, 8);
+ vint32m1_t delta_pos = __riscv_vmv_v_x_i32m1(1, 8);
+ vint32m1_t delta = __riscv_vmerge_vvm_i32m1(delta_neg, delta_pos, mask, 8);
+
+ // Load qs.
+ vuint8m1_t qs = __riscv_vle8_v_u8m1(x[i].qs, 32);
+
+ // Prepare the indices.
+ const uint64_t shift = 0x0009000600030000;
+ vuint16m2_t qh_shift = __riscv_vreinterpret_v_u64m2_u16m2(__riscv_vmv_v_x_u64m2(shift, 8));
+ vuint16m2_t qh_gather_index = __riscv_vreinterpret_v_i16m2_u16m2(
+ __riscv_vdiv_vx_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vid_v_u16m2(32)), 4, 32));
+ vuint16m2_t qh_ext = __riscv_vlmul_ext_v_u16m1_u16m2(__riscv_vlmul_ext_v_u16mf2_u16m1(qh));
+ vuint16m2_t qh_index = __riscv_vrgather_vv_u16m2(qh_ext, qh_gather_index, 32);
+ qh_index = __riscv_vsrl_vv_u16m2(qh_index, qh_shift, 32);
+ qh_index = __riscv_vand_vx_u16m2(qh_index, 7, 32);
+ qh_index = __riscv_vsll_vx_u16m2(qh_index, 8, 32);
+ qh_index = __riscv_vor_vv_u16m2(qh_index, __riscv_vzext_vf2_u16m2(qs, 32), 32);
+ vuint16m2_t index = __riscv_vsll_vx_u16m2(qh_index, 3, 32);
+
+ // Final lsums.
+ int32_t lsums_s[8];
+ vint32m1_t one_scalar = __riscv_vmv_v_x_i32m1(0, 1);
+
+ // Sub-blocks 1-4
+ {
+ vuint16m1_t grid_index0 = __riscv_vget_v_u16m2_u16m1(index, 0);
+ vint8m4_t grid0 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, grid_index0, 16));
+ vint8m4_t q80 = __riscv_vle8_v_i8m4(y[i].qs, 128);
+ vint16m8_t lsum0 = __riscv_vwmul_vv_i16m8(grid0, q80, 128);
+ lsums_s[0] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 0), one_scalar, 32));
+ lsums_s[1] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 1), one_scalar, 32));
+ lsums_s[2] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 2), one_scalar, 32));
+ lsums_s[3] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 3), one_scalar, 32));
+ }
+ __asm__ __volatile__("" ::: "memory");
+ // Sub-blocks 5-8
+ {
+ vuint16m1_t grid_index1 = __riscv_vget_v_u16m2_u16m1(index, 1);
+ vint8m4_t grid1 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, grid_index1, 16));
+ vint8m4_t q81 = __riscv_vle8_v_i8m4(&y[i].qs[128], 128);
+ vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(grid1, q81, 128);
+ lsums_s[4] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 0), one_scalar, 32));
+ lsums_s[5] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 1), one_scalar, 32));
+ lsums_s[6] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 2), one_scalar, 32));
+ lsums_s[7] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 3), one_scalar, 32));
+ }
+ __asm__ __volatile__("" ::: "memory");
+ vint32m1_t lsums = __riscv_vle32_v_i32m1(&lsums_s[0], 8);
+
+ // Calculate the bsums.
+ vint16m1_t bsums_0 = __riscv_vle16_v_i16m1(y[i].bsums, 16);
+ const vuint32m1_t bsums_i32 = __riscv_vreinterpret_v_u16m1_u32m1(__riscv_vreinterpret_v_i16m1_u16m1(bsums_0));
+ const vint16mf2_t bsums_i32_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(bsums_i32, 0, 8));
+ const vint16mf2_t bsums_i32_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(bsums_i32, 16, 8));
+ const vint32m1_t bsums = __riscv_vwadd_vv_i32m1(bsums_i32_0, bsums_i32_1, 8);
+
+ // Accumulation.
+ vint32m1_t sumi_v = __riscv_vmul_vv_i32m1(ls, lsums, 8);
+ vint32m1_t sumi1_v = __riscv_vmul_vv_i32m1(__riscv_vmul_vv_i32m1(ls, delta, 8), bsums, 8);
+
+ // Update sumf.
+ int sumi = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m1_i32m1(sumi_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8));
+ int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m1_i32m1(sumi1_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8));
+ sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined __riscv_v_intrinsic
+ switch (__riscv_vlenb() * 8) {
+ case 256:
+ ggml_vec_dot_iq1_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ default:
+ ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ }
+#else
+ ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
+static void ggml_vec_dot_iq1_m_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_iq1_m * GGML_RESTRICT x = vx;
+ const block_q8_K * GGML_RESTRICT y = vy;
+
+ const int nb = n / QK_K;
+
+ iq1m_scale_t scale;
+ float sumf = 0.0f;
+ for (int i = 0; i < nb; ++i) {
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint16_t * sc = (const uint16_t *)x[i].scales;
+
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+
+ // Accumulators.
+ vint32m2_t acc1 = __riscv_vmv_v_x_i32m2(0, 16);
+ vint32m2_t acc2 = __riscv_vmv_v_x_i32m2(0, 16);
+
+ // We process 4 sub-blocks together.
+ for (int ib = 0; ib < QK_K/128; ib++) {
+ // Load qh for 4 sub-blocks.
+ const vuint8mf4_t qh_8 = __riscv_vle8_v_u8mf4(qh, 8);
+ const vuint16mf2_t qh_16_lo = __riscv_vzext_vf2_u16mf2(qh_8, 8);
+ const vuint16mf2_t qh_16_hi = __riscv_vsll_vx_u16mf2(qh_16_lo, 8, 8);
+ const vuint16m1_t qhb = __riscv_vzext_vf2_u16m1(
+ __riscv_vreinterpret_v_u16mf2_u8mf2(__riscv_vor_vv_u16mf2(qh_16_lo, qh_16_hi, 8)), 16);
+ qh += 8;
+
+ // Prepare grid indices.
+ const vuint16m1_t qsb = __riscv_vzext_vf2_u16m1(__riscv_vle8_v_u8mf2(&qs[0], 16), 16);
+ const vuint16m1_t shift = __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00040008, 8));
+ vuint16m1_t index = __riscv_vor_vv_u16m1(qsb, __riscv_vand_vx_u16m1(__riscv_vsll_vv_u16m1(qhb, shift, 16), 0x700, 16), 16);
+ index = __riscv_vsll_vx_u16m1(index, 3, 16);
+ qs += 16;
+
+ // Load the grid.
+ const vint8m4_t iq1b = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vreinterpret_v_u64m4_i64m4(
+ __riscv_vluxei16_v_u64m4(iq1s_grid, index, 16)));
+
+ // Prepare the deltas.
+ const vbool16_t mask = __riscv_vmsgtu_vx_u16m1_b16(
+ __riscv_vand_vv_u16m1(qhb, __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00800008, 8)), 16), 0, 16);
+ const vint64m4_t delta_pos = __riscv_vmv_v_x_i64m4(0x0101010101010101, 16);
+ const vint64m4_t delta_neg = __riscv_vmv_v_x_i64m4(0xffffffffffffffff, 16);
+ const vint8m4_t delta = __riscv_vreinterpret_v_i64m4_i8m4(
+ __riscv_vmerge_vvm_i64m4(delta_pos, delta_neg, mask, 16));
+
+ // Load q8 for sub-blocks.
+ const vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 128);
+ q8 += 128;
+
+ // Calculate the lsums.
+ const vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(iq1b, q8b, 128);
+ const vint16m8_t lsum2 = __riscv_vwmul_vv_i16m8(delta, q8b, 128);
+
+ // Prepare the scales.
+ const int16_t ls_0_0 = 2*((sc[0] >> 0) & 0x7) + 1;
+ const int16_t ls_0_1 = 2*((sc[0] >> 3) & 0x7) + 1;
+ const int16_t ls_1_0 = 2*((sc[0] >> 6) & 0x7) + 1;
+ const int16_t ls_1_1 = 2*((sc[0] >> 9) & 0x7) + 1;
+ const int16_t ls_2_0 = 2*((sc[1] >> 0) & 0x7) + 1;
+ const int16_t ls_2_1 = 2*((sc[1] >> 3) & 0x7) + 1;
+ const int16_t ls_3_0 = 2*((sc[1] >> 6) & 0x7) + 1;
+ const int16_t ls_3_1 = 2*((sc[1] >> 9) & 0x7) + 1;
+ sc += 2;
+
+ // Accumulate in acc0 and acc1 for each sub-block.
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0_0, __riscv_vget_v_i16m8_i16m1(lsum1, 0), 16);
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0_1, __riscv_vget_v_i16m8_i16m1(lsum1, 1), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0_0, __riscv_vget_v_i16m8_i16m1(lsum2, 0), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0_1, __riscv_vget_v_i16m8_i16m1(lsum2, 1), 16);
+ //
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_1_0, __riscv_vget_v_i16m8_i16m1(lsum1, 2), 16);
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_1_1, __riscv_vget_v_i16m8_i16m1(lsum1, 3), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_1_0, __riscv_vget_v_i16m8_i16m1(lsum2, 2), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_1_1, __riscv_vget_v_i16m8_i16m1(lsum2, 3), 16);
+ //
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_2_0, __riscv_vget_v_i16m8_i16m1(lsum1, 4), 16);
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_2_1, __riscv_vget_v_i16m8_i16m1(lsum1, 5), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_2_0, __riscv_vget_v_i16m8_i16m1(lsum2, 4), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_2_1, __riscv_vget_v_i16m8_i16m1(lsum2, 5), 16);
+ //
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_3_0, __riscv_vget_v_i16m8_i16m1(lsum1, 6), 16);
+ acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_3_1, __riscv_vget_v_i16m8_i16m1(lsum1, 7), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_3_0, __riscv_vget_v_i16m8_i16m1(lsum2, 6), 16);
+ acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_3_1, __riscv_vget_v_i16m8_i16m1(lsum2, 7), 16);
+ }
+
+ // Reduce and accumulate in `sumf`.
+ vint32m1_t one = __riscv_vmv_v_x_i32m1(0, 1);
+ int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc1, one, 16));
+ int sumi2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc2, one, 16));
+ sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (sumi1 + IQ1M_DELTA * sumi2);
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined __riscv_v_intrinsic
+ switch (__riscv_vlenb() * 8) {
+ case 256:
+ ggml_vec_dot_iq1_m_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ default:
+ ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+ break;
+ }
+#else
+ ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
diff --git a/ggml/src/ggml-cpu/binary-ops.cpp b/ggml/src/ggml-cpu/binary-ops.cpp
index 14f5b43ae0..75e3829001 100644
--- a/ggml/src/ggml-cpu/binary-ops.cpp
+++ b/ggml/src/ggml-cpu/binary-ops.cpp
@@ -59,11 +59,7 @@ static void apply_binary_op(const ggml_compute_params * params, ggml_tensor * ds
GGML_ASSERT(nb00 == sizeof(src0_t));
const auto [ir0, ir1] = get_thread_range(params, src0);
- const bool is_src1_contiguous = (nb10 == sizeof(src1_t));
-
- if (!is_src1_contiguous) { // broadcast not implemented yet for non-contiguous
- GGML_ASSERT(ggml_are_same_shape(src0, src1));
- }
+ const bool is_src1_contiguous_rows = ggml_is_contiguous_rows(src1);
#ifdef GGML_USE_ACCELERATE
vDSP_fn_t vDSP_op = nullptr;
@@ -94,7 +90,7 @@ static void apply_binary_op(const ggml_compute_params * params, ggml_tensor * ds
const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
const src1_t * src1_ptr = (const src1_t *) ((const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
- if (is_src1_contiguous) {
+ if (is_src1_contiguous_rows) {
// src1 is broadcastable across src0 and dst in i1, i2, i3
const int64_t nr0 = ne00 / ne10;
diff --git a/ggml/src/ggml-cpu/common.h b/ggml/src/ggml-cpu/common.h
index 1057b5bb15..abbadc359c 100644
--- a/ggml/src/ggml-cpu/common.h
+++ b/ggml/src/ggml-cpu/common.h
@@ -6,8 +6,8 @@
#include "ggml-impl.h"
#include "simd-mappings.h"
-#define GGML_FA_TILE_Q 32
-#define GGML_FA_TILE_KV 16
+#define GGML_FA_TILE_Q 64
+#define GGML_FA_TILE_KV 64
#ifdef __cplusplus
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index b003fe13fd..64eb01a4e1 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -2874,8 +2874,8 @@ struct ggml_cplan ggml_graph_plan(
const int64_t DV = node->src[2]->ne[0];
// Tiled flash attention scratch (tile sizes defined in common.h)
- // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + padding
- size_t prefill = sizeof(float)*(GGML_FA_TILE_Q*DK + 2*GGML_FA_TILE_Q*GGML_FA_TILE_KV + GGML_FA_TILE_Q*DV + GGML_FA_TILE_KV*DV)*n_tasks;
+ // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding
+ size_t prefill = sizeof(float)*(GGML_FA_TILE_Q*DK + 2*GGML_FA_TILE_Q*GGML_FA_TILE_KV + GGML_FA_TILE_Q*DV + GGML_FA_TILE_KV*DV + GGML_FA_TILE_KV*DK)*n_tasks;
// Decode path: n_kv_chunks = n_tasks (one chunk per thread)
// Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ
@@ -2947,7 +2947,11 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
/*.use_ref =*/ cplan->use_ref,
};
- GGML_PRINT_DEBUG("thread #%d compute-start cplan %p last-graph %d \n", state->ith, cplan, state->last_graph);
+#ifdef GGML_USE_OPENMP
+ GGML_PRINT_DEBUG("thread #%d compute-start cplan %p\n", state->ith, (const void *)cplan);
+#else
+ GGML_PRINT_DEBUG("thread #%d compute-start cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph);
+#endif
for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {
struct ggml_tensor * node = cgraph->nodes[node_n];
@@ -2974,7 +2978,11 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
}
}
- GGML_PRINT_DEBUG("thread #%d compute-done cplan %p last-graph %d \n", state->ith, cplan, state->last_graph);
+#ifdef GGML_USE_OPENMP
+ GGML_PRINT_DEBUG("thread #%d compute-done cplan %p\n", state->ith, (const void *)cplan);
+#else
+ GGML_PRINT_DEBUG("thread #%d compute-done cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph);
+#endif
ggml_barrier(state->threadpool);
diff --git a/ggml/src/ggml-cpu/llamafile/sgemm-ppc.h b/ggml/src/ggml-cpu/llamafile/sgemm-ppc.h
deleted file mode 100644
index a707868728..0000000000
--- a/ggml/src/ggml-cpu/llamafile/sgemm-ppc.h
+++ /dev/null
@@ -1,333 +0,0 @@
-#pragma once
-
-typedef vector unsigned char vec_t;
-typedef __vector_quad acc_t;
-
-template
-class tinyBLAS_Q0_PPC {
- public:
- tinyBLAS_Q0_PPC(int64_t k,
- const TA *A, int64_t lda,
- const block_q8_0 *B, int64_t ldb,
- float *C, int64_t ldc,
- int ith, int nth);
-
- void matmul(int64_t m, int64_t n);
- void matmul_tiled_q0(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) {
- vec_t A_pack[mc*kc*2];
- vec_t B_pack[nc*kc*2];
- int comparray[mc*kc];
- constexpr bool is_Ablock_q4 = std::is_same_v;
- int64_t ytiles = m / mc;
- int64_t xtiles = n / nc;
- int64_t tiles = xtiles * ytiles;
- int64_t duty = (tiles + nth - 1) / nth;
- int64_t start = duty * ith;
- int64_t end = start + duty;
- if (end > tiles) {
- end = tiles;
- }
- for (int64_t job = start; job < end; ++job) {
- int64_t ii = (job / xtiles) * mc;
- int64_t jj = (job % xtiles) * nc;
- for (int64_t kk = 0; kk < k; kk += kc) {
- if constexpr(is_Ablock_q4) {
- packNormalInt4_large(A + ii*lda + kk, lda, mc, 4, (int8_t*)A_pack, comparray);
- } else {
- packNormal_large(A + ii*lda + kk, lda, mc, 8, (int8_t*)A_pack, false, comparray);
- }
- packNormal_large(B + jj*ldb + kk, ldb, nc, 8, (uint8_t*)B_pack, true);
- KERNEL_Q0(ii, jj, mc, nc, kc, kk, A_pack, B_pack, comparray);
- }
- }
- }
-
- private:
- inline void save_res(int ii, int jj, int idx, vector float* fin_res, int RM=4, int RN=4) {
- for (int I = 0; I < RM; I++) {
- for (int J = 0; J < RN; J++) {
- *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&fin_res[idx+I]+J);
- }
- }
- }
-
- inline void add_save_res(int ii, int jj, int idx, vector float* fin_res, int RM=4, int RN=4) {
- for (int I = 0; I < RM; I++) {
- for (int J = 0; J < RN; J++) {
- float * c_ptr = (float *)(C+ii+((jj+J)*ldc)+I);
- *c_ptr += *((float*)&fin_res[idx+I]+J);
- }
- }
- }
-
- template
- inline void compute(acc_t* ACC, int c_idx, int s_idx, ArrayType& comparray, vector float* vs, vector float* fin_res) {
- vector signed int vec_C[4];
- vector float CA[4] = {0};
- vector float res[4] = {0};
- __builtin_mma_disassemble_acc(vec_C, ACC);
- for (int i = 0; i < 4; i++) {
- CA[i] = vec_splats((float)(((double)comparray[c_idx+i]) * -128.0));
- res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);
- fin_res[s_idx+i] = vec_madd(res[i], vs[s_idx+i], fin_res[s_idx+i]);
- }
- }
-
- inline void process_q4_elements(vector signed char (&c)[2], int* ca) {
- const vector signed char lowMask = vec_splats((signed char)0xF);
- const vector unsigned char v4 = vec_splats((unsigned char)0x4);
- const vector signed char v8 = vec_splats((signed char)0x8);
- vector signed int vsum = {0};
- vector signed int vsum2 = {0};
- c[0] = vec_and(c[1], lowMask);
- c[1] = vec_sr(c[1], v4);
- c[0] = vec_sub(c[0], v8);
- c[1] = vec_sub(c[1], v8);
- vsum = vec_sum4s(c[0], vsum);
- vsum2 = vec_sum4s(c[1], vsum2);
- vsum = vec_add(vsum, vsum2);
- *(ca) = vsum[0] + vsum[1] + vsum[2] + vsum[3];
- }
-
- template
- inline void vector_permute_store(V2 &s1, V2 &s2, V2 &s3, V2 &s4, V1 *vecOffset, bool flip) {
- vector unsigned char swiz1 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
- vector unsigned char swiz2 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
- vector unsigned char swiz3 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};
- vector unsigned char swiz4 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
- V2 t1, t2, t3, t4, t5, t6, t7, t8;
- vector unsigned char xor_vector;
- uint8_t flip_vec = 0x80;
- xor_vector = vec_splats(flip_vec);
- t1 = vec_perm(s1, s2, swiz1);
- t2 = vec_perm(s1, s2, swiz2);
- t3 = vec_perm(s3, s4, swiz1);
- t4 = vec_perm(s3, s4, swiz2);
- t5 = vec_perm(t1, t3, swiz3);
- t6 = vec_perm(t1, t3, swiz4);
- t7 = vec_perm(t2, t4, swiz3);
- t8 = vec_perm(t2, t4, swiz4);
- if (flip == true) {
- t5 = vec_xor(t5, xor_vector);
- t6 = vec_xor(t6, xor_vector);
- t7 = vec_xor(t7, xor_vector);
- t8 = vec_xor(t8, xor_vector);
- }
- vec_xst(t5, 0, vecOffset);
- vec_xst(t6, 0, vecOffset+16);
- vec_xst(t7, 0, vecOffset+32);
- vec_xst(t8, 0, vecOffset+48);
- }
-
- template
- inline void kernel(int64_t ii, int64_t jj) {
- if constexpr(RM == 4 && RN == 8) {
- KERNEL_4x8(ii,jj);
- } else if constexpr(RM == 8 && RN == 4) {
- KERNEL_8x4(ii,jj);
- } else if constexpr(RM == 8 && RN == 8) {
- KERNEL_8x8(ii,jj);
- } else {
- assert(false && "RN/RM values not supported");
- }
- }
- template
- void packNormalInt4(const TA* a, int64_t lda, int rows, int cols, int8_t* vec, std::array& comparray);
- template
- void packNormal(const block_q8_0* a, int64_t lda, int rows, int cols, VA* vec, bool flip);
- void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n);
- void KERNEL_4x8(int64_t ii, int64_t jj);
- void KERNEL_8x4(int64_t ii, int64_t jj);
- void KERNEL_8x8(int64_t ii, int64_t jj);
- void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN);
- template
- void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n);
-
- void compute_scale(int64_t ii, int64_t jj, int blk, vector float* vs){
- for (int I = 0; I<8; I++) {
- float a_scale = unhalf((A+((ii+I)*lda)+blk)->d);
- for (int J = 0; J<4; J++) {
- *((float*)&vs[I]+J) = (a_scale * unhalf((B+((jj+J)*ldb)+blk)->d));
- *((float*)&vs[I+8]+J) = (a_scale * unhalf((B+((jj+J+4)*ldb)+blk)->d));
- }
- }
- }
-
- inline void process_q8_elements(const int8_t *qs, int *ca) {
- vector signed char c1 = vec_xl(0, qs);
- vector signed char c2 = vec_xl(16, qs);
- vector signed int vsum1 = {0};
- vector signed int vsum2 = {0};
- vsum1 = vec_sum4s(c1, vsum1);
- vsum2 = vec_sum4s(c2, vsum2);
- vector signed int vsum = vec_add(vsum1, vsum2);
- *ca = vsum[0] + vsum[1] + vsum[2] + vsum[3];
- }
-
- template
- void packNormal_large(const block_q8_0* a, int64_t lda, int rows, int cols, VA* vec, bool flip, int* comparray=nullptr) {
- int64_t i, j;
- block_q8_0 *aoffset = NULL;
- VA *vecOffset = NULL;
- block_q8_0* aoffsets[8];
- __vector_pair arr[8];
- VB c[8][2] = {0};
- VB c1[8] = {0}; VB c2[8] = {0};
- aoffset = const_cast(a);
- vecOffset = vec;
- j = (rows >> 3);
- int index = 0;
- if (j > 0) {
- do {
- for (int it = 0; it < 8; it++)
- aoffsets[it] = aoffset + it*lda;
- aoffset += 8 * lda;
- for (int blk = 0; blk < kc; blk++) {
- for (int it = 0; it < 8; it++) {
- arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)(aoffsets[it]+blk)->qs);
- __builtin_vsx_disassemble_pair(c[it], &arr[it]);
- c1[it] = c[it][0];
- c2[it] = c[it][1];
- if (comparray){
- process_q8_elements((aoffsets[it]+ blk)->qs, &comparray[index + 8*blk + it]);
- }
- }
- vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);
- vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset+64, flip);
- vector_permute_store(c1[4], c1[5], c1[6], c1[7], vecOffset+128, flip);
- vector_permute_store(c2[4], c2[5], c2[6], c2[7], vecOffset+192, flip);
- vecOffset += 256;
- }
- j--;
- index += 8*kc;
- } while(j > 0);
- }
-
- }
-
- void packNormalInt4_large(const TA* a, int64_t lda, int rows, int cols, int8_t* vec, int*comparray) {
- int64_t i, j;
- TA *aoffset = NULL;
- int8_t *vecOffset = NULL;
- TA *aoffset1 = NULL, *aoffset2 = NULL, *aoffset3 = NULL, *aoffset4 = NULL;
- TA *aoffset5 = NULL, *aoffset6 = NULL, *aoffset7 = NULL, *aoffset8 = NULL;
- vector signed char c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2] = {0};
- vector signed char c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2] = {0};
- aoffset = const_cast(a);
- vecOffset = vec;
- int index = 0;
- j = (rows >> 3);
- if (j > 0) {
- do {
- aoffset1 = aoffset;
- aoffset2 = aoffset1 + lda;
- aoffset3 = aoffset2 + lda;
- aoffset4 = aoffset3 + lda;
- aoffset5 = aoffset4 + lda;
- aoffset6 = aoffset5 + lda;
- aoffset7 = aoffset6 + lda;
- aoffset8 = aoffset7 + lda;
- aoffset += 8 * lda;
- for (int blk = 0; blk < kc; blk++) {
- c1[1] = reinterpret_cast(vec_xl(0, (aoffset1+blk)->qs));
- c2[1] = reinterpret_cast(vec_xl(0, (aoffset2+blk)->qs));
- c3[1] = reinterpret_cast(vec_xl(0, (aoffset3+blk)->qs));
- c4[1] = reinterpret_cast(vec_xl(0, (aoffset4+blk)->qs));
- c5[1] = reinterpret_cast(vec_xl(0, (aoffset5+blk)->qs));
- c6[1] = reinterpret_cast(vec_xl(0, (aoffset6+blk)->qs));
- c7[1] = reinterpret_cast(vec_xl(0, (aoffset7+blk)->qs));
- c8[1] = reinterpret_cast(vec_xl(0, (aoffset8+blk)->qs));
-
- process_q4_elements(c1, &comparray[index + 8*blk+0]);
- process_q4_elements(c2, &comparray[index + 8*blk+1]);
- process_q4_elements(c3, &comparray[index + 8*blk+2]);
- process_q4_elements(c4, &comparray[index + 8*blk+3]);
- process_q4_elements(c5, &comparray[index + 8*blk+4]);
- process_q4_elements(c6, &comparray[index + 8*blk+5]);
- process_q4_elements(c7, &comparray[index + 8*blk+6]);
- process_q4_elements(c8, &comparray[index + 8*blk+7]);
- vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);
- vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset+64, false);
- vector_permute_store(c5[0], c6[0], c7[0], c8[0], vecOffset+128, false);
- vector_permute_store(c5[1], c6[1], c7[1], c8[1], vecOffset+192, false);
- vecOffset += 256;
- }
- j--;
- index += 8*kc;
- } while (j > 0);
- }
- }
-
- void KERNEL_Q0(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, int64_t l, vec_t *vec_A, vec_t *vec_B, int *comparray) {
- acc_t acc[8];
- for (int i = 0; i < mc ; i += 8) {
- for (int j = 0; j < nc; j += 8) {
- vector float fin_res[16] = {0};
- vector float vs[16] = {0};
- for (int64_t kk = 0; kk < kc; kk+=2) {
- for (int x = 0; x < 8; x++) {
- __builtin_mma_xxsetaccz(&acc[x]);
- }
- int A_block_idx = (i/8)*(16*kc) + kk*16;
- int B_block_idx = (j/8)*(16*kc)+ kk*16;
- vec_t *A_block = &vec_A[A_block_idx];
- vec_t *B_block = &vec_B[B_block_idx];
- for (int x = 0; x < 8; x++) {
- __builtin_mma_xvi8ger4pp(&acc[0], A_block[x], B_block[x]);
- __builtin_mma_xvi8ger4pp(&acc[1], A_block[x + 8], B_block[x]);
- __builtin_mma_xvi8ger4pp(&acc[2], A_block[x], B_block[x+8]);
- __builtin_mma_xvi8ger4pp(&acc[3], A_block[x+8], B_block[x+8]);
- }
- compute_scale(ii+i, jj+j, l+kk, vs);
- int c_index = (i/8)*(8*kc)+ kk*8;
- int* c_block = &comparray[c_index];
- compute(&acc[0], 0, 0, c_block, vs, fin_res);
- compute(&acc[1], 4, 4, c_block, vs, fin_res);
- compute(&acc[2], 0, 8, c_block, vs, fin_res);
- compute(&acc[3], 4, 12, c_block, vs, fin_res);
-
- A_block_idx = (i/8)*(16*kc) + (kk+1)*16;
- B_block_idx = (j/8)*(16*kc)+ (kk+1)*16;
- A_block = &vec_A[A_block_idx];
- B_block = &vec_B[B_block_idx];
- for (int x = 0; x < 8; x++) {
- __builtin_mma_xvi8ger4pp(&acc[4], A_block[x], B_block[x]);
- __builtin_mma_xvi8ger4pp(&acc[5], A_block[x + 8], B_block[x]);
- __builtin_mma_xvi8ger4pp(&acc[6], A_block[x], B_block[x+8]);
- __builtin_mma_xvi8ger4pp(&acc[7], A_block[x+8], B_block[x+8]);
- }
- compute_scale(ii+i, jj+j, l+kk+1, vs);
- c_index = (i/8)*(8*kc)+ (kk+1)*8;
- c_block = &comparray[c_index];
- compute(&acc[4], 0, 0, c_block, vs, fin_res);
- compute(&acc[5], 4, 4, c_block, vs, fin_res);
- compute(&acc[6], 0, 8, c_block, vs, fin_res);
- compute(&acc[7], 4, 12, c_block, vs, fin_res);
-
- }
- if (l == 0) {
- save_res(ii+i, jj+j, 0, fin_res);
- save_res(ii+i+4, jj+j, 4, fin_res);
- save_res(ii+i, jj+j+4, 8, fin_res);
- save_res(ii+i+4, jj+j+4, 12, fin_res);
- } else {
- add_save_res(ii+i, jj+j, 0, fin_res);
- add_save_res(ii+i+4, jj+j, 4, fin_res);
- add_save_res(ii+i, jj+j+4, 8, fin_res);
- add_save_res(ii+i+4, jj+j+4, 12, fin_res);
- }
- }
- }
- }
-
- const TA *const A;
- const block_q8_0 *const B;
- float *C;
- const int64_t k;
- int64_t kc;
- const int64_t lda;
- const int64_t ldb;
- const int64_t ldc;
- const int ith;
- const int nth;
-};
diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
index 8f980c16b9..da412fd009 100644
--- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp
+++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
@@ -121,7 +121,8 @@ inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vec_mul(x, y); }
#endif
#if defined(__MMA__)
-#include "sgemm-ppc.h"
+typedef vector unsigned char vec_t;
+typedef __vector_quad acc_t;
#endif
////////////////////////////////////////////////////////////////////////////////////////////////////
// VECTORIZED FUSED MULTIPLY ADD
@@ -2153,7 +2154,7 @@ class tinyBLAS_HP16_PPC {
packNormal((B+(jj*ldb)+l), ldb, 8, 4, (uint8_t*)vec_B);
for (int x = 0; x < 4; x++) {
mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]);
- mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]);
+ mma_instr::outer_product(&acc_1, vec_A[x+4], vec_B[x]);
}
}
SAVE_ACC(&acc_0, ii, jj);
@@ -2301,43 +2302,299 @@ class tinyBLAS_HP16_PPC {
const int nth;
};
- template
- tinyBLAS_Q0_PPC::tinyBLAS_Q0_PPC(int64_t k,
- const TA *A, int64_t lda,
- const block_q8_0 *B, int64_t ldb,
- float *C, int64_t ldc,
- int ith, int nth)
+template
+class tinyBLAS_Q0_PPC {
+ public:
+ tinyBLAS_Q0_PPC(int64_t k,
+ const TA * A, int64_t lda,
+ const block_q8_0 * B, int64_t ldb,
+ float * C, int64_t ldc,
+ int ith, int nth)
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
- kc = 64;
}
- template
- void tinyBLAS_Q0_PPC::matmul(int64_t m, int64_t n) {
- int mc = 64; int nc = 64;
- if (n % 8 == 0 && n < nc) {
- nc = n;
- mc = 32 ;
- kc = 32;
+ void matmul(int64_t m, int64_t n) {
+ const int64_t mc = 64;
+ const int64_t kc = 64;
+ int64_t nc = 64;
+ int64_t n_aligned = 0;
+ if (n % 64 == 0) {
+ n_aligned = n;
+ } else if (n == 4) {
+ n_aligned = 4;
+ } else if (n < 64) {
+ n_aligned = (n / 8) * 8;
+ } else {
+ n_aligned = (n / 64) * 64;
}
- const bool is_aligned = ((m & (mc - 1)) == 0) & ((n & (nc - 1)) == 0) & ((k & (kc - 1)) == 0);
- if (is_aligned) {
- this->matmul_tiled_q0(m, n, mc, nc, kc);
+
+ if (n_aligned > 0) {
+ if (n_aligned % 64 == 0) nc = 64;
+ else if (n_aligned == n) nc = n;
+ else if (n_aligned % 32 == 0) nc = 32;
+ else if (n_aligned % 24 == 0) nc = 24;
+ else if (n_aligned % 16 == 0) nc = 16;
+ else nc = 8;
+ }
+ bool can_use_tiled = n_aligned > 0 && (m % mc == 0) && (k % kc == 0);
+ if (can_use_tiled) {
+ matmul_tiled(m, n_aligned, mc, nc, kc);
+ if (n > n_aligned) {
+ mnpack(0, m, n_aligned, n);
+ }
} else {
mnpack(0, m, 0, n);
}
}
- template
- template
- void tinyBLAS_Q0_PPC::packNormalInt4(const TA* a, int64_t lda, int rows, int cols, int8_t* vec, std::array& comparray) {
+ private:
+ inline void save_res(int ii, int jj, int idx, vector float * fin_res, int RM = 4, int RN = 4) {
+ for (int I = 0; I < RM; I++) {
+ for (int J = 0; J < RN; J++) {
+ *((float *)(C + ii + ((jj + J) * ldc) + I)) = *((float *)&fin_res[idx + I] + J);
+ }
+ }
+ }
+
+ inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) {
+ vec_t vec_C[4];
+ __builtin_mma_disassemble_acc(vec_C, ACC);
+ for (int I = 0; I < 4; I++) {
+ for (int J = 0; J < 4; J++) {
+ *((float *)(C + ii + ((jj + J) * ldc) + I)) = *((float *)&vec_C[I] + J);
+ }
+ }
+ }
+
+ inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) {
+ vec_t vec_C[4];
+ __builtin_mma_disassemble_acc(vec_C, ACC);
+ for (int I = 0; I < 4; I++) {
+ for (int J = 0; J < 4; J++) {
+ float * c_ptr = (float *)(C + ii+ ((jj + J) * ldc) + I);
+ *c_ptr += *((float *)&vec_C[I] + J);
+ }
+ }
+ }
+
+ template
+ inline void compute(acc_t * ACC, int c_idx, int s_idx, ArrayType & comparray, vector float * vs, vector float * fin_res) {
+ vector signed int vec_C[4];
+ vector float CA[4] = {0};
+ vector float res[4] = {0};
+ __builtin_mma_disassemble_acc(vec_C, ACC);
+ for (int i = 0; i < 4; i++) {
+ CA[i] = vec_splats((float)(((double)comparray[c_idx + i]) * -128.0));
+ res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);
+ fin_res[s_idx + i] = vec_madd(res[i], vs[s_idx + i], fin_res[s_idx + i]);
+ }
+ }
+
+ inline void process_q4_elements(vector signed char (&c)[2], int * ca) {
+ const vector signed char lowMask = vec_splats((signed char)0xF);
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
+ const vector signed char v8 = vec_splats((signed char)0x8);
+ vector signed int vsum = {0};
+ vector signed int vsum2 = {0};
+ c[0] = vec_and(c[1], lowMask);
+ c[1] = vec_sr(c[1], v4);
+ c[0] = vec_sub(c[0], v8);
+ c[1] = vec_sub(c[1], v8);
+ vsum = vec_sum4s(c[0], vsum);
+ vsum2 = vec_sum4s(c[1], vsum2);
+ vsum = vec_add(vsum, vsum2);
+ *(ca) = vsum[0] + vsum[1] + vsum[2] + vsum[3];
+ }
+
+ template
+ inline void vector_permute_store(V2 & s1, V2 & s2, V2 & s3, V2 & s4, V1 * vecOffset, bool flip) {
+ vector unsigned char swiz1 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+ vector unsigned char swiz2 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+ vector unsigned char swiz3 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};
+ vector unsigned char swiz4 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
+ V2 t1, t2, t3, t4, t5, t6, t7, t8;
+ vector unsigned char xor_vector;
+ uint8_t flip_vec = 0x80;
+ xor_vector = vec_splats(flip_vec);
+ t1 = vec_perm(s1, s2, swiz1);
+ t2 = vec_perm(s1, s2, swiz2);
+ t3 = vec_perm(s3, s4, swiz1);
+ t4 = vec_perm(s3, s4, swiz2);
+ t5 = vec_perm(t1, t3, swiz3);
+ t6 = vec_perm(t1, t3, swiz4);
+ t7 = vec_perm(t2, t4, swiz3);
+ t8 = vec_perm(t2, t4, swiz4);
+ if (flip == true) {
+ t5 = vec_xor(t5, xor_vector);
+ t6 = vec_xor(t6, xor_vector);
+ t7 = vec_xor(t7, xor_vector);
+ t8 = vec_xor(t8, xor_vector);
+ }
+ vec_xst(t5, 0, vecOffset);
+ vec_xst(t6, 0, vecOffset + 16);
+ vec_xst(t7, 0, vecOffset + 32);
+ vec_xst(t8, 0, vecOffset + 48);
+ }
+
+ inline void unpack_q4_to_q8(vector signed char packed, vector signed char & lo, vector signed char & hi) {
+ const vector signed char lowMask = vec_splats((signed char)0x0F);
+ const vector signed char v8 = vec_splats((signed char)0x08);
+ const vector unsigned char v4 = vec_splats((unsigned char)4);
+ lo = vec_and(packed, lowMask);
+ hi = vec_sr(packed, v4);
+ lo = vec_sub(lo, v8);
+ hi = vec_sub(hi, v8);
+ }
+
+ inline void vector_permute_store_fp16(vec_t * c, unsigned char * vecOffset) {
+ vec_t t[8], s[8];
+ vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
+ vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
+ vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+ vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+ for (int i = 0; i < 4; i += 2) {
+ t[i + 0] = vec_perm(c[i + 0], c[i + 1], swiz1);
+ t[i + 1] = vec_perm(c[i + 0], c[i + 1], swiz2);
+ }
+ for (int i = 4; i < 8; i += 2) {
+ t[i + 0] = vec_perm(c[i + 0], c[i + 1], swiz1);
+ t[i + 1] = vec_perm(c[i + 0], c[i + 1], swiz2);
+ }
+ s[0] = vec_perm(t[0], t[2], swiz3);
+ s[1] = vec_perm(t[0], t[2], swiz4);
+ s[2] = vec_perm(t[1], t[3], swiz3);
+ s[3] = vec_perm(t[1], t[3], swiz4);
+ s[4] = vec_perm(t[4], t[6], swiz3);
+ s[5] = vec_perm(t[4], t[6], swiz4);
+ s[6] = vec_perm(t[5], t[7], swiz3);
+ s[7] = vec_perm(t[5], t[7], swiz4);
+ for (int i = 0; i < 8; ++i) {
+ vec_xst(s[i], 0, (vec_t *)(vecOffset + i * 16));
+ }
+ }
+
+ static inline void convert_and_scale_q8(vector signed char raw, vector float v_scale, vector unsigned short & out_hi, vector unsigned short & out_lo) {
+ vector signed short i16_hi = vec_unpackh(raw);
+ vector signed short i16_lo = vec_unpackl(raw);
+
+ vector float f_hi_h = vec_ctf(vec_unpackh(i16_hi), 0);
+ vector float f_hi_l = vec_ctf(vec_unpackl(i16_hi), 0);
+ vector float f_lo_h = vec_ctf(vec_unpackh(i16_lo), 0);
+ vector float f_lo_l = vec_ctf(vec_unpackl(i16_lo), 0);
+ out_hi = vec_pack_to_short_fp32(vec_mul(f_hi_h, v_scale), vec_mul(f_hi_l, v_scale));
+ out_lo = vec_pack_to_short_fp32(vec_mul(f_lo_h, v_scale), vec_mul(f_lo_l, v_scale));
+ }
+
+ void packNormal_q4_fp16(const block_q4_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {
+ unsigned char * vecOffset = vec;
+ for (int i = 0; i < rows; i += 8) {
+ const block_q4_0 * rows_base[8];
+ for (int r = 0; r < 8; r++) {
+ rows_base[r] = a + (i + r) * lda;
+ }
+ for (int blk = 0; blk < blocks; blk++) {
+ vector unsigned short hp_res[8][4];
+ for (int r = 0; r < 8; r++) {
+ const block_q4_0 * current_blk = rows_base[r] + blk;
+ vector float v_scale = vec_extract_fp32_from_shorth(vec_splats(current_blk->d));
+ vector signed char v_qs = reinterpret_cast(vec_xl(0, current_blk->qs));
+ vector signed char c1, c2;
+ unpack_q4_to_q8(v_qs, c1, c2);
+ convert_and_scale_q8(c1, v_scale, hp_res[r][0], hp_res[r][1]);
+ convert_and_scale_q8(c2, v_scale, hp_res[r][2], hp_res[r][3]);
+ }
+ for (int c = 0; c < 4; c++) {
+ vector unsigned char c_arr[8];
+ for (int r = 0; r < 8; r++) {
+ c_arr[r] = (vector unsigned char)hp_res[r][c];
+ }
+ vector_permute_store_fp16((vec_t *)c_arr, vecOffset);
+ vecOffset += 128;
+ }
+ }
+ }
+ }
+
+ template
+ static inline void pack_q8_block(const block_q8_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {
+ unsigned char * vecOffset = vec;
+ const vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
+ const vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
+ const vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+ const vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+
+ for (int i = 0; i < rows; i += chunk_size) {
+ const block_q8_0 * rows_base[chunk_size];
+ for (int r = 0; r < chunk_size; r++) {
+ rows_base[r] = a + (i + r) * lda;
+ }
+ for (int blk = 0; blk < blocks; blk++) {
+ vector unsigned short hp_res[chunk_size][4];
+ for (int r = 0; r < chunk_size; r++) {
+ const block_q8_0 * b = rows_base[r] + blk;
+ vector float v_scale = vec_extract_fp32_from_shorth(vec_splats(b->d));
+ vector signed char c[2];
+ __vector_pair pair = __builtin_vsx_lxvp(0, (__vector_pair *)b->qs);
+ __builtin_vsx_disassemble_pair(c, & pair);
+ convert_and_scale_q8(c[0], v_scale, hp_res[r][0], hp_res[r][1]);
+ convert_and_scale_q8(c[1], v_scale, hp_res[r][2], hp_res[r][3]);
+ }
+ for (int col = 0; col < 4; col++) {
+ if constexpr (chunk_size == 8) {
+ vec_t t[8];
+ t[0] = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz1);
+ t[1] = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz2);
+ t[2] = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz1);
+ t[3] = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz2);
+ t[4] = vec_perm((vec_t)hp_res[4][col], (vec_t)hp_res[5][col], swiz1);
+ t[5] = vec_perm((vec_t)hp_res[4][col], (vec_t)hp_res[5][col], swiz2);
+ t[6] = vec_perm((vec_t)hp_res[6][col], (vec_t)hp_res[7][col], swiz1);
+ t[7] = vec_perm((vec_t)hp_res[6][col], (vec_t)hp_res[7][col], swiz2);
+
+ vec_xst(vec_perm(t[0], t[2], swiz3), 0, (vec_t *)(vecOffset + 0));
+ vec_xst(vec_perm(t[0], t[2], swiz4), 0, (vec_t *)(vecOffset + 16));
+ vec_xst(vec_perm(t[1], t[3], swiz3), 0, (vec_t *)(vecOffset + 32));
+ vec_xst(vec_perm(t[1], t[3], swiz4), 0, (vec_t *)(vecOffset + 48));
+ vec_xst(vec_perm(t[4], t[6], swiz3), 0, (vec_t *)(vecOffset + 64));
+ vec_xst(vec_perm(t[4], t[6], swiz4), 0, (vec_t *)(vecOffset + 80));
+ vec_xst(vec_perm(t[5], t[7], swiz3), 0, (vec_t *)(vecOffset + 96));
+ vec_xst(vec_perm(t[5], t[7], swiz4), 0, (vec_t *)(vecOffset + 112));
+ vecOffset += 128;
+ } else {
+ vec_t t0 = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz1);
+ vec_t t1 = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz2);
+ vec_t t2 = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz1);
+ vec_t t3 = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz2);
+
+ vec_xst(vec_perm(t0, t2, swiz3), 0, (vec_t *)(vecOffset + 0));
+ vec_xst(vec_perm(t0, t2, swiz4), 0, (vec_t *)(vecOffset + 16));
+ vec_xst(vec_perm(t1, t3, swiz3), 0, (vec_t *)(vecOffset + 32));
+ vec_xst(vec_perm(t1, t3, swiz4), 0, (vec_t *)(vecOffset + 48));
+ vecOffset += 64;
+ }
+ }
+ }
+ }
+ }
+
+ void packNormal_q8_fp16(const block_q8_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {
+ if (rows == 4) {
+ pack_q8_block<4>(a, lda, rows, blocks, vec);
+ } else {
+ pack_q8_block<8>(a, lda, rows, blocks, vec);
+ }
+ }
+
+ template
+ void packNormalInt4(const TA * a, int64_t lda, int rows, int cols, int8_t * vec, std::array & comparray) {
int64_t i, j;
- TA *aoffset = NULL;
- int8_t *vecOffset = NULL;
- TA *aoffset1 = NULL, *aoffset2 = NULL, *aoffset3 = NULL, *aoffset4 = NULL;
- TA *aoffset5 = NULL, *aoffset6 = NULL, *aoffset7 = NULL, *aoffset8 = NULL;
+ TA * aoffset = NULL;
+ int8_t * vecOffset = NULL;
+ TA * aoffset1 = NULL, * aoffset2 = NULL, * aoffset3 = NULL, * aoffset4 = NULL;
+ TA * aoffset5 = NULL, * aoffset6 = NULL, * aoffset7 = NULL, * aoffset8 = NULL;
vector signed char c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2] = {0};
vector signed char c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2] = {0};
- aoffset = const_cast(a);
+ aoffset = const_cast(a);
vecOffset = vec;
j = (rows >> 3);
if (j > 0) {
@@ -2363,18 +2620,18 @@ class tinyBLAS_HP16_PPC {
c7[1] = reinterpret_cast(vec_xl(0, aoffset7->qs));
c8[1] = reinterpret_cast(vec_xl(0, aoffset8->qs));
- process_q4_elements(c1, &comparray[0]);
- process_q4_elements(c2, &comparray[1]);
- process_q4_elements(c3, &comparray[2]);
- process_q4_elements(c4, &comparray[3]);
- process_q4_elements(c5, &comparray[4]);
- process_q4_elements(c6, &comparray[5]);
- process_q4_elements(c7, &comparray[6]);
- process_q4_elements(c8, &comparray[7]);
+ process_q4_elements(c1, & comparray[0]);
+ process_q4_elements(c2, & comparray[1]);
+ process_q4_elements(c3, & comparray[2]);
+ process_q4_elements(c4, & comparray[3]);
+ process_q4_elements(c5, & comparray[4]);
+ process_q4_elements(c6, & comparray[5]);
+ process_q4_elements(c7, & comparray[6]);
+ process_q4_elements(c8, & comparray[7]);
vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);
- vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset+64, false);
- vector_permute_store(c5[0], c6[0], c7[0], c8[0], vecOffset+128, false);
- vector_permute_store(c5[1], c6[1], c7[1], c8[1], vecOffset+192, false);
+ vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);
+ vector_permute_store(c5[0], c6[0], c7[0], c8[0], vecOffset + 128, false);
+ vector_permute_store(c5[1], c6[1], c7[1], c8[1], vecOffset + 192, false);
aoffset1 += lda;
aoffset2 += lda;
aoffset3 += lda;
@@ -2405,12 +2662,12 @@ class tinyBLAS_HP16_PPC {
c3[1] = reinterpret_cast(vec_xl(0, aoffset3->qs));
c4[1] = reinterpret_cast(vec_xl(0, aoffset4->qs));
- process_q4_elements(c1, &comparray[0]);
- process_q4_elements(c2, &comparray[1]);
- process_q4_elements(c3, &comparray[2]);
- process_q4_elements(c4, &comparray[3]);
+ process_q4_elements(c1, & comparray[0]);
+ process_q4_elements(c2, & comparray[1]);
+ process_q4_elements(c3, & comparray[2]);
+ process_q4_elements(c4, & comparray[3]);
vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);
- vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset+64, false);
+ vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);
aoffset1 += lda;
aoffset2 += lda;
aoffset3 += lda;
@@ -2434,12 +2691,12 @@ class tinyBLAS_HP16_PPC {
case 1: c1[1] = reinterpret_cast(vec_xl(0, aoffset1->qs));
break;
}
- process_q4_elements(c1, &comparray[0]);
- process_q4_elements(c2, &comparray[1]);
- process_q4_elements(c3, &comparray[2]);
- process_q4_elements(c4, &comparray[3]);
+ process_q4_elements(c1, & comparray[0]);
+ process_q4_elements(c2, & comparray[1]);
+ process_q4_elements(c3, & comparray[2]);
+ process_q4_elements(c4, & comparray[3]);
vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);
- vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset+64, false);
+ vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);
aoffset1 += lda;
aoffset2 += lda;
aoffset3 += lda;
@@ -2450,39 +2707,38 @@ class tinyBLAS_HP16_PPC {
}
}
- template
template
- void tinyBLAS_Q0_PPC::packNormal(const block_q8_0* a, int64_t lda, int rows, int cols, VA* vec, bool flip) {
+ void packNormal(const block_q8_0 * a, int64_t lda, int rows, int cols, VA * vec, bool flip) {
int64_t i, j;
- block_q8_0 *aoffset = NULL;
- VA *vecOffset = NULL;
- block_q8_0* aoffsets[8];
+ block_q8_0 * aoffset = NULL;
+ VA * vecOffset = NULL;
+ block_q8_0 * aoffsets[8];
__vector_pair arr[8];
VB c[8][2] = {0};
VB c1[8] = {0}; VB c2[8] = {0};
- aoffset = const_cast(a);
+ aoffset = const_cast(a);
vecOffset = vec;
j = (rows >> 3);
if (j > 0) {
do {
aoffsets[0] = aoffset;
for (int it = 1; it < 8; it++)
- aoffsets[it] = aoffsets[it-1] + lda;
+ aoffsets[it] = aoffsets[it - 1] + lda;
aoffset += 8 * lda;
i = (cols >> 3);
if (i > 0) {
do {
for (int it = 0; it < 8; it++) {
- arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[it]->qs);
- __builtin_vsx_disassemble_pair(c[it], &arr[it]);
+ arr[it] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[it]->qs);
+ __builtin_vsx_disassemble_pair(c[it], & arr[it]);
c1[it] = c[it][0];
c2[it] = c[it][1];
}
vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);
- vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset+64, flip);
- vector_permute_store(c1[4], c1[5], c1[6], c1[7], vecOffset+128, flip);
- vector_permute_store(c2[4], c2[5], c2[6], c2[7], vecOffset+192, flip);
+ vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);
+ vector_permute_store(c1[4], c1[5], c1[6], c1[7], vecOffset + 128, flip);
+ vector_permute_store(c2[4], c2[5], c2[6], c2[7], vecOffset + 192, flip);
for (int it = 0; it < 8; it++)
aoffsets[it] += lda;
vecOffset += 256;
@@ -2501,13 +2757,13 @@ class tinyBLAS_HP16_PPC {
if (i > 0) {
do {
for (int it = 0; it < 4; it++) {
- arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[it]->qs);
- __builtin_vsx_disassemble_pair(c[it], &arr[it]);
+ arr[it] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[it]->qs);
+ __builtin_vsx_disassemble_pair(c[it], & arr[it]);
c1[it] = c[it][0];
c2[it] = c[it][1];
}
vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);
- vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset+64, flip);
+ vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);
for (int it = 0; it < 4; it++) {
aoffsets[it] += lda;
}
@@ -2520,24 +2776,24 @@ class tinyBLAS_HP16_PPC {
if (rows & 3) {
aoffsets[0] = aoffset;
for (int it = 1; it < 3; it++ )
- aoffsets[it] = aoffsets[it-1] + lda;
+ aoffsets[it] = aoffsets[it - 1] + lda;
i = (cols >> 3);
if (i > 0) {
do {
switch(rows) {
- case 3: arr[2] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[2]->qs);
- __builtin_vsx_disassemble_pair(c[2], &arr[2]);
+ case 3: arr[2] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[2]->qs);
+ __builtin_vsx_disassemble_pair(c[2], & arr[2]);
c1[2] = c[2][0]; c2[2] = c[2][1];
- case 2: arr[1] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[1]->qs);
- __builtin_vsx_disassemble_pair(c[1], &arr[1]);
+ case 2: arr[1] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[1]->qs);
+ __builtin_vsx_disassemble_pair(c[1], & arr[1]);
c1[1] = c[1][0]; c2[1] = c[1][1];
- case 1: arr[0] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[0]->qs);
- __builtin_vsx_disassemble_pair(c[0], &arr[0]);
+ case 1: arr[0] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[0]->qs);
+ __builtin_vsx_disassemble_pair(c[0], & arr[0]);
c1[0] = c[0][0]; c2[0] = c[0][1];
break;
}
vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);
- vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset+64, flip);
+ vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);
for (int it = 0; it < 3; it++)
aoffsets[it] += lda;
vecOffset += 128;
@@ -2547,8 +2803,7 @@ class tinyBLAS_HP16_PPC {
}
}
- template
- void tinyBLAS_Q0_PPC::mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
int m_rem = MIN(m - m0, 16);
int n_rem = MIN(n - n0, 16);
@@ -2585,8 +2840,7 @@ class tinyBLAS_HP16_PPC {
}
- template
- void tinyBLAS_Q0_PPC::KERNEL_4x8(int64_t ii, int64_t jj) {
+ void KERNEL_4x8(int64_t ii, int64_t jj) {
vec_t vec_A[8], vec_B[16] = {0};
acc_t acc_0, acc_1;
std::array comparray {};
@@ -2594,26 +2848,26 @@ class tinyBLAS_HP16_PPC {
vector float vs[8] = {0};
bool isAblock_q4 = std::is_same_v;
for (int l = 0; l < k; l++) {
- __builtin_mma_xxsetaccz(&acc_0);
- __builtin_mma_xxsetaccz(&acc_1);
+ __builtin_mma_xxsetaccz(& acc_0);
+ __builtin_mma_xxsetaccz(& acc_1);
if (std::is_same_v) {
- packNormalInt4<4>((A+(ii*lda)+l), lda, 4, 4, (int8_t*)vec_A, comparray);
+ packNormalInt4<4>((A + (ii * lda) + l), lda, 4, 4, (int8_t *)vec_A, comparray);
} else {
- packNormal((const block_q8_0*)(A+(ii*lda)+l), lda, 4, 8, (int8_t*)vec_A, false);
+ packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 4, 8, (int8_t *)vec_A, false);
}
- packNormal((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B, true);
+ packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true);
for(int x = 0; x < 8; x++) {
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
- __builtin_mma_xvi8ger4pp(&acc_1, vec_A[x], vec_B[x+8]);
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);
+ __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x], vec_B[x+8]);
}
for (int I = 0; I<4; I++) {
for (int J = 0; J<4; J++) {
- *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
- *((float*)&vs[I+4]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J+4)*ldb)+l)->d));
+ *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));
+ *((float *)& vs[I + 4] + J) = (unhalf((A +((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d));
}
}
if (!isAblock_q4) {
- auto aoffset = A+(ii*lda)+l;
+ auto aoffset = A + (ii * lda) + l;
for (int i = 0; i < 4; i++) {
comparray[i] = 0;
int ca = 0;
@@ -2624,15 +2878,14 @@ class tinyBLAS_HP16_PPC {
aoffset += lda;
}
}
- compute(&acc_0, 0, 0, comparray, vs, fin_res);
- compute(&acc_1, 0, 4, comparray, vs, fin_res);
+ compute(& acc_0, 0, 0, comparray, vs, fin_res);
+ compute(& acc_1, 0, 4, comparray, vs, fin_res);
}
save_res(ii, jj, 0, fin_res);
- save_res(ii, jj+4, 4, fin_res);
+ save_res(ii, jj + 4, 4, fin_res);
}
- template
- void tinyBLAS_Q0_PPC::KERNEL_8x4(int64_t ii, int64_t jj) {
+ void KERNEL_8x4(int64_t ii, int64_t jj) {
vec_t vec_A[16], vec_B[8] = {0};
acc_t acc_0, acc_1;
std::array comparray {};
@@ -2640,25 +2893,25 @@ class tinyBLAS_HP16_PPC {
vector float vs[8] = {0};
bool isAblock_q4 = std::is_same_v;
for (int l = 0; l < k; l++) {
- __builtin_mma_xxsetaccz(&acc_0);
- __builtin_mma_xxsetaccz(&acc_1);
+ __builtin_mma_xxsetaccz(& acc_0);
+ __builtin_mma_xxsetaccz(& acc_1);
if (std::is_same_v) {
- packNormalInt4<8>((A+(ii*lda)+l), lda, 8, 4, (int8_t*)vec_A, comparray);
+ packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray);
} else {
- packNormal((const block_q8_0*)(A+(ii*lda)+l), lda, 8, 8, (int8_t*)vec_A, false);
+ packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false);
}
- packNormal((B+(jj*ldb)+l), ldb, 4, 8, (uint8_t*)vec_B, true);
+ packNormal((B + (jj * ldb) + l), ldb, 4, 8, (uint8_t *)vec_B, true);
for(int x = 0; x < 8; x++) {
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
- __builtin_mma_xvi8ger4pp(&acc_1, vec_A[x+8], vec_B[x]);
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);
+ __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x + 8], vec_B[x]);
}
- for (int I = 0; I<8; I++) {
- for (int J = 0; J<4; J++) {
- *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
+ for (int I = 0; I < 8; I++) {
+ for (int J = 0; J < 4; J++) {
+ *((float *)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));
}
}
if (!isAblock_q4) {
- auto aoffset = A+(ii*lda)+l;
+ auto aoffset = A + (ii * lda) + l;
for (int i = 0; i < 8; i++) {
comparray[i] = 0;
int ca = 0;
@@ -2669,15 +2922,14 @@ class tinyBLAS_HP16_PPC {
aoffset += lda;
}
}
- compute(&acc_0, 0, 0, comparray, vs, fin_res);
- compute(&acc_1, 4, 4, comparray, vs, fin_res);
+ compute(& acc_0, 0, 0, comparray, vs, fin_res);
+ compute(& acc_1, 4, 4, comparray, vs, fin_res);
}
save_res(ii, jj, 0, fin_res);
- save_res(ii+4, jj, 4, fin_res);
+ save_res(ii + 4, jj, 4, fin_res);
}
- template
- void tinyBLAS_Q0_PPC::KERNEL_8x8(int64_t ii, int64_t jj) {
+ void KERNEL_8x8(int64_t ii, int64_t jj) {
vec_t vec_A[16], vec_B[16] = {0};
acc_t acc_0, acc_1, acc_2, acc_3;
acc_t acc_4, acc_5, acc_6, acc_7;
@@ -2686,30 +2938,30 @@ class tinyBLAS_HP16_PPC {
vector float vs[16] = {0};
bool isAblock_q4 = std::is_same_v;
for (int l = 0; l < k; l++) {
- __builtin_mma_xxsetaccz(&acc_0);
- __builtin_mma_xxsetaccz(&acc_1);
- __builtin_mma_xxsetaccz(&acc_2);
- __builtin_mma_xxsetaccz(&acc_3);
+ __builtin_mma_xxsetaccz(& acc_0);
+ __builtin_mma_xxsetaccz(& acc_1);
+ __builtin_mma_xxsetaccz(& acc_2);
+ __builtin_mma_xxsetaccz(& acc_3);
if (std::is_same_v) {
- packNormalInt4<8>((A+(ii*lda)+l), lda, 8, 4, (int8_t*)vec_A, comparray);
+ packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray);
} else {
- packNormal((const block_q8_0*)(A+(ii*lda)+l), lda, 8, 8, (int8_t*)vec_A, false);
+ packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false);
}
- packNormal((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B, true);
+ packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true);
for(int x = 0; x < 8; x++) {
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
- __builtin_mma_xvi8ger4pp(&acc_1, vec_A[x+8], vec_B[x]);
- __builtin_mma_xvi8ger4pp(&acc_2, vec_A[x], vec_B[x+8]);
- __builtin_mma_xvi8ger4pp(&acc_3, vec_A[x+8], vec_B[x+8]);
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);
+ __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x + 8], vec_B[x]);
+ __builtin_mma_xvi8ger4pp(& acc_2, vec_A[x], vec_B[x + 8]);
+ __builtin_mma_xvi8ger4pp(& acc_3, vec_A[x + 8], vec_B[x + 8]);
}
- for (int I = 0; I<8; I++) {
- for (int J = 0; J<4; J++) {
- *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
- *((float*)&vs[I+8]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J+4)*ldb)+l)->d));
+ for (int I = 0; I < 8 ; I++) {
+ for (int J = 0; J < 4; J++) {
+ *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));
+ *((float *)& vs[I + 8] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d));
}
}
if (!isAblock_q4) {
- auto aoffset = A+(ii*lda)+l;
+ auto aoffset = A + (ii * lda) + l;
for (int i = 0; i < 8; i++) {
comparray[i] = 0;
int ca = 0;
@@ -2720,19 +2972,99 @@ class tinyBLAS_HP16_PPC {
aoffset += lda;
}
}
- compute(&acc_0, 0, 0, comparray, vs, fin_res);
- compute(&acc_1, 4, 4, comparray, vs, fin_res);
- compute(&acc_2, 0, 8, comparray, vs, fin_res);
- compute(&acc_3, 4, 12, comparray, vs, fin_res);
+ compute(& acc_0, 0, 0, comparray, vs, fin_res);
+ compute(& acc_1, 4, 4, comparray, vs, fin_res);
+ compute(& acc_2, 0, 8, comparray, vs, fin_res);
+ compute(& acc_3, 4, 12, comparray, vs, fin_res);
}
save_res(ii, jj, 0, fin_res);
- save_res(ii+4, jj, 4, fin_res);
- save_res(ii, jj+4, 8, fin_res);
- save_res(ii+4, jj+4, 12, fin_res);
+ save_res(ii + 4, jj, 4, fin_res);
+ save_res(ii, jj + 4, 8, fin_res);
+ save_res(ii + 4, jj + 4, 12, fin_res);
}
- template
- void tinyBLAS_Q0_PPC::gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {
+ void KERNEL_Q0(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, int64_t l, vec_t * vec_A, vec_t * vec_B) {
+ acc_t acc[8];
+ for (int i = 0; i < mc ; i += 16) {
+ for (int j = 0; j < nc; j += 8) {
+ int A0_base = (i / 16) * (2 * 32 * kc);
+ int B0_base = (j / 8) * (32 * kc);
+ for (int x = 0; x < 8; x++) {
+ __builtin_mma_xxsetaccz(&acc[x]);
+ }
+ for (int64_t kk = 0; kk < kc; kk++) {
+ int A0_block_idx = A0_base + kk * 32;
+ int B0_block_idx = B0_base + kk * 32;
+ int A1_block_idx = A0_block_idx + 32 * kc;
+ int B1_block_idx = B0_block_idx + 32 * kc;
+ vec_t * A0_block = & vec_A[A0_block_idx];
+ vec_t * B0_block = & vec_B[B0_block_idx];
+ vec_t * A1_block = & vec_A[A1_block_idx];
+ for (int it = 0; it < 4; it++) {
+ for (int x = 0; x < 4; x++) {
+ __builtin_mma_xvf16ger2pp(& acc[0], A0_block[8 * it + x], B0_block[8 * it + x]);
+ __builtin_mma_xvf16ger2pp(& acc[1], A0_block[8 * it + x], B0_block[8 * it + x + 4]);
+ __builtin_mma_xvf16ger2pp(& acc[2], A0_block[8 * it + x + 4], B0_block[8 * it + x]);
+ __builtin_mma_xvf16ger2pp(& acc[3], A0_block[8 * it + x + 4], B0_block[8 * it + x + 4]);
+ __builtin_mma_xvf16ger2pp(& acc[4], A1_block[8 * it + x], B0_block[8 * it + x]);
+ __builtin_mma_xvf16ger2pp(& acc[5], A1_block[8 * it + x], B0_block[8 * it+ x + 4]);
+ __builtin_mma_xvf16ger2pp(& acc[6], A1_block[8 * it + x + 4], B0_block[8 * it + x]);
+ __builtin_mma_xvf16ger2pp(& acc[7], A1_block[8 * it + x + 4], B0_block[8 * it + x + 4]);
+ }
+ }
+ }
+ if (l == 0) {
+ save_acc(& acc[0], ii + i, jj + j);
+ save_acc(& acc[1], ii + i, jj + j + 4);
+ save_acc(& acc[2], ii + i + 4, jj + j);
+ save_acc(& acc[3], ii + i + 4, jj + j + 4);
+ save_acc(& acc[4], ii + i + 8, jj + j);
+ save_acc(& acc[5], ii + i + 8, jj + j + 4);
+ save_acc(& acc[6], ii + i + 12, jj + j);
+ save_acc(& acc[7], ii + i + 12, jj + j + 4);
+ } else {
+ add_save_acc(& acc[0], ii + i, jj + j);
+ add_save_acc(& acc[1], ii + i, jj + j + 4);
+ add_save_acc(& acc[2], ii + i + 4, jj + j);
+ add_save_acc(& acc[3], ii + i + 4, jj + j + 4);
+ add_save_acc(& acc[4], ii + i + 8, jj + j);
+ add_save_acc(& acc[5], ii + i + 8, jj + j + 4);
+ add_save_acc(& acc[6], ii + i + 12, jj + j);
+ add_save_acc(& acc[7], ii + i + 12, jj + j + 4);
+ }
+ }
+ }
+ }
+
+ void matmul_tiled(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) {
+ vec_t A_pack[mc * kc * 4];
+ vec_t B_pack[nc * kc * 4];
+ constexpr bool is_Ablock_q4 = std::is_same_v;
+ int64_t ytiles = m / mc;
+ int64_t xtiles = n / nc;
+ int64_t tiles = xtiles * ytiles;
+ int64_t duty = (tiles + nth - 1) / nth;
+ int64_t start = duty * ith;
+ int64_t end = start + duty;
+ if (end > tiles) {
+ end = tiles;
+ }
+ for (int64_t job = start; job < end; ++job) {
+ int64_t ii = (job / xtiles) * mc;
+ int64_t jj = (job % xtiles) * nc;
+ for (int64_t kk = 0; kk < k; kk += kc) {
+ if constexpr(is_Ablock_q4) {
+ packNormal_q4_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack);
+ } else {
+ packNormal_q8_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack);
+ }
+ packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, kc, (uint8_t *)B_pack);
+ KERNEL_Q0(ii, jj, mc, nc, kc, kk, A_pack, B_pack);
+ }
+ }
+ }
+
+ void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {
int64_t ytiles = (m - m0) / RM;
int64_t xtiles = (n - n0) / RN;
int64_t tiles = xtiles * ytiles;
@@ -2754,32 +3086,32 @@ class tinyBLAS_HP16_PPC {
vector float fin_res[4] = {0};
vector float vs[4] = {0};
vector float CA[4] = {0};
- __builtin_prefetch((A+(ii*lda)+0)->qs, 0, 1); // prefetch first value
- __builtin_prefetch((B+(jj*ldb)+0)->qs, 0, 1); // prefetch first value
+ __builtin_prefetch((A + (ii * lda) + 0)->qs, 0, 1); // prefetch first value
+ __builtin_prefetch((B + (jj * ldb) + 0)->qs, 0, 1); // prefetch first value
for (int l = 0; l < k; l++) {
- __builtin_prefetch((A+(ii*lda)+(l+1))->qs, 0, 1); // prefetch one loop ahead
- __builtin_prefetch((B+(jj*ldb)+(l+1))->qs, 0, 1); // prefetch one loop ahead
- __builtin_mma_xxsetaccz(&acc_0);
+ __builtin_prefetch((A + (ii * lda) + (l + 1))->qs, 0, 1); // prefetch one loop ahead
+ __builtin_prefetch((B + (jj * ldb) + (l + 1))->qs, 0, 1); // prefetch one loop ahead
+ __builtin_mma_xxsetaccz(& acc_0);
if (isAblock_q4) {
- packNormalInt4<4>((A+(ii*lda)+l), lda, RM, 4, (int8_t*)vec_A, comparray);
+ packNormalInt4<4>((A + (ii * lda) + l), lda, RM, 4, (int8_t *)vec_A, comparray);
} else {
- packNormal((const block_q8_0*)(A+(ii*lda)+l), lda, RM, 8, (int8_t*)vec_A, false);
+ packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, RM, 8, (int8_t *)vec_A, false);
}
- packNormal((B+(jj*ldb)+l), ldb, RN, 8, (uint8_t*)vec_B, true);
- for(int x = 0; x < 8; x+=4) {
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x+1], vec_B[x+1]);
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x+2], vec_B[x+2]);
- __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x+3], vec_B[x+3]);
+ packNormal((B + (jj * ldb) + l), ldb, RN, 8, (uint8_t *)vec_B, true);
+ for (int x = 0; x < 8; x += 4) {
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 1], vec_B[x + 1]);
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 2], vec_B[x + 2]);
+ __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 3], vec_B[x + 3]);
}
- for (int I = 0; Id) * unhalf((B+((jj+J)*ldb)+l)->d));
+ for (int I = 0; I < RM; I++) {
+ for (int J = 0; J < RN; J++) {
+ *((float*)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));
}
}
- __builtin_mma_disassemble_acc(vec_C, &acc_0);
+ __builtin_mma_disassemble_acc(vec_C, & acc_0);
if (!isAblock_q4) {
- auto aoffset = A+(ii*lda)+l;
+ auto aoffset = A + (ii * lda) + l;
for (int i = 0; i < RM; i++) {
comparray[i] = 0;
int ca = 0;
@@ -2800,9 +3132,21 @@ class tinyBLAS_HP16_PPC {
}
}
- template
+ template
+ inline void kernel(int64_t ii, int64_t jj) {
+ if constexpr(RM == 4 && RN == 8) {
+ KERNEL_4x8(ii,jj);
+ } else if constexpr(RM == 8 && RN == 4) {
+ KERNEL_8x4(ii,jj);
+ } else if constexpr(RM == 8 && RN == 8) {
+ KERNEL_8x8(ii,jj);
+ } else {
+ assert(false && "RN/RM values not supported");
+ }
+ }
+
template
- NOINLINE void tinyBLAS_Q0_PPC::gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
int64_t ytiles = (m - m0) / RM;
int64_t xtiles = (n - n0) / RN;
int64_t tiles = xtiles * ytiles;
@@ -2814,12 +3158,20 @@ class tinyBLAS_HP16_PPC {
for (int64_t job = start; job < end; ++job) {
int64_t ii = m0 + job / xtiles * RM;
int64_t jj = n0 + job % xtiles * RN;
- this->kernel(ii, jj);
+ kernel(ii, jj);
}
}
-
-template class tinyBLAS_Q0_PPC;
-template class tinyBLAS_Q0_PPC;
+ const TA * const A;
+ const block_q8_0 * const B;
+ float * C;
+ const int64_t k;
+ int64_t kc;
+ const int64_t lda;
+ const int64_t ldb;
+ const int64_t ldc;
+ const int ith;
+ const int nth;
+};
class tinyBLAS_PPC {
public:
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index ce15b18ce0..b7a70e06f1 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -3,6 +3,7 @@
#include "ggml-cpu.h"
#include "ggml-impl.h"
#include "binary-ops.h"
+#include "simd-gemm.h"
#include "ggml.h"
#include "unary-ops.h"
#include "vec.h"
@@ -2096,10 +2097,14 @@ static void ggml_compute_forward_gelu_f32(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2113,10 +2118,14 @@ static void ggml_compute_forward_gelu_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_gelu_f32(nc,
- (float *) ((char *) dst->data + i1*( dst->nb[1])),
- (float *) ((char *) src0->data + i1*(src0->nb[1])));
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2135,10 +2144,14 @@ static void ggml_compute_forward_gelu_f16(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2152,10 +2165,14 @@ static void ggml_compute_forward_gelu_f16(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_gelu_f16(nc,
- (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),
- (ggml_fp16_t *) ((char *) src0->data + i1*(src0->nb[1])));
+ (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2276,10 +2293,14 @@ static void ggml_compute_forward_gelu_erf_f32(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2293,10 +2314,14 @@ static void ggml_compute_forward_gelu_erf_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_gelu_erf_f32(nc,
- (float *) ((char *) dst->data + i1*( dst->nb[1])),
- (float *) ((char *) src0->data + i1*(src0->nb[1])));
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2315,10 +2340,14 @@ static void ggml_compute_forward_gelu_erf_f16(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2332,10 +2361,14 @@ static void ggml_compute_forward_gelu_erf_f16(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_gelu_erf_f16(nc,
- (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),
- (ggml_fp16_t *) ((char *) src0->data + i1*(src0->nb[1])));
+ (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2379,10 +2412,14 @@ static void ggml_compute_forward_gelu_quick_f32(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2396,10 +2433,14 @@ static void ggml_compute_forward_gelu_quick_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_gelu_quick_f32(nc,
- (float *) ((char *) dst->data + i1*( dst->nb[1])),
- (float *) ((char *) src0->data + i1*(src0->nb[1])));
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2418,10 +2459,14 @@ static void ggml_compute_forward_gelu_quick_f16(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2435,10 +2480,14 @@ static void ggml_compute_forward_gelu_quick_f16(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_gelu_quick_f16(nc,
- (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),
- (ggml_fp16_t *) ((char *) src0->data + i1*(src0->nb[1])));
+ (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2482,10 +2531,14 @@ static void ggml_compute_forward_silu_f32(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2499,10 +2552,14 @@ static void ggml_compute_forward_silu_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_silu_f32(nc,
- (float *) ((char *) dst->data + i1*( dst->nb[1])),
- (float *) ((char *) src0->data + i1*(src0->nb[1])));
+ (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -2521,10 +2578,14 @@ static void ggml_compute_forward_silu_f16(
const ggml_tensor * src0 = dst->src[0];
- assert(ggml_is_contiguous_1(src0));
- assert(ggml_is_contiguous_1(dst));
+ assert(ggml_is_contiguous_rows(src0));
assert(ggml_are_same_shape(src0, dst));
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
const int ith = params->ith;
const int nth = params->nth;
@@ -2538,10 +2599,14 @@ static void ggml_compute_forward_silu_f16(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ const int i3 = ir/(ne02*ne01);
+ const int i2 = (ir - i3*ne02*ne01)/ne01;
+ const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
+
ggml_vec_silu_f16(nc,
- (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),
- (ggml_fp16_t *) ((char *) src0->data + i1*(src0->nb[1])));
+ (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
+ (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
@@ -7629,8 +7694,7 @@ static void ggml_compute_forward_pad_f32(
const ggml_tensor * src0 = dst->src[0];
- GGML_ASSERT(src0->nb[0] == sizeof(float));
- GGML_ASSERT( dst->nb[0] == sizeof(float));
+ assert(dst->nb[0] == sizeof(float));
const int ith = params->ith;
const int nth = params->nth;
@@ -8326,10 +8390,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
GGML_ASSERT(k->type == v->type);
const ggml_type kv_type = k->type;
- const auto * kv_type_traits_cpu = ggml_get_type_traits_cpu(kv_type);
- const ggml_from_float_t kv_from_float = kv_type_traits_cpu->from_float;
- const ggml_vec_dot_t kv_vec_dot = kv_type_traits_cpu->vec_dot;
- const size_t kv_type_size = ggml_type_size(kv_type);
// broadcast factors
const int64_t rk2 = neq2/nek2;
@@ -8361,8 +8421,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
static constexpr int Q_TILE_SZ = ggml_fa_tile_config::Q;
static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV;
- GGML_ASSERT(nek1 % KV_TILE_SZ == 0 && "KV sequence length must be divisible by KV_TILE_SZ");
-
int ir = ir0;
while (ir < ir1) {
// q indices for the start of this tile
@@ -8389,18 +8447,20 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
}
// Per-thread scratch layout:
- // Q_q: Q_TILE_SZ * DK (converted Q tile in KV type)
+ // Q_q: Q_TILE_SZ * DK (converted Q tile — F32 for GEMM, KV type for scalar)
// KQ: Q_TILE_SZ * KV_TILE_SZ (attention scores in float)
// mask: Q_TILE_SZ * KV_TILE_SZ (mask in float)
// VKQ32: Q_TILE_SZ * DV (FP32 output accumulator)
- // V32: KV_TILE_SZ * DV (F32 buffer for V tile - used for f166 conversion)
- float * base = (float *) params->wdata + ith*(Q_TILE_SZ*DK + 2*Q_TILE_SZ*KV_TILE_SZ + Q_TILE_SZ*DV + KV_TILE_SZ*DV + CACHE_LINE_SIZE_F32);
+ // V32: KV_TILE_SZ * DV (F32 buffer for V tile)
+ // K_f32: KV_TILE_SZ * DK (F32 buffer for K tile — GEMM path)
+ float * base = (float *) params->wdata + ith*(Q_TILE_SZ*DK + 2*Q_TILE_SZ*KV_TILE_SZ + Q_TILE_SZ*DV + KV_TILE_SZ*DV + KV_TILE_SZ*DK + CACHE_LINE_SIZE_F32);
void * Q_q = base;
float * KQ = (float *)((char *)base + Q_TILE_SZ * DK * sizeof(float));
float * mask32 = KQ + Q_TILE_SZ * KV_TILE_SZ;
float * VKQ32 = mask32 + Q_TILE_SZ * KV_TILE_SZ;
- float * V32 = VKQ32 + Q_TILE_SZ * DV; // F32 buffer for V tile
+ float * V32 = VKQ32 + Q_TILE_SZ * DV;
+ float * K_f32 = V32 + KV_TILE_SZ * DV;
memset(VKQ32, 0, Q_TILE_SZ * DV * sizeof(float));
memset(mask32, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
@@ -8413,28 +8473,38 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
const int iv3 = iq3 / rv3;
const int iv2 = iq2 / rv2;
- for (int tq = 0; tq < tile_rows; tq++) {
- const float * pq = (const float *) ((char *) q->data + ((iq1 + tq)*nbq1 + iq2*nbq2 + iq3*nbq3));
- kv_from_float(pq, (char *)Q_q + tq * DK * kv_type_size, DK);
- }
- // Zero-pad remaining rows
- for (int tq = tile_rows; tq < Q_TILE_SZ; tq++) {
- memset((char *)Q_q + tq * DK * kv_type_size, 0, DK * kv_type_size);
+ {
+ float * Q_f32 = (float *)Q_q;
+ for (int tq = 0; tq < tile_rows; tq++) {
+ const float * pq = (const float *) ((char *) q->data + ((iq1 + tq)*nbq1 + iq2*nbq2 + iq3*nbq3));
+ memcpy(Q_f32 + tq * DK, pq, DK * sizeof(float));
+ }
+ for (int tq = tile_rows; tq < Q_TILE_SZ; tq++) {
+ memset(Q_f32 + tq * DK, 0, DK * sizeof(float));
+ }
}
+ memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float));
+ memset(V32, 0, KV_TILE_SZ * DV * sizeof(float));
+
for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) {
+ const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic);
// skip the tile entirely if all the masks are -inf
if (mask) {
bool can_skip = true;
for (int tq = 0; tq < tile_rows; tq++) {
const ggml_fp16_t * mp_row = (const ggml_fp16_t *)((const char *) mask->data + (iq1 + tq)*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]);
- for (int tk = 0; tk < KV_TILE_SZ; tk++) {
+ for (int tk = 0; tk < kv_tile; tk++) {
mask32[tq * KV_TILE_SZ + tk] = slope * GGML_CPU_FP16_TO_FP32(mp_row[ic + tk]);
if (mask32[tq * KV_TILE_SZ + tk] != -INFINITY) {
can_skip = false;
}
}
+ // Pad remaining mask entries with -inf
+ for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {
+ mask32[tq * KV_TILE_SZ + tk] = -INFINITY;
+ }
}
if (can_skip) {
@@ -8442,13 +8512,32 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
}
}
- for (int tq = 0; tq < Q_TILE_SZ; tq++) {
- const void * q_row = (const char *)Q_q + tq * DK * kv_type_size;
- for (int tk = 0; tk < KV_TILE_SZ; tk++) {
- const void * k_row = (const char *) k->data + ((ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3);
- float s;
- kv_vec_dot(DK, &s, 0, k_row, 0, q_row, 0, 1);
- KQ[tq * KV_TILE_SZ + tk] = s * scale;
+ // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim)
+ // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns
+ for (int tk = 0; tk < kv_tile; tk++) {
+ const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
+ if (kv_type == GGML_TYPE_F16) {
+ const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
+ for (int64_t dk = 0; dk < DK; dk++) {
+ K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);
+ }
+ } else {
+ const float * k_f32_src = (const float *)k_data;
+ for (int64_t dk = 0; dk < DK; dk++) {
+ K_f32[dk * KV_TILE_SZ + tk] = k_f32_src[dk];
+ }
+ }
+ }
+ memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
+ simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);
+ ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);
+
+ // Set padded KQ entries to -inf so softmax gives them zero weight
+ if (kv_tile < KV_TILE_SZ) {
+ for (int tq = 0; tq < Q_TILE_SZ; tq++) {
+ for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {
+ KQ[tq * KV_TILE_SZ + tk] = -INFINITY;
+ }
}
}
@@ -8488,33 +8577,22 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
S[tq] += ggml_vec_soft_max_f32(KV_TILE_SZ, kq_row, kq_row, Mnew);
}
- // Convert V tile to F32 first (if F16), then do MAD
- // On x86, ggml_vec_mad_f16 internall converts F16<->F32 on every load/store, so pre-converting is faster.
- // TODO: on ARM, native f16 should be faster
- if (kv_type == GGML_TYPE_F16) {
- for (int tk = 0; tk < KV_TILE_SZ; tk++) {
- const ggml_fp16_t * v_row = (const ggml_fp16_t *)((const char *) v->data + ((ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3));
- ggml_fp16_to_fp32_row(v_row, V32 + tk * DV, DV);
- }
- for (int tq = 0; tq < Q_TILE_SZ; tq++) {
- if (skip[tq]) continue;
- float * vkq_row = VKQ32 + tq * DV;
- for (int tk = 0; tk < KV_TILE_SZ; tk++) {
- const float p = KQ[tq * KV_TILE_SZ + tk];
- ggml_vec_mad_f32(DV, vkq_row, V32 + tk * DV, p);
- }
- }
- } else {
- for (int tq = 0; tq < Q_TILE_SZ; tq++) {
- if (skip[tq]) continue;
- float * vkq_row = VKQ32 + tq * DV;
- for (int tk = 0; tk < KV_TILE_SZ; tk++) {
- const float p = KQ[tq * KV_TILE_SZ + tk];
- const float * v_row = (const float *)((const char *) v->data + ((ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3));
- ggml_vec_mad_f32(DV, vkq_row, v_row, p);
- }
+ // V accumulation: VKQ32 += softmax(KQ) * V
+ // Pack V tile to contiguous F32, zero-padded
+ for (int tk = 0; tk < kv_tile; tk++) {
+ const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
+ if (kv_type == GGML_TYPE_F16) {
+ ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
+ } else {
+ memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
}
}
+ for (int tq = 0; tq < Q_TILE_SZ; tq++) {
+ if (skip[tq]) {
+ memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));
+ }
+ }
+ simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);
}
// sinks (apply only to valid rows in the tile)
@@ -8731,15 +8809,15 @@ static void ggml_compute_forward_flash_attn_ext_f16(
const int64_t dr = (nr + nchunk - 1) / nchunk;
- static constexpr int64_t KV_TILE_SZ = ggml_fa_tile_config::KV;
static constexpr int64_t Q_TILE_SZ = ggml_fa_tile_config::Q;
- const bool use_tiled = !use_ref &&
+ bool use_tiled = !use_ref &&
(q->type == GGML_TYPE_F32 &&
kv_is_f32_or_f16 &&
k->type == v->type &&
- nek1 % KV_TILE_SZ == 0 &&
neq1 >= Q_TILE_SZ);
-
+#ifdef GGML_SIMD
+ use_tiled &= (DV % GGML_F32_EPR == 0);
+#endif
int current_chunk = ith;
while (current_chunk < nchunk) {
diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
index 24e8ab4618..f94426ddd7 100644
--- a/ggml/src/ggml-cpu/repack.cpp
+++ b/ggml/src/ggml-cpu/repack.cpp
@@ -256,6 +256,200 @@ template <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_K>(const float * GGML_RESTR
ggml_quantize_mat_q8_K_4x8(x, vy, n_per_row);
}
+template
+static void ggml_gemv_q6_K_NxM_q8_K_generic_impl(int n,
+ float * GGML_RESTRICT s,
+ size_t bs,
+ const void * GGML_RESTRICT vx,
+ const void * GGML_RESTRICT vy,
+ int nr,
+ int nc) {
+ constexpr int blocklen = M;
+ constexpr int ncols_interleaved = N;
+ const int qk = QK_K;
+ const int nb = n / qk;
+ const int blocks_per_half = 64 / blocklen;
+
+ assert(n % qk == 0);
+ assert(nc % ncols_interleaved == 0);
+
+ UNUSED(bs);
+ UNUSED(nr);
+
+ float sumf[8];
+
+ const block_q8_K * a_ptr = (const block_q8_K *) vy;
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);
+
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumf[j] = 0.0f;
+ }
+
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ const int base_l = (k / blocks_per_half) * 128 + (k % blocks_per_half) * blocklen;
+ const int base_h = base_l + 64;
+
+ const int scale_idx_l = base_l / 16;
+ const int scale_idx_h = base_h / 16;
+
+ const int qh_shift_l = ((base_l % 128) / 32) * 2;
+ const int qh_shift_h = ((base_h % 128) / 32) * 2;
+
+ const int qh_half_l = (base_l / 128) * 32;
+ const int qh_half_h = (base_h / 128) * 32;
+
+ for (int j = 0; j < ncols_interleaved; j++) {
+ const int8_t scale_l = b_ptr[l].scales[scale_idx_l * ncols_interleaved + j];
+ const int8_t scale_h = b_ptr[l].scales[scale_idx_h * ncols_interleaved + j];
+
+ int sumi_l = 0;
+ int sumi_h = 0;
+
+ for (int i = 0; i < blocklen; i++) {
+ const int ql_pos = k * ncols_interleaved * blocklen + j * blocklen + i;
+ const int l_4 = b_ptr[l].ql[ql_pos] & 0xF;
+ const int hi_4 = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;
+
+ const int qh_idx_l = qh_half_l + ((base_l + i) % 32);
+ const int qh_chunk_l = qh_idx_l / blocklen;
+ const int qh_pos_l = qh_idx_l % blocklen;
+ const int qh_offset_l = qh_chunk_l * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_l;
+ const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;
+
+ const int qh_idx_h = qh_half_h + ((base_h + i) % 32);
+ const int qh_chunk_h = qh_idx_h / blocklen;
+ const int qh_pos_h = qh_idx_h % blocklen;
+ const int qh_offset_h = qh_chunk_h * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_h;
+ const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;
+
+ const int q_l = ((hi_2_l << 4) | l_4) - 32;
+ const int q_h = ((hi_2_h << 4) | hi_4) - 32;
+
+ const int8_t a_l = a_ptr[l].qs[base_l + i];
+ const int8_t a_h = a_ptr[l].qs[base_h + i];
+
+ sumi_l += q_l * a_l;
+ sumi_h += q_h * a_h;
+ }
+
+ sumf[j] +=
+ (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;
+ }
+ }
+ }
+
+ for (int j = 0; j < ncols_interleaved; j++) {
+ s[x * ncols_interleaved + j] = sumf[j];
+ }
+ }
+}
+
+template
+static void ggml_gemm_q6_K_NxM_q8_K_generic_impl(int n,
+ float * GGML_RESTRICT s,
+ size_t bs,
+ const void * GGML_RESTRICT vx,
+ const void * GGML_RESTRICT vy,
+ int nr,
+ int nc) {
+ constexpr int blocklen = M;
+ constexpr int ncols_interleaved = N;
+ const int qk = QK_K;
+ const int nb = n / qk;
+ const int blocks_per_half = 64 / blocklen;
+ const int q8_half_stride = 512;
+ const int q8_low_high_step = 256;
+
+ assert(n % qk == 0);
+ assert(nr % 4 == 0);
+ assert(nc % ncols_interleaved == 0);
+
+ UNUSED(bs);
+
+ float sumf[4][8];
+
+ for (int y = 0; y < nr / 4; y++) {
+ const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);
+ for (int x = 0; x < nc / ncols_interleaved; x++) {
+ const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);
+
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ sumf[m][j] = 0.0f;
+ }
+ }
+
+ for (int l = 0; l < nb; l++) {
+ for (int k = 0; k < (qk / (2 * blocklen)); k++) {
+ const int base_l = (k / blocks_per_half) * 128 + (k % blocks_per_half) * blocklen;
+ const int base_h = base_l + 64;
+
+ const int scale_idx_l = base_l / 16;
+ const int scale_idx_h = base_h / 16;
+
+ const int qh_shift_l = ((base_l % 128) / 32) * 2;
+ const int qh_shift_h = ((base_h % 128) / 32) * 2;
+
+ const int qh_half_l = (base_l / 128) * 32;
+ const int qh_half_h = (base_h / 128) * 32;
+
+ const int q8_base = (k / blocks_per_half) * q8_half_stride + (k % blocks_per_half) * (blocklen * 4);
+
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ const int8_t scale_l = b_ptr[l].scales[scale_idx_l * ncols_interleaved + j];
+ const int8_t scale_h = b_ptr[l].scales[scale_idx_h * ncols_interleaved + j];
+
+ int sumi_l = 0;
+ int sumi_h = 0;
+
+ for (int i = 0; i < blocklen; i++) {
+ const int ql_pos = k * ncols_interleaved * blocklen + j * blocklen + i;
+ const int l_4 = b_ptr[l].ql[ql_pos] & 0xF;
+ const int hi_4 = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;
+
+ const int qh_idx_l = qh_half_l + ((base_l + i) % 32);
+ const int qh_chunk_l = qh_idx_l / blocklen;
+ const int qh_pos_l = qh_idx_l % blocklen;
+ const int qh_offset_l =
+ qh_chunk_l * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_l;
+ const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;
+
+ const int qh_idx_h = qh_half_h + ((base_h + i) % 32);
+ const int qh_chunk_h = qh_idx_h / blocklen;
+ const int qh_pos_h = qh_idx_h % blocklen;
+ const int qh_offset_h =
+ qh_chunk_h * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_h;
+ const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;
+
+ const int q_l = ((hi_2_l << 4) | l_4) - 32;
+ const int q_h = ((hi_2_h << 4) | hi_4) - 32;
+
+ const int8_t q8_l = a_ptr[l].qs[q8_base + m * blocklen + i];
+ const int8_t q8_h = a_ptr[l].qs[q8_base + m * blocklen + i + q8_low_high_step];
+
+ sumi_l += q_l * q8_l;
+ sumi_h += q_h * q8_h;
+ }
+
+ sumf[m][j] += (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) *
+ a_ptr[l].d[m];
+ }
+ }
+ }
+ }
+
+ for (int m = 0; m < 4; m++) {
+ for (int j = 0; j < ncols_interleaved; j++) {
+ s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
+ }
+ }
+ }
+ }
+}
+
extern "C" {
void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
@@ -704,94 +898,12 @@ void ggml_gemv_q5_K_8x8_q8_K_generic(int n,
}
+void ggml_gemv_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
+ ggml_gemv_q6_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);
+}
+
void ggml_gemv_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
- constexpr int qk = QK_K;
- const int nb = n / qk;
- const int ncols_interleaved = 8;
- const int blocklen = 8;
-
- assert(n % qk == 0);
- assert(nc % ncols_interleaved == 0);
-
- UNUSED(bs);
- UNUSED(nr);
-
- float sumf[8];
-
- const block_q8_K * a_ptr = (const block_q8_K *) vy;
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);
-
- for (int j = 0; j < ncols_interleaved; j++) {
- sumf[j] = 0.0f;
- }
-
- for (int l = 0; l < nb; l++) {
-
-
- for (int k = 0; k < 16; k++) {
- // k = 0.. 7 weights 0-63 low, 64-127 high
- // k = 8..15 weights 128-191 low, 192-255 high
- const int base_l = (k / 8) * 128 + (k % 8) * 8;
- const int base_h = base_l + 64;
-
- const int scale_idx_l = base_l / 16;
- const int scale_idx_h = base_h / 16;
-
- // Bit shift cycles 0,2,4,6 for each 32-value group within a 128-value half
- const int qh_shift_l = ((base_l % 128) / 32) * 2;
- const int qh_shift_h = ((base_h % 128) / 32) * 2;
-
- // qh_half: offset to the correct 32-byte half (0 or 32)
- const int qh_half_l = (base_l / 128) * 32;
- const int qh_half_h = (base_h / 128) * 32;
-
- for (int j = 0; j < ncols_interleaved; j++) {
- // Interleaved scales
- const int8_t scale_l = b_ptr[l].scales[scale_idx_l * 8 + j];
- const int8_t scale_h = b_ptr[l].scales[scale_idx_h * 8 + j];
-
- int sumi_l = 0;
- int sumi_h = 0;
-
- for (int i = 0; i < blocklen; i++) {
- const int ql_pos = k * 64 + j * 8 + i;
- const int l_4 = b_ptr[l].ql[ql_pos] & 0xF;
- const int hi_4 = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;
-
- // qh indexing with 8-byte interleaving (like q5_K)
- const int qh_byte_l = qh_half_l + ((base_l + i) % 32);
- const int qh_chunk_l = qh_byte_l / 8;
- const int qh_pos_l = qh_byte_l % 8;
- const int qh_offset_l = qh_chunk_l * 64 + j * 8 + qh_pos_l;
- const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;
-
- const int qh_byte_h = qh_half_h + ((base_h + i) % 32);
- const int qh_chunk_h = qh_byte_h / 8;
- const int qh_pos_h = qh_byte_h % 8;
- const int qh_offset_h = qh_chunk_h * 64 + j * 8 + qh_pos_h;
- const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;
-
- const int q_l = ((hi_2_l << 4) | l_4) - 32;
- const int q_h = ((hi_2_h << 4) | hi_4) - 32;
-
- const int8_t a_l = a_ptr[l].qs[base_l + i];
- const int8_t a_h = a_ptr[l].qs[base_h + i];
-
- sumi_l += q_l * a_l;
- sumi_h += q_h * a_h;
- }
-
- sumf[j] +=
- (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;
- }
- }
- }
-
- for (int j = 0; j < ncols_interleaved; j++) {
- s[x * ncols_interleaved + j] = sumf[j];
- }
- }
+ ggml_gemv_q6_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);
}
void ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
@@ -1485,109 +1597,12 @@ void ggml_gemm_q5_K_8x8_q8_K_generic(int n,
}
}
-void ggml_gemm_q6_K_8x8_q8_K_generic(int n,
- float * GGML_RESTRICT s,
- size_t bs,
- const void * GGML_RESTRICT vx,
- const void * GGML_RESTRICT vy,
- int nr,
- int nc) {
- const int qk = QK_K;
- const int nb = n / qk;
- const int ncols_interleaved = 8;
- const int blocklen = 8;
+void ggml_gemm_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
+ ggml_gemm_q6_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);
+}
- assert(n % qk == 0);
- assert(nr % 4 == 0);
- assert(nc % ncols_interleaved == 0);
-
- UNUSED(bs);
-
- float sumf[4][8];
-
- for (int y = 0; y < nr / 4; y++) {
- const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);
-
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumf[m][j] = 0.0f;
- }
- }
-
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < 16; k++) {
- // k = 0.. 7 weights 0-63 low, 64-127 high
- // k = 8..15 weights 128-191 low, 192-255 high
- const int base_l = (k / 8) * 128 + (k % 8) * 8;
- const int base_h = base_l + 64;
-
- const int scale_idx_l = base_l / 16;
- const int scale_idx_h = base_h / 16;
-
- // Bit shift cycles 0,2,4,6 for each 32-value group within a 128-value half
- const int qh_shift_l = ((base_l % 128) / 32) * 2;
- const int qh_shift_h = ((base_h % 128) / 32) * 2;
-
- // qh_half: offset to the correct 32-byte half (0 or 32)
- const int qh_half_l = (base_l / 128) * 32;
- const int qh_half_h = (base_h / 128) * 32;
-
- // Activation base indices for q8_Kx4 interleaved format
- // Layout: 128-value halves (k/8), then 8-value sub-blocks (k%8) with stride 32
- const int q8_base = (k / 8) * 512 + (k % 8) * 32;
-
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- // Interleaved scales
- const int8_t scale_l = b_ptr[l].scales[scale_idx_l * 8 + j];
- const int8_t scale_h = b_ptr[l].scales[scale_idx_h * 8 + j];
-
- int sumi_l = 0;
- int sumi_h = 0;
-
- for (int i = 0; i < blocklen; i++) {
- const int ql_pos = k * 64 + j * 8 + i;
- const int l_4 = b_ptr[l].ql[ql_pos] & 0xF;
- const int hi_4 = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;
-
- const int qh_idx_l = qh_half_l + ((base_l + i) % 32);
- const int qh_chunk_l = qh_idx_l / 8;
- const int qh_pos_l = qh_idx_l % 8;
- const int qh_offset_l = qh_chunk_l * 64 + j * 8 + qh_pos_l;
- const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;
-
- const int qh_idx_h = qh_half_h + ((base_h + i) % 32);
- const int qh_chunk_h = qh_idx_h / 8;
- const int qh_pos_h = qh_idx_h % 8;
- const int qh_offset_h = qh_chunk_h * 64 + j * 8 + qh_pos_h;
- const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;
-
- const int q_l = ((hi_2_l << 4) | l_4) - 32;
- const int q_h = ((hi_2_h << 4) | hi_4) - 32;
-
- const int8_t q8_l = a_ptr[l].qs[q8_base + m * 8 + i];
- const int8_t q8_h = a_ptr[l].qs[q8_base + m * 8 + i + 256];
-
- sumi_l += q_l * q8_l;
- sumi_h += q_h * q8_h;
- }
-
- sumf[m][j] += (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) *
- a_ptr[l].d[m];
- }
- }
- }
- }
-
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
- }
- }
- }
- }
+void ggml_gemm_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
+ ggml_gemm_q6_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);
}
void ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
@@ -1901,9 +1916,10 @@ static block_q4_Kx8 make_block_q4_Kx8(block_q4_K * in, unsigned int blck_size_in
int src_offset = (i / 8) * blck_size_interleave;
int dst_offset = i * blck_size_interleave;
+ // buffer large enough for the max interleave block size (8 bytes)
uint64_t elems;
- memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));
- memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));
+ memcpy(&elems, &in[src_id].qs[src_offset], blck_size_interleave);
+ memcpy(&out.qs[dst_offset], &elems, blck_size_interleave);
}
// The below logic is designed so as to unpack and rearrange scales and mins values in Q4_K
@@ -2097,18 +2113,18 @@ static block_q6_Kx8 make_block_q6_Kx8(block_q6_K * in, unsigned int blck_size_in
}
const int end_ls = QK_K * 4 / blck_size_interleave;
- // Interleave Q6_K quants by taking 8 bytes at a time
+ // Interleave Q6_K quants by taking blck_size_interleave bytes at a time
for (int i = 0; i < end_ls; ++i) {
int src_id = i % n_blocks;
int src_offset = (i / n_blocks) * blck_size_interleave;
int dst_offset = i * blck_size_interleave;
uint64_t elem_ls;
- memcpy(&elem_ls, &in[src_id].ql[src_offset], sizeof(uint64_t));
- memcpy(&out.ql[dst_offset], &elem_ls, sizeof(uint64_t));
+ memcpy(&elem_ls, &in[src_id].ql[src_offset], blck_size_interleave);
+ memcpy(&out.ql[dst_offset], &elem_ls, blck_size_interleave);
}
- // Interleave high bits using same 8-byte pattern as low bits
+ // Interleave high bits using same chunk size as low bits
const int end_hs = end_ls / 2;
for (int i = 0; i < end_hs; ++i) {
int src_id = i % n_blocks;
@@ -2116,8 +2132,8 @@ static block_q6_Kx8 make_block_q6_Kx8(block_q6_K * in, unsigned int blck_size_in
int dst_offset = i * blck_size_interleave;
uint64_t elem_hs;
- memcpy(&elem_hs, &in[src_id].qh[src_offset], sizeof(uint64_t));
- memcpy(&out.qh[dst_offset], &elem_hs, sizeof(uint64_t));
+ memcpy(&elem_hs, &in[src_id].qh[src_offset], blck_size_interleave);
+ memcpy(&out.qh[dst_offset], &elem_hs, blck_size_interleave);
}
// The below logic is designed so as to unpack and rearrange scales in Q6_K
@@ -2262,7 +2278,7 @@ static int repack_q5_K_to_q5_K_8_bl(struct ggml_tensor * t,
static int repack_q6_K_to_q6_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {
GGML_ASSERT(t->type == GGML_TYPE_Q6_K);
- GGML_ASSERT(interleave_block == 8);
+ GGML_ASSERT(interleave_block == 4 || interleave_block == 8);
constexpr int nrows_interleaved = 8;
block_q6_Kx8 * dst = (block_q6_Kx8 *)t->data;
@@ -2511,6 +2527,10 @@ template <> int repack(struct ggml_tensor * t, const void * da
return repack_q5_K_to_q5_K_8_bl(t, 8, data, data_size);
}
+template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {
+ return repack_q6_K_to_q6_K_8_bl(t, 4, data, data_size);
+}
+
template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {
return repack_q6_K_to_q6_K_8_bl(t, 8, data, data_size);
}
@@ -2575,6 +2595,10 @@ template <> void gemv(int n, float * s, size_t
ggml_gemv_q5_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);
}
+template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {
+ ggml_gemv_q6_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);
+}
+
template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {
ggml_gemv_q6_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);
}
@@ -2634,6 +2658,10 @@ template <> void gemm(int n, float * s, size_t
ggml_gemm_q5_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);
}
+template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {
+ ggml_gemm_q6_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);
+}
+
template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {
ggml_gemm_q6_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);
}
@@ -3043,6 +3071,7 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
static const ggml::cpu::repack::tensor_traits q5_K_8x8_q8_K;
// instance for Q6_K
+ static const ggml::cpu::repack::tensor_traits q6_K_8x4_q8_K;
static const ggml::cpu::repack::tensor_traits q6_K_8x8_q8_K;
// instance for Q2
@@ -3107,6 +3136,11 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
return &q6_K_8x8_q8_K;
}
}
+ if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {
+ if (cur->ne[1] % 8 == 0) {
+ return &q6_K_8x4_q8_K;
+ }
+ }
} else if (cur->type == GGML_TYPE_IQ4_NL) {
if (ggml_cpu_has_avx2()) {
if (cur->ne[1] % 8 == 0) {
diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h
index 855320eeeb..39b6b48238 100644
--- a/ggml/src/ggml-cpu/repack.h
+++ b/ggml/src/ggml-cpu/repack.h
@@ -112,6 +112,7 @@ void ggml_gemv_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
void ggml_gemv_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_q5_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_q6_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
@@ -122,6 +123,7 @@ void ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
void ggml_gemm_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_q5_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_q6_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
@@ -142,6 +144,7 @@ void ggml_gemv_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs,
void ggml_gemv_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
@@ -152,6 +155,7 @@ void ggml_gemm_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs,
void ggml_gemm_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
void ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
diff --git a/ggml/src/ggml-cpu/simd-gemm.h b/ggml/src/ggml-cpu/simd-gemm.h
new file mode 100644
index 0000000000..78d663e593
--- /dev/null
+++ b/ggml/src/ggml-cpu/simd-gemm.h
@@ -0,0 +1,136 @@
+#pragma once
+
+// Computes C[M x N] += A[M x K] * B[K x N]
+
+#include "simd-mappings.h"
+
+// TODO: add support for sizeless vector types
+#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)
+
+// TODO: untested on avx512
+// These are in units of GGML_F32_EPR
+#if defined(__AVX512F__) || defined (__ARM_NEON__)
+ static constexpr int GEMM_RM = 4;
+ static constexpr int GEMM_RN = 4; // 16+4+1 = 25/32
+#elif defined(__AVX2__) || defined(__AVX__)
+ static constexpr int GEMM_RM = 6;
+ static constexpr int GEMM_RN = 2; // 12+2+1 = 15/16
+#else
+ static constexpr int GEMM_RM = 2;
+ static constexpr int GEMM_RN = 2;
+#endif
+
+template
+static inline void simd_gemm_ukernel(
+ float * GGML_RESTRICT C,
+ const float * GGML_RESTRICT A,
+ const float * GGML_RESTRICT B,
+ int K, int N)
+{
+ static constexpr int KN = GGML_F32_EPR;
+
+ GGML_F32_VEC acc[RM][RN];
+ for (int64_t i = 0; i < RM; i++) {
+ for (int r = 0; r < RN; r++) {
+ acc[i][r] = GGML_F32_VEC_LOAD(C + i * N + r * KN);
+ }
+ }
+
+ for (int64_t kk = 0; kk < K; kk++) {
+ GGML_F32_VEC Bv[RN];
+ for (int r = 0; r < RN; r++) {
+ Bv[r] = GGML_F32_VEC_LOAD(B + kk * N + r * KN);
+ }
+ for (int64_t i = 0; i < RM; i++) {
+ GGML_F32_VEC p = GGML_F32_VEC_SET1(A[i * K + kk]);
+ for (int r = 0; r < RN; r++) {
+ acc[i][r] = GGML_F32_VEC_FMA(acc[i][r], Bv[r], p);
+ }
+ }
+ }
+
+ for (int64_t i = 0; i < RM; i++) {
+ for (int r = 0; r < RN; r++) {
+ GGML_F32_VEC_STORE(C + i * N + r * KN, acc[i][r]);
+ }
+ }
+}
+
+// C[M x N] += A[M x K] * B[K x N]
+static void simd_gemm(
+ float * GGML_RESTRICT C,
+ const float * GGML_RESTRICT A,
+ const float * GGML_RESTRICT B,
+ int M, int K, int N)
+{
+ static constexpr int KN = GGML_F32_EPR;
+
+ int64_t ii = 0;
+ for (; ii + GEMM_RM <= M; ii += GEMM_RM) {
+ int64_t jj = 0;
+ for (; jj + GEMM_RN * KN <= N; jj += GEMM_RN * KN) {
+ simd_gemm_ukernel(C + jj, A, B + jj, K, N);
+ }
+ for (; jj + KN <= N; jj += KN) {
+ simd_gemm_ukernel(C + jj, A, B + jj, K, N);
+ }
+ for (; jj < N; jj++) {
+ for (int64_t i = 0; i < GEMM_RM; i++) {
+ float a = C[i * N + jj];
+ for (int64_t kk = 0; kk < K; kk++) {
+ a += A[i + kk] * B[kk * N + jj];
+ }
+ C[i * N + jj] = a;
+ }
+ }
+
+ A += GEMM_RM * K;
+ C += GEMM_RM * N;
+ }
+
+ // Tail rows: one at a time
+ for (; ii < M; ii++) {
+ int64_t jj = 0;
+ for (; jj + GEMM_RN * KN <= N; jj += GEMM_RN * KN) {
+ simd_gemm_ukernel<1, GEMM_RN>(C + jj, A, B + jj, K, N);
+ }
+ for (; jj + KN <= N; jj += KN) {
+ simd_gemm_ukernel<1, 1>(C + jj, A, B + jj, K, N);
+ }
+ for (; jj < N; jj++) {
+ float a = C[jj];
+ for (int64_t kk = 0; kk < K; kk++) {
+ a += A[kk] * B[kk * N + jj];
+ }
+ C[jj] = a;
+ }
+
+ A += K;
+ C += N;
+ }
+}
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#else // scalar path
+
+static void simd_gemm(
+ float * GGML_RESTRICT C,
+ const float * GGML_RESTRICT A,
+ const float * GGML_RESTRICT B,
+ int M, int K, int N)
+{
+ for (int64_t i = 0; i < M; i++) {
+ for (int64_t j = 0; j < N; j++) {
+ float sum = C[i * N + j];
+ for (int64_t kk = 0; kk < K; kk++) {
+ sum += A[i * K + kk] * B[kk * N + j];
+ }
+ C[i * N + j] = sum;
+ }
+ }
+}
+
+#endif // GGML_SIMD
diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h
index 630e506542..22de55700d 100644
--- a/ggml/src/ggml-cpu/simd-mappings.h
+++ b/ggml/src/ggml-cpu/simd-mappings.h
@@ -1160,6 +1160,14 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {
float32x4_t tmp = x[0] + vec_reve(x[0]); \
res = tmp[0] + tmp[1]; \
}
+#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \
+{ \
+ float32x4_t v = vec_add(vec_add(s0, s1), \
+ vec_add(s2, s3)); \
+ v = vec_add(v, vec_sld(v, v, 8)); \
+ v = vec_add(v, vec_sld(v, v, 4)); \
+ res += (ggml_float)vec_extract(v, 0); \
+}
#define GGML_F32_VEC GGML_F32x4
#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
@@ -1209,6 +1217,24 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) {
#define GGML_F16_VEC_MUL GGML_F32x4_MUL
#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE
+// BF16 s390x
+#define GGML_BF16_STEP 16
+#define GGML_BF16_EPR 8
+
+#define GGML_BF16x8 __vector unsigned short
+#define GGML_BF16x8_ZERO vec_splats((unsigned short)0)
+#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))
+
+#define GGML_BF16_VEC GGML_BF16x8
+#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO
+#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD
+#define GGML_BF16_TO_F32_LO(v) ((float32x4_t) vec_mergel((v), GGML_BF16_VEC_ZERO))
+#define GGML_BF16_TO_F32_HI(v) ((float32x4_t) vec_mergeh((v), GGML_BF16_VEC_ZERO))
+#define GGML_BF16_FMA_LO(acc, x, y) \
+ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))
+#define GGML_BF16_FMA_HI(acc, x, y) \
+ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))
+
#elif defined(__riscv_v_intrinsic)
// compatible with vlen >= 128
diff --git a/ggml/src/ggml-cpu/unary-ops.cpp b/ggml/src/ggml-cpu/unary-ops.cpp
index 1d9873ad0f..1d8344436f 100644
--- a/ggml/src/ggml-cpu/unary-ops.cpp
+++ b/ggml/src/ggml-cpu/unary-ops.cpp
@@ -111,7 +111,7 @@ template
static void apply_unary_op(const ggml_compute_params * params, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
- GGML_ASSERT(ggml_is_contiguous_1(src0) && ggml_is_contiguous_1(dst) && ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst) && ggml_are_same_shape(src0, dst));
GGML_TENSOR_UNARY_OP_LOCALS
diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp
index 8708cd4e92..d0e4001338 100644
--- a/ggml/src/ggml-cpu/vec.cpp
+++ b/ggml/src/ggml-cpu/vec.cpp
@@ -236,8 +236,7 @@ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t *
vfloat32m1_t redsum = __riscv_vfredusum_vs_f32m4_f32m1(vsum0, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);
sumf += __riscv_vfmv_f_s_f32m1_f32(redsum);
-#endif
-#if defined(__POWER9_VECTOR__)
+#elif defined(__POWER9_VECTOR__) || defined(__VXE__) || defined(__VXE2__)
const int np = (n & ~(GGML_BF16_STEP - 1));
if (np > 0) {
GGML_F32_VEC sum[4] = {GGML_F32_VEC_ZERO};
diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt
index d313c1ac9a..262f88204e 100644
--- a/ggml/src/ggml-cuda/CMakeLists.txt
+++ b/ggml/src/ggml-cuda/CMakeLists.txt
@@ -64,7 +64,7 @@ if (CUDAToolkit_FOUND)
FetchContent_Declare(
CCCL
GIT_REPOSITORY https://github.com/nvidia/cccl.git
- GIT_TAG v3.2.0-rc2
+ GIT_TAG v3.2.0
GIT_SHALLOW TRUE
)
diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu
index 0e6d777b1e..7339fe0c07 100644
--- a/ggml/src/ggml-cuda/binbcast.cu
+++ b/ggml/src/ggml-cuda/binbcast.cu
@@ -39,13 +39,16 @@ static __global__ void k_bin_bcast(const src0_t * src0,
const uint3 ne11,
const uint3 ne12,
const uint3 ne13,
- /*int s0, */ const int s1,
+ /*const int s0,*/
+ const int s1,
const int s2,
const int s3,
- /*int s00,*/ const int s01,
+ const int s00,
+ const int s01,
const int s02,
const int s03,
- /*int s10,*/ const int s11,
+ const int s10,
+ const int s11,
const int s12,
const int s13,
src1_ptrs... src1s) {
@@ -72,11 +75,11 @@ static __global__ void k_bin_bcast(const src0_t * src0,
for (int i0 = i0s; i0 < ne0; i0 += blockDim.x * gridDim.x) {
const uint32_t i10 = fastmodulo(i0, ne10);
- float result = src0_row ? (float) src0_row[i0] : 0.0f;
+ float result = src0_row ? (float) src0_row[i0*s00] : 0.0f;
if constexpr (sizeof...(src1_ptrs) > 0) {
- result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10])));
+ result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10])));
} else {
- result = bin_op(result, (float)src1[i_src1 + i10]);
+ result = bin_op(result, (float)src1[i_src1 + i10*s10]);
}
dst_row[i0] = (dst_t) result;
@@ -101,13 +104,16 @@ static __global__ void k_bin_bcast_unravel(const src0_t * src0,
const uint3 ne11,
const uint3 ne12,
const uint3 ne13,
- /*int s0, */ const int s1,
+ /*const int s0,*/
+ const int s1,
const int s2,
const int s3,
- /*int s00,*/ const int s01,
+ const int s00,
+ const int s01,
const int s02,
const int s03,
- /*int s10,*/ const int s11,
+ const int s10,
+ const int s11,
const int s12,
const int s13,
src1_ptrs... src1s) {
@@ -135,11 +141,11 @@ static __global__ void k_bin_bcast_unravel(const src0_t * src0,
const int i10 = fastmodulo(i0, ne10);
- float result = src0_row ? (float) src0_row[i0] : 0.0f;
+ float result = src0_row ? (float) src0_row[i0*s00] : 0.0f;
if constexpr (sizeof...(src1_ptrs) > 0) {
- result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10])));
+ result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10])));
} else {
- result = bin_op(result, (float)src1[i_src1 + i10]);
+ result = bin_op(result, (float)src1[i_src1 + i10*s10]);
}
dst_row[i0] = (dst_t) result;
@@ -179,7 +185,7 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor *
cnb[3] *= cne[3];
};
- if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) {
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && !ggml_is_permuted(src0) && !ggml_is_permuted(src1)) {
for (int i = 0; i < 4; i++) {
if (nr[i] != 1) {
break;
@@ -221,7 +227,7 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor *
size_t nb12 = cnb1[2];
size_t nb13 = cnb1[3];
- size_t s0 = nb0 / sizeof(dst_t);
+ //size_t s0 = nb0 / sizeof(dst_t);
size_t s1 = nb1 / sizeof(dst_t);
size_t s2 = nb2 / sizeof(dst_t);
size_t s3 = nb3 / sizeof(dst_t);
@@ -251,10 +257,6 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor *
GGML_ASSERT(nb12 % sizeof(src1_t) == 0);
GGML_ASSERT(nb13 % sizeof(src1_t) == 0);
- GGML_ASSERT(s0 == 1);
- GGML_ASSERT(s00 == 1);
- GGML_ASSERT(s10 == 1);
-
const int block_size = 128;
int64_t hne0 = std::max(ne0 / 2LL, 1LL);
@@ -284,31 +286,31 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor *
k_bin_bcast_unravel<<>>(
src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv, ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11,
ne12, ne13,
- /* s0, */ s1, s2, s3,
- /* s00,*/ s01, s02, s03,
- /* s10,*/ s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);
+ /*s0,*/ s1, s2, s3,
+ s00, s01, s02, s03,
+ s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);
} else {
k_bin_bcast_unravel
<<>>(src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv,
ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11, ne12, ne13,
- /* s0, */ s1, s2, s3,
- /* s00,*/ s01, s02, s03,
- /* s10,*/ s11, s12, s13);
+ /*s0,*/ s1, s2, s3,
+ s00, s01, s02, s03,
+ s10, s11, s12, s13);
}
} else {
const uint3 ne3_fastdiv = init_fastdiv_values((uint32_t) ne3);
if constexpr (sizeof...(I) > 0) {
k_bin_bcast<<>>(
src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13,
- /* s0, */ s1, s2, s3,
- /* s00,*/ s01, s02, s03,
- /* s10,*/ s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);
+ /*s0,*/ s1, s2, s3,
+ s00 ,s01, s02, s03,
+ s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);
} else {
k_bin_bcast<<>>(
src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13,
- /* s0, */ s1, s2, s3,
- /* s00,*/ s01, s02, s03,
- /* s10,*/ s11, s12, s13);
+ /*s0,*/ s1, s2, s3,
+ s00, s01, s02, s03,
+ s10, s11, s12, s13);
}
}
}
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
index a3256d59dd..36d8a3aaab 100644
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -1149,8 +1149,7 @@ struct ggml_cuda_graph {
size_t num_nodes = 0;
std::vector nodes;
bool disable_due_to_gpu_arch = false;
- bool disable_due_to_too_many_updates = false;
- int number_consecutive_updates = 0;
+ bool warmup_complete = false;
std::vector props;
// these are extra tensors (inputs) that participate in the ggml graph but are not nodes
@@ -1159,21 +1158,9 @@ struct ggml_cuda_graph {
// ref: https://github.com/ggml-org/llama.cpp/pull/19165
std::vector extra;
- void record_update(bool use_graph, bool update_required) {
- if (use_graph && update_required) {
- number_consecutive_updates++;
- } else {
- number_consecutive_updates = 0;
- }
- if (number_consecutive_updates >= 4) {
- GGML_LOG_DEBUG("%s: disabling CUDA graphs due to too many consecutive updates\n", __func__);
- disable_due_to_too_many_updates = true;
- }
- }
-
bool is_enabled() const {
static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr);
- return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env || disable_due_to_too_many_updates);
+ return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env);
}
#endif
};
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
index ba3d4eeb88..09b6d5db6a 100644
--- a/ggml/src/ggml-cuda/convert.cu
+++ b/ggml/src/ggml-cuda/convert.cu
@@ -7,7 +7,8 @@
template
static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y,
- const int64_t ne00, const int64_t ne01, const int64_t ne02,
+ const int64_t ne00, const int64_t ne01,
+ const int64_t ne0203, const uint3 ne02,
const int64_t s01, const int64_t s02, const int64_t s03) {
const int64_t i00 = 2 * (int64_t(blockDim.x)*blockIdx.x + threadIdx.x);
@@ -16,23 +17,27 @@ static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __
}
const int64_t i01 = blockIdx.y;
- const int64_t i02 = blockIdx.z % ne02;
- const int64_t i03 = blockIdx.z / ne02;
- const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;
+ for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {
+ const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);
+ const int64_t i02 = dm.y;
+ const int64_t i03 = dm.x;
- const int64_t ib = ibx0 + i00/qk; // block index
- const int64_t iqs = (i00%qk)/qr; // quant index
- const int64_t iybs = i00 - i00%qk; // y block start index
- const int64_t y_offset = qr == 1 ? 1 : qk/2;
+ const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;
- // dequantize
- float2 v;
- dequantize_kernel(vx, ib, iqs, v);
+ const int64_t ib = ibx0 + i00/qk; // block index
+ const int64_t iqs = (i00%qk)/qr; // quant index
+ const int64_t iybs = i00 - i00%qk; // y block start index
+ const int64_t y_offset = qr == 1 ? 1 : qk/2;
- const int64_t iy0 = ((i03*ne02 + i02)*ne01 + i01)*ne00 + iybs + iqs;
- y[iy0 + 0] = ggml_cuda_cast(v.x);
- y[iy0 + y_offset] = ggml_cuda_cast(v.y);
+ // dequantize
+ float2 v;
+ dequantize_kernel(vx, ib, iqs, v);
+
+ const int64_t iy0 = (i0203*ne01 + i01)*ne00 + iybs + iqs;
+ y[iy0 + 0] = ggml_cuda_cast(v.x);
+ y[iy0 + y_offset] = ggml_cuda_cast(v.y);
+ }
}
template