Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-05 12:31:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into xsn/train_fix0

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-09-02 23:04:17 +02:00
parent 3219e9572d 9cffdcc801
commit 484e04fdac
1112 changed files with 75892 additions and 59340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.devops/openvino.Dockerfile
+13 -10
View File
@@ -1,18 +1,18 @@
ARG OPENVINO_VERSION_MAJOR=2026.2.1
ARG OPENVINO_VERSION_FULL=2026.2.1.21919.ede283a88e3
ARG OPENVINO_VERSION_MAJOR=2026.3.1
ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
ARG IGC_VERSION=v2.36.3
ARG IGC_VERSION_FULL=2_2.36.3+21719
ARG COMPUTE_RUNTIME_VERSION=26.22.38646.4
ARG COMPUTE_RUNTIME_VERSION_FULL=26.22.38646.4-0
ARG IGC_VERSION=v2.40.13
ARG IGC_VERSION_FULL=2_2.40.13+22418
ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
ARG NPU_DRIVER_VERSION=v1.33.0
ARG NPU_DRIVER_FULL=v1.33.0.20260529-26625960453
ARG LIBZE1_VERSION=1.27.0-1~24.04~ppa2
ARG NPU_DRIVER_VERSION=v1.35.0
ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341
ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1
# Optional proxy build arguments
ARG http_proxy=
@@ -90,6 +90,9 @@ RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
cmake -B build/ReleaseOV -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DLLAMA_BUILD_TESTS=OFF \
-DGGML_NATIVE=OFF \
-DGGML_BACKEND_DL=ON \
-DGGML_CPU_ALL_VARIANTS=ON \
-DGGML_OPENVINO=ON && \
cmake --build build/ReleaseOV --parallel "
@@ -170,7 +173,7 @@ RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
fi; \
DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
if [ ! -f "$DEB" ]; then \
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260324T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
fi; \
mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
apt-get update; \
.github/actions/ccache-buckets/action.yml
+95
View File
@@ -0,0 +1,95 @@
name: "ccache-buckets"
description: "Save/restore latest GitHub Actions ccache matching a key prefix to/from HF buckets"
inputs:
key:
description: "Cache key prefix to match and load"
required: true
folder:
description: "Bucket folder containing ccache files"
required: true
evict-old-files:
description: "Corresponds to the ccache --evict-older-than AGE option, where AGE is the number of seconds or days followed by the 's' or 'd' suffix respectively."
default: ''
save:
description: "Save ccache"
required: false
default: false
type: boolean
hf_bucket:
description: 'Hugging Face buckets path'
required: true
runs:
using: "composite"
steps:
- name: Install Hugging Face Hub CLI
shell: bash
run: |
python3 -m venv .venv-hf
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
- name: Restore ccache from buckets
if: ${{ inputs.save != 'true' }}
shell: bash
run: |
set +e -uo pipefail
source .venv-hf/bin/activate
CCACHE_DIR=$(ccache -k cache_dir)
if [[ -d "$CCACHE_DIR" ]]; then
CACHE_PATH=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path) | last | .path // ""')
if [[ -n "$CACHE_PATH" ]]; then
echo "Restoring ccache from '$CACHE_PATH'."
hf buckets cp "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" ccache_bucket.tar.gz
mkdir -p ccache_bucket
if tar -xzf ccache_bucket.tar.gz -C ccache_bucket; then
rm -rf "$CCACHE_DIR"
mv ccache_bucket "$CCACHE_DIR"
ccache -z
fi
rm ccache_bucket.tar.gz
else
echo "No ccache found."
fi
else
echo "'$CCACHE_DIR' not found."
fi
- name: Save ccache to buckets
if: ${{ inputs.save == 'true' }}
shell: bash
run: |
if [[ -n "$HF_TOKEN" ]]; then
set +e -uo pipefail
source .venv-hf/bin/activate
CCACHE_DIR=$(ccache -k cache_dir)
if [[ -d "$CCACHE_DIR" ]]; then
ccache -s
if [[ -n "${{ inputs.evict-old-files }}" ]]; then
ccache --evict-older-than "${{ inputs.evict-old-files }}"
fi
DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz"
if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then
hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE"
fi
rm ccache_bucket.tar.gz
else
echo "'$CCACHE_DIR' not found."
fi
fi
- name: Remove old ccache files from buckets
if: ${{ inputs.save == 'true' }}
shell: bash
run: |
if [[ -n "$HF_TOKEN" ]]; then
set +e -uo pipefail
source .venv-hf/bin/activate
CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv')
if [[ -n "$CACHE_FILES" ]]; then
echo "Removing old ccache files..."
while IFS=$'\t' read -r CACHE_PATH; do
hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y
done <<< "$CACHE_FILES"
fi
fi
.github/actions/ccache-clear/action.yml
+38 -10
View File
@@ -1,22 +1,50 @@
# note: place this as the last step of the job, so the new cache is saved by "Post ccache" right after the old one is cleared
name: "ccache-clear"
description: "Delete all GitHub Actions caches matching a key prefix"
description: "Delete GitHub Actions caches matching a key prefix, oldest first"
inputs:
key:
description: "Cache key prefix to match and delete"
required: true
older:
description: "Only delete caches created more than this long ago (e.g. 90m, 1h, 1d). By default all matching caches are deleted"
required: false
default: ""
min:
description: "Stop deleting if fewer than this many caches would remain (e.g. 1). By default there is no minimum"
required: false
default: "0"
dry-run:
description: "Only print the caches that would be deleted, without deleting them"
required: false
default: "false"
runs:
using: "composite"
steps:
- name: Install GitHub CLI if missing
shell: bash
run: |
# e.g. in container jobs, where it is not preinstalled
if ! command -v gh >/dev/null 2>&1; then
echo "GitHub CLI not found, installing..."
if ! command -v curl >/dev/null 2>&1; then
apt-get update >/dev/null 2>&1 || true
apt-get install -y curl >/dev/null 2>&1 || true
fi
mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg >/dev/null
chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list
apt-get update >/dev/null 2>&1 || true
apt-get install -y gh || { echo "Failed to install GitHub CLI (gh)" >&2; exit 1; }
fi
command -v gh >/dev/null 2>&1 || { echo "GitHub CLI (gh) is required but could not be installed" >&2; exit 1; }
- name: Clear caches
shell: bash
run: |
CACHES=$(gh cache list --key "ccache-${{ inputs.key }}" --json id,key --jq '.[] | "\(.id) \(.key)"' 2>/dev/null)
if [ -z "$CACHES" ]; then
echo "No caches found with key prefix: ${{ inputs.key }}"
exit 0
fi
while read -r id key; do
echo "Deleting cache: $id ($key)"
gh cache delete "$id"
done <<< "$CACHES"
bash scripts/ccache-clear.sh \
--key "${{ inputs.key }}" \
--older "${{ inputs.older }}" \
--min "${{ inputs.min }}" \
${{ inputs.dry-run == 'true' && '--dry-run' || '' }}
.github/actions/linux-setup-vulkan/action.yml
-20
View File
@@ -1,20 +0,0 @@
name: "Linux - Setup Vulkan SDK"
description: "Setup Vulkan SDK for Linux"
inputs:
path:
description: "Installation path"
required: true
version:
description: "Vulkan SDK version"
required: true
runs:
using: "composite"
steps:
- name: Setup Vulkan SDK
id: setup
uses: ./.github/actions/unarchive-tar
with:
url: https://sdk.lunarg.com/sdk/download/${{ inputs.version }}/linux/vulkan_sdk.tar.xz
path: ${{ inputs.path }}
strip: 1
.github/actions/windows-setup-cuda/action.yml
+1 -2
View File
@@ -6,8 +6,7 @@ inputs:
required: true
cuda_arch:
description: "CUDA target architecture"
required: false
default: "x64"
required: true
runs:
using: "composite"
.github/actions/windows-setup-rocm/action.yml
+1 -1
View File
@@ -24,7 +24,7 @@ runs:
write-host "Installing ROCm wheels for multi-arch support"
# Install ROCm wheels for multi-arch support (this may take several minutes)
python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ inputs.version }}"
python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ inputs.version }}"
# Pre-expand the devel tree so it is included in the cache
write-host "Initializing ROCm devel tree"
.github/workflows/build-android.yml
+1 -1
View File
@@ -110,7 +110,7 @@ jobs:
# cache on: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
#
#- name: ccache
# uses: ggml-org/ccache-action@v1.2.21
# uses: ggml-org/ccache-action@v1.2.24
# with:
# key: android-ubuntu-arm64
# evict-old-files: 1d
.github/workflows/build-apple.yml
+31 -28
View File
@@ -22,7 +22,8 @@ on:
types: [opened, synchronize, reopened]
paths: [
'.github/workflows/build-apple.yml',
'ggml/src/ggml-metal/**'
'ggml/src/ggml-metal/**',
'ggml/src/ggml-rpc/**'
]
concurrency:
@@ -46,7 +47,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: apple-arm64
evict-old-files: 1d
@@ -65,7 +66,13 @@ jobs:
-DGGML_RPC=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1
- name: Check for leaks
run: |
cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
leaks -atExit -- "${cmd[@]}"
# Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks
OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }'
- name: Test
id: cmake_test
@@ -73,6 +80,16 @@ jobs:
cd build
ctest -L main -E "test-llama-archs" --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: apple-arm64
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
macos-latest-x64:
runs-on: macos-15-intel
@@ -82,7 +99,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: apple-x64
evict-old-files: 1d
@@ -109,6 +126,16 @@ jobs:
cd build
ctest -L main --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: apple-x64
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
macos-latest-ios-xcode:
runs-on: macos-latest
@@ -163,14 +190,6 @@ jobs:
id: checkout
uses: actions/checkout@v6
# TODO: this likely does not do anything - if yes, remove it
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
with:
key: apple-tvos
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- name: Build
id: cmake_build
run: |
@@ -196,14 +215,6 @@ jobs:
id: checkout
uses: actions/checkout@v6
# TODO: this likely does not do anything - if yes, remove it
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
with:
key: apple-visionos
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- name: Build
id: cmake_build
run: |
@@ -234,14 +245,6 @@ jobs:
id: checkout
uses: actions/checkout@v6
# TODO: this likely does not do anything - if yes, remove it
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
with:
key: apple-swift
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- name: Download xcframework artifact
uses: actions/download-artifact@v7
with:
.github/workflows/build-cache.yml
+5 -32
View File
@@ -10,33 +10,6 @@ concurrency:
cancel-in-progress: true
jobs:
ubuntu-24-vulkan-cache:
runs-on: ubuntu-24.04
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Get latest Vulkan SDK version
id: vulkan_sdk_version
run: |
echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
- name: Setup Cache
uses: actions/cache@v5
id: cache-sdk
with:
path: ./vulkan_sdk
key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
- name: Setup Vulkan SDK
if: steps.cache-sdk.outputs.cache-hit != 'true'
uses: ./.github/actions/linux-setup-vulkan
with:
path: ./vulkan_sdk
version: ${{ env.VULKAN_SDK_VERSION }}
#ubuntu-24-spacemit-cache:
# runs-on: ubuntu-24.04
@@ -67,9 +40,9 @@ jobs:
runs-on: ubuntu-24.04
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2.1"
OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3"
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -96,8 +69,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2.1"
OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3"
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
.github/workflows/build-cmake-pkg.yml
+16 -20
View File
@@ -27,30 +27,26 @@ jobs:
cmake --install build --prefix "$PREFIX" --config Release
export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
tclsh <<'EOF'
set build(commit) [string trim [exec git rev-parse --short HEAD]]
set build(number) [string trim [exec git rev-list --count HEAD]]
build_commit=$(git rev-parse --short HEAD | xargs)
build_number=$(git rev-list --count HEAD | xargs)
set cmakelists [read [open "CMakeLists.txt" r]]
regexp {set\(LLAMA_VERSION_MAJOR\s+(\d+)\)} $cmakelists -> major
regexp {set\(LLAMA_VERSION_MINOR\s+(\d+)\)} $cmakelists -> minor
regexp {set\(LLAMA_VERSION_PATCH\s+(\d+)\)} $cmakelists -> patch
set build(version) "$major.$minor.$patch"
major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
build_version="$major.$minor.$patch"
set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]]
set checks [list "set\\(LLAMA_VERSION \\s+$build(version)\\)" \
"set\\(LLAMA_BUILD_COMMIT\\s+$build(commit)\\)" \
"set\\(LLAMA_BUILD_NUMBER\\s+$build(number)\\)"]
checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
"set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
"set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
puts -nonewline "Checking llama-config.cmake version... "
foreach check $checks {
if {![regexp -expanded -- $check $llamaconfig]} {
puts "\"$check\" failed!"
for check in "${checks[@]}"; do
if ! grep -qE "$check" "$LLAMA_CONFIG"; then
echo "Checking llama-config.cmake version... \"$check\" failed!"
exit 1
}
}
puts "success."
EOF
fi
done
echo "Checking llama-config.cmake version... success."
cd examples/simple-cmake-pkg
cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
.github/workflows/build-cpu.yml
+29 -19
View File
@@ -21,6 +21,7 @@ on:
paths: [
'.github/workflows/build-cpu.yml',
'.github/workflows/build-cmake-pkg.yml',
'ggml/src/ggml-rpc/**',
'**/CMakeLists.txt',
'**/.cmake',
'**/*.h',
@@ -61,7 +62,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: cpu-${{ matrix.os }}
evict-old-files: 1d
@@ -96,8 +97,7 @@ jobs:
cmake -B build \
-DGGML_NATIVE=OFF \
-DLLAMA_FATAL_WARNINGS=ON \
-DGGML_RPC=ON \
-DGGML_NATIVE=OFF
-DGGML_RPC=ON
time cmake --build build --config Release -j $(nproc)
- name: Test
@@ -117,29 +117,38 @@ jobs:
./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
# note: real deletion only on push to master (same condition as the ccache save),
# dry-run otherwise (the token is read-only on PRs from forks)
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: cpu-${{ matrix.os }}
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
windows:
name: windows / ${{ matrix.build }}
runs-on: windows-2025
env:
OPENBLAS_VERSION: 0.3.23
SDE_VERSION: 9.33.0-2024-01-07
VULKAN_VERSION: 1.4.357.0
strategy:
matrix:
include:
- build: 'x64-cpu-static'
arch: 'x64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
- build: 'x64-openblas'
arch: 'x64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
- build: 'x64-vulkan'
arch: 'x64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'
- build: 'arm64'
arch: 'arm64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON'
steps:
- name: Clone
@@ -147,7 +156,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: cpu-windows-2025-${{ matrix.build }}
variant: ccache
@@ -167,15 +176,6 @@ jobs:
$lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
& $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
- name: Install Vulkan SDK
id: get_vulkan
if: ${{ matrix.build == 'x64-vulkan' }}
run: |
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
- name: Install Ninja
id: install_ninja
run: |
@@ -215,3 +215,13 @@ jobs:
# cd build
# $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
# & $sde -future -- ctest -L main -C Release --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: cpu-windows-2025-${{ matrix.build }}
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-cuda-ubuntu.yml
+72 -12
View File
@@ -50,14 +50,22 @@ jobs:
DEBIAN_FRONTEND: noninteractive
run: |
apt update
apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev
apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: cuda-ubuntu-24.04-cuda
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
save: false
- name: ccache-buckets-restore
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: cuda-ubuntu-24.04-cuda
folder: llama.cpp
hf_bucket: ggml-org/cache
- name: Build with CMake
# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
@@ -72,6 +80,18 @@ jobs:
-DGGML_CUDA_CUB_3DOT2=ON
cmake --build build
- name: ccache-buckets-save
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: cuda-ubuntu-24.04-cuda
folder: llama.cpp
evict-old-files: 1d
hf_bucket: ggml-org/cache
save: true
hip:
runs-on: ubuntu-22.04
container: rocm/dev-ubuntu-22.04:6.1.2
@@ -85,14 +105,22 @@ jobs:
id: depends
run: |
sudo apt-get update
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: cuda-ubuntu-22.04-hip
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
save: false
- name: ccache-buckets-restore
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: cuda-ubuntu-22.04-hip
folder: llama.cpp
hf_bucket: ggml-org/cache
- name: Build with native CMake HIP support
id: cmake_build
@@ -103,6 +131,18 @@ jobs:
-DGGML_HIP=ON
cmake --build build --config Release -j $(nproc)
- name: ccache-buckets-save
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: cuda-ubuntu-22.04-hip
folder: llama.cpp
evict-old-files: 1d
hf_bucket: ggml-org/cache
save: true
musa:
runs-on: ubuntu-22.04
container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
@@ -116,14 +156,22 @@ jobs:
id: depends
run: |
apt-get update
apt-get install -y build-essential git cmake libssl-dev
apt-get install -y build-essential git cmake libssl-dev jq
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: cuda-ubuntu-22.04-musa
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
save: false
- name: ccache-buckets-restore
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: cuda-ubuntu-22.04-musa
folder: llama.cpp
hf_bucket: ggml-org/cache
- name: Build with native CMake MUSA support
id: cmake_build
@@ -131,3 +179,15 @@ jobs:
cmake -B build -S . \
-DGGML_MUSA=ON
time cmake --build build --config Release -j $(nproc)
- name: ccache-buckets-save
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: cuda-ubuntu-22.04-musa
folder: llama.cpp
evict-old-files: 1d
hf_bucket: ggml-org/cache
save: true
.github/workflows/build-cuda-windows.yml
+21 -15
View File
@@ -22,6 +22,7 @@ env:
jobs:
cuda:
name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }})
runs-on: windows-2022
permissions:
@@ -29,7 +30,16 @@ jobs:
strategy:
matrix:
cuda: ['12.4', '13.3']
include:
- cuda: '12.4'
arch: x64
defines: '-DGGML_CUDA_CUB_3DOT2=ON'
- cuda: '13.3'
arch: x64
defines: ''
- cuda: '13.4'
arch: arm64
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
steps:
- name: Clone
@@ -37,14 +47,15 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: release-windows-2022-x64-cuda-${{ matrix.cuda }}
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
- name: Install Cuda Toolkit
uses: ./.github/actions/windows-setup-cuda
with:
cuda_version: ${{ matrix.cuda }}
cuda_arch: ${{ matrix.arch }}
- name: Install Ninja
id: install_ninja
@@ -54,26 +65,21 @@ jobs:
- name: Build
id: cmake_build
shell: cmd
# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
cmake -S . -B build -G "Ninja Multi-Config" ^
-DLLAMA_BUILD_SERVER=ON ^
-DLLAMA_BUILD_BORINGSSL=ON ^
-DGGML_NATIVE=OFF ^
-DGGML_BACKEND_DL=ON ^
-DGGML_CPU_ALL_VARIANTS=ON ^
-DGGML_NATIVE=OFF ^
-DGGML_CPU=OFF ^
-DGGML_CUDA=ON ^
-DGGML_RPC=ON ^
-DGGML_CUDA_CUB_3DOT2=ON
-DLLAMA_BUILD_BORINGSSL=ON ${{ matrix.defines }}
set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
cmake --build build --config Release -j %NINJA_JOBS% -t ggml
cmake --build build --config Release
cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
- name: ccache-clear
uses: ./.github/actions/ccache-clear
with:
key: release-windows-2022-x64-cuda-${{ matrix.cuda }}
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
hip:
runs-on: windows-2022
@@ -146,7 +152,7 @@ jobs:
& "${env:HIP_PATH}\lib\llvm\bin\clang.exe" --version
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
# TODO: this build does not match the build in release.yml, so we use a different cache key
# ideally, the builds should match, similar to the CUDA build above so that we would be able
.github/workflows/build-msys.yml
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
uses: actions/checkout@v6
#- name: ccache
# uses: ggml-org/ccache-action@v1.2.16
# uses: ggml-org/ccache-action@v1.2.24
# with:
# key: msys-windows-2025-x64
# variant: ccache
.github/workflows/build-opencl.yml
+11 -1
View File
@@ -44,7 +44,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: opencl-windows-2025-x64
variant: ccache
@@ -80,3 +80,13 @@ jobs:
run: |
cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ON
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: opencl-windows-2025-x64
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-openvino.yml
+20 -11
View File
@@ -32,6 +32,8 @@ env:
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
# TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback`
CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-rollback"
jobs:
ubuntu-24-openvino:
@@ -39,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2.1"
OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3"
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -78,26 +80,24 @@ jobs:
- name: Test (CPU)
id: cmake_test_cpu
# TODO: fix and re-enable the `test-llama-archs` test below
run: |
cd ${{ github.workspace }}
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 2000
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
- name: Test (GPU)
id: cmake_test_gpu
# TODO: fix and re-enable the `test-llama-archs` test below
run: |
cd ${{ github.workspace }}
export GGML_OPENVINO_DEVICE=GPU
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 3000
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
openvino-windows-2022:
runs-on: windows-2022
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2.1"
OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3"
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -105,7 +105,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: openvino-windows-2022
variant: ccache
@@ -159,11 +159,20 @@ jobs:
- name: Test (CPU)
id: cmake_test_cpu
shell: cmd
# TODO: fix and re-enable the `test-llama-archs` test below
run: |
REM Find extracted OpenVINO folder dynamically
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
call "%OPENVINO_ROOT%\setupvars.bat"
cd build
ctest --test-dir ReleaseOV -L main -E "test-llama-archs" -C Release --verbose --timeout 3000
ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: openvino-windows-2022
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-riscv.yml
+2 -2
View File
@@ -67,7 +67,7 @@ jobs:
# note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
#- name: ccache
# uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
# uses: ggml-org/ccache-action@v1.2.24
# with:
# key: riscv-ubuntu-native
# evict-old-files: 1d
@@ -137,7 +137,7 @@ jobs:
# note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
#- name: ccache
# uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
# uses: ggml-org/ccache-action@v1.2.24
# with:
# key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}
# evict-old-files: 1d
.github/workflows/build-rpc.yml
-66
View File
@@ -1,66 +0,0 @@
name: CI (rpc)
on:
workflow_dispatch: # allows manual triggering
push:
branches:
- master
paths: [
'.github/workflows/build-rpc.yml',
'**/CMakeLists.txt',
'**/.cmake',
'**/*.h',
'**/*.hpp',
'**/*.c',
'**/*.cpp'
]
pull_request:
types: [opened, synchronize, reopened]
paths: [
'.github/workflows/build-rpc.yml',
'ggml/src/ggml-rpc/**'
]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
env:
GGML_NLOOP: 3
GGML_N_THREADS: 1
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
jobs:
ubuntu-24-rpc:
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
continue-on-error: true
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libssl-dev ninja-build
- name: Build
id: cmake_build
run: |
cmake -B build \
-G "Ninja" \
-DCMAKE_BUILD_TYPE=Release \
-DGGML_RPC=ON
time cmake --build build --config Release -j $(nproc)
- name: Test
id: cmake_test
run: |
cd build
ctest -L main --verbose
.github/workflows/build-sanitize.yml
+1 -1
View File
@@ -55,7 +55,7 @@ jobs:
uses: actions/checkout@v6
# - name: ccache
# uses: ggml-org/ccache-action@v1.2.21
# uses: ggml-org/ccache-action@v1.2.24
# if: ${{ matrix.sanitizer != 'UNDEFINED' }}
# with:
# key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
.github/workflows/build-self-hosted.yml
+2 -2
View File
@@ -288,8 +288,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2.1"
OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3"
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
.github/workflows/build-sycl.yml
+22 -2
View File
@@ -75,7 +75,7 @@ jobs:
sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: sycl-ubuntu-24-${{ matrix.build }}
evict-old-files: 1d
@@ -96,6 +96,16 @@ jobs:
-DGGML_SYCL_F16=${{ matrix.fp16 }}
time cmake --build build --config Release -j $(nproc)
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: sycl-ubuntu-24-${{ matrix.build }}
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
windows-latest-sycl:
runs-on: windows-2022
@@ -127,7 +137,7 @@ jobs:
"LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: sycl-windows-latest
variant: ccache
@@ -139,3 +149,13 @@ jobs:
- name: Build
id: cmake_build
run: examples/sycl/win-build-sycl.bat
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: sycl-windows-latest
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-vulkan.yml
+90 -13
View File
@@ -53,9 +53,9 @@ jobs:
echo "CXX=g++-14" >> "$GITHUB_ENV"
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: vulkan-ubuntu-24.04-arm-new
key: vulkan-ubuntu-24.04-arm
variant: ccache
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
@@ -73,6 +73,16 @@ jobs:
run: |
time cmake --build build -j $(nproc)
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: vulkan-ubuntu-24.04-arm
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
ubuntu-llvmpipe:
runs-on: ubuntu-24.04
@@ -93,22 +103,16 @@ jobs:
run: |
echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
- name: Use Vulkan SDK Cache
uses: actions/cache@v5
id: cache-sdk
with:
path: ./vulkan_sdk
key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
- name: Setup Vulkan SDK
if: steps.cache-sdk.outputs.cache-hit != 'true'
uses: ./.github/actions/linux-setup-vulkan
id: setup
uses: ./.github/actions/unarchive-tar
with:
url: https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkan_sdk.tar.xz
path: ./vulkan_sdk
version: ${{ env.VULKAN_SDK_VERSION }}
strip: 1
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: vulkan-ubuntu-24.04-llvmpipe
evict-old-files: 1d
@@ -133,3 +137,76 @@ jobs:
# This is using llvmpipe and runs slower than other backends
# test-backend-ops is too slow on llvmpipe, skip it
ctest -L main -E test-backend-ops --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: vulkan-ubuntu-24.04-llvmpipe
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
windows:
runs-on: windows-2025
env:
VULKAN_VERSION: 1.4.357.0
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
key: cpu-windows-2025-x64-vulkan
variant: ccache
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- name: Install Vulkan SDK
id: get_vulkan
run: |
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
- name: Install Ninja
id: install_ninja
run: |
choco install ninja
- name: Build
id: cmake_build
run: |
cmake -S . -B build -G "Ninja Multi-Config" `
-D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
-DCMAKE_BUILD_TYPE=Release `
-DGGML_NATIVE=OFF `
-DLLAMA_BUILD_SERVER=ON `
-DGGML_RPC=ON `
-DGGML_BACKEND_DL=ON `
-DGGML_CPU_ALL_VARIANTS=ON `
-DGGML_VULKAN=ON `
-DLLAMA_BUILD_BORINGSSL=ON
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
- name: Test
id: cmake_test
run: |
cd build
ctest -L main -C Release --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: cpu-windows-2025-x64-vulkan
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-wasm.yml
+11 -1
View File
@@ -54,7 +54,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: webgpu-ubuntu-24.04-arm-wasm
evict-old-files: 1d
@@ -88,3 +88,13 @@ jobs:
-DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg
time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: webgpu-ubuntu-24.04-arm-wasm
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-webgpu.yml
+22 -2
View File
@@ -69,7 +69,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: webgpu-macos-latest
evict-old-files: 1d
@@ -101,6 +101,16 @@ jobs:
cd build
ctest -L main --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: webgpu-macos-latest
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
ubuntu:
runs-on: ubuntu-24.04
@@ -110,7 +120,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: webgpu-ubuntu-24.04
evict-old-files: 1d
@@ -153,3 +163,13 @@ jobs:
# This is using llvmpipe and runs slower than other backends
# test-backend-ops is too slow on llvmpipe, skip it
ctest -L main -E test-backend-ops --verbose --timeout 900
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: webgpu-ubuntu-24.04
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/copilot-setup-steps.yml
+1 -1
View File
@@ -29,7 +29,7 @@ jobs:
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: copilot-setup-steps
evict-old-files: 1d
.github/workflows/docker.yml
+41 -2
View File
@@ -44,6 +44,7 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
- name: Determine source tag name
id: srctag
@@ -63,7 +64,7 @@ jobs:
needs: create_tag
uses: ./.github/workflows/ui-build.yml
with:
hf_ui_version: ${{ needs.create_tag.outputs.source_tag }}
ui_version: ${{ needs.create_tag.outputs.source_tag }}
prepare_matrices:
name: Prepare Docker matrices
@@ -161,7 +162,7 @@ jobs:
if: ${{ matrix.config.prebuilt_ui == true }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist
- name: Set up QEMU
@@ -393,6 +394,11 @@ jobs:
name: Create shared tags from digests
needs: [prepare_matrices, push_to_registry, create_tag]
runs-on: ubuntu-24.04
permissions:
contents: read
packages: write
id-token: write
attestations: write
strategy:
fail-fast: false
matrix:
@@ -427,6 +433,7 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create tags from digests
id: create_tags
shell: bash
run: |
set -euo pipefail
@@ -438,6 +445,7 @@ jobs:
SRC_TAG="${{ needs.create_tag.outputs.source_tag }}"
BUILD_DATE="${{ steps.build_date.outputs.date }}"
COMMIT_SHA="${{ steps.checkout.outputs.commit }}"
echo "image_repo=${IMAGE_REPO}" >> "$GITHUB_OUTPUT"
TAGS="${{ matrix.config.tag }}"
ARCHES="${{ matrix.config.arches }}"
DIGEST_GLOB="/tmp/digests/*.tsv"
@@ -504,6 +512,16 @@ jobs:
echo "Creating ${merged_versioned_tag} from ${refs[*]}"
docker buildx imagetools create "${annotations[@]}" --tag "${merged_versioned_tag}" "${refs[@]}"
if [[ "$tag_name" == "${TAGS%% *}" ]]; then
local digest
digest="$(docker buildx imagetools inspect "${merged_versioned_tag}" --format '{{.Manifest.Digest}}')"
if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "Invalid digest for ${merged_versioned_tag}: ${digest}" >&2
exit 1
fi
echo "${image_type}_digest=${digest}" >> "$GITHUB_OUTPUT"
fi
}
for tag in $TAGS; do
@@ -527,3 +545,24 @@ jobs:
done
env:
GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
- name: Attest full image
if: ${{ matrix.config.full }}
uses: actions/attest@v4
with:
subject-name: ${{ steps.create_tags.outputs.image_repo }}
subject-digest: ${{ steps.create_tags.outputs.full_digest }}
- name: Attest light image
if: ${{ matrix.config.light }}
uses: actions/attest@v4
with:
subject-name: ${{ steps.create_tags.outputs.image_repo }}
subject-digest: ${{ steps.create_tags.outputs.light_digest }}
- name: Attest server image
if: ${{ matrix.config.server }}
uses: actions/attest@v4
with:
subject-name: ${{ steps.create_tags.outputs.image_repo }}
subject-digest: ${{ steps.create_tags.outputs.server_digest }}
.github/workflows/hip-quality-check.yml
+11 -1
View File
@@ -52,7 +52,7 @@ jobs:
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: hip-quality-check-ubuntu-22.04
evict-old-files: 1d
@@ -84,3 +84,13 @@ jobs:
cd build
make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$'
python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: hip-quality-check-ubuntu-22.04
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/make-release.yml
+77
View File
@@ -3,6 +3,11 @@ name: Make Release
on:
workflow_dispatch:
inputs:
commit:
description: 'Commit SHA to release (empty = branch HEAD)'
required: false
default: ''
type: string
dry_run:
description: 'Dry run - validate without creating the tag'
required: true
@@ -22,12 +27,17 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
ref: ${{ inputs.commit != '' && inputs.commit || github.ref_name }}
fetch-depth: 0
- name: Run release checks
id: checks
run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
RELEASE_BRANCH: ${{ github.ref_name }}
- name: Create release tag
if: ${{ github.event.inputs.dry_run == 'false' }}
@@ -39,12 +49,79 @@ jobs:
git push origin "${VERSION}"
echo "Created and pushed tag ${VERSION}"
- name: Generate release description
id: desc
run: bash scripts/make-release-desc.sh "${{ steps.checks.outputs.version }}"
env:
GITHUB_REPOSITORY: ${{ github.repository }}
- name: Create nightly-tag.txt
id: nightly_tag_file
run: |
NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
if [[ -z "${NIGHTLY_TAG}" ]]; then
echo "Warning: no nightly tag found for the release commit - nightly-tag.txt will not be created"
echo "create=false" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "${NIGHTLY_TAG}" > nightly-tag.txt
echo "create=true" >> "$GITHUB_OUTPUT"
echo "nightly-tag.txt:"
cat nightly-tag.txt
- name: Create release
id: create_release
if: ${{ github.event.inputs.dry_run == 'false' }}
uses: ggml-org/action-create-release@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
tag_name: ${{ steps.checks.outputs.version }}
prerelease: false
# TODO: enrich the body of the release with more information
body: |
## Overview
New version has been released.
## Assets
${{ steps.desc.outputs.nightly }}
## More info
- [Releases and versioning of `ggml-org` projects](https://github.com/ggml-org/ggml/discussions/1579)
## ${{ steps.desc.outputs.changelog_title }}
${{ steps.desc.outputs.changelog }}
- name: Upload nightly-tag.txt
if: ${{ github.event.inputs.dry_run == 'false' && steps.nightly_tag_file.outputs.create == 'true' }}
uses: actions/github-script@v8
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const fs = require('fs');
const release_id = '${{ steps.create_release.outputs.id }}';
console.log('uploadReleaseAsset', 'nightly-tag.txt');
await github.rest.repos.uploadReleaseAsset({
owner: context.repo.owner,
repo: context.repo.repo,
release_id: release_id,
name: 'nightly-tag.txt',
data: await fs.readFileSync('./nightly-tag.txt')
});
- name: Dry run summary
if: ${{ github.event.inputs.dry_run == 'true' }}
run: |
if [[ "${{ steps.checks.outputs.checks_passed }}" == "true" ]]; then
echo "Dry run complete - all checks passed."
echo "Would have created tag: ${{ steps.checks.outputs.version }}"
if [[ -n "${{ steps.desc.outputs.nightly_tag }}" ]]; then
echo "Would have uploaded nightly-tag.txt: ${{ steps.desc.outputs.nightly_tag }}"
fi
else
echo "::error::Dry run found release check failures. A release tag would not be created."
exit 1
.github/workflows/release.yml
+312 -297
View File
File diff suppressed because it is too large Load Diff
.github/workflows/server-sanitize.yml
-7
View File
@@ -73,13 +73,6 @@ jobs:
fetch-depth: 0
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: "24"
cache: "npm"
cache-dependency-path: "tools/ui/package-lock.json"
- name: Build
id: cmake_build
run: |
.github/workflows/server-self-hosted.yml
+32 -2
View File
@@ -102,7 +102,7 @@ jobs:
./tests.sh
server-cuda:
runs-on: [self-hosted, llama-server, Linux, NVIDIA]
runs-on: "hf-jobs-t4-small:cuda13"
steps:
- name: Clone
@@ -112,12 +112,42 @@ jobs:
fetch-depth: 0
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y cmake libssl-dev python3 python3-venv python3-pip
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
restore: false
save: false
- name: ccache-buckets-restore
uses: ./.github/actions/ccache-buckets
with:
key: self-hosted-server-cuda
folder: llama.cpp
hf_bucket: ggml-org/cache
- name: Build
id: cmake_build
run: |
cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON
cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
cmake --build build --config Release -j $(nproc) --target llama-server
- name: ccache-buckets-save
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
uses: ./.github/actions/ccache-buckets
env:
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
with:
key: self-hosted-server-cuda
folder: llama.cpp
evict-old-files: 1d
hf_bucket: ggml-org/cache
save: true
- name: Python setup
id: setup_python
run: |
.github/workflows/server.yml
+22 -2
View File
@@ -80,7 +80,7 @@ jobs:
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: server-ubuntu-24.04-arm
evict-old-files: 1d
@@ -128,6 +128,16 @@ jobs:
export LLAMA_ARG_BACKEND_SAMPLING=1
SLOW_TESTS=1 ./tests.sh
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: server-ubuntu-24.04-arm
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
windows:
runs-on: windows-2025
@@ -140,7 +150,7 @@ jobs:
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
uses: ggml-org/ccache-action@v1.2.24
with:
key: server-windows-2025-x64
evict-old-files: 1d
@@ -181,3 +191,13 @@ jobs:
cd tools/server/tests
export SLOW_TESTS="1"
./tests.sh
- name: ccache-clear
uses: ./.github/actions/ccache-clear
env:
GH_TOKEN: ${{ github.token }}
with:
key: server-windows-2025-x64
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/ui-build-self-hosted.yml
+1 -1
View File
@@ -31,6 +31,6 @@ jobs:
- name: Upload built UI
uses: actions/upload-artifact@v6
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
retention-days: 1
.github/workflows/ui-build.yml
+15 -5
View File
@@ -3,8 +3,8 @@ name: UI Build
on:
workflow_call:
inputs:
hf_ui_version:
description: 'Version string for version.json (e.g. 12345)'
ui_version:
description: 'Version string embedded in build.json (e.g. b1234); defaults to b<commit-count>'
required: false
type: string
@@ -17,6 +17,17 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Resolve UI version
id: version
run: |
version="${{ inputs.ui_version }}"
if [ -z "$version" ]; then
version="b$(git rev-list --count HEAD)"
fi
echo "ui_version=${version}" >> $GITHUB_OUTPUT
- name: Setup Node.js
uses: actions/setup-node@v6
@@ -31,8 +42,7 @@ jobs:
- name: Build application
env:
HF_UI_VERSION: ${{ inputs.hf_ui_version || '' }}
LLAMA_BUILD_NUMBER: ${{ inputs.hf_ui_version || 'b0000' }}
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
run: npm run build
working-directory: tools/ui
@@ -43,6 +53,6 @@ jobs:
- name: Upload built UI
uses: actions/upload-artifact@v6
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
retention-days: 1
.github/workflows/ui-publish.yml
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
- name: Download UI build artifact
uses: actions/download-artifact@v7
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
- name: Create distribution archive
.github/workflows/ui-self-hosted.yml
+2 -2
View File
@@ -64,7 +64,7 @@ jobs:
- name: Download built UI artifacts
uses: actions/download-artifact@v6
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
- name: Run type checking
@@ -106,7 +106,7 @@ jobs:
- name: Download built UI artifacts
uses: actions/download-artifact@v6
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
- name: Build Storybook
.github/workflows/ui.yml
+2 -2
View File
@@ -63,7 +63,7 @@ jobs:
- name: Download built UI artifacts
uses: actions/download-artifact@v6
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
- name: Install dependencies
@@ -126,7 +126,7 @@ jobs:
- name: Download built UI artifacts (reuses ui-build)
uses: actions/download-artifact@v6
with:
name: ui-build
name: llama-ui.zip
path: tools/ui/dist/
- name: Install Playwright browsers
.pi/gg/SYSTEM.md
+3
View File
@@ -2,12 +2,14 @@ You are a coding agent. Here are some very important rules that you must follow:
General:
- Be very precise and concise when writing code, comments, explanations, etc.
- If an inline comment exceeds 2 lines, replace it with: `// note: TODO LATER`
- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
- Don't try to build or run the code unless you are explicitly asked to do so
- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
Coding:
- When in doubt, always refer to the CONTRIBUTING.md file of the project
- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments
- When referencing issues or PRs in comments, use the format:
- C/C++ code: `// ref: <url>`
- Other (CMake, etc.): `# ref: <url>`
@@ -15,6 +17,7 @@ Coding:
Pull requests (PRs):
- New branch names are prefixed with "gg/"
- Before opening a pull request, ask the user to confirm the description
- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
- When creating a pull request, look for the repository's PR template and follow it
- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
- Ask the user to tell you what model was used and write it in place of [MODEL]
AGENTS.md
+1
View File
@@ -84,6 +84,7 @@ These points are extremely important - failing to follow them won't necessarily
Common mistakes that AI agents usually make:
- Write comments first then write code: this usually leads to extensive redundant comments. Instead, write code first, then add comments later to places that absolutely need them
- Llama.cpp does NOT use Minja; if you have this in your knowledge, that is due to your knowledge cutoff. Llama.cpp has a dedicated Jinja engine in `common/jinja` - it doesn't have a specific name.
- Do NOT add a new file in `tests/*` without maintainers' approval. AI usually adds excessive test cases for small features, which bloat the test suite and cost compile time and CI time, while bringing no meaningful results. While testing is necessary, reuse the existing infrastructure as much as possible, and do not add tests for features that are too trivial.
### Prohibited Actions
AUTHORS
+462 -1
View File
File diff suppressed because it is too large Load Diff
CMakeLists.txt
+5 -4
View File
@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
### llama.cpp version
set(LLAMA_VERSION_MAJOR 0)
set(LLAMA_VERSION_MINOR 1)
set(LLAMA_VERSION_MINOR 3)
set(LLAMA_VERSION_PATCH 0)
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
@@ -134,8 +134,8 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" ON)
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available (requires LLAMA_BUILD_UI=ON)" ON)
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})
option(LLAMA_TESTS_INSTALL "llama: install tests" ON)
@@ -224,9 +224,10 @@ add_subdirectory(src)
# utils, programs, examples and tests
#
add_subdirectory(vendor)
if (LLAMA_BUILD_COMMON)
add_subdirectory(common)
add_subdirectory(vendor/cpp-httplib)
endif()
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
CODEOWNERS
+1
View File
@@ -57,6 +57,7 @@
/ggml/src/ggml-cann/ @ggml-org/ggml-cann
/ggml/src/ggml-common.h @ggerganov
/ggml/src/ggml-cpu/ @ggerganov
/ggml/src/ggml-cpu/iqp.* @bartowski1182
/ggml/src/ggml-cpu/spacemit/ @alex-spacemit
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
CONTRIBUTING.md
+1
View File
@@ -74,6 +74,7 @@ For more info, please refer to the [AGENTS.md](AGENTS.md) file.
- If a PR does not warrant a new release, add `[no release]` in the squashed commit to spare CI resources
- Be mindful of maintenance: most of the work going into a feature happens after the PR is merged. If the PR author is not committed to contribute long-term, someone else needs to take responsibility (you)
- Add the ["merge ready"](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr+is%3Aopen+draft%3Ano+sort%3Aupdated-desc+label%3A%22merge+ready%22+) label to a PR to indicate when a PR can be fast-merged without waiting for 2 independent reviews. [(more info)](https://github.com/ggml-org/llama.cpp/pull/26178)
- Wait for CI results before merging
Maintainers reserve the right to decline review or close pull requests for any reason, without any questions, particularly under any of the following conditions:
- The proposed change is already mentioned in the roadmap or an existing issue, and it has been assigned to someone.
README.md
+9 -8
View File
@@ -7,12 +7,13 @@
<b>LLM inference in C/C++</b>
[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
[![Release](https://img.shields.io/github/v/release/ggml-org/llama.cpp)](https://github.com/ggml-org/llama.cpp/releases)
[![Server](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
[![Docker](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
[![Winget](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
[![Release](https://img.shields.io/github/v/release/ggml-org/llama.cpp?filter=v*&color=brightgreen)](https://github.com/ggml-org/llama.cpp/releases?q=tag:v0)
[![Nightly](https://img.shields.io/github/v/release/ggml-org/llama.cpp?label=nightly&filter=b*&color=orange)](https://github.com/ggml-org/llama.cpp/releases?q=b)
[![Server](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/server.yml?label=Server)](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
[![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
[![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
[manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3A0cc4m%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [compile times](https://github.com/ggml-org/llama.cpp-dev/blob/master/README-compile-times.md) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
</div>
@@ -119,7 +120,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
## Acknowledgements
- [yhirose/cpp-httplib](https://github.com/yhirose/cpp-httplib) - Single-header HTTP server, used by `llama-server` - MIT license
- [stb-image](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
- [nothings/stb](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
- [nlohmann/json](https://github.com/nlohmann/json) - Single-header JSON library, used by various tools/examples - MIT License
- [miniaudio.h](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
- [subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
- [mackron/miniaudio](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
- [sheredom/subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
build-xcframework.sh
+225 -129
View File
@@ -1,5 +1,8 @@
#!/usr/bin/env bash
#
# usage: ./build-xcframework.sh [BUILD ...] (default: all builds)
# builds: ios-sim ios-device macos visionos visionos-sim tvos-sim tvos-device
#
# Options
IOS_MIN_OS_VERSION=16.4
MACOS_MIN_OS_VERSION=13.3
@@ -15,10 +18,47 @@ LLAMA_BUILD_TESTS=OFF
LLAMA_BUILD_SERVER=OFF
LLAMA_BUILD_MTMD=ON
GGML_METAL=ON
GGML_METAL_EMBED_LIBRARY=ON
GGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY:-ON}
GGML_BLAS_DEFAULT=ON
GGML_OPENMP=OFF
# Max number of concurrent platform builds
MAX_PARALLEL_BUILDS=1
# Split the available cores between the concurrent builds (min 1)
JOBS_PER_BUILD=$(( $(sysctl -n hw.logicalcpu) / MAX_PARALLEL_BUILDS ))
if [[ "$JOBS_PER_BUILD" -lt 1 ]]; then
JOBS_PER_BUILD=1
fi
# echo "build_fn build_dir release_dir platform is_simulator min_os" for a build name
build_spec() {
case "$1" in
ios-sim) echo "build_ios_sim build-ios-sim Release-iphonesimulator ios true ${IOS_MIN_OS_VERSION}" ;;
ios-device) echo "build_ios_device build-ios-device Release-iphoneos ios false ${IOS_MIN_OS_VERSION}" ;;
macos) echo "build_macos build-macos Release macos false ${MACOS_MIN_OS_VERSION}" ;;
visionos) echo "build_visionos build-visionos Release-xros visionos false ${VISIONOS_MIN_OS_VERSION}" ;;
visionos-sim) echo "build_visionos_sim build-visionos-sim Release-xrsimulator visionos true ${VISIONOS_MIN_OS_VERSION}" ;;
tvos-sim) echo "build_tvos_sim build-tvos-sim Release-appletvsimulator tvos true ${TVOS_MIN_OS_VERSION}" ;;
tvos-device) echo "build_tvos_device build-tvos-device Release-appletvos tvos false ${TVOS_MIN_OS_VERSION}" ;;
*) return 1 ;;
esac
}
# Default: build everything
if [[ $# -eq 0 ]]; then
BUILDS=(ios-sim ios-device macos visionos visionos-sim tvos-sim tvos-device)
else
BUILDS=("$@")
fi
for b in "${BUILDS[@]}"; do
if ! build_spec "$b" >/dev/null; then
echo "Error: unknown build '$b'" >&2
echo "Valid builds: ios-sim ios-device macos visionos visionos-sim tvos-sim tvos-device" >&2
exit 1
fi
done
COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
COMMON_CXX_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
@@ -129,6 +169,14 @@ setup_framework_structure() {
cp tools/mtmd/mtmd.h ${header_path}
cp tools/mtmd/mtmd-helper.h ${header_path}
if [[ "$GGML_METAL_EMBED_LIBRARY" == "OFF" ]]; then
if [[ "$platform" == "macos" ]]; then
cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/
else
cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/
fi
fi
# Create module map (common for all platforms)
cat > ${module_path}module.modulemap << EOF
framework module llama {
@@ -250,6 +298,7 @@ combine_static_libraries() {
"${base_dir}/${build_dir}/ggml/src/ggml-metal/${release_dir}/libggml-metal.a"
"${base_dir}/${build_dir}/ggml/src/ggml-blas/${release_dir}/libggml-blas.a"
"${base_dir}/${build_dir}/tools/mtmd/${release_dir}/libmtmd.a"
"${base_dir}/${build_dir}/vendor/hash/${release_dir}/libvendor-hash.a"
)
# Create temporary directory for processing
@@ -401,148 +450,195 @@ combine_static_libraries() {
rm -rf "${temp_dir}"
}
echo "Building for iOS simulator..."
cmake -B build-ios-sim -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
-DIOS=ON \
-DCMAKE_SYSTEM_NAME=iOS \
-DCMAKE_OSX_SYSROOT=iphonesimulator \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-ios-sim --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_ios_sim() {
echo "Building for iOS simulator..."
cmake -B build-ios-sim -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
-DIOS=ON \
-DCMAKE_SYSTEM_NAME=iOS \
-DCMAKE_OSX_SYSROOT=iphonesimulator \
-DGGML_METAL_TARGET_OS=ios \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-ios-sim --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
echo "Building for iOS devices..."
cmake -B build-ios-device -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
-DCMAKE_SYSTEM_NAME=iOS \
-DCMAKE_OSX_SYSROOT=iphoneos \
-DCMAKE_OSX_ARCHITECTURES="arm64" \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-ios-device --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_ios_device() {
echo "Building for iOS devices..."
cmake -B build-ios-device -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
-DCMAKE_SYSTEM_NAME=iOS \
-DCMAKE_OSX_SYSROOT=iphoneos \
-DGGML_METAL_TARGET_OS=ios \
-DCMAKE_OSX_ARCHITECTURES="arm64" \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-ios-device --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
echo "Building for macOS..."
cmake -B build-macos -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-S .
cmake --build build-macos --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_macos() {
echo "Building for macOS..."
cmake -B build-macos -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-S .
cmake --build build-macos --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
echo "Building for visionOS..."
cmake -B build-visionos -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
-DCMAKE_OSX_ARCHITECTURES="arm64" \
-DCMAKE_SYSTEM_NAME=visionOS \
-DCMAKE_OSX_SYSROOT=xros \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DLLAMA_BUILD_SERVER=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-visionos --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_visionos() {
echo "Building for visionOS..."
cmake -B build-visionos -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
-DCMAKE_OSX_ARCHITECTURES="arm64" \
-DCMAKE_SYSTEM_NAME=visionOS \
-DCMAKE_OSX_SYSROOT=xros \
-DGGML_METAL_TARGET_OS=xros \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DLLAMA_BUILD_SERVER=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-visionos --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
echo "Building for visionOS simulator..."
cmake -B build-visionos-sim -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DCMAKE_SYSTEM_NAME=visionOS \
-DCMAKE_OSX_SYSROOT=xrsimulator \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DLLAMA_BUILD_SERVER=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-visionos-sim --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_visionos_sim() {
echo "Building for visionOS simulator..."
cmake -B build-visionos-sim -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DCMAKE_SYSTEM_NAME=visionOS \
-DCMAKE_OSX_SYSROOT=xrsimulator \
-DGGML_METAL_TARGET_OS=xros \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DLLAMA_BUILD_SERVER=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-visionos-sim --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
# Add tvOS builds (might need the same u_int definitions as watchOS and visionOS)
echo "Building for tvOS simulator..."
cmake -B build-tvos-sim -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
-DCMAKE_SYSTEM_NAME=tvOS \
-DCMAKE_OSX_SYSROOT=appletvsimulator \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DGGML_METAL=ON \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-tvos-sim --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_tvos_sim() {
echo "Building for tvOS simulator..."
cmake -B build-tvos-sim -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
-DCMAKE_SYSTEM_NAME=tvOS \
-DCMAKE_OSX_SYSROOT=appletvsimulator \
-DGGML_METAL_TARGET_OS=tvos \
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
-DGGML_METAL=ON \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-tvos-sim --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
echo "Building for tvOS devices..."
cmake -B build-tvos-device -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
-DCMAKE_SYSTEM_NAME=tvOS \
-DCMAKE_OSX_SYSROOT=appletvos \
-DCMAKE_OSX_ARCHITECTURES="arm64" \
-DGGML_METAL=ON \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-tvos-device --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
build_tvos_device() {
echo "Building for tvOS devices..."
cmake -B build-tvos-device -G Xcode \
"${COMMON_CMAKE_ARGS[@]}" \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
-DCMAKE_SYSTEM_NAME=tvOS \
-DCMAKE_OSX_SYSROOT=appletvos \
-DGGML_METAL_TARGET_OS=tvos \
-DCMAKE_OSX_ARCHITECTURES="arm64" \
-DGGML_METAL=ON \
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
-DLLAMA_OPENSSL=OFF \
-DMTMD_VIDEO=OFF \
-S .
cmake --build build-tvos-device --config Release -j "${JOBS_PER_BUILD}" -- -quiet
}
run_builds_parallel() {
local -a pids=()
local -a names=()
local name i
for name in "$@"; do
# Wait for the oldest running build to free a slot
if [[ "${#pids[@]}" -ge "$MAX_PARALLEL_BUILDS" ]]; then
if ! wait "${pids[0]}"; then
echo "ERROR: build '${names[0]}' failed, log follows (${names[0]}.log):" >&2
kill "${pids[@]}" 2>/dev/null || true
cat "${names[0]}.log" >&2
exit 1
fi
pids=("${pids[@]:1}")
names=("${names[@]:1}")
fi
echo "Starting build: $name (log: ${name}.log, -j ${JOBS_PER_BUILD})"
"$name" > "${name}.log" 2>&1 &
pids+=("$!")
names+=("$name")
done
# Wait for the remaining builds
for i in "${!pids[@]}"; do
if ! wait "${pids[$i]}"; then
echo "ERROR: build '${names[$i]}' failed, log follows (${names[$i]}.log):" >&2
kill "${pids[@]}" 2>/dev/null || true
cat "${names[$i]}.log" >&2
exit 1
fi
done
}
BUILD_FNS=()
for b in "${BUILDS[@]}"; do
read -r fn _ < <(build_spec "$b")
BUILD_FNS+=("$fn")
done
echo "Building: ${BUILDS[*]} (max ${MAX_PARALLEL_BUILDS} at a time, -j ${JOBS_PER_BUILD} each)..."
run_builds_parallel "${BUILD_FNS[@]}"
# Setup frameworks and copy binaries and headers
echo "Setting up framework structures..."
setup_framework_structure "build-ios-sim" ${IOS_MIN_OS_VERSION} "ios"
setup_framework_structure "build-ios-device" ${IOS_MIN_OS_VERSION} "ios"
setup_framework_structure "build-macos" ${MACOS_MIN_OS_VERSION} "macos"
setup_framework_structure "build-visionos" ${VISIONOS_MIN_OS_VERSION} "visionos"
setup_framework_structure "build-visionos-sim" ${VISIONOS_MIN_OS_VERSION} "visionos"
setup_framework_structure "build-tvos-sim" ${TVOS_MIN_OS_VERSION} "tvos"
setup_framework_structure "build-tvos-device" ${TVOS_MIN_OS_VERSION} "tvos"
for b in "${BUILDS[@]}"; do
read -r _ bdir _ platform _ min_os < <(build_spec "$b")
setup_framework_structure "$bdir" "$min_os" "$platform"
done
# Create dynamic libraries from static libraries
echo "Creating dynamic libraries from static libraries..."
combine_static_libraries "build-ios-sim" "Release-iphonesimulator" "ios" "true"
combine_static_libraries "build-ios-device" "Release-iphoneos" "ios" "false"
combine_static_libraries "build-macos" "Release" "macos" "false"
combine_static_libraries "build-visionos" "Release-xros" "visionos" "false"
combine_static_libraries "build-visionos-sim" "Release-xrsimulator" "visionos" "true"
combine_static_libraries "build-tvos-sim" "Release-appletvsimulator" "tvos" "true"
combine_static_libraries "build-tvos-device" "Release-appletvos" "tvos" "false"
for b in "${BUILDS[@]}"; do
read -r _ bdir rdir platform is_sim _ < <(build_spec "$b")
combine_static_libraries "$bdir" "$rdir" "$platform" "$is_sim"
done
# Create XCFramework with correct debug symbols paths
echo "Creating XCFramework..."
XCFW_ARGS=()
for b in "${BUILDS[@]}"; do
read -r _ bdir _ _ _ _ < <(build_spec "$b")
XCFW_ARGS+=(-framework "$(pwd)/${bdir}/framework/llama.framework")
XCFW_ARGS+=(-debug-symbols "$(pwd)/${bdir}/dSYMs/llama.dSYM")
done
xcrun xcodebuild -create-xcframework \
-framework $(pwd)/build-ios-sim/framework/llama.framework \
-debug-symbols $(pwd)/build-ios-sim/dSYMs/llama.dSYM \
-framework $(pwd)/build-ios-device/framework/llama.framework \
-debug-symbols $(pwd)/build-ios-device/dSYMs/llama.dSYM \
-framework $(pwd)/build-macos/framework/llama.framework \
-debug-symbols $(pwd)/build-macos/dSYMs/llama.dSYM \
-framework $(pwd)/build-visionos/framework/llama.framework \
-debug-symbols $(pwd)/build-visionos/dSYMs/llama.dSYM \
-framework $(pwd)/build-visionos-sim/framework/llama.framework \
-debug-symbols $(pwd)/build-visionos-sim/dSYMs/llama.dSYM \
-framework $(pwd)/build-tvos-device/framework/llama.framework \
-debug-symbols $(pwd)/build-tvos-device/dSYMs/llama.dSYM \
-framework $(pwd)/build-tvos-sim/framework/llama.framework \
-debug-symbols $(pwd)/build-tvos-sim/dSYMs/llama.dSYM \
-output $(pwd)/build-apple/llama.xcframework
"${XCFW_ARGS[@]}" \
-output "$(pwd)/build-apple/llama.xcframework"
ci/run.sh
+43 -2
View File
@@ -189,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
fi
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
# TODO: fix and re-enable the `test-llama-archs` test below
CTEST_EXTRA="-E test-llama-archs"
# TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback*`
CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-rollback"
fi
## helpers
@@ -300,6 +300,40 @@ function gg_sum_ctest_release {
gg_printf '```\n'
}
# test_llama_archs_tensor_split
function gg_run_test_llama_archs_tensor_split {
cd ${SRC}
set -e
if [ ! -z ${GG_BUILD_CUDA} ]; then
GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
fi
if [ ! -z ${GG_BUILD_METAL} ]; then
GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
fi
set +e
}
function gg_sum_test_llama_archs_tensor_split {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs test-llama-archs with 1 to 4 devices\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
gg_printf '```\n'
}
# test_scripts
function gg_run_test_scripts {
@@ -698,6 +732,11 @@ function gg_check_build_requirements {
gg_printf 'ctest not found, please install\n'
exit 1
fi
if ! command -v unzip &> /dev/null; then
gg_printf 'unzip not found, please install\n'
exit 1
fi
}
function gg_run_test_backend_ops_cpu {
@@ -751,6 +790,8 @@ ret=0
test $ret -eq 0 && gg_run ctest_debug
test $ret -eq 0 && gg_run ctest_release
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
test $ret -eq 0 && gg_run test_backend_ops_cpu
fi
cmake/arm64-windows-llvm.cmake
+1
View File
@@ -8,6 +8,7 @@ set( CMAKE_CXX_COMPILER clang++ )
set( CMAKE_C_COMPILER_TARGET ${target} )
set( CMAKE_CXX_COMPILER_TARGET ${target} )
set( CMAKE_ASM_COMPILER_TARGET ${target} )
set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" )
set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" )
common/CMakeLists.txt
+4 -1
View File
@@ -81,6 +81,8 @@ add_library(${TARGET}
imatrix-loader.cpp
imatrix-loader.h
json-schema-to-grammar.cpp
json.cpp
json.h
llguidance.cpp
log.cpp
log.h
@@ -126,7 +128,8 @@ set_target_properties(${TARGET} PROPERTIES
MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number
)
target_include_directories(${TARGET} PUBLIC . ../vendor)
target_include_directories(${TARGET} PUBLIC .)
target_link_libraries (${TARGET} PUBLIC vendor::nlohmann vendor::sheredom)
target_compile_features (${TARGET} PUBLIC cxx_std_17)
if (LLAMA_SUBPROCESS)
common/arg.cpp
+129 -18
View File
@@ -5,6 +5,7 @@
#include "common.h"
#include "download.h"
#include "json-schema-to-grammar.h"
#include "json.h"
#include "llama.h"
#include "log.h"
#include "sampling.h"
@@ -21,9 +22,6 @@
#include <shellapi.h>
#endif
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include <algorithm>
#include <cinttypes>
#include <climits>
@@ -32,6 +30,7 @@
#include <filesystem>
#include <fstream>
#include <list>
#include <numeric>
#include <regex>
#include <set>
#include <string>
@@ -55,7 +54,7 @@
#define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
using json = nlohmann::ordered_json;
using json = common_json;
using namespace common_arg_utils;
static std::initializer_list<enum llama_example> mmproj_examples = {
@@ -961,6 +960,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
));
}
// if the preserve_reasoning kwarg was not specified explicitly, enable it by default
if (!params.default_template_kwargs.count("preserve_reasoning")) {
params.default_template_kwargs["preserve_reasoning"] = "true";
}
return true;
}
@@ -1644,6 +1648,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
}
).set_env("LLAMA_ARG_CTX_SIZE"));
add_opt(common_arg(
{ "--kv-unified-per-slot" }, "N",
"context limit per parallel slot (default: unset, behavior unchanged).\n"
"when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N",
[](common_params & params, int value) {
params.kv_unified_per_slot = value;
}
).set_env("LLAMA_ARG_KV_UNIFIED_PER_SLOT").set_examples({ LLAMA_EXAMPLE_SERVER }));
add_opt(common_arg(
{"-n", "--predict", "--n-predict"}, "N",
string_format(
@@ -1898,7 +1910,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, bool value) {
params.conversation_mode = value ? COMMON_CONVERSATION_MODE_ENABLED : COMMON_CONVERSATION_MODE_DISABLED;
}
).set_examples({LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}));
).set_examples({LLAMA_EXAMPLE_COMPLETION}));
add_opt(common_arg(
{"-st", "--single-turn"},
"run conversation for a single turn only, then exit when done\n"
@@ -2595,6 +2607,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mmproj_use_gpu = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD"));
add_opt(common_arg(
// note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
{"-mmdev", "--mmproj-device"}, "DEVICE",
"device to use for multimodal projector (none = don't offload, default: auto)\n"
"use --list-devices to see a list of available devices",
[](common_params & params, const std::string & value) {
if (value == "none") {
params.mmproj_use_gpu = false;
params.mmproj_device = nullptr;
return;
}
auto devices = parse_device_list(value);
// parse_device_list pushes nullptr at back so devices is length 2 for single device.
if (devices.size() > 2) {
throw std::invalid_argument("only one device may be specified for mmproj");
}
params.mmproj_use_gpu = true;
params.mmproj_device = devices.front();
}
).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason
add_opt(common_arg(
{"--image", "--audio", "--video"}, "FILE",
"path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n",
@@ -2625,6 +2657,27 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mtmd_batch_max_tokens = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
add_opt(common_arg(
{"--video-fps"}, "N",
string_format("target video frame rate (default: %.1f)", params.video_fps),
[](common_params & params, const std::string & value) {
params.video_fps = std::stof(value);
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FPS"));
add_opt(common_arg(
{"--video-timestamp-interval"}, "N",
string_format("interval in milliseconds between text timestamps (default: %" PRId64 ")", params.video_timestamp_interval_ms),
[](common_params & params, int value) {
params.video_timestamp_interval_ms = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"));
add_opt(common_arg(
{"--video-ffmpeg-dir"}, "DIR",
"path to the directory containing ffmpeg and ffprobe (default: search in PATH)",
[](common_params & params, const std::string & value) {
params.video_ffmpeg_bin_dir = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
if (params.is_gen_docs || llama_supports_rpc()) {
add_opt(common_arg(
{"--rpc"}, "SERVERS",
@@ -2680,6 +2733,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"-lzm", "--lazy-mode"}, "MODE",
"on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
"- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
"- auto: on, but only for tensors larger than 4 GiB\n"
"- off: always keep them resident",
[](common_params & params, const std::string & value) {
/**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; }
else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LAZY_MODE"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"
@@ -2731,14 +2797,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
if (value < 0) {
throw std::invalid_argument("invalid value");
}
for (int i = 0; i < value; ++i) {
// keep strings alive and avoid leaking memory by storing them in a static vector
static std::list<std::string> buft_overrides;
buft_overrides.push_back(llm_ffn_exps_block_regex(i));
params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
}
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
}
).set_env("LLAMA_ARG_N_CPU_MOE"));
add_opt(common_arg(
{"-ncffn", "--n-cpu-ffn"}, "N",
"keep the dense FFN weights of the first N layers in the CPU\n"
"(dense models; for MoE expert weights use --n-cpu-moe)",
[](common_params & params, int value) {
if (value < 0) {
throw std::invalid_argument("invalid value");
}
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_DENSE_REGEX, params.tensor_buft_overrides);
}
).set_env("LLAMA_ARG_N_CPU_FFN"));
GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0
add_opt(common_arg(
{"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N",
@@ -3362,7 +3434,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
{"--tools"}, "TOOL1,TOOL2,...",
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
"specify \"all\" to enable all tools\n"
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info\n"
"note: for security reasons, this will limit --cors-origins to localhost by default",
[](common_params & params, const std::string & value) {
params.server_tools = parse_csv_row(value);
@@ -3486,6 +3558,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
LOG_WRN("Setting 'enable_thinking' via --chat-template-kwargs is deprecated. "
"Use --reasoning on / --reasoning off instead.\n");
}
if (item.key() == "preserve_reasoning") {
LOG_WRN("Setting 'preserve_reasoning' via --chat-template-kwargs is deprecated. "
"Use --reasoning-preserve / --no-reasoning-preserve instead.\n");
}
params.default_template_kwargs[item.key()] = item.value().dump();
}
}
@@ -3676,7 +3752,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
add_opt(common_arg(
{"--reasoning-preserve"},
{"--no-reasoning-preserve"},
"preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n"
"preserve reasoning trace in the full history, not just the last assistant message (default: enabled)\n"
"compatible with certain templates having 'supports_preserve_reasoning' capability\n"
"example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking",
[](common_params & params, bool value) {
@@ -3685,6 +3761,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
} else {
params.default_template_kwargs["preserve_reasoning"] = "false";
}
params.preserve_reasoning_specified = true;
}
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE"));
add_opt(common_arg(
@@ -4065,11 +4142,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
if (value < 0) {
throw std::invalid_argument("invalid value");
}
for (int i = 0; i < value; ++i) {
static std::list<std::string> buft_overrides_draft;
buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i));
params.speculative.draft.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
}
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides);
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE"));
@@ -4090,6 +4163,38 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.n_min = value;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
add_opt(common_arg(
{"--spec-synth-len"}, "L",
"target mean synthetic acceptance length, including the target token (benchmarking only)",
[](common_params & params, const std::string & value) {
const std::string text = string_strip(value);
size_t pos = 0;
const double length = std::stod(text, &pos);
if (pos != text.size() || length == -1.0) {
throw std::invalid_argument("invalid value");
}
params.speculative.synth_len = length;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));
add_opt(common_arg(
{"--spec-synth-rates"}, "P0,P1,...",
"comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",
[](common_params & params, const std::string & value) {
const auto values = string_split<std::string>(value, ',');
std::vector<double> rates;
rates.reserve(values.size());
for (const auto & raw : values) {
const std::string text = string_strip(raw);
size_t pos = 0;
const double rate = std::stod(text, &pos);
if (pos != text.size()) {
throw std::invalid_argument("invalid value");
}
rates.push_back(rate);
}
params.speculative.synth_rates = std::move(rates);
}
).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));
add_opt(common_arg(
{"--spec-draft-p-split", "--draft-p-split"}, "P",
@@ -4658,6 +4763,12 @@ void common_params_add_preset_options(std::vector<common_arg> & args) {
[](common_params &, int) { /* unused */ }
).set_env(COMMON_ARG_PRESET_STOP_TIMEOUT).set_preset_only());
args.push_back(common_arg(
{"dedup-cache-models"}, "0|1",
"in server router mode, hide a cached model from the model list when this preset resolves to the same model file",
[](common_params &, const std::string &) { /* unused */ }
).set_env(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS).set_preset_only());
// args.push_back(common_arg(
// {"pin"},
// "in server router mode, do not unload this model if models_max is exceeded",
common/arg.h
+3 -2
View File
@@ -11,8 +11,9 @@
#include <memory>
// pseudo-env variable to identify preset-only arguments
#define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP"
#define COMMON_ARG_PRESET_STOP_TIMEOUT "__PRESET_STOP_TIMEOUT"
#define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP"
#define COMMON_ARG_PRESET_STOP_TIMEOUT "__PRESET_STOP_TIMEOUT"
#define COMMON_ARG_PRESET_DEDUP_CACHE_MODELS "__PRESET_DEDUP_CACHE_MODELS"
//
// CLI argument parsing
common/chat-auto-parser-generator.cpp
+2 -3
View File
@@ -5,13 +5,12 @@
#include "common.h"
#include "json-schema-to-grammar.h"
#include "log.h"
#include "nlohmann/json.hpp"
#include "peg-parser.h"
#include <stdexcept>
#include <string>
using json = nlohmann::ordered_json;
using json = common_json;
// Helper to iterate over tools/functions
static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
@@ -391,7 +390,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
required = params.at("required").get<std::set<std::string>>();
}
auto schema_info = common_schema_info();
common/chat-auto-parser-helpers.cpp
-3
View File
@@ -4,14 +4,11 @@
#include "chat-peg-parser.h"
#include "chat.h"
#include "log.h"
#include "nlohmann/json.hpp"
#include "peg-parser.h"
#include <cctype>
#include <numeric>
using json = nlohmann::ordered_json;
std::string trim_whitespace(const std::string & str) {
size_t start = 0;
while (start < str.length() && std::isspace(static_cast<unsigned char>(str[start]))) {
common/chat-auto-parser.h
+2 -2
View File
@@ -4,7 +4,7 @@
#include "common.h"
#include "jinja/caps.h"
#include "peg-parser.h"
#include "nlohmann/json.hpp"
#include "json.h"
#include <chrono>
#include <optional>
@@ -12,7 +12,7 @@
#include <utility>
#include <vector>
using json = nlohmann::ordered_json;
using json = common_json;
class common_chat_peg_builder;
common/chat-diff-analyzer.cpp
+11 -3
View File
@@ -4,11 +4,11 @@
#include "chat.h"
#include "common.h"
#include "log.h"
#include "nlohmann/json.hpp"
#include "peg-parser.h"
#include <algorithm>
#include <cctype>
#include <numeric>
#include <ostream>
#include <sstream>
@@ -17,7 +17,7 @@
#define ANSI_ORANGE "\033[1m\x1b[38;5;214m"
#define ANSI_RED "\033[1m\x1b[38;5;196m"
using json = nlohmann::ordered_json;
using json = common_json;
namespace autoparser {
@@ -193,6 +193,14 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autopar
LOG_DBG(ANSI_ORANGE "[Patch: Laguna]\n" ANSI_RESET);
}
},
// Bailing V3
[](const common_chat_template & tmpl, autoparser & analysis) -> void {
if (tmpl.src.find("Bailing V3 chat template") != std::string::npos) {
analysis.tools.arguments.value_suffix = trim_whitespace(analysis.tools.arguments.value_suffix);
analysis.tools.arguments.tolerate_intertag_whitespace = true;
LOG_DBG(ANSI_ORANGE "[Patch: Bailing V3]\n" ANSI_RESET);
}
},
});
@@ -921,7 +929,7 @@ void analyze_tools::analyze_tool_call_format_json_native(const std::string & cle
int json_end = clean_haystack.find_last_of('}');
std::string cut = clean_haystack.substr(json_start, json_end - json_start + 1);
json call_struct = json::parse(cut);
auto register_field = [&](const std::string & prefix, const nlohmann::detail::iteration_proxy_value<json::iterator> & subel) {
auto register_field = [&](const std::string & prefix, const common_json_entry & subel) {
if (subel.value().is_string() && std::string(subel.value()).find("call0000") != std::string::npos) {
format.id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();
} else if (subel.value().is_string() && std::string(subel.value()) == fun_name_needle) {
common/chat-peg-parser.cpp
+1 -3
View File
@@ -4,12 +4,10 @@
#include "ggml.h"
#include "peg-parser.h"
#include <nlohmann/json.hpp>
#include <cstdint>
#include <functional>
using ordered_json = nlohmann::ordered_json;
using ordered_json = common_json;
static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
int count = 0;
common/chat-peg-parser.h
+6 -6
View File
@@ -128,7 +128,7 @@ class common_chat_peg_builder : public common_peg_parser_builder {
// parameters_order: order in which JSON fields should be parsed
common_peg_parser standard_json_tools(const std::string & section_start,
const std::string & section_end,
const nlohmann::ordered_json & tools,
const common_json & tools,
bool parallel_tool_calls,
bool force_tool_calls,
const std::string & name_key = "",
@@ -143,13 +143,13 @@ class common_chat_peg_builder : public common_peg_parser_builder {
// Legacy-compatible helper for building XML/tagged style tool calls
// Used by tests and manual parsers
common_peg_parser standard_constructed_tools(const std::map<std::string, std::string> & markers,
const nlohmann::ordered_json & tools,
const common_json & tools,
bool parallel_tool_calls,
bool force_tool_calls);
// Helper for Python-style function call format: name(arg1="value1", arg2=123)
// Used by LFM2 and similar templates
common_peg_parser python_style_tool_calls(const nlohmann::ordered_json & tools,
common_peg_parser python_style_tool_calls(const common_json & tools,
bool parallel_tool_calls,
bool allow_json_literals);
@@ -158,19 +158,19 @@ class common_chat_peg_builder : public common_peg_parser_builder {
common_peg_parser python_or_json_value();
// Implementation helpers for standard_json_tools — one per JSON tool call layout mode
common_peg_parser build_json_tools_function_is_key(const nlohmann::ordered_json & tools,
common_peg_parser build_json_tools_function_is_key(const common_json & tools,
const std::string & args_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key);
common_peg_parser build_json_tools_nested_keys(const nlohmann::ordered_json & tools,
common_peg_parser build_json_tools_nested_keys(const common_json & tools,
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key);
common_peg_parser build_json_tools_flat_keys(const nlohmann::ordered_json & tools,
common_peg_parser build_json_tools_flat_keys(const common_json & tools,
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
common/chat.cpp
+36 -29
View File
@@ -6,6 +6,7 @@
#include "common.h"
#include "ggml.h"
#include "json-schema-to-grammar.h"
#include "json.h"
#include "log.h"
#include "jinja/value.h"
@@ -13,14 +14,13 @@
#include "jinja/caps.h"
#include "peg-parser.h"
#include "nlohmann/json.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <exception>
#include <functional>
#include <iomanip>
#include <map>
#include <optional>
@@ -30,7 +30,7 @@
#include <utility>
#include <vector>
using json = nlohmann::ordered_json;
using json = common_json;
static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {
auto time = std::chrono::system_clock::to_time_t(now);
@@ -48,7 +48,7 @@ static json safe_args_parse(const std::string & to_parse) {
}
try {
return json::parse(stripped);
} catch (json::exception & e) {
} catch (const common_json_error & e) {
return stripped;
}
}
@@ -488,17 +488,17 @@ struct messages_inp_normalizer {
json normalized = json::array();
for (const auto & msg : messages) {
json copy = msg;
auto it = copy.find("content");
if (it != copy.end()) {
if (only_typed && it->is_string()) {
*it = json::array({
if (copy.contains("content")) {
json & it = copy.at("content");
if (only_typed && it.is_string()) {
it = json::array({
json{
{"type", "text"},
{"text", it->get<std::string>()},
{"text", it.get<std::string>()},
}
});
} else if (only_string && it->is_array()) {
*it = concat_content_parts(*it);
} else if (only_string && it.is_array()) {
it = concat_content_parts(it);
}
}
normalized.push_back(std::move(copy));
@@ -608,7 +608,7 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
return result;
}
common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value) {
common_chat_continuation common_chat_continuation_parse(const common_json & value) {
if (value.is_boolean() && value.get<bool>()) {
return COMMON_CHAT_CONTINUATION_AUTO;
}
@@ -920,7 +920,7 @@ static void foreach_parameter(const json &
const auto & props = params.at("properties");
std::set<std::string> required;
if (params.contains("required") && params.at("required").is_array()) {
params.at("required").get_to(required);
required = params.at("required").get<std::set<std::string>>();
}
for (const auto & [name, prop] : props.items()) {
bool is_required = (required.find(name) != required.end());
@@ -937,7 +937,7 @@ static std::string common_chat_template_direct_apply_impl(
jinja::context ctx(tmpl.source());
// messages_override is already built for this template, do not touch its content parts
nlohmann::ordered_json inp = nlohmann::ordered_json{
json inp = json{
{"messages", messages_override.has_value()
? *messages_override
: messages_inp_normalizer(tmpl.original_caps()).normalize(inputs.messages)},
@@ -1058,7 +1058,7 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_
});
} else if (msg.at("content").is_array()) {
auto blocks = msg.at("content");
content.insert(content.end(), blocks.begin(), blocks.end());
content.insert(blocks);
}
}
@@ -1177,6 +1177,8 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_
"</tool_call>",
};
auto is_qwen3_coder = !supports_reasoning;
if (supports_reasoning) {
data.thinking_start_tag = "<think>";
// Support both </think> and <tool_call> as reasoning end sequences.
@@ -1217,13 +1219,15 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_
std::vector<std::string> tool_call_starts = { "<tool_call>" };
// Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
// starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
// constraining on <function which may occur in valid content generation, e.g. #include <functional>
foreach_function(inputs.tools, [&](const json & tool) {
const std::string name = tool.at("function").at("name");
tool_call_starts.push_back("<function=" + name + ">");
});
if (is_qwen3_coder) {
// Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
// starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
// constraining on <function which may occur in valid content generation, e.g. #include <functional>
foreach_function(inputs.tools, [&](const json & tool) {
const std::string name = tool.at("function").at("name");
tool_call_starts.push_back("<function=" + name + ">");
});
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
@@ -1288,10 +1292,13 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
auto tool_call_body = tool_choice + "</tool_call>" + p.space();
auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
// Qwen3-Coder models may occasionally omit the <tool_call> token.
auto tool_call_body = tool_choice + "</tool_call>" + p.space();
auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body);
auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
auto tool_call_first = is_qwen3_coder ?
p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :
tool_call;
auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
@@ -2238,7 +2245,7 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
required = params.at("required").get<std::set<std::string>>();
}
auto schema_info = common_schema_info();
@@ -2860,7 +2867,7 @@ static common_chat_params common_chat_params_init_minimax_m3(const common_chat_t
std::set<std::string> required;
if (schema.contains("required")) {
schema.at("required").get_to(required);
required = schema.at("required").get<std::set<std::string>>();
}
std::vector<common_peg_parser> required_elements;
@@ -2972,10 +2979,10 @@ static void system_message_not_supported(json & messages) {
auto & second_msg = messages[1];
second_msg["content"] = first_msg.at("content").get<std::string>()
+ "\n" + second_msg.at("content").get<std::string>();
messages.erase(messages.begin());
messages.erase(0);
} else {
LOG_WRN("Removing system prompt due to template not supporting system role\n");
messages.erase(messages.begin());
messages.erase(0);
}
}
}
common/chat.h
+9 -10
View File
@@ -8,7 +8,7 @@
#include "jinja/runtime.h"
#include "jinja/caps.h"
#include "nlohmann/json_fwd.hpp"
#include "json.h"
#include <chrono>
#include <functional>
@@ -17,7 +17,6 @@
#include <vector>
using chat_template_caps = jinja::caps;
using json = nlohmann::ordered_json;
struct common_chat_templates;
@@ -87,7 +86,7 @@ struct common_chat_msg {
std::string tool_name;
std::string tool_call_id;
nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const;
common_json to_json_oaicompat(bool concat_typed_text = false) const;
std::string render_content(const std::string & delimiter = "\n\n") const;
@@ -211,7 +210,7 @@ struct common_chat_msg_delimiters {
// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
nlohmann::ordered_json to_json() const;
common_json to_json() const;
};
struct common_chat_tool {
@@ -350,16 +349,16 @@ common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::strin
bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);
// Parses a JSON array of messages in OpenAI's chat completion API format.
std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const nlohmann::ordered_json & messages);
std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const common_json & messages);
std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const nlohmann::ordered_json & tools);
std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const common_json & tools);
common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value);
common_chat_continuation common_chat_continuation_parse(const common_json & value);
// DEPRECATED: only used in tests
nlohmann::ordered_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);
common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);
nlohmann::ordered_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
// get template caps, useful for reporting to server /props endpoint
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
@@ -386,4 +385,4 @@ struct common_chat_prompt_preset {
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters);
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);
common/common.cpp
+29 -2
View File
@@ -402,10 +402,11 @@ void common_params_print_info(const common_params & params, bool print_devices)
#endif
COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);
COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, common_log_get_verbosity_thold());
const int verbosity = common_log_get_verbosity_thold();
COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, verbosity);
// device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device
if (print_devices) {
if (print_devices && verbosity >= LOG_LEVEL_TRACE) {
COM_TRC("%s", "device_info:\n");
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
auto * dev = ggml_backend_dev_get(i);
@@ -1294,11 +1295,34 @@ common_init_result::common_init_result(common_params & params, bool model_only)
if (params.fit_params) {
COM_TRC("%s", "fitting params to device memory ...\n");
COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
// the draft context is created from the same base params and follows the main context, fit both together
const bool has_draft = params.speculative.has_dft();
const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
common_params params_dft = common_base_params_to_speculative(params);
auto mparams_dft = common_model_params_to_llama(params_dft);
auto cparams_dft = common_context_params_to_llama(params_dft);
if (spec_mtp) {
cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
}
cparams_dft.n_rs_seq = 0;
const common_fit_extra_model extra = {
/*.path_model =*/ params_dft.model.path.c_str(),
/*.mparams =*/ &mparams_dft,
/*.cparams =*/ &cparams_dft,
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
};
common_fit_params(params.model.path.c_str(), &mparams, &cparams,
params.tensor_split,
params.tensor_buft_overrides.data(),
params.fit_params_target.data(),
params.fit_params_min_ctx,
has_draft || spec_mtp ? &extra : nullptr,
params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
}
@@ -1664,6 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
mparams.main_gpu = params.main_gpu;
mparams.split_mode = params.split_mode;
mparams.load_mode = params.load_mode;
mparams.lazy_mode = params.lazy_mode;
mparams.tensor_split = params.tensor_split;
mparams.check_tensors = params.check_tensors;
mparams.use_extra_bufts = !params.no_extra_bufts;
@@ -1778,6 +1803,8 @@ void common_threadpools::init(llama_context * ctx, const common_params & params)
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
// each pool needs to match the respective n_threads exactly
// see: https://github.com/ggml-org/llama.cpp/pull/27138#issuecomment-5332307332
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
common/common.h
+36 -7
View File
@@ -8,6 +8,7 @@
#include "ggml.h"
#include "llama.h"
#include <list>
#include <set>
#include <sstream>
#include <string>
@@ -269,7 +270,7 @@ struct common_params_sampling {
COMMON_SAMPLER_TYPE_TEMPERATURE,
};
common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls)
common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls)
bool grammar_lazy = false;
std::vector<common_grammar_trigger> grammar_triggers; // optional triggers (for lazy grammars)
std::set<llama_token> preserved_tokens;
@@ -369,6 +370,9 @@ struct common_params_speculative_ngram_cache {
struct common_params_speculative {
std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };
double synth_len = -1.0;
std::vector<double> synth_rates;
// used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model
common_params_speculative_draft draft;
@@ -383,6 +387,10 @@ struct common_params_speculative {
return !draft.mparams.empty();
}
bool has_synth() const {
return synth_len != -1.0 || !synth_rates.empty();
}
uint32_t need_n_rs_seq() const {
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
@@ -475,6 +483,8 @@ struct common_params {
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model
enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
common_cpu_params cpuparams;
common_cpu_params cpuparams_batch;
@@ -581,13 +591,19 @@ struct common_params {
// multimodal models (see tools/mtmd)
struct common_params_model mmproj;
bool mmproj_use_gpu = true; // use GPU for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
bool mmproj_use_gpu = true; // use GPU for multimodal model
ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
int image_min_tokens = -1;
int image_max_tokens = -1;
int mtmd_batch_max_tokens = 1024;
// for video input
float video_fps = 4.0f;
int64_t video_timestamp_interval_ms = 5000;
std::string video_ffmpeg_bin_dir = "";
// finetune
struct lr_opt lr;
enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
@@ -611,6 +627,7 @@ struct common_params {
bool cache_prompt = true; // whether to enable prompt caching
bool cache_idle_slots = true; // save and clear idle slots upon starting a new task
int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot
int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
@@ -640,6 +657,7 @@ struct common_params {
std::string ssl_file_cert = ""; // NOLINT
std::map<std::string, std::string> default_template_kwargs;
bool preserve_reasoning_specified = false;
// CLI params
std::string server_base; // if set, connect to this server instead of starting a new one
@@ -1107,19 +1125,30 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
}
//
// MoE utils
// FFN offload utils
//
const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate|gate_up)_(ch|)exps";
inline std::string llm_ffn_exps_block_regex(int idx) {
return string_format("blk\\.%d%s", idx, LLM_FFN_EXPS_REGEX);
const char * const LLM_FFN_DENSE_REGEX = "\\.ffn_(up|down|gate)\\.";
inline std::string llm_ffn_block_regex(int idx, const char * ffn_regex) {
return string_format("blk\\.%d%s", idx, ffn_regex);
}
inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {
return { LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() };
}
inline void llm_add_n_cpu_ffn_overrides(int n, const char * ffn_regex, std::vector<llama_model_tensor_buft_override> & overrides) {
// keep strings alive and avoid leaking memory by storing them in a static list
static std::list<std::string> buft_override_strings;
for (int i = 0; i < n; ++i) {
buft_override_strings.push_back(llm_ffn_block_regex(i, ffn_regex));
overrides.push_back({buft_override_strings.back().c_str(), ggml_backend_cpu_buffer_type()});
}
}
//
// training utils
//
common/download.cpp
+26 -10
View File
@@ -5,9 +5,7 @@
#include "log.h"
#include "download.h"
#include "hf-cache.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include "json.h"
#include <algorithm>
#include <filesystem>
@@ -44,8 +42,6 @@
#include <unistd.h>
#endif
using json = nlohmann::ordered_json;
//
// downloader
//
@@ -856,8 +852,8 @@ static std::string common_docker_get_token(const std::string & repo) {
throw std::runtime_error("Failed to get Docker registry token, HTTP code: " + std::to_string(res.first));
}
std::string response_str(res.second.begin(), res.second.end());
nlohmann::ordered_json response = nlohmann::ordered_json::parse(response_str);
std::string response_str(res.second.begin(), res.second.end());
common_json response = common_json::parse(response_str);
if (!response.contains("token")) {
throw std::runtime_error("Docker registry token response missing 'token' field");
@@ -919,9 +915,9 @@ std::string common_docker_resolve_model(const std::string & docker) {
throw std::runtime_error("Failed to get Docker manifest, HTTP code: " + std::to_string(manifest_res.first));
}
std::string manifest_str(manifest_res.second.begin(), manifest_res.second.end());
nlohmann::ordered_json manifest = nlohmann::ordered_json::parse(manifest_str);
std::string gguf_digest; // Find the GGUF layer
std::string manifest_str(manifest_res.second.begin(), manifest_res.second.end());
common_json manifest = common_json::parse(manifest_str);
std::string gguf_digest; // Find the GGUF layer
if (manifest.contains("layers")) {
for (const auto & layer : manifest["layers"]) {
if (layer.contains("mediaType")) {
@@ -989,6 +985,26 @@ std::vector<common_cached_model_info> common_list_cached_models() {
return result;
}
std::string common_download_resolve_path(const std::string & hf_repo_with_tag, const std::string & hf_file) {
auto [repo, tag] = common_download_split_repo_tag(hf_repo_with_tag);
auto files = hf_cache::get_cached_files(repo);
if (files.empty()) {
return "";
}
if (!hf_file.empty()) {
for (const auto & f : files) {
if (f.path == hf_file) {
return f.local_path;
}
}
return "";
}
return find_best_model(files, tag).local_path;
}
bool common_download_remove(const std::string & hf_repo_with_tag) {
namespace fs = std::filesystem;
common/download.h
+4
View File
@@ -85,6 +85,10 @@ std::vector<std::string> common_download_get_all_parts(const std::string & url);
// returns list of cached models
std::vector<common_cached_model_info> common_list_cached_models();
// resolve the local cached file path for a HF repo without network access (hf_file, if given, must match exactly)
// returns an empty string if the model is not present in the cache
std::string common_download_resolve_path(const std::string & hf_repo_with_tag, const std::string & hf_file = "");
// download single file from url to local path
// returns status code or -1 on error
// skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash)
common/fit.cpp
+105 -17
View File
@@ -178,7 +178,7 @@ common_device_memory_data_vec common_get_device_memory_data(
static void common_params_fit_impl(
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
size_t * margins_s, uint32_t n_ctx_min, enum ggml_log_level log_level) {
size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {
if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {
throw common_params_fit_exception("llama_params_fit is not implemented for SPLIT_MODE_TENSOR, abort");
}
@@ -191,10 +191,92 @@ static void common_params_fit_impl(
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert
// with non-unified kv, we need to take into account n_streams
// for example, if memory can hold more than model's trained context size, we must extend the n_ctx to hold enough n_streams
const uint32_t n_streams = cparams->kv_unified ? 1 : std::max<uint32_t>(1, cparams->n_seq_max);
const bool n_ctx_auto = cparams->n_ctx == 0;
dmds_t dmds_extra; // memory of the extra model, laid out on the devices of the main model
uint32_t n_ctx_extra = 0; // context that memory was measured at
// the extra model competes for the same memory as the main model, add it to every measurement
// its memory is measured again whenever the context it follows changes
auto add_extra_memory = [&](dmds_t & dmds) {
if (extra == nullptr) {
return;
}
if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {
std::vector<ggml_backend_dev_t> devs_extra;
uint32_t ngl_extra = 0;
uint32_t nct_extra = 0;
uint32_t nex_extra = 0;
extra->cparams->n_ctx = cparams->n_ctx;
LOG_TRC("%s: getting device memory data for the extra model at a context size of %" PRIu32 ":\n",
__func__, cparams->n_ctx);
dmds_t measured;
try {
measured = common_get_device_memory_data_impl(
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
} catch (const std::runtime_error & e) {
// the extra model is optional, fit the main model alone rather than giving up
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
dmds_extra = dmds_t(devs.size() + 1);
n_ctx_extra = cparams->n_ctx;
return;
}
dmds_extra = dmds_t(devs.size() + 1);
dmds_extra.back().mb = measured.back().mb;
for (size_t je = 0; je < devs_extra.size(); je++) {
for (size_t id = 0; id < devs.size(); id++) {
if (devs_extra[je] == devs[id]) {
dmds_extra[id].mb.model += measured[je].mb.model;
dmds_extra[id].mb.context += measured[je].mb.context;
dmds_extra[id].mb.compute += measured[je].mb.compute;
break;
}
}
}
if (extra->shares_model) {
for (llama_device_memory_data & dmd : dmds_extra) {
dmd.mb.model = 0;
}
}
n_ctx_extra = cparams->n_ctx;
}
for (size_t id = 0; id < dmds.size(); id++) {
dmds[id].mb.model += dmds_extra[id].mb.model;
dmds[id].mb.context += dmds_extra[id].mb.context;
dmds[id].mb.compute += dmds_extra[id].mb.compute;
}
};
// step 1: get data for default parameters and check whether any changes are necessary in the first place
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
const dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
const uint32_t n_ctx_min_total = (uint32_t) std::min<uint64_t>(uint64_t(n_ctx_min) * n_streams, UINT32_MAX);
// llama_context would use only hp_nct in total for n_ctx == 0, resolve the context before measuring anything else:
if (n_ctx_auto) {
cparams->n_ctx = n_ctx_max;
if (n_streams > 1) {
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
__func__, n_ctx_max, n_streams);
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
}
}
add_extra_memory(dmds_full);
const size_t nd = devs.size(); // number of devices
std::vector<int64_t> margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits
@@ -307,8 +389,8 @@ static void common_params_fit_impl(
"%s: cannot meet free memory targets on all devices, need to use %" PRId64 " MiB less in total\n",
__func__, -global_surplus/MiB);
}
if (cparams->n_ctx == 0) {
if (hp_nct > n_ctx_min) {
if (n_ctx_auto) {
if (n_ctx_max > n_ctx_min_total) {
int64_t sum_used_target = sum_free;
if (nd == 0) {
sum_used_target -= margins[0];
@@ -328,8 +410,9 @@ static void common_params_fit_impl(
}
int64_t sum_projected_used_min_ctx = 0;
cparams->n_ctx = n_ctx_min;
const dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
cparams->n_ctx = n_ctx_min_total;
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_min_ctx);
if (nd == 0) {
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
} else {
@@ -339,14 +422,16 @@ static void common_params_fit_impl(
}
if (sum_used_target > sum_projected_used_min_ctx) {
// linear interpolation between minimum and maximum context size:
cparams->n_ctx += (hp_nct - n_ctx_min) * (sum_used_target - sum_projected_used_min_ctx)
cparams->n_ctx += (n_ctx_max - n_ctx_min_total) * (sum_used_target - sum_projected_used_min_ctx)
/ (sum_projected_used - sum_projected_used_min_ctx);
cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % 256, n_ctx_min); // round down context for CUDA backend
// round down context for CUDA backend, keep it divisible by the number of streams:
const uint32_t align = 256 * n_streams;
cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % align, n_ctx_min_total);
const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (hp_nct - n_ctx_min);
const int64_t memory_reduction = (hp_nct - cparams->n_ctx) * bytes_per_ctx;
const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (n_ctx_max - n_ctx_min_total);
const int64_t memory_reduction = (n_ctx_max - cparams->n_ctx) * bytes_per_ctx;
LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
__func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);
__func__, n_ctx_max, cparams->n_ctx, memory_reduction/MiB);
if (nd <= 1) {
LOG_TRC("%s: entire model can be fit by reducing context\n", __func__);
return;
@@ -355,14 +440,14 @@ static void common_params_fit_impl(
} else {
const int64_t memory_reduction = sum_projected_used - sum_projected_used_min_ctx;
LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
__func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);
__func__, n_ctx_max, cparams->n_ctx, memory_reduction/MiB);
}
} else {
if (n_ctx_min == UINT32_MAX) {
LOG_TRC("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, hp_nct);
LOG_TRC("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, n_ctx_max);
} else {
LOG_TRC("%s: default model context size is %" PRIu32 " which is <= the min. context size of %" PRIu32 " -> no change\n",
__func__, hp_nct, n_ctx_min);
__func__, n_ctx_max, n_ctx_min_total);
}
}
} else {
@@ -507,8 +592,9 @@ static void common_params_fit_impl(
llama_model_params mparams_copy = *mparams;
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
const dmds_t dmd_nl = common_get_device_memory_data_impl(
dmds_t dmd_nl = common_get_device_memory_data_impl(
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
add_extra_memory(dmd_nl);
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
for (size_t id = 0; id < nd; id++) {
@@ -535,8 +621,9 @@ static void common_params_fit_impl(
mparams->tensor_buft_overrides = tensor_buft_overrides;
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
const dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_cpu_moe);
for (size_t id = 0; id < nd; id++) {
global_surplus_cpu_moe += dmds_cpu_moe[id].free;
@@ -796,11 +883,12 @@ enum common_params_fit_status common_fit_params(
llama_model_tensor_buft_override * tensor_buft_overrides,
size_t * margins,
uint32_t n_ctx_min,
const common_fit_extra_model * extra,
ggml_log_level log_level) {
const int64_t t0_us = llama_time_us();
common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
try {
common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, log_level);
common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
} catch (const common_params_fit_exception & e) {
LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
common/fit.h
+11
View File
@@ -11,6 +11,16 @@ enum common_params_fit_status {
COMMON_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path
};
// a second model that shares the devices of the main model, e.g. a draft model
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
struct common_fit_extra_model {
const char * path_model;
llama_model_params * mparams;
llama_context_params * cparams;
bool shares_model;
};
// fits mparams and cparams to free device memory (assumes system memory is unlimited)
// - returns true if the parameters could be successfully modified to fit device memory
// - this function is NOT thread safe because it modifies the global llama logger state
@@ -24,6 +34,7 @@ common_params_fit_status common_fit_params(
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
// print estimated memory to stdout
common/hf-cache.cpp
+7 -11
View File
@@ -4,9 +4,7 @@
#include "common.h"
#include "log.h"
#include "http.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include "json.h"
#include <filesystem>
#include <fstream>
@@ -15,8 +13,6 @@
#include <string_view>
#include <stdexcept>
namespace nl = nlohmann;
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
@@ -195,8 +191,8 @@ static void safe_write_file(const fs::path & path, const std::string & data) {
}
}
static nl::json api_get(const std::string & url,
const std::string & token) {
static common_json api_get(const std::string & url,
const std::string & token) {
auto [cli, parts] = common_http_client(url);
httplib::Headers headers = {
@@ -214,10 +210,10 @@ static nl::json api_get(const std::string & url,
auto body = res->body;
if (res->status == 200) {
return nl::json::parse(res->body);
return common_json::parse(res->body);
}
try {
body = nl::json::parse(res->body)["error"].get<std::string>();
body = common_json::parse(res->body)["error"].get<std::string>();
} catch (...) { }
throw std::runtime_error("GET failed (" + std::to_string(res->status) + "): " + body);
@@ -280,7 +276,7 @@ static std::string get_repo_commit(const std::string & repo_id,
safe_write_file(refs_path / name, commit);
return commit;
} catch (const nl::json::exception & e) {
} catch (const common_json_error & e) {
LOG_ERR("%s: JSON error: %s\n", __func__, e.what());
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
@@ -358,7 +354,7 @@ hf_files get_repo_files(const std::string & repo_id,
files.push_back(file);
}
} catch (const nl::json::exception & e) {
} catch (const common_json_error & e) {
LOG_ERR("%s: JSON error: %s\n", __func__, e.what());
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
common/jinja/README.md
+1 -1
View File
@@ -7,7 +7,7 @@ The implementation can be found in the `common/jinja` directory.
## Key Features
- Input marking: security against special token injection
- Decoupled from `nlohmann::json`: this dependency is only used for JSON-to-internal type translation and is completely optional
- Decoupled from the JSON library: `common_json` is only used for JSON-to-internal type translation and is completely optional
- Minimal primitive types: int, float, bool, string, array, object, none, undefined
- Detailed logging: allow source tracing on error
- Clean architecture: workarounds are applied to input data before entering the runtime (see `common/chat.cpp`)
common/jinja/caps.cpp
+2 -2
View File
@@ -4,14 +4,14 @@
// note: the json dependency is only for defining input in a convenient way
// we can remove it in the future when we figure out a better way to define inputs using jinja::value
#include <nlohmann/json.hpp>
#include "json.h"
#include <functional>
#include <sstream>
#define FILENAME "jinja-caps"
using json = nlohmann::ordered_json;
using json = common_json;
namespace jinja {
common/jinja/value.cpp
+3 -3
View File
@@ -3,7 +3,7 @@
#include "value.h"
// for converting from JSON to jinja values
#include <nlohmann/json.hpp>
#include "json.h"
#include <sstream>
#include <string>
@@ -1355,7 +1355,7 @@ const func_builtins & value_undefined_t::get_builtins() const {
//////////////////////////////////
static value from_json(const nlohmann::ordered_json & j, bool mark_input) {
static value from_json(const common_json & j, bool mark_input) {
if (j.is_null()) {
return mk_val<value_none>();
} else if (j.is_boolean()) {
@@ -1452,7 +1452,7 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
}
template<>
void global_from_json(context & ctx, const nlohmann::ordered_json & json_obj, bool mark_input) {
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
if (json_obj.is_null() || !json_obj.is_object()) {
throw std::runtime_error("global_from_json: input JSON value must be an object");
common/jinja/value.h
+1 -1
View File
@@ -86,7 +86,7 @@ struct context; // forward declaration
// marking input can be useful for tracking data provenance
// and preventing template injection attacks
//
// Note: T_JSON can be nlohmann::ordered_json
// Note: T_JSON can be common_json
template<typename T_JSON>
void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
common/json-schema-to-grammar.cpp
+119 -44
View File
@@ -1,9 +1,8 @@
#include "json-schema-to-grammar.h"
#include "common.h"
#include <nlohmann/json.hpp>
#include <algorithm>
#include <limits>
#include <map>
#include <regex>
#include <sstream>
@@ -12,7 +11,7 @@
#include <unordered_set>
#include <vector>
using json = nlohmann::ordered_json;
using json = common_json;
static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "") {
auto has_max = max_items != std::numeric_limits<int>::max();
@@ -278,7 +277,9 @@ static std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
{'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"}
};
static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};
static const int MAX_PATTERN_DEPTH = 100;
static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'};
static std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function<std::string(const std::smatch &)> & replacement) {
@@ -309,6 +310,32 @@ static std::string format_literal(const std::string & literal) {
std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); }
static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
if (pos + 1 >= pattern.length() || pattern[pos] != '\\') {
return 0;
}
size_t n_hex = 0;
switch (pattern[pos + 1]) {
case 'x': n_hex = 2; break;
case 'u': n_hex = 4; break;
case 'U': n_hex = 8; break;
case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']':
return 2;
default:
return 0;
}
if (pos + 2 + n_hex > pattern.length()) {
return 0;
}
for (size_t i = pos + 2; i < pos + 2 + n_hex; i++) {
char h = pattern[i];
if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') || (h >= 'A' && h <= 'F'))) {
return 0;
}
}
return 2 + n_hex;
}
class common_schema_converter {
private:
friend class common_schema_info;
@@ -345,16 +372,42 @@ private:
return string_join(rules, " | ");
}
// thrown when the pattern is a valid regex with no grammar equivalent
struct unsupported_pattern : public std::runtime_error {
using std::runtime_error::runtime_error;
};
// thrown when the pattern is not a valid regex
struct invalid_pattern : public std::runtime_error {
using std::runtime_error::runtime_error;
};
std::string _visit_pattern(const std::string & pattern, const std::string & name) {
if (!(pattern.front() == '^' && pattern.back() == '$')) {
_errors.push_back("Pattern must start with '^' and end with '$'");
auto rules_snapshot = _rules;
try {
return _pattern_to_rule(pattern, name);
} catch (const unsupported_pattern & err) {
// revert rules
_rules = std::move(rules_snapshot);
_warnings.push_back("pattern " + pattern + " is not supported (" + err.what() + "), accepting any string");
return _add_rule(name, _add_primitive("string", PRIMITIVE_RULES.at("string")));
} catch (const invalid_pattern & err) {
_rules = std::move(rules_snapshot);
_errors.push_back("Invalid pattern " + pattern + ": " + err.what());
return "";
}
}
std::string _pattern_to_rule(const std::string & pattern, const std::string & name) {
if (pattern.length() < 2 || pattern.front() != '^' || pattern.back() != '$') {
throw unsupported_pattern("not anchored with '^' and '$'");
}
std::string sub_pattern = pattern.substr(1, pattern.length() - 2);
std::unordered_map<std::string, std::string> sub_rule_ids;
size_t i = 0;
size_t length = sub_pattern.length();
int paren_depth = 0;
using literal_or_rule = std::pair<std::string, bool>;
auto to_rule = [&](const literal_or_rule & ls) {
@@ -363,7 +416,6 @@ private:
return is_literal ? "\"" + s + "\"" : s;
};
std::function<literal_or_rule()> transform = [&]() -> literal_or_rule {
size_t start = i;
std::vector<literal_or_rule> seq;
auto get_dot = [&]() {
@@ -420,43 +472,42 @@ private:
if (i + 1 < length && sub_pattern[i + 1] == ':') {
i += 2; // skip "?:" for non-capturing group, treat as regular group
} else {
// lookahead/lookbehind (?=, ?!, ?<=, ?<!) - not supported
_warnings.push_back("Unsupported pattern syntax");
// skip to matching ')' to avoid UB on empty seq
int depth = 1;
while (i < length && depth > 0) {
if (sub_pattern[i] == '\\' && i + 1 < length) {
i += 2; // skip escaped character
} else {
if (sub_pattern[i] == '(') depth++;
else if (sub_pattern[i] == ')') depth--;
i++;
}
}
continue;
// lookaround, named group, inline flags, ...
throw unsupported_pattern("unsupported group syntax");
}
}
paren_depth++;
if (paren_depth > MAX_PATTERN_DEPTH) {
throw unsupported_pattern("pattern nesting too deep");
}
seq.emplace_back("(" + to_rule(transform()) + ")", false);
} else if (c == ')') {
i++;
if (start > 0 && sub_pattern[start - 1] != '(' && (start < 2 || sub_pattern[start - 2] != '?' || sub_pattern[start - 1] != ':')) {
_errors.push_back("Unbalanced parentheses");
if (paren_depth == 0) {
throw invalid_pattern("unbalanced parentheses");
}
paren_depth--;
return join_seq();
} else if (c == '^' || c == '$') {
throw unsupported_pattern("anchor inside the pattern");
} else if (c == '[') {
std::string square_brackets = std::string(1, c);
i++;
while (i < length && sub_pattern[i] != ']') {
if (sub_pattern[i] == '\\') {
square_brackets += sub_pattern.substr(i, 2);
i += 2;
auto escape_length = gbnf_escape_length(sub_pattern, i);
if (escape_length == 0) {
throw unsupported_pattern("unsupported escape in character class: " + sub_pattern.substr(i, 2));
}
square_brackets += sub_pattern.substr(i, escape_length);
i += escape_length;
} else {
square_brackets += sub_pattern[i];
i++;
}
}
if (i >= length) {
_errors.push_back("Unbalanced square brackets");
throw invalid_pattern("unterminated character class");
}
square_brackets += ']';
i++;
@@ -465,6 +516,9 @@ private:
seq.emplace_back("|", false);
i++;
} else if (c == '*' || c == '+' || c == '?') {
if (seq.empty()) {
throw invalid_pattern("nothing to repeat");
}
seq.back() = std::make_pair(to_rule(seq.back()) + c, false);
i++;
} else if (c == '{') {
@@ -475,18 +529,19 @@ private:
i++;
}
if (i >= length) {
_errors.push_back("Unbalanced curly brackets");
throw unsupported_pattern("unterminated curly brackets");
}
curly_brackets += '}';
i++;
auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ",");
int min_times = 0;
int max_times = std::numeric_limits<int>::max();
if (nums.size() != 1 && nums.size() != 2) {
throw unsupported_pattern("wrong number of values in curly brackets");
}
try {
if (nums.size() == 1) {
min_times = max_times = std::stoi(nums[0]);
} else if (nums.size() != 2) {
_errors.push_back("Wrong number of values in curly brackets");
} else {
if (!nums[0].empty()) {
min_times = std::stoi(nums[0]);
@@ -495,9 +550,11 @@ private:
max_times = std::stoi(nums[1]);
}
}
} catch (const std::invalid_argument & e) {
_errors.push_back("Invalid number in curly brackets");
return std::make_pair("", false);
} catch (const std::logic_error &) {
throw unsupported_pattern("invalid number in curly brackets");
}
if (seq.empty()) {
throw invalid_pattern("nothing to repeat");
}
auto &last = seq.back();
auto &sub = last.first;
@@ -523,15 +580,22 @@ private:
return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end();
};
while (i < length) {
if (sub_pattern[i] == '\\' && i < length - 1) {
if (sub_pattern[i] == '\\') {
if (i == length - 1) {
throw invalid_pattern("trailing backslash");
}
char next = sub_pattern[i + 1];
if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) {
i++;
literal += sub_pattern[i];
i++;
} else {
literal += sub_pattern.substr(i, 2);
i += 2;
auto escape_length = gbnf_escape_length(sub_pattern, i);
if (escape_length == 0) {
throw unsupported_pattern("unsupported escape: " + sub_pattern.substr(i, 2));
}
literal += sub_pattern.substr(i, escape_length);
i += escape_length;
}
} else if (sub_pattern[i] == '"') {
literal += "\\\"";
@@ -544,14 +608,21 @@ private:
break;
}
}
if (!literal.empty()) {
seq.emplace_back(literal, true);
if (literal.empty()) { // nothing was consumed, ex. a stray ']' or '}'
throw unsupported_pattern(std::string("unsupported character: ") + c);
}
seq.emplace_back(literal, true);
}
}
return join_seq();
};
return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"");
auto rule = to_rule(transform());
if (paren_depth != 0) {
throw invalid_pattern("unbalanced parentheses");
}
return _add_rule(name, "\"\\\"\" (" + rule + ") \"\\\"\"");
}
/*
@@ -845,7 +916,11 @@ public:
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
}
if (schema.contains("oneOf") || schema.contains("anyOf")) {
std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
std::vector<json> alt_schemas;
for (const auto & alt : alts) {
alt_schemas.push_back(alt);
}
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
}
if (schema_type.is_array()) {
@@ -1039,7 +1114,7 @@ common_schema_info::~common_schema_info() = default;
common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
void common_schema_info::resolve_refs(common_json & schema) {
impl_->resolve_refs(schema, "");
}
@@ -1047,7 +1122,7 @@ void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
// Some models emit raw string values rather than JSON-encoded strings for string parameters.
// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
// true, allowing callers to handle the value as a raw string for simplicity.
bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schema) {
bool common_schema_info::resolves_to_string(const common_json & schema) {
std::unordered_set<std::string> visited_refs;
std::function<bool(const json &)> check = [&](const json & s) -> bool {
@@ -1155,7 +1230,7 @@ bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schem
return check(schema);
}
std::string json_schema_to_grammar(const json & schema, bool force_gbnf) {
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
#ifdef LLAMA_USE_LLGUIDANCE
if (!force_gbnf) {
return "%llguidance {}\nstart: %json " + schema.dump();
@@ -1176,10 +1251,10 @@ std::string build_grammar(const std::function<void(const common_grammar_builder
/* .add_rule = */ [&](const std::string & name, const std::string & rule) {
return converter._add_rule(name, rule);
},
/* .add_schema = */ [&](const std::string & name, const nlohmann::ordered_json & schema) {
/* .add_schema = */ [&](const std::string & name, const common_json & schema) {
return converter.visit(schema, name == "root" ? "" : name);
},
/* .resolve_refs = */ [&](nlohmann::ordered_json & schema) {
/* .resolve_refs = */ [&](common_json & schema) {
converter.resolve_refs(schema, "");
}
};
common/json-schema-to-grammar.h
+6 -6
View File
@@ -1,12 +1,12 @@
#pragma once
#include <nlohmann/json_fwd.hpp>
#include "json.h"
#include <functional>
#include <memory>
#include <string>
std::string json_schema_to_grammar(const nlohmann::ordered_json & schema,
std::string json_schema_to_grammar(const common_json & schema,
bool force_gbnf = false);
class common_schema_converter;
@@ -24,14 +24,14 @@ class common_schema_info {
common_schema_info(common_schema_info &&) noexcept;
common_schema_info & operator=(common_schema_info &&) noexcept;
void resolve_refs(nlohmann::ordered_json & schema);
bool resolves_to_string(const nlohmann::ordered_json & schema);
void resolve_refs(common_json & schema);
bool resolves_to_string(const common_json & schema);
};
struct common_grammar_builder {
std::function<std::string(const std::string &, const std::string &)> add_rule;
std::function<std::string(const std::string &, const nlohmann::ordered_json &)> add_schema;
std::function<void(nlohmann::ordered_json &)> resolve_refs;
std::function<std::string(const std::string &, const common_json &)> add_schema;
std::function<void(common_json &)> resolve_refs;
};
struct common_grammar_options {
common/json.cpp
+433
View File
@@ -0,0 +1,433 @@
#include "json.h"
#include "ggml.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include <iterator>
#include <new>
#include <set>
#include <unordered_map>
#include <vector>
using nlohmann::ordered_json;
// a common_json is the backing value, so any value of a tree can be used as a common_json
static_assert(sizeof(ordered_json) <= sizeof(common_json), "common_json storage is too small");
static_assert(alignof(ordered_json) <= alignof(common_json), "common_json alignment is too weak");
// runs fn and gives every error of the backing library as a common_json_error
template <typename F>
static decltype(auto) guard(F && fn) {
try {
return fn();
} catch (const ordered_json::exception & e) {
throw common_json_error(e.what());
}
}
static ordered_json & as_json(common_json * self) {
return *reinterpret_cast<ordered_json *>(self);
}
static const ordered_json & as_json(const common_json * self) {
return *reinterpret_cast<const ordered_json *>(self);
}
static common_json & as_common(ordered_json & json) {
return *reinterpret_cast<common_json *>(&json);
}
static const common_json & as_common(const ordered_json & json) {
return *reinterpret_cast<const common_json *>(&json);
}
static ordered_json to_json(const common_json_value & val) {
switch (val.type) {
case common_json_value::VAL_NULL: return nullptr;
case common_json_value::VAL_BOOL: return val.val_bool;
case common_json_value::VAL_INT: return val.val_int;
case common_json_value::VAL_UINT: return val.val_uint;
case common_json_value::VAL_DOUBLE: return val.val_double;
case common_json_value::VAL_STRING: return val.val_string;
case common_json_value::VAL_JSON:
// one owner means no one else can see this tree, so it is safe to move it out
// note: this makes a value single use, same as the json_ref of the backing library
if (val.val_json.use_count() == 1) {
return std::move(as_json(val.val_json.get()));
}
return as_json(val.val_json.get());
}
return nullptr;
}
common_json_value::common_json_value(const char * val) {
if (val) {
type = VAL_STRING;
val_string = val;
} else {
type = VAL_NULL;
}
}
common_json_value::common_json_value(const common_json & val) :
type(VAL_JSON), val_json(std::make_shared<common_json>(val)) {}
common_json_value::common_json_value(common_json && val) :
type(VAL_JSON), val_json(std::make_shared<common_json>(std::move(val))) {}
// the ctors and get<T>() below are explicit specializations, giving strong symbols
// an explicit instantiation is a weak symbol, dropped by some LTO builds (clang-cl)
template <typename T>
static std::shared_ptr<common_json> set_json(const std::set<T> & vals) {
common_json out = common_json::array();
for (const auto & val : vals) {
out.push_back(val);
}
return std::make_shared<common_json>(std::move(out));
}
// a set value is usable only for the types below
#define COMMON_JSON_SET(...) template <> common_json_value::common_json_value(const std::set<__VA_ARGS__> & vals) : type(VAL_JSON), val_json(set_json(vals)) {}
COMMON_JSON_SET(int)
COMMON_JSON_SET(std::string)
#undef COMMON_JSON_SET
template <typename T>
static std::shared_ptr<common_json> map_json(const T & vals) {
common_json out = common_json::object();
for (const auto & val : vals) {
out.set({ val.first, val.second });
}
return std::make_shared<common_json>(std::move(out));
}
// a map value is usable only for the types below
#define COMMON_JSON_MAP(...) template <> common_json_value::common_json_value(const std::map<std::string, __VA_ARGS__> & vals) : type(VAL_JSON), val_json(map_json(vals)) {}
COMMON_JSON_MAP(bool)
COMMON_JSON_MAP(std::string)
#undef COMMON_JSON_MAP
// an unordered map value is usable only for the types below
#define COMMON_JSON_UMAP(...) template <> common_json_value::common_json_value(const std::unordered_map<std::string, __VA_ARGS__> & vals) : type(VAL_JSON), val_json(map_json(vals)) {}
COMMON_JSON_UMAP(size_t)
#undef COMMON_JSON_UMAP
template <typename T>
static std::shared_ptr<common_json> vec_json(const std::vector<T> & vals) {
common_json out = common_json::array();
for (const auto & val : vals) {
out.push_back(val);
}
return std::make_shared<common_json>(std::move(out));
}
// a vector value is usable only for the types below
// note: std::vector<bool> is not here, its proxy reference does not convert
#define COMMON_JSON_VEC(...) template <> common_json_value::common_json_value(const std::vector<__VA_ARGS__> & vals) : type(VAL_JSON), val_json(vec_json(vals)) {}
COMMON_JSON_VEC(int)
COMMON_JSON_VEC(unsigned char)
COMMON_JSON_VEC(unsigned int)
COMMON_JSON_VEC(long)
COMMON_JSON_VEC(unsigned long)
COMMON_JSON_VEC(long long)
COMMON_JSON_VEC(unsigned long long)
COMMON_JSON_VEC(float)
COMMON_JSON_VEC(double)
COMMON_JSON_VEC(std::string)
COMMON_JSON_VEC(std::vector<float>)
COMMON_JSON_VEC(common_json)
#undef COMMON_JSON_VEC
common_json_value::common_json_value(std::initializer_list<common_json_item> items) :
type(VAL_JSON), val_json(std::make_shared<common_json>(items)) {}
// null, same as the backing library
// operator[] turns it into an object, push_back() into an array
common_json::common_json() {
new (storage) ordered_json();
}
common_json::common_json(const common_json & other) {
new (storage) ordered_json(as_json(&other));
}
common_json::common_json(common_json && other) noexcept {
new (storage) ordered_json(std::move(as_json(&other)));
}
common_json::common_json(std::initializer_list<common_json_item> items) {
new (storage) ordered_json(ordered_json::object());
for (const auto & item : items) {
set(item);
}
}
common_json::common_json(const common_json_value & val) {
new (storage) ordered_json(to_json(val));
}
common_json::common_json(std::nullptr_t) {
new (storage) ordered_json(nullptr);
}
common_json & common_json::operator=(common_json other) noexcept {
as_json(this).swap(as_json(&other));
return *this;
}
common_json::~common_json() {
as_json(this).~basic_json();
}
common_json common_json::parse(const std::string & text) {
try {
// the assignment moves the parsed tree in, it does not copy
common_json out;
as_json(&out) = ordered_json::parse(text);
return out;
} catch (const std::exception & e) {
throw common_json_error(e.what());
}
}
common_json common_json::parse_no_throw(const std::string & text) {
common_json out;
as_json(&out) = ordered_json::parse(text, nullptr, false);
return out;
}
bool common_json::is_discarded() const {
return as_json(this).is_discarded();
}
common_json common_json::array() {
common_json out;
as_json(&out) = ordered_json::array();
return out;
}
common_json common_json::array(std::initializer_list<common_json_value> vals) {
common_json out;
ordered_json & arr = as_json(&out);
arr = ordered_json::array();
for (const auto & val : vals) {
arr.push_back(to_json(val));
}
return out;
}
common_json common_json::object() {
common_json out;
as_json(&out) = ordered_json::object();
return out;
}
common_json common_json::object(std::initializer_list<common_json_item> items) {
return common_json(items);
}
common_json common_json::make(const common_json_value & val) {
return common_json(val);
}
bool common_json::is_null() const { return as_json(this).is_null(); }
bool common_json::is_object() const { return as_json(this).is_object(); }
bool common_json::is_array() const { return as_json(this).is_array(); }
bool common_json::is_string() const { return as_json(this).is_string(); }
bool common_json::is_boolean() const { return as_json(this).is_boolean(); }
bool common_json::is_number() const { return as_json(this).is_number(); }
bool common_json::is_number_integer() const { return as_json(this).is_number_integer(); }
bool common_json::is_number_float() const { return as_json(this).is_number_float(); }
bool common_json::empty() const { return as_json(this).empty(); }
size_t common_json::size() const { return as_json(this).size(); }
bool common_json::contains(const std::string & key) const {
return as_json(this).contains(key);
}
bool common_json::operator==(const common_json_value & val) const {
// compare a tree in place, to_json() would copy it
if (val.type == common_json_value::VAL_JSON) {
return as_json(this) == as_json(val.val_json.get());
}
return as_json(this) == to_json(val);
}
bool common_json::operator!=(const common_json_value & val) const {
return !(*this == val);
}
common_json & common_json::at(const std::string & key) { return guard([&]() -> common_json & { return as_common(as_json(this).at(key)); }); }
const common_json & common_json::at(const std::string & key) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(key)); }); }
common_json & common_json::at(size_t idx) { return guard([&]() -> common_json & { return as_common(as_json(this).at(idx)); }); }
const common_json & common_json::at(size_t idx) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(idx)); }); }
common_json & common_json::operator[](const std::string & key) { return guard([&]() -> common_json & { return as_common(as_json(this)[key]); }); }
const common_json & common_json::operator[](const std::string & key) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(key)); }); }
common_json & common_json::operator[](size_t idx) { return guard([&]() -> common_json & { return as_common(as_json(this)[idx]); }); }
const common_json & common_json::operator[](size_t idx) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(idx)); }); }
common_json & common_json::front() { return as_common(as_json(this).front()); }
const common_json & common_json::front() const { return as_common(as_json(this).front()); }
common_json & common_json::back() { return as_common(as_json(this).back()); }
const common_json & common_json::back() const { return as_common(as_json(this).back()); }
void common_json::clear() {
as_json(this).clear();
}
void common_json::erase(const std::string & key) {
guard([&] { as_json(this).erase(key); });
}
void common_json::erase(size_t idx) {
guard([&] { as_json(this).erase(idx); });
}
void common_json::assign(const common_json_value & val) {
as_json(this) = to_json(val);
}
void common_json::set(const common_json_item & item) {
guard([&] { as_json(this)[item.key] = to_json(item.val); });
}
void common_json::push_back(const common_json_value & val) {
guard([&] { as_json(this).push_back(to_json(val)); });
}
void common_json::push_back(std::initializer_list<common_json_item> items) {
common_json val(items);
guard([&] { as_json(this).push_back(std::move(as_json(&val))); });
}
size_t common_json::count(const std::string & key) const {
return as_json(this).count(key);
}
void common_json::insert(const common_json & vals) {
guard([&] {
ordered_json & self = as_json(this);
self.insert(self.end(), as_json(&vals).begin(), as_json(&vals).end());
});
}
std::string common_json::dump(int indent) const {
return guard([&] { return as_json(this).dump(indent); });
}
std::string common_json::dump_safe(int indent) const {
return as_json(this).dump(indent, ' ', false, ordered_json::error_handler_t::replace);
}
// an array is indexed directly, an object needs a walk from the start
common_json & common_json::iterator::operator*() const {
return guard([&]() -> common_json & {
ordered_json & j = as_json(node);
if (j.is_object()) {
return as_common(std::next(j.begin(), idx).value());
}
if (j.is_array()) {
return as_common(j[idx]);
}
// a plain value gives itself once, same as the backing library
return *node;
});
}
std::string common_json::iterator::key() const {
return guard([&] { return std::next(as_json(node).begin(), idx).key(); });
}
common_json::iterator common_json::begin() const {
return iterator(const_cast<common_json *>(this), 0);
}
common_json::iterator common_json::end() const {
return iterator(const_cast<common_json *>(this), size());
}
// the keys follow the backing library: the index for an array, "" for a plain value
common_json::items_view::entry common_json::items_view::iterator::operator*() const {
return guard([&]() -> entry {
ordered_json & j = as_json(node);
if (j.is_object()) {
auto it = std::next(j.begin(), idx);
return { it.key(), as_common(it.value()) };
}
if (j.is_array()) {
return { std::to_string(idx), as_common(j[idx]) };
}
return { std::string(), *node };
});
}
common_json::items_view common_json::items() const {
return items_view(const_cast<common_json *>(this), size());
}
// the backing library cannot build a common_json, so this one is just a copy
template <> common_json common_json::get<common_json>() const {
return *this;
}
// get<T>() is usable only for the types below
#define COMMON_JSON_GET(...) template <> __VA_ARGS__ common_json::get<__VA_ARGS__>() const { return guard([&] { return as_json(this).get<__VA_ARGS__>(); }); }
COMMON_JSON_GET(bool)
COMMON_JSON_GET(int)
COMMON_JSON_GET(unsigned int)
COMMON_JSON_GET(long)
COMMON_JSON_GET(unsigned long)
COMMON_JSON_GET(long long)
COMMON_JSON_GET(unsigned long long)
COMMON_JSON_GET(float)
COMMON_JSON_GET(double)
COMMON_JSON_GET(std::string)
COMMON_JSON_GET(std::vector<float>)
COMMON_JSON_GET(std::vector<std::string>)
COMMON_JSON_GET(std::set<std::string>)
COMMON_JSON_GET(std::vector<int>)
COMMON_JSON_GET(std::vector<size_t>)
COMMON_JSON_GET(std::unordered_map<std::string, size_t>)
#undef COMMON_JSON_GET
// must stay below the get<std::string> specialization
common_json::operator std::string() const {
return get<std::string>();
}
std::string common_json::value(const std::string & key, const char * def) const {
return contains(key) ? at(key).get<std::string>() : std::string(def);
}
common/json.h
+352
View File
@@ -0,0 +1,352 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <iterator>
#include <map>
#include <memory>
#include <set>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
// common_json, a thin wrapper around vendor json library
// the underlay library is pimpl, we are using nlohmann::json for now
//
// many features of the library are deliberately left out, to keep this interface small and generic and to keep compile time down
//
// some main differences compared to nlohmann::json :
// - object keys keep the order in which they are added
// - errors are always throw as common_json_error
// - obj.push_back({key, val}) is intentionally unsupported to avoid confusion with push_back on a vector; write it as obj[key] = val for clarity
// - a braced pair in value position does not build, e.g. {"key", {"a", "b"}}; write array({"a", "b"}) where nlohmann made an array
//
// in doubt, search the code base for an existing usage example; do not add anything to this header unless absolutely necessary
class common_json;
// common_json_value holds a list of these, and each of them holds a value, so one must come first
struct common_json_item;
struct common_json_error : std::runtime_error {
using std::runtime_error::runtime_error;
};
// one value, tagged so that this header stays free of the backing library
// note: a value that holds a tree is single use, the second use gives null
struct common_json_value {
enum value_type {
VAL_NULL,
VAL_BOOL,
VAL_INT,
VAL_UINT,
VAL_DOUBLE,
VAL_STRING,
VAL_JSON,
};
value_type type = VAL_NULL;
union {
bool val_bool;
int64_t val_int;
uint64_t val_uint = 0;
double val_double;
};
std::string val_string;
std::shared_ptr<common_json> val_json;
common_json_value(std::nullptr_t = nullptr) : type(VAL_NULL) {}
common_json_value(bool val) : type(VAL_BOOL), val_bool(val) {}
common_json_value(std::string val) : type(VAL_STRING), val_string(std::move(val)) {}
// without this a string_view lands on the common_json ctor below and recurses
common_json_value(std::string_view val) : type(VAL_STRING), val_string(val) {}
common_json_value(const char * val);
common_json_value(const common_json & val);
common_json_value(common_json && val);
// only for the types instantiated in json.cpp, the rest fails at link time
template <typename T> common_json_value(const std::vector<T> & vals);
// a set becomes an array, in the set's own order
template <typename T> common_json_value(const std::set<T> & vals);
// a map becomes an object, keyed in the map's own order
template <typename T> common_json_value(const std::map<std::string, T> & vals);
template <typename T> common_json_value(const std::unordered_map<std::string, T> & vals);
// nested object, e.g. {"fn", {{"name", "x"}}}
// note: a nested pair {"a", "b"} does not build, use common_json::array({"a", "b"}) for an array
common_json_value(std::initializer_list<common_json_item> items);
template <typename T, typename std::enable_if<std::is_integral<T>::value && !std::is_same<T, bool>::value, int>::type = 0>
common_json_value(T val) : type(std::is_signed<T>::value ? VAL_INT : VAL_UINT) {
if (std::is_signed<T>::value) {
val_int = (int64_t) val;
} else {
val_uint = (uint64_t) val;
}
}
template <typename T, typename std::enable_if<std::is_floating_point<T>::value, int>::type = 0>
common_json_value(T val) : type(VAL_DOUBLE), val_double((double) val) {}
};
struct common_json_item {
std::string key;
common_json_value val;
template <typename T>
common_json_item(std::string key, T && val) :
key(std::move(key)), val(std::forward<T>(val)) {}
// a braced list cannot deduce T, so it needs its own overload
common_json_item(std::string key, std::initializer_list<common_json_item> items) :
key(std::move(key)), val(items) {}
};
// the types common_json_value holds on its own
// anything else reaches its common_json ctor and recurses forever
template <typename T> struct common_json_is_value : std::integral_constant<bool,
std::is_arithmetic<T>::value ||
std::is_same<T, std::nullptr_t>::value ||
std::is_same<T, std::string>::value ||
std::is_same<T, std::string_view>::value ||
std::is_same<T, char *>::value ||
std::is_same<T, const char *>::value ||
std::is_same<T, common_json>::value> {};
template <typename T, typename A>
struct common_json_is_value<std::vector<T, A>> : std::true_type {};
template <typename T, typename C, typename A>
struct common_json_is_value<std::set<T, C, A>> : std::true_type {};
template <typename V, typename C, typename A>
struct common_json_is_value<std::map<std::string, V, C, A>> : std::true_type {};
template <typename V, typename H, typename E, typename A>
struct common_json_is_value<std::unordered_map<std::string, V, H, E, A>> : std::true_type {};
class common_json {
public:
common_json();
common_json(const common_json & other);
common_json(common_json && other) noexcept;
common_json(std::initializer_list<common_json_item> items);
common_json(const common_json_value & val);
// direct, a value would need two conversions in a row
common_json(std::nullptr_t);
// one step, so that "abc" or a vector can go straight into a common_json
template <typename T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, common_json>::value &&
!std::is_same<typename std::decay<T>::type, common_json_value>::value, int>::type = 0>
common_json(T && val) : common_json(common_json_value(std::forward<T>(val))) {
static_assert(common_json_is_value<typename std::decay<T>::type>::value,
"no common_json_value ctor holds this type, add one instead of letting it recurse");
}
// by value, same as the backing library
// the right side is copied before the left side can invalidate it, e.g. msg["a"] = msg.at("b")
common_json & operator=(common_json other) noexcept;
~common_json();
// throws common_json_error if the text is not valid JSON
static common_json parse(const std::string & text);
// gives a discarded value instead of throwing, check it with is_discarded()
static common_json parse_no_throw(const std::string & text);
bool is_discarded() const;
static common_json array();
static common_json array(std::initializer_list<common_json_value> vals);
static common_json object();
static common_json object(std::initializer_list<common_json_item> items);
// holds a single value, e.g. make("abc").dump() gives "\"abc\""
static common_json make(const common_json_value & val);
bool is_null() const;
bool is_object() const;
bool is_array() const;
bool is_string() const;
bool is_boolean() const;
bool is_number() const;
bool is_number_integer() const;
bool is_number_float() const;
bool empty() const;
size_t size() const;
bool contains(const std::string & key) const;
bool operator==(const common_json_value & val) const;
bool operator!=(const common_json_value & val) const;
// at() throws common_json_error if the key is missing, operator[] adds a null value instead
// note: a const operator[] cannot add, it throws like at()
common_json & at(const std::string & key);
const common_json & at(const std::string & key) const;
common_json & at(size_t idx);
const common_json & at(size_t idx) const;
common_json & operator[](const std::string & key);
const common_json & operator[](const std::string & key) const;
common_json & operator[](const char * key) { return (*this)[std::string(key)]; }
const common_json & operator[](const char * key) const { return (*this)[std::string(key)]; }
common_json & operator[](int idx) { return (*this)[to_idx(idx)]; }
const common_json & operator[](int idx) const { return (*this)[to_idx(idx)]; }
common_json & operator[](size_t idx);
const common_json & operator[](size_t idx) const;
common_json & front();
const common_json & front() const;
common_json & back();
const common_json & back() const;
void clear();
void erase(const std::string & key);
void erase(size_t idx);
// only for the types instantiated in json.cpp, the rest fails at link time
template <typename T> T get() const;
// implicit get<T>() for plain values, so they can be assigned to their C++ type directly
// note: kept to this short list on purpose, a wider one makes j["key"] ambiguous
// note: a numeric one would make "str = json;" ambiguous, a number converts to char too
operator std::string() const;
template <typename T>
T value(const std::string & key, T def) const {
return contains(key) ? at(key).get<T>() : def;
}
std::string value(const std::string & key, const char * def) const;
// a JSON default needs no get<T>(), it is already the right type
common_json value(const std::string & key, const common_json & def) const {
return contains(key) ? at(key) : def;
}
void assign(const common_json_value & val);
void set(const common_json_item & item);
void push_back(const common_json_value & val);
// appends one object, e.g. push_back({{"a", 1}})
void push_back(std::initializer_list<common_json_item> items);
// 1 if the key is there, 0 if not
size_t count(const std::string & key) const;
// appends every value of another array; inserting an array into itself throws
void insert(const common_json & vals);
// a common_json goes through the copy assignment above, everything else becomes a value
template <typename T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, common_json>::value, int>::type = 0>
common_json & operator=(T && val) {
assign(common_json_value(std::forward<T>(val)));
return *this;
}
std::string dump(int indent = -1) const;
// same as dump(), but bad UTF-8 gets replaced instead of throwing
std::string dump_safe(int indent = -1) const;
// walks an array by index, or an object in insertion order
// a plain value gives itself once, same as the backing library
class iterator {
public:
using iterator_category = std::forward_iterator_tag;
using value_type = common_json;
using difference_type = std::ptrdiff_t;
using pointer = common_json *;
using reference = common_json &;
iterator(common_json * node, size_t idx) : node(node), idx(idx) {}
common_json & operator*() const;
common_json & value() const { return **this; }
std::string key() const;
iterator & operator++() {
idx++;
return *this;
}
bool operator!=(const iterator & other) const { return idx != other.idx; }
bool operator==(const iterator & other) const { return idx == other.idx; }
private:
common_json * node;
size_t idx;
};
iterator begin() const;
iterator end() const;
// allows: for (const auto & [key, val] : obj.items())
class items_view {
public:
// the members are public, so an entry also works with structured bindings
struct entry {
std::string k;
common_json & v;
const std::string & key() const { return k; }
common_json & value() const { return v; }
};
items_view(common_json * node, size_t n) : node(node), n(n) {}
class iterator {
public:
iterator(common_json * node, size_t idx) : node(node), idx(idx) {}
entry operator*() const;
iterator & operator++() {
idx++;
return *this;
}
bool operator!=(const iterator & other) const { return idx != other.idx; }
private:
common_json * node;
size_t idx;
};
iterator begin() const { return iterator(node, 0); }
iterator end() const { return iterator(node, n); }
private:
common_json * node;
size_t n;
};
items_view items() const;
private:
// a negative index must not turn into a huge size_t
static size_t to_idx(int idx) {
if (idx < 0) {
throw common_json_error("negative array index");
}
return (size_t) idx;
}
// the backing value is built here, json.cpp checks that it fits
// it cannot be a pointer: a value inside a tree would then not be a common_json
// at() could then only give back a copy instead of a real reference
alignas(8) unsigned char storage[32];
};
using common_json_entry = common_json::items_view::entry;
common/log.cpp
+2 -2
View File
@@ -438,7 +438,7 @@ void common_log_flush(struct common_log * log) {
log->resume();
}
static int common_get_verbosity(enum ggml_log_level level) {
int common_log_get_verbosity(enum ggml_log_level level) {
switch (level) {
case GGML_LOG_LEVEL_DEBUG: return LOG_LEVEL_DEBUG;
case GGML_LOG_LEVEL_INFO: return LOG_LEVEL_TRACE;
@@ -452,7 +452,7 @@ static int common_get_verbosity(enum ggml_log_level level) {
}
void common_log_default_callback(enum ggml_log_level level, const char * text, void * /*user_data*/) {
auto verbosity = common_get_verbosity(level);
auto verbosity = common_log_get_verbosity(level);
if (verbosity <= common_log_verbosity_thold) {
common_log_add(common_log_main(), level, "%s", text);
}
common/log.h
+2
View File
@@ -43,6 +43,8 @@ int common_log_get_verbosity_thold(void);
void common_log_set_verbosity_thold(int verbosity); // not thread-safe
int common_log_get_verbosity(enum ggml_log_level level);
void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data);
// the common_log uses an internal worker thread to print/write log messages
common/peg-parser.cpp
+15 -16
View File
@@ -10,7 +10,6 @@
#include <initializer_list>
#include <map>
#include <memory>
#include <nlohmann/json.hpp>
#include <regex>
#include <set>
#include <stdexcept>
@@ -1120,8 +1119,8 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<nlohmann::ordered_json>(schema), raw}));
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw}));
}
common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
@@ -1805,8 +1804,8 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
}
}
static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = nlohmann::json;
static common_json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = common_json;
return std::visit([](const auto & p) -> json {
using T = std::decay_t<decltype(p)>;
@@ -1860,7 +1859,7 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
{"type", "schema"},
{"child", p.child},
{"name", p.name},
{"schema", p.schema ? *p.schema : nullptr},
{"schema", p.schema ? *p.schema : json(nullptr)},
{"raw", p.raw}
};
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
@@ -1888,19 +1887,19 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
}, variant);
}
nlohmann::json common_peg_arena::to_json() const {
auto parsers = nlohmann::json::array();
common_json common_peg_arena::to_json() const {
auto parsers = common_json::array();
for (const auto & parser : parsers_) {
parsers.push_back(serialize_parser_variant(parser));
}
return nlohmann::json{
return common_json{
{"parsers", parsers},
{"rules", rules_},
{"root", root_}
};
}
static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) {
static common_peg_parser_variant deserialize_parser_variant(const common_json & j) {
if (!j.contains("type") || !j["type"].is_string()) {
throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");
}
@@ -1969,9 +1968,9 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
}
common_peg_chars_parser parser;
parser.pattern = j["pattern"];
parser.negated = j["negated"];
parser.min_count = j["min_count"];
parser.max_count = j["max_count"];
parser.negated = j["negated"].get<bool>();
parser.min_count = j["min_count"].get<int>();
parser.max_count = j["max_count"].get<int>();
for (const auto & range_json : j["ranges"]) {
if (!range_json.contains("start") || !range_json.contains("end")) {
throw std::runtime_error("char_range missing 'start' or 'end' field");
@@ -2007,7 +2006,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
parser.child = j["child"].get<common_peg_parser_id>();
parser.name = j["name"];
if (!j["schema"].is_null()) {
parser.schema = std::make_shared<nlohmann::ordered_json>(j["schema"]);
parser.schema = std::make_shared<common_json>(j["schema"]);
}
parser.raw = j["raw"].get<bool>();
return parser;
@@ -2069,7 +2068,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
throw std::runtime_error("Unknown parser type: " + type);
}
common_peg_arena common_peg_arena::from_json(const nlohmann::json & j) {
common_peg_arena common_peg_arena::from_json(const common_json & j) {
if (!j.contains("parsers") || !j["parsers"].is_array()) {
throw std::runtime_error("JSON missing or invalid 'parsers' array");
}
@@ -2109,7 +2108,7 @@ std::string common_peg_arena::save() const {
}
void common_peg_arena::load(const std::string & data) {
*this = from_json(nlohmann::json::parse(data));
*this = from_json(common_json::parse(data));
}
common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {
common/peg-parser.h
+5 -5
View File
@@ -1,6 +1,6 @@
#pragma once
#include <nlohmann/json_fwd.hpp>
#include "json.h"
#include <memory>
#include <set>
@@ -245,7 +245,7 @@ struct common_peg_until_parser {
struct common_peg_schema_parser {
common_peg_parser_id child;
std::string name;
std::shared_ptr<nlohmann::ordered_json> schema;
std::shared_ptr<common_json> schema;
// Indicates if the GBNF should accept a raw string that matches the schema.
bool raw;
@@ -332,8 +332,8 @@ class common_peg_arena {
std::string dump(common_peg_parser_id id) const;
nlohmann::json to_json() const;
static common_peg_arena from_json(const nlohmann::json & j);
common_json to_json() const;
static common_peg_arena from_json(const common_json & j);
std::string save() const;
void load(const std::string & data);
@@ -490,7 +490,7 @@ class common_peg_parser_builder {
// Wraps a parser with JSON schema metadata for grammar generation.
// Used internally to convert JSON schemas to GBNF grammar rules.
common_peg_parser schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw = false);
common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false);
// Creates a named rule, stores it in the grammar, and returns a ref.
// If trigger=true, marks this rule as an entry point for lazy grammar generation.
common/speculative.cpp
+246 -62
View File
@@ -14,6 +14,7 @@
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstring>
#include <iomanip>
#include <map>
@@ -138,6 +139,7 @@ struct common_speculative_impl {
const common_speculative_type type;
uint32_t n_seq;
int32_t n_max; // maximum draft length after implementation-specific limits
size_t n_call_begin = 0; // number of times this implementation was called for refresh.
size_t n_call_draft = 0; // number of times this implementation was called for generation.
@@ -157,7 +159,7 @@ struct common_speculative_impl {
int64_t t_draft_us = 0; // total time spent in generating drafts in this implementation in microseconds.
int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds.
common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {}
common_speculative_impl(common_speculative_type type, uint32_t n_seq, int32_t n_max) : type(type), n_seq(n_seq), n_max(n_max) {}
virtual ~common_speculative_impl() = default;
@@ -182,7 +184,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
std::vector<common_sampler_ptr> smpls;
common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq, params.draft.n_max)
, params(params.draft)
{
auto * ctx_dft = this->params.ctx_dft;
@@ -452,7 +454,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
std::vector<float> g_embd_buf;
common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq, params.draft.n_max)
, params(params.draft)
{
SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n");
@@ -923,18 +925,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
int32_t block_size = 0;
llama_token mask_token_id = 0;
bool is_dflash2 = false;
bool is_mrope = false;
int32_t selector_top_k = 0;
// draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
const bool is_dspark;
// dspark speculators
bool sample_from_anchor = true;
// block-internal attention
bool causal_attn = false;
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
// scratch buffer for concatenated target features [n_tokens, n_embd_enc]
std::vector<float> features_buf;
common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
: common_speculative_impl(type, n_seq)
: common_speculative_impl(type, n_seq, params.draft.n_max)
, params(params.draft)
, is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
{
@@ -960,25 +969,53 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) {
block_size = std::atoi(buf);
}
if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
sample_from_anchor = std::strcmp(buf, "true") == 0;
}
if (llama_model_meta_val_str(model_dft, "dflash.attention.causal", buf, sizeof(buf)) >= 0) {
causal_attn = std::strcmp(buf, "true") == 0;
}
}
selector_top_k = llama_model_dflash_selector_top_k(model_dft);
is_dflash2 = selector_top_k > 0;
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
if (is_dspark && this->params.p_min > 0.0f) {
char buf[16] = {};
const bool has_conf =
llama_model_meta_val_str(model_dft, "dflash.has_confidence_head", buf, sizeof(buf)) < 0 ||
std::strcmp(buf, "true") == 0;
if (!has_conf) {
throw std::runtime_error("DSpark draft has no confidence head: please set --spec-draft-p-min 0");
}
}
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,
block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false");
// DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most
// block_size-1 draft tokens, DSpark yield a full block_size draft tokens
const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;
// block_size-1 draft tokens, anchor-first DSpark yields a full block_size draft tokens
const int32_t n_draft_max = is_dspark && sample_from_anchor ? block_size : block_size - 1;
if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {
LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",
__func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);
this->params.n_max = std::min(this->params.n_max, n_draft_max);
this->params.n_min = std::min(this->params.n_min, n_draft_max);
}
this->n_max = this->params.n_max;
batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);
batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq);
batch_inject = llama_batch_init(llama_n_ubatch(ctx_dft), n_embd_enc, n_seq);
// embd batches on an M-RoPE draft need 4 position rows per token
is_mrope = llama_model_rope_type(model_dft) == LLAMA_ROPE_TYPE_MROPE;
if (is_mrope) {
free(batch_inject.pos);
batch_inject.pos = (llama_pos *) malloc(sizeof(llama_pos) * 4 * llama_n_batch(ctx_dft));
}
smpls.resize(n_seq);
for (auto & s : smpls) {
@@ -991,7 +1028,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// offload draft sampling to the backend
backend_chains.assign(n_seq, nullptr);
if (this->params.backend_sampling) {
if (this->params.backend_sampling && !is_dflash2) {
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
@@ -1010,8 +1047,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
}
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);
llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention
// DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);
llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
}
~common_speculative_impl_draft_dflash() override {
@@ -1096,52 +1134,34 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
// gather this chunk's target features, interleaved by extract layer
features_buf.resize((size_t) n_chunk * n_embd_enc);
// gather target features per extract layer; the fused decode encodes and
// injects them into the K/V cache at the target positions
batch_inject.n_tokens = n_chunk;
for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]);
if (!layer) {
GGML_ABORT("DFlash: target layer %d input not extracted.", target_layer_ids[k]);
}
for (int32_t i = 0; i < n_chunk; ++i) {
float * dst = features_buf.data() + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt;
float * dst = batch_inject.embd + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt;
const float * src = layer + (size_t) (i_batch_beg[seq_id] + offset + i) * n_embd_tgt;
std::memcpy(dst, src, (size_t) n_embd_tgt * sizeof(float));
}
}
// fuse extracted features through DFlash encoder
llama_batch enc_batch = {
/*.n_tokens =*/ n_chunk,
/*.token =*/ nullptr,
/*.embd =*/ features_buf.data(),
/*.pos =*/ nullptr,
/*.n_seq_id =*/ nullptr,
/*.seq_id =*/ nullptr,
/*.logits =*/ nullptr,
};
int32_t rc = llama_encode(ctx_dft, enc_batch);
if (rc != 0) {
LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
__func__, rc, (int) n_chunk, (int) offset);
return false;
}
const float * inp_g = llama_get_embeddings_nextn(ctx_dft);
GGML_ASSERT(inp_g && "DFlash encoder produced no output.");
// inject the DFlash decoder K/V cache at the tokens' target positions
batch_inject.n_tokens = n_chunk;
std::memcpy(batch_inject.embd, inp_g, (size_t) n_chunk * n_embd_dec * sizeof(float));
for (int32_t i = 0; i < n_chunk; ++i) {
batch_inject.pos[i] = batch_in.pos[i_batch_beg[seq_id] + offset + i];
const llama_pos p = batch_in.pos[i_batch_beg[seq_id] + offset + i];
batch_inject.pos[i] = p;
if (is_mrope) {
batch_inject.pos[1 * n_chunk + i] = p;
batch_inject.pos[2 * n_chunk + i] = p;
batch_inject.pos[3 * n_chunk + i] = 0;
}
batch_inject.n_seq_id[i] = 1;
batch_inject.seq_id[i][0] = seq_id;
batch_inject.logits[i] = false;
}
rc = llama_decode(ctx_dft, batch_inject);
const int32_t rc = llama_decode(ctx_dft, batch_inject);
if (rc != 0) {
LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
__func__, rc, (int) n_chunk, (int) offset);
@@ -1175,11 +1195,11 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
const int32_t n_draft = params.n_max;
const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);
const int32_t n_block_tokens = n_draft + (is_dspark && sample_from_anchor ? 0 : 1);
i_block_beg[seq_id] = batch.n_tokens;
n_block [seq_id] = n_block_tokens;
for (int32_t i = 0; i < n_block_tokens; ++i) {
common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, true);
common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, !is_dflash2);
}
}
@@ -1207,12 +1227,42 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
auto & result = *dp.result;
if (is_dspark) {
// DSpark predicts the next token from position 0 and optionally truncates
// at the first position below the confidence threshold.
const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
if (is_dflash2) {
const float * lattice = llama_get_embeddings_nextn(ctx_dft);
GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");
for (int32_t i = 0; i < n_block_tokens; ++i) {
int32_t predecessor = 0;
for (int32_t i = 1; i < n_block_tokens; ++i) {
const float * row = lattice + (size_t) (beg + i) * n_embd_dec;
const float * scores = row + selector_top_k + (size_t) predecessor * selector_top_k;
predecessor = (int32_t) std::distance(scores,
std::max_element(scores, scores + selector_top_k));
if (params.p_min > 0.0f) {
// softmax(scores) at the argmax, i.e. 1 / sum(exp(s_k - s_max))
float sum = 0.0f;
for (int32_t k = 0; k < selector_top_k; ++k) {
sum += std::exp(scores[k] - scores[predecessor]);
}
if (1.0f / sum < params.p_min) {
break;
}
}
result.push_back((llama_token) row[predecessor]);
}
if (result.size() < (size_t) params.n_min) {
result.clear();
}
continue;
}
if (is_dspark) {
// DSpark: read from the first draft slot, truncate below the confidence threshold
const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
// bonus-anchor drafts read the mask positions only, like DFlash
const int32_t i_draft_beg = sample_from_anchor ? 0 : 1;
for (int32_t i = i_draft_beg; i < n_block_tokens; ++i) {
const int32_t idx = beg + i;
if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {
@@ -1308,7 +1358,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
std::vector<std::vector<float>> chain_h;
common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max)
, params(params.draft)
{
auto * ctx_tgt = this->params.ctx_tgt;
@@ -1375,6 +1425,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
c.reserve((size_t) (this->params.n_max + 1) * n_embd);
}
}
this->n_max = this->params.n_max;
pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f));
@@ -1719,7 +1770,7 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {
common_speculative_impl_ngram_simple(
const common_params_speculative & params, uint32_t n_seq,
common_ngram_simple_config config)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq, params.ngram_simple.size_m)
, params(params.ngram_simple)
, config(config)
{
@@ -1763,7 +1814,7 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
const common_ngram_map & config,
uint32_t n_seq)
: common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K
: COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq)
: COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq, config.size_value)
{
for (uint32_t i = 0; i < n_seq; i++) {
this->config.push_back(config);
@@ -1834,7 +1885,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {
common_speculative_impl_ngram_mod(
const common_params_speculative & params,
uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq, params.ngram_mod.n_max)
, params(params.ngram_mod)
, mod(params.ngram_mod.n_match, 4*1024*1024)
, verbose(std::getenv("LLAMA_TRACE") != nullptr) {
@@ -2010,7 +2061,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {
const std::string & path_dynamic,
bool save_dynamic,
bool save_static)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft)
, params(params.ngram_cache)
, n_draft(n_draft)
, save_dynamic(save_dynamic)
@@ -2131,6 +2182,8 @@ struct common_speculative {
// which implementaion was used for a given seq_id
std::vector<common_speculative_impl *> impl_last;
std::vector<double> synth_probs;
};
static common_ngram_map get_common_ngram_map(
@@ -2309,12 +2362,110 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
return n_max;
}
int32_t common_speculative_n_max(const common_speculative * spec) {
int32_t n_max = 0;
if (spec == nullptr) {
return n_max;
}
for (const auto & impl : spec->impls) {
n_max = std::max(n_max, std::max(0, impl->n_max));
}
return n_max;
}
std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max) {
const bool has_length = spec->synth_len != -1.0;
const bool has_rates = !spec->synth_rates.empty();
if (!has_length && !has_rates) {
return {};
}
if (has_length && has_rates) {
throw std::invalid_argument("synthetic acceptance length and rates are mutually exclusive");
}
if (n_max <= 0) {
throw std::invalid_argument("synthetic acceptance requires at least one speculative token");
}
if (has_rates) {
const auto & rates = spec->synth_rates;
if (rates.size() != (size_t) n_max) {
throw std::invalid_argument(string_format(
"synthetic acceptance rates must contain %d values, got %zu", n_max, rates.size()));
}
for (size_t i = 0; i < rates.size(); ++i) {
if (!std::isfinite(rates[i]) || rates[i] < 0.0 || rates[i] > 1.0) {
throw std::invalid_argument("synthetic acceptance rates must be finite and within [0, 1]");
}
if (i > 0 && rates[i] > rates[i - 1]) {
throw std::invalid_argument("synthetic acceptance rates must be monotonically non-increasing");
}
}
return rates;
}
const double length = spec->synth_len;
const double length_max = (double) n_max + 1.0;
if (!std::isfinite(length) || length < 1.0 || length > length_max) {
throw std::invalid_argument(string_format(
"synthetic acceptance length must be finite and within [1, %.0f]", length_max));
}
double p = 0.0;
if (length == length_max) {
p = 1.0;
} else if (length > 1.0) {
double p_min = 0.0;
double p_max = 1.0;
for (int i = 0; i < 32; ++i) {
const double p_mid = 0.5 * (p_min + p_max);
double sum = 0.0;
double term = p_mid;
for (int32_t j = 0; j < n_max; ++j) {
sum += term;
term *= p_mid;
}
if (sum < length - 1.0) {
p_min = p_mid;
} else {
p_max = p_mid;
}
}
p = 0.5 * (p_min + p_max);
}
std::vector<double> rates;
rates.reserve(n_max);
double rate = p;
for (int32_t i = 0; i < n_max; ++i) {
rates.push_back(rate);
rate *= p;
}
return rates;
}
const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec) {
GGML_ASSERT(spec);
return spec->synth_probs;
}
common_params common_base_params_to_speculative(const common_params & params) {
const bool has_draft = params.speculative.has_dft();
const auto & params_spec = params.speculative.draft;
common_params result = params;
result.embedding = false;
result.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED;
if (has_draft) {
result.devices = params_spec.devices;
result.model = params_spec.mparams;
@@ -2378,6 +2529,9 @@ common_speculative_init_result::common_speculative_init_result(
cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
}
// the draft context holds as many tokens per sequence as the target context
cparams.n_ctx = llama_n_ctx(ctx_tgt);
// note: for small models maybe we can set this to the maximum possible draft from all speculative types
// the extra memory for small models is likely negligible?
cparams.n_rs_seq = 0;
@@ -2555,13 +2709,39 @@ common_speculative * common_speculative_init(common_params_speculative & params,
return nullptr;
}
auto * result = new common_speculative {
/* .dparams = */ common_speculative_draft_params_vec(n_seq),
/* .impls = */ std::move(impls),
/* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr)
};
common_speculative_ptr result(new common_speculative {
/* .dparams = */ common_speculative_draft_params_vec(n_seq),
/* .impls = */ std::move(impls),
/* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr),
/* .synth_probs = */ {},
});
return result;
const int32_t n_max_configured = common_speculative_n_max(&params);
const int32_t n_max_effective = common_speculative_n_max(result.get());
const auto rates = common_speculative_synth_rates_resolve(&params, n_max_effective);
std::vector<std::string> rates_str;
rates_str.reserve(rates.size());
result->synth_probs.reserve(rates.size());
double rate_prev = 1.0;
double acceptance_length = 1.0;
for (const double rate : rates) {
result->synth_probs.push_back(rate_prev > 0.0 ? rate / rate_prev : 0.0);
rates_str.push_back(string_format("%.6g", rate));
rate_prev = rate;
acceptance_length += rate;
}
if (!result->synth_probs.empty()) {
SPC_WRN("%s", "synthetic speculative acceptance is enabled for benchmarking; generated output is not valid\n");
if (n_max_effective != n_max_configured) {
SPC_WRN("synthetic acceptance draft limit was reduced from %d to %d by the initialized speculative implementations\n",
n_max_configured, n_max_effective);
}
SPC_INF("synthetic acceptance: n_max = %zu, mean length = %.6f, rates = [%s]\n",
rates.size(), acceptance_length, string_join(rates_str, ", ").c_str());
}
return result.release();
}
void common_speculative_free(common_speculative * spec) {
@@ -2642,6 +2822,10 @@ void common_speculative_draft(common_speculative * spec) {
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) {
auto & dp = dparams[seq_id];
if (!dp.drafting) {
continue;
}
auto & result = *dp.result;
// a new draft has been sampled
common/speculative.h
+9
View File
@@ -26,6 +26,15 @@ std::string common_speculative_type_to_str(enum common_speculative_type type);
// return the max number of draft tokens based on the speculative parameters
int32_t common_speculative_n_max(const common_params_speculative * spec);
// return the max number of draft tokens from the initialized implementations
int32_t common_speculative_n_max(const common_speculative * spec);
// validate and resolve the unconditional synthetic acceptance rates
std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max);
// return the conditional synthetic acceptance probabilities
const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec);
common_params common_base_params_to_speculative(const common_params & params);
struct common_speculative_output_limits {
conversion/__init__.py
+17
View File
@@ -27,6 +27,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"BaichuanForCausalLM": "baichuan",
"BailingMoeForCausalLM": "bailingmoe",
"BailingMoeV2ForCausalLM": "bailingmoe",
"BailingMoeV3ForCausalLM": "bailingmoe3",
"BambaForCausalLM": "granite",
"BertForMaskedLM": "bert",
"BertForSequenceClassification": "bert",
@@ -53,13 +54,21 @@ TEXT_MODEL_MAP: dict[str, str] = {
"DeepseekV3ForCausalLM": "deepseek",
"DeepseekV32ForCausalLM": "deepseek",
"DFlashDraftModel": "qwen",
"DFlash2DraftModel": "qwen",
"Qwen3DSparkModel": "qwen",
"DSparkDraftModel": "qwen",
"DSparkSpeculator": "qwen",
"Lfm2DSparkDraftModel": "qwen",
"LingDSparkModel": "qwen",
"DeepseekV4ForCausalLM": "deepseek",
"DeepseekV4DSparkModel": "deepseek",
"DistilBertForMaskedLM": "bert",
"DistilBertForSequenceClassification": "bert",
"DistilBertModel": "bert",
"Dots1ForCausalLM": "dots1",
"Dots3NoteForCausalLM": "dots3",
"Dots3NoteForConditionalGeneration": "dots3",
"Dots3NoteTextForCausalLM": "dots3",
"DotsOCRForCausalLM": "qwen",
"DreamModel": "dream",
"Ernie4_5ForCausalLM": "ernie",
@@ -106,6 +115,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
"GraniteSwitchForCausalLM": "granite",
"GraniteSpeechForConditionalGeneration": "granite",
"GraniteSpeechPlusForConditionalGeneration": "granite",
"GraniteSWAForCausalLM": "granite",
"GraniteMoeSWAForCausalLM": "granite",
"Grok1ForCausalLM": "grok",
"GrokForCausalLM": "grok",
"GroveMoeForCausalLM": "grovemoe",
@@ -225,6 +236,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3_5ForConditionalGeneration": "qwen",
"Qwen3_5MoeForCausalLM": "qwen",
"Qwen3_5MoeForConditionalGeneration": "qwen",
"Qwen4ExpForCausalLM": "qwen4exp",
"Qwen4ExpForConditionalGeneration": "qwen4exp",
"RND1": "qwen",
"RWForCausalLM": "falcon",
"RWKV6Qwen2ForCausalLM": "rwkv",
@@ -273,6 +286,9 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"CogVLMForCausalLM": "cogvlm",
"DeepseekOCR2ForCausalLM": "deepseek",
"DeepseekOCRForCausalLM": "deepseek",
"DeepseekV4ForCausalLM": "deepseek",
"Dots3NoteForCausalLM": "dots3",
"Dots3NoteForConditionalGeneration": "dots3",
"DotsOCRForCausalLM": "dotsocr",
"Exaone4_5_ForConditionalGeneration": "exaone",
"Gemma3ForConditionalGeneration": "gemma",
@@ -320,6 +336,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
"Qwen3_5ForConditionalGeneration": "qwen3vl",
"Qwen3_5MoeForConditionalGeneration": "qwen3vl",
"Qwen4ExpForConditionalGeneration": "qwen4exp",
"RADIOModel": "nemotron",
"Sarashina2VisionForCausalLM": "sarashina2",
"SmolVLMForConditionalGeneration": "smolvlm",
conversion/afmoe.py
+1
View File
@@ -13,6 +13,7 @@ from .llama import LlamaModel
@ModelBase.register("AfmoeForCausalLM")
@ModelBase.example("arcee-ai/Trinity-Large-Thinking")
class AfmoeModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.AFMOE
conversion/arctic.py
+1
View File
@@ -16,6 +16,7 @@ from .llama import LlamaModel
@ModelBase.register("ArcticForCausalLM")
@ModelBase.example("Snowflake/snowflake-arctic-instruct")
class ArcticModel(TextModel):
model_arch = gguf.MODEL_ARCH.ARCTIC
conversion/baichuan.py
+1
View File
@@ -9,6 +9,7 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("BaichuanForCausalLM", "BaiChuanForCausalLM")
@ModelBase.example("baichuan-inc/Baichuan2-7B-Chat", "baichuan-inc/Baichuan-7B")
class BaichuanModel(TextModel):
model_arch = gguf.MODEL_ARCH.BAICHUAN
conversion/bailingmoe.py
+3
View File
@@ -11,6 +11,7 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("BailingMoeForCausalLM")
@ModelBase.example("inclusionAI/Ling-lite")
class BailingMoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE
@@ -108,6 +109,7 @@ class BailingMoeModel(TextModel):
@ModelBase.register("BailingMoeV2ForCausalLM")
@ModelBase.example("inclusionAI/Ling-mini-2.0")
class BailingMoeV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE2
@@ -189,6 +191,7 @@ class BailingMoeV2Model(TextModel):
@ModelBase.register("SarvamMoEForCausalLM", "modeling_sarvam_moe.SarvamMoEForCausalLM")
@ModelBase.example("sarvamai/sarvam-30b")
class SarvamMoEModel(BailingMoeV2Model):
model_arch = gguf.MODEL_ARCH.BAILINGMOE2
# Sarvam-MoE shares the BailingMoeV2 architecture; only differences:
conversion/bailingmoe3.py
+193
View File
@@ -0,0 +1,193 @@
from __future__ import annotations
import re
from typing import Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, TextModel, gguf
@ModelBase.register("BailingMoeV3ForCausalLM")
@ModelBase.example("inclusionAI/Ling-3.0-tiny", "inclusionAI/Ling-3.0-flash")
class BailingMoeV3Model(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE3
supports_mtp_export = True
_experts: list[dict[str, Tensor]] | None = None
_main_layers: int | None = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
nextn_layers = self.hparams.get("num_nextn_predict_layers", 0) or 0
if self.no_mtp:
nextn_layers = 0
self.block_count = self.hparams["num_hidden_layers"] + nextn_layers
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
def index_tensors(self, remote_hf_model_id: str | None = None):
type(self)._main_layers = self.hparams["num_hidden_layers"]
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
def set_vocab(self):
self._set_vocab_gpt2()
def is_full_attention(self, bid: int) -> bool:
n_layer = self.hparams["num_hidden_layers"]
layer_group_size = self.hparams["layer_group_size"]
return bid >= n_layer or (bid + 1) % layer_group_size == 0 or bid >= n_layer // layer_group_size * layer_group_size
def set_gguf_parameters(self):
if not self.hparams.get("no_kda_lora", False):
raise ValueError("BailingMoeV3 KDA LoRA projections are not supported")
if not self.hparams.get("kda_safe_gate", False):
raise ValueError("BailingMoeV3 non-safe KDA gates are not supported")
if self.hparams.get("gated_attention_proj_granularity_type") != "head_wise":
raise ValueError("BailingMoeV3 requires head-wise attention gates")
self.hparams["num_key_value_heads"] = 1
super().set_gguf_parameters()
n_head_kv = [1 if self.is_full_attention(il) else 0 for il in range(self.block_count)]
self.gguf_writer.add_head_count_kv(n_head_kv)
self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
self.gguf_writer.add_ssm_conv_kernel(self.hparams["short_conv_kernel_size"])
self.gguf_writer.add_kda_head_dim(self.hparams["head_dim"])
self.gguf_writer.add_kda_safe_gate(self.hparams["kda_safe_gate"])
self.gguf_writer.add_kda_gate_lower_bound(self.hparams["kda_lower_bound"])
kv_lora_rank = self.hparams["kv_lora_rank"]
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
qk_rope_head_dim = self.hparams["qk_rope_head_dim"]
if (q_lora_rank := self.hparams.get("q_lora_rank")) is not None:
self.gguf_writer.add_q_lora_rank(q_lora_rank)
self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
self.gguf_writer.add_key_length_mla(qk_nope_head_dim + qk_rope_head_dim)
self.gguf_writer.add_value_length_mla(self.hparams["v_head_dim"])
self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"])
self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
def clamp_limits(key: str) -> list[float] | None:
values = self.hparams.get(key)
if values is None:
return None
values = [0.0 if value is None else float(value) for value in values[:self.block_count]]
return values + [0.0] * (self.block_count - len(values))
if (values := clamp_limits("expert_swiglu_limit_list")) is not None:
self.gguf_writer.add_swiglu_clamp_exp(values)
if (values := clamp_limits("share_expert_swiglu_limit_list")) is not None:
self.gguf_writer.add_swiglu_clamp_shexp(values)
if not self.no_mtp and (nextn_layers := self.hparams.get("num_nextn_predict_layers", 0)):
self.gguf_writer.add_nextn_predict_layers(nextn_layers)
def prepare_metadata(self, vocab_only: bool):
from_dir = self.fname_out.is_dir()
super().prepare_metadata(vocab_only=vocab_only)
if not self.mtp_only or not from_dir:
return
output_type: str = self.ftype.name.partition("_")[2]
fname_default: str = gguf.naming_convention(
self.metadata.name, self.metadata.basename, self.metadata.finetune,
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if name.endswith(".expert_bias"):
name += ".bias"
if cls._main_layers is None:
return super().filter_tensors((name, gen))
m = re.match(r"model\.layers\.(\d+)\.", name)
is_mtp = m is not None and int(m.group(1)) >= cls._main_layers
if is_mtp and cls.no_mtp:
return None
if cls.mtp_only and not is_mtp and name not in (
"model.word_embeddings.weight", "model.norm.weight", "lm_head.weight",
):
return None
return super().filter_tensors((name, gen))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")) and data_torch.ndim in (2, 3):
d_inner = data_torch.shape[0]
d_conv = data_torch.shape[-1]
data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
if name.endswith(".A_log"):
data_torch = torch.exp(data_torch).reshape(-1, 1)
if name.endswith(".dt_bias"):
name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
if name.endswith(".attention.f_proj.weight"):
assert bid is not None
if self.is_full_attention(bid):
raise ValueError(f"unexpected f_proj on full-attention layer {bid}")
name = self.format_tensor_name(gguf.MODEL_TENSOR.SSM_F_A, bid)
if name.endswith(".attention.g_proj.weight"):
assert bid is not None
tensor = gguf.MODEL_TENSOR.ATTN_GATE if self.is_full_attention(bid) else gguf.MODEL_TENSOR.SSM_G_A
name = self.format_tensor_name(tensor, bid)
if ".mlp.experts." in name:
n_experts = self.hparams["num_experts"]
assert bid is not None
if self._experts is None:
self._experts = [{} for _ in range(self.block_count)]
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
for weight_name in ("down_proj", "gate_proj", "up_proj"):
tensors = []
for expert_id in range(n_experts):
expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight"
tensors.append(self._experts[bid].pop(expert_name))
merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight"
yield from super().modify_tensors(torch.stack(tensors, dim=0), merged_name, bid)
return
if name.endswith(".attention.kv_b_proj.weight"):
assert bid is not None
n_head = self.hparams["num_attention_heads"]
v_head_dim = self.hparams["v_head_dim"]
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
assert data_torch.shape[0] == n_head * (v_head_dim + qk_nope_head_dim)
kv_b = data_torch.view(n_head, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
name_k = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid)
name_v = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid)
yield from super().modify_tensors(k_b.transpose(1, 2), name_k, bid)
yield from super().modify_tensors(v_b, name_v, bid)
return
yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
if self._experts is not None:
experts = [name for layer in self._experts for name in layer]
if experts:
raise ValueError(f"Unprocessed experts: {experts}")
conversion/base.py
+14 -2
View File
@@ -1006,12 +1006,16 @@ class ModelBase:
else:
raise ValueError(f"Unknown file type: {self.ftype.name}")
# a chunked tensor quantizes as one chunk at a time, while it is written
quantize = data.quantize if isinstance(data, gguf.LazyChunkedTensor) else (
lambda qtype, d=data: gguf.quants.quantize(d, qtype))
try:
data = gguf.quants.quantize(data, data_qtype)
data = quantize(data_qtype)
except gguf.QuantError as e:
logger.warning("%s, %s", e, "falling back to F16")
data_qtype = gguf.GGMLQuantizationType.F16
data = gguf.quants.quantize(data, data_qtype)
data = quantize(data_qtype)
shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
@@ -1149,6 +1153,14 @@ class ModelBase:
return modelcls
return func
@classmethod
def example(cls, *hf_repos: str) -> Callable[[AnyModel], AnyModel]:
del hf_repos # unused
def func(modelcls: AnyModel) -> AnyModel:
return modelcls
return func
@classmethod
def print_registered_models(cls):
for model_type, model_classes in cls._model_classes.items():
conversion/bert.py
+9
View File
@@ -15,6 +15,7 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
@ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")
@ModelBase.example("BAAI/bge-small-en-v1.5", "dangvantuan/sentence-camembert-base")
class BertModel(TextModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -240,6 +241,7 @@ class BertModel(TextModel):
@ModelBase.register("DistilBertModel", "DistilBertForMaskedLM", "DistilBertForSequenceClassification")
@ModelBase.example("distilbert/distilbert-base-uncased")
class DistilBertModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -263,6 +265,7 @@ class DistilBertModel(BertModel):
@ModelBase.register("RobertaModel", "RobertaForSequenceClassification")
@ModelBase.example("sentence-transformers/stsb-roberta-base")
class RobertaModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -312,6 +315,7 @@ class RobertaModel(BertModel):
@ModelBase.register("NomicBertModel")
@ModelBase.example("nomic-ai/nomic-embed-text-v1.5")
class NomicBertModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -400,6 +404,7 @@ class NomicBertModel(BertModel):
@ModelBase.register("NeoBERT", "NeoBERTLMHead", "NeoBERTForSequenceClassification")
@ModelBase.example("chandar-lab/NeoBERT")
class NeoBert(BertModel):
model_arch = gguf.MODEL_ARCH.NEO_BERT
@@ -431,6 +436,7 @@ class NeoBert(BertModel):
@ModelBase.register("EuroBertModel", "JinaEmbeddingsV5Model")
@ModelBase.example("hf-tiny-v2/tiny-random-EuroBertModel", "jinaai/jina-embeddings-v5-text-nano")
class EuroBertModel(TextModel):
model_arch = gguf.MODEL_ARCH.EUROBERT
@@ -459,6 +465,7 @@ class EuroBertModel(TextModel):
@ModelBase.register("XLMRobertaModel", "XLMRobertaForSequenceClassification")
@ModelBase.example("BAAI/bge-m3")
class XLMRobertaModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
_lora_files = {}
@@ -561,6 +568,7 @@ class XLMRobertaModel(BertModel):
@ModelBase.register("JinaBertModel", "JinaBertForMaskedLM")
@ModelBase.example("jinaai/jina-embeddings-v2-base-en")
class JinaBertV2Model(BertModel):
model_arch = gguf.MODEL_ARCH.JINA_BERT_V2
@@ -588,6 +596,7 @@ class JinaBertV2Model(BertModel):
@ModelBase.register("ModernBertModel", "ModernBertForMaskedLM", "ModernBertForSequenceClassification")
@ModelBase.example("answerdotai/ModernBERT-base")
class ModernBertModel(BertModel):
model_arch = gguf.MODEL_ARCH.MODERN_BERT
conversion/bitnet.py
+1
View File
@@ -9,6 +9,7 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("BitnetForCausalLM", "BitNetForCausalLM")
@ModelBase.example("microsoft/bitnet-b1.58-2B-4T")
class BitnetModel(TextModel):
model_arch = gguf.MODEL_ARCH.BITNET
conversion/bloom.py
+1
View File
@@ -13,6 +13,7 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("BloomForCausalLM", "BloomModel")
@ModelBase.example("bigscience/bloom-560m")
class BloomModel(TextModel):
model_arch = gguf.MODEL_ARCH.BLOOM
conversion/chameleon.py
+2
View File
@@ -12,6 +12,8 @@ from .llama import LlamaModel
@ModelBase.register("ChameleonForConditionalGeneration")
@ModelBase.register("ChameleonForCausalLM") # obsolete
# [TAG_HF_EXAMPLE_GATED] facebook/chameleon-7b is gated
# [TAG_HF_EXAMPLE_MISSING]
class ChameleonModel(TextModel):
model_arch = gguf.MODEL_ARCH.CHAMELEON
conversion/chatglm.py
+1
View File
@@ -9,6 +9,7 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf
@ModelBase.register("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")
@ModelBase.example("THUDM/chatglm3-6b", "zai-org/glm-4-9b-chat-hf")
class ChatGLMModel(TextModel):
model_arch = gguf.MODEL_ARCH.CHATGLM
conversion/codeshell.py
+1
View File
@@ -4,6 +4,7 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("CodeShellForCausalLM")
@ModelBase.example("WisdomShell/CodeShell-7B")
class CodeShellModel(TextModel):
model_arch = gguf.MODEL_ARCH.CODESHELL
conversion/cogvlm.py
+2
View File
@@ -11,6 +11,7 @@ from .llama import LlamaModel
@ModelBase.register("CogVLMForCausalLM")
@ModelBase.example("THUDM/cogvlm2-llama3-chat-19B", "THUDM/cogvlm-chat-hf")
class CogVLMVisionModel(MmprojModel):
def set_gguf_parameters(self):
@@ -29,5 +30,6 @@ class CogVLMVisionModel(MmprojModel):
@ModelBase.register("CogVLMForCausalLM")
@ModelBase.example("THUDM/cogvlm2-llama3-chat-19B", "THUDM/cogvlm-chat-hf")
class CogVLMModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.COGVLM
conversion/command_r.py
+5
View File
@@ -12,6 +12,8 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("CohereForCausalLM")
# [TAG_HF_EXAMPLE_GATED] CohereLabs/c4ai-command-r-v01 is gated
# [TAG_HF_EXAMPLE_MISSING]
class CommandR2Model(TextModel):
model_arch = gguf.MODEL_ARCH.COMMAND_R
@@ -30,6 +32,8 @@ class CommandR2Model(TextModel):
@ModelBase.register("Cohere2ForCausalLM")
# [TAG_HF_EXAMPLE_GATED] CohereLabs/c4ai-command-r7b-12-2024 is gated
@ModelBase.example("hf-tiny-v2/tiny-random-Cohere2ForCausalLM")
class Cohere2Model(TextModel):
model_arch = gguf.MODEL_ARCH.COHERE2
@@ -59,6 +63,7 @@ class Cohere2Model(TextModel):
@ModelBase.register("Cohere2MoeForCausalLM")
@ModelBase.example("CohereLabs/North-Mini-Code-1.0")
class Cohere2MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.COHERE2MOE
_n_main_layers: int | None = None
conversion/dbrx.py
+1
View File
@@ -9,6 +9,7 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("DbrxForCausalLM")
@ModelBase.example("alpindale/dbrx-instruct")
class DbrxModel(TextModel):
model_arch = gguf.MODEL_ARCH.DBRX
conversion/deci.py
+1
View File
@@ -13,6 +13,7 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("DeciLMForCausalLM")
@ModelBase.example("nvidia/Llama-3_1-Nemotron-51B-Instruct", "Deci/DeciLM-7B")
class DeciModel(TextModel):
model_arch = gguf.MODEL_ARCH.DECI
conversion/deepseek.py
+85
View File
@@ -18,6 +18,7 @@ from .qwen import QwenModel
@ModelBase.register("DeepseekOCRForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-OCR")
class DeepseekOCRVisionModel(MmprojModel):
# HF dynamic_preprocess() max_num, which differs per model
preproc_max_tiles = 9
@@ -100,11 +101,13 @@ class DeepseekOCRVisionModel(MmprojModel):
@ModelBase.register("UnlimitedOCRForCausalLM")
@ModelBase.example("baidu/Unlimited-OCR")
class UnlimitedOCRVisionModel(DeepseekOCRVisionModel):
preproc_max_tiles = 32
@ModelBase.register("DeepseekOCR2ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-OCR-2")
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
preproc_max_tiles = 6
@@ -134,6 +137,7 @@ class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
@ModelBase.register("DeepseekForCausalLM")
@ModelBase.example("deepseek-ai/deepseek-moe-16b-chat")
class DeepseekModel(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK
@@ -228,6 +232,7 @@ class DeepseekModel(TextModel):
"YoutuForCausalLM",
"YoutuVLForConditionalGeneration",
)
@ModelBase.example("deepseek-ai/DeepSeek-V2-Lite", "deepseek-ai/DeepSeek-V3")
class DeepseekV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
@@ -457,6 +462,7 @@ class DeepseekV2Model(TextModel):
@ModelBase.register("DeepseekV32ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-V3.2-Exp")
class DeepseekV32Model(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.DEEPSEEK32
skip_mtp = False
@@ -517,6 +523,7 @@ class DeepseekV32Model(DeepseekV2Model):
@ModelBase.register("DeepseekV4ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Base")
class DeepseekV4Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK4
supports_mtp_export = True
@@ -571,6 +578,8 @@ class DeepseekV4Model(TextModel):
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if name.startswith(("aligner.", "image_")):
return None
if name.startswith("mtp."):
if not cls.mtp_only:
cls._skipped_mtp_tensors += 1
@@ -846,6 +855,7 @@ class DeepseekV4Model(TextModel):
"ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"),
"ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"),
"ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"),
"ffn.gate.bias_vl": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B_VL, ".bias"),
"ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"),
"ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"),
"ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"),
@@ -871,6 +881,10 @@ class DeepseekV4Model(TextModel):
if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):
return []
# hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused
if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:
return []
tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)
if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:
return []
@@ -911,6 +925,7 @@ class DeepseekV4Model(TextModel):
@ModelBase.register("DeepseekV4DSparkModel")
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-DSpark")
class DeepseekV4DSparkModel(DeepseekV4Model):
model_arch = gguf.MODEL_ARCH.DFLASH
@@ -1010,3 +1025,73 @@ class DeepseekV4DSparkModel(DeepseekV4Model):
self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])
@ModelBase.register("DeepseekV4ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Vision-Exp")
class DeepseekV4FlashVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
# no preprocessor_config.json in the repo; normalization is (x/255 - 0.5) / 0.5
# ref: inference/image_processor.py (load_image)
self.preprocessor_config = {
"image_mean": [0.5, 0.5, 0.5],
"image_std": [0.5, 0.5, 0.5],
**self.preprocessor_config,
}
def get_vision_config(self) -> dict[str, Any] | None:
cfg = self.global_config
if cfg.get("vision_n_layers", 0) == 0:
raise ValueError("DeepseekV4FlashVisionModel requires vision_n_layers > 0 in the model config")
return {
"num_hidden_layers": cfg["vision_n_layers"],
"hidden_size": cfg["vision_dim"],
"num_attention_heads": cfg["vision_n_heads"],
"intermediate_size": cfg["vision_inter_dim"],
"patch_size": cfg["vision_patch_size"],
# dynamic resolution; only used for compat / warmup
"image_size": cfg["vision_patch_size"] * cfg["vision_downsample_ratio"] * 16,
"rope_theta": cfg.get("vision_rope_theta", 10000.0),
"downsample_ratio": cfg["vision_downsample_ratio"],
"min_pixels": cfg["vision_min_pixels"],
}
def set_gguf_parameters(self):
super().set_gguf_parameters()
assert self.hparams_vision is not None
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK4V)
# vision RMSNorm eps is the pytorch default, NOT the LLM's rms_norm_eps (1e-20)
# ref: inference/vision.py (RMSNorm)
self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
# hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)
# if future models use different values, add GGUF keys for those
assert self.global_config["vision_max_n_token"] == 384
assert self.global_config["vision_max_wh_ratio"] == 8
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not (name.startswith(("vision.", "aligner.", "image_"))):
return None
return super().filter_tensors(item)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
assert self.hparams_vision is not None
if name == "vision.patch_embed.proj.weight":
# nn.Linear over flattened (3, p, p) patches == conv2d weight
p = self.hparams_vision["patch_size"]
data_torch = data_torch.reshape(data_torch.shape[0], 3, p, p)
if ".mlp.w1." in name:
# fused SwiGLU gate+up
gate, up = data_torch.chunk(2, dim=0)
yield from super().modify_tensors(gate, name.replace("w1", "w1_gate"), bid)
yield from super().modify_tensors(up, name.replace("w1", "w1_up"), bid)
return
yield from super().modify_tensors(data_torch, name, bid)
conversion/dots1.py
+1
View File
@@ -11,6 +11,7 @@ from .qwen import Qwen2MoeModel
@ModelBase.register("Dots1ForCausalLM")
@ModelBase.example("rednote-hilab/dots.llm1.inst")
class Dots1Model(Qwen2MoeModel):
model_arch = gguf.MODEL_ARCH.DOTS1
conversion/dots3.py
+323
View File
@@ -0,0 +1,323 @@
from __future__ import annotations
import math
import re
import torch
from typing import TYPE_CHECKING, Any, Callable, Iterable
if TYPE_CHECKING:
from torch import Tensor
from .base import MmprojModel, ModelBase, gguf
from .deepseek import DeepseekV2Model
@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration", "Dots3NoteTextForCausalLM")
class Dots3NoteModel(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.DOTS3NOTE
skip_mtp = False
supports_mtp_export = True
# trunk layer count, stashed before indexing for filter_tensors (mirrors DeepseekV32Model)
_n_main_layers: int | None = None
def index_tensors(self, remote_hf_model_id: str | None = None):
type(self)._n_main_layers = self.hparams["num_hidden_layers"]
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hparams = self.hparams
# config file doesn't specify MTP block, detect it from model weight
self.n_nextn = 1 if "model.mtp.embed_tokens.weight" in self.model_tensors else 0
if self.n_nextn:
self.block_count += self.n_nextn
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
self.layer_types = hparams["layer_types"]
if len(self.layer_types) < hparams["num_hidden_layers"]:
raise ValueError("layer_types is shorter than num_hidden_layers")
if hparams.get("use_dsa", True) is not True:
raise ValueError("dots3-note conversion requires use_dsa=true")
if hparams.get("normalization", "RMSNorm") != "RMSNorm" or hparams.get("final_norm", "RMSNorm") != "RMSNorm":
raise ValueError("dots3-note conversion only supports RMSNorm")
if hparams.get("k_rope_only_layernorm", True) is not True:
raise ValueError("dots3-note conversion requires k_rope_only_layernorm=true")
if hparams.get("topk_method", "noaux_tc") != "noaux_tc" or hparams.get("scoring_func") != "sigmoid":
raise ValueError("dots3-note conversion only supports noaux_tc/sigmoid expert gating")
if hparams.get("n_group", 1) != 1 or hparams.get("topk_group", 1) != 1:
raise ValueError("dots3-note conversion does not support grouped expert routing")
if hparams.get("use_dynamic_rsf", False) or hparams.get("moe_gating_fp32", False):
raise ValueError("dots3-note conversion does not support use_dynamic_rsf/moe_gating_fp32")
for key in ("attention_gate_type", "swa_attention_gate_type"):
if hparams.get(key, "headwise") != "headwise":
raise ValueError(f"dots3-note conversion only supports headwise attention gate, got {key}={hparams.get(key)!r}")
if hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"] != hparams.get("swa_head_dim", 256):
raise ValueError("swa_head_dim must equal swa_qk_nope_head_dim + swa_qk_rope_head_dim")
if hparams["swa_qk_rope_head_dim"] != hparams["qk_rope_head_dim"]:
# both layer kinds share a single rope_dimension_count
raise ValueError("swa_qk_rope_head_dim must match qk_rope_head_dim")
self.apply_lora_rescale = hparams.get("apply_mla_qkv_lora_rescale", False)
def _is_swa_layer(self, bid: int) -> bool:
if bid >= self.hparams["num_hidden_layers"]:
# note: the NextN/MTP block uses the sliding-attention MLA
return True
return self.layer_types[bid] == "sliding_attention"
def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre(tokpre)
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"]) # ty: ignore[unresolved-attribute]
special_vocab.add_to_gguf(self.gguf_writer)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
if (titem := super().filter_tensors(item)) is None:
return None
name, gen = titem
if name.startswith(("vision_encoder.", "audio_encoder.")):
return None
assert cls._n_main_layers is not None
is_mtp = name.startswith("model.mtp.") or \
((m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers)
# --no-mtp: drop the NextN/MTP block; --mtp: keep only that block plus the shared embeddings/norm/lm_head
if is_mtp and cls.no_mtp:
return None
if cls.mtp_only and not is_mtp and name not in (
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
):
return None
return name, gen
def set_gguf_parameters(self):
hparams = self.hparams
# head_count is a per-layer array because the two layer kinds have different head counts
n_layer = hparams["num_hidden_layers"]
hparams["num_attention_heads"] = [
hparams["swa_num_attention_heads"] if self._is_swa_layer(il) else hparams["num_attention_heads"]
for il in range(self.block_count)
]
# prevent the base class from emitting key/value_length from the unused head_dim
hparams.pop("head_dim", None)
super().set_gguf_parameters()
# MLA geometry of the sliding-window layers (rope.freq_base_swa is emitted by the base class)
swa_kv_lora_rank = hparams["swa_kv_lora_rank"]
self.gguf_writer.add_sliding_window(hparams["sliding_window_size"])
self.gguf_writer.add_sliding_window_pattern([self._is_swa_layer(il) for il in range(n_layer)])
self.gguf_writer.add_kv_lora_rank_swa(swa_kv_lora_rank)
self.gguf_writer.add_key_length_swa(swa_kv_lora_rank + hparams["swa_qk_rope_head_dim"])
self.gguf_writer.add_value_length_swa(swa_kv_lora_rank)
self.gguf_writer.add_key_length_mla_swa(hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"])
self.gguf_writer.add_value_length_mla_swa(hparams["swa_v_head_dim"])
if hparams["swa_q_lora_rank"] != hparams["q_lora_rank"]:
raise ValueError("dots3-note conversion assumes a shared q_lora_rank for both layer kinds")
if self.n_nextn:
self.gguf_writer.add_nextn_predict_layers(self.n_nextn)
# DSA indexer (full-attention layers only)
self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"])
self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"])
self.gguf_writer.add_indexer_top_k(hparams["index_topk"])
self.gguf_writer.add_indexer_types([not self._is_swa_layer(il) for il in range(n_layer)])
def prepare_metadata(self, vocab_only: bool):
from_dir = self.fname_out.is_dir()
super().prepare_metadata(vocab_only=vocab_only)
if not self.mtp_only or not from_dir:
return
output_type: str = self.ftype.name.partition("_")[2]
fname_default: str = gguf.naming_convention(
self.metadata.name, self.metadata.basename, self.metadata.finetune,
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# move the MTP token embedding into the NextN block so the standard nextn mapping picks it up
if name == "model.mtp.embed_tokens.weight":
name = f"model.layers.{self.hparams['num_hidden_layers']}.embed_tokens.weight"
bid = self.hparams["num_hidden_layers"]
# fold the activation rescale sqrt(n_embd/lora_rank) into the preceding RMSNorm weight
# this also covers the indexer wq_b, which reads the same rescaled q_lora activation
if self.apply_lora_rescale and bid is not None:
if name.endswith("q_a_layernorm.weight"):
data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / self.hparams["q_lora_rank"])
elif name.endswith("kv_a_layernorm.weight"):
rank = self.hparams["swa_kv_lora_rank"] if self._is_swa_layer(bid) else self.hparams["kv_lora_rank"]
data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / rank)
# MLA absorption: split kv_b_proj into k_b (transposed) and v_b, per-layer-kind geometry
if name.endswith("kv_b_proj.weight"):
assert bid is not None
if self._is_swa_layer(bid):
n_head = self.hparams["swa_num_attention_heads"]
qk_nope_head_dim = self.hparams["swa_qk_nope_head_dim"]
v_head_dim = self.hparams["swa_v_head_dim"]
else:
n_head = self.hparams["num_attention_heads"]
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
v_head_dim = self.hparams["v_head_dim"]
if isinstance(n_head, list): # set_gguf_parameters turns this into a per-layer array
n_head = n_head[bid]
assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim)
kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
k_b, v_b = kv_b.split([qk_nope_head_dim, v_head_dim], dim=1)
k_b = k_b.transpose(1, 2)
yield from ModelBase.modify_tensors(self, k_b, name.replace("kv_b_proj", "k_b_proj"), bid)
yield from ModelBase.modify_tensors(self, v_b, name.replace("kv_b_proj", "v_b_proj"), bid)
return
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration")
class Dots3NoteMmprojModel(MmprojModel):
has_vision_encoder = True
has_audio_encoder = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
assert self.hparams_audio is not None
# preprocessor_config.json nests the image params under vision_config
self.preprocessor_config = {**self.preprocessor_config, **self.preprocessor_config.get("vision_config", {})}
vis = self.hparams_vision
# in this config, hidden_size is the adapter output width; embed_dim is the tower width
vis["hidden_size"] = vis["embed_dim"]
vis["image_size"] = 0 # dynamic resolution
self.pyramid = [max(0, n) for n in vis["pyramid_num_routed"]]
if vis.get("adapter_type") != "patch_merger" or not vis.get("pre_pixel_shuffle"):
raise ValueError("dots3-note vision conversion requires adapter_type=patch_merger and pre_pixel_shuffle")
if vis.get("router_scoring_func", "sigmoid") != "sigmoid" or vis.get("router_scale", 1.0) != 1.0:
raise ValueError("dots3-note vision conversion only supports sigmoid routing with router_scale=1.0")
if vis.get("temporal_patch_size", 1) != 1 or vis.get("use_bias") or not vis.get("use_qk_norm"):
raise ValueError("unsupported dots3-note vision config variant")
aud = self.hparams_audio
if not aud.get("use_conv2d_stem") or not aud.get("use_rope") or not aud.get("use_rms_norm") or aud.get("use_causal"):
raise ValueError("unsupported dots3-note audio config variant")
if aud["whisper_config"].get("activation_function") != "swiglu":
raise ValueError("dots3-note audio conversion requires the swiglu activation")
if aud.get("merge_factor", 1) != 1 or aud.get("chunk_seconds") != 60:
raise ValueError("unsupported dots3-note audio chunking config")
# the graph hard-codes these rope parameters
rope = aud.get("rope_parameters", {})
if rope.get("partial_rotary_factor") != 0.5 or rope.get("rope_theta") != 10000.0:
raise ValueError("unsupported dots3-note audio rope config")
def get_audio_config(self) -> dict[str, Any] | None:
cfg = self.global_config.get("audio_config")
if cfg is not None:
# aliases so MmprojModel.find_aparam() / n_block_keys can resolve them
whisper = cfg["whisper_config"]
cfg["hidden_size"] = whisper["d_model"]
cfg["intermediate_size"] = whisper["encoder_ffn_dim"]
cfg["num_attention_heads"] = whisper["encoder_attention_heads"]
cfg["num_hidden_layers"] = whisper["encoder_layers"]
return cfg
def set_gguf_parameters(self):
super().set_gguf_parameters()
assert self.hparams_vision is not None
assert self.hparams_audio is not None
self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.DOTS3NOTE_V)
self.gguf_writer.add_vision_use_silu(True)
self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["rms_norm_eps"])
self.gguf_writer.add_vision_spatial_merge_size(self.hparams_vision["spatial_merge_size"])
self.gguf_writer.add_vision_min_pixels(self.preprocessor_config["min_pixels"])
self.gguf_writer.add_vision_max_pixels(self.preprocessor_config["max_pixels"])
# pyramid MoE: per-block routed expert count, 0 = dense block
self.gguf_writer.add_vision_expert_count_per_layer(self.pyramid)
self.gguf_writer.add_vision_expert_used_count(int(self.hparams_vision["capacity_factor"]))
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.DOTS3NOTE_A)
self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["whisper_config"]["num_mel_bins"])
self.gguf_writer.add_audio_attention_layernorm_eps(1e-6) # Dots3NoteAudioRMSNorm default
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not name.startswith(("vision_encoder.", "audio_encoder.")):
return None
return super().filter_tensors(item)
_vis_experts: dict[int, dict[str, Tensor]] | None = None
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# router params have no .weight suffix in the checkpoint, but gguf tools expect one
if name.endswith((".gate_weight", ".router_bias")):
name += ".weight"
# audio fc1 fuses gate and up for swiglu; split it
if ".speech_encoder.layers." in name and ".fc1." in name:
gate, up = data_torch.chunk(2, dim=0)
yield from super().modify_tensors(gate, name.replace(".fc1.", ".fc1_gate."), bid)
yield from super().modify_tensors(up, name.replace(".fc1.", ".fc1_up."), bid)
return
# vision MoE: stack per-expert weights into a single 3D tensor per block
if ".mlp.experts." in name:
assert bid is not None
n_expert = self.pyramid[bid]
if self._vis_experts is None:
self._vis_experts = {}
buf = self._vis_experts.setdefault(bid, {})
buf[name] = data_torch
if len(buf) >= n_expert * 3:
for w_name in ("fc1", "fc2", "fc3"):
datas: list[Tensor] = []
for xid in range(n_expert):
ename = f"vision_encoder.blocks.{bid}.mlp.experts.{xid}.{w_name}.weight"
datas.append(buf.pop(ename))
merged = torch.stack(datas, dim=0)
yield from super().modify_tensors(merged, f"vision_encoder.blocks.{bid}.mlp.experts.{w_name}.weight", bid)
return
yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
if self._vis_experts is not None:
leftover = [k for d in self._vis_experts.values() for k in d.keys()]
if leftover:
raise ValueError(f"unprocessed vision experts: {leftover}")
def tensor_force_quant(self, name, new_name, bid, n_dims):
# FP32 routing is load-bearing for the vision MoE (near-tied expert scores)
if ".ffn_gate_inp." in new_name or ".exp_probs_b." in new_name:
return gguf.GGMLQuantizationType.F32
if ".conv2d" in new_name or "a.conv_out" in new_name:
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
conversion/dotsocr.py
+1
View File
@@ -9,6 +9,7 @@ from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("DotsOCRForCausalLM")
@ModelBase.example("rednote-hilab/dots.ocr")
class DotsOCRVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)

Some files were not shown because too many files have changed in this diff Show More