mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-18 02:32:40 +02:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 01818e4956 | |||
| 0021a77de0 | |||
| 058df671b2 | |||
| 087f94d82e | |||
| 533b18257b | |||
| ed1c3a20f5 | |||
| d8df12ebc4 | |||
| b75ecd1971 | |||
| 60eeeb6082 | |||
| 39be55c97e | |||
| 34af94cd9a | |||
| 666f8898a2 | |||
| 805984d676 | |||
| 9cd719af21 | |||
| 7077abbe14 | |||
| d83f72d463 | |||
| 9f0d017efb |
@@ -1,20 +0,0 @@
|
||||
name: "Linux - Setup Vulkan SDK"
|
||||
description: "Setup Vulkan SDK for Linux"
|
||||
inputs:
|
||||
path:
|
||||
description: "Installation path"
|
||||
required: true
|
||||
version:
|
||||
description: "Vulkan SDK version"
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Setup Vulkan SDK
|
||||
id: setup
|
||||
uses: ./.github/actions/unarchive-tar
|
||||
with:
|
||||
url: https://sdk.lunarg.com/sdk/download/${{ inputs.version }}/linux/vulkan_sdk.tar.xz
|
||||
path: ${{ inputs.path }}
|
||||
strip: 1
|
||||
@@ -10,33 +10,6 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
ubuntu-24-vulkan-cache:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Get latest Vulkan SDK version
|
||||
id: vulkan_sdk_version
|
||||
run: |
|
||||
echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Setup Cache
|
||||
uses: actions/cache@v5
|
||||
id: cache-sdk
|
||||
with:
|
||||
path: ./vulkan_sdk
|
||||
key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
|
||||
|
||||
- name: Setup Vulkan SDK
|
||||
if: steps.cache-sdk.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/linux-setup-vulkan
|
||||
with:
|
||||
path: ./vulkan_sdk
|
||||
version: ${{ env.VULKAN_SDK_VERSION }}
|
||||
|
||||
#ubuntu-24-spacemit-cache:
|
||||
# runs-on: ubuntu-24.04
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/build-cpu.yml',
|
||||
'.github/workflows/build-cmake-pkg.yml',
|
||||
'ggml/src/ggml-rpc/**',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
@@ -123,7 +124,6 @@ jobs:
|
||||
env:
|
||||
OPENBLAS_VERSION: 0.3.23
|
||||
SDE_VERSION: 9.33.0-2024-01-07
|
||||
VULKAN_VERSION: 1.4.357.0
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -134,9 +134,6 @@ jobs:
|
||||
- build: 'x64-openblas'
|
||||
arch: 'x64'
|
||||
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
|
||||
- build: 'x64-vulkan'
|
||||
arch: 'x64'
|
||||
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'
|
||||
- build: 'arm64'
|
||||
arch: 'arm64'
|
||||
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'
|
||||
@@ -167,15 +164,6 @@ jobs:
|
||||
$lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
|
||||
& $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
|
||||
|
||||
- name: Install Vulkan SDK
|
||||
id: get_vulkan
|
||||
if: ${{ matrix.build == 'x64-vulkan' }}
|
||||
run: |
|
||||
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
|
||||
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
|
||||
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
|
||||
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
|
||||
|
||||
- name: Install Ninja
|
||||
id: install_ninja
|
||||
run: |
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
name: CI (rpc)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-rpc.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-rpc.yml',
|
||||
'ggml/src/ggml-rpc/**'
|
||||
]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-24-rpc:
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential libssl-dev ninja-build
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-G "Ninja" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose
|
||||
@@ -93,19 +93,13 @@ jobs:
|
||||
run: |
|
||||
echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Use Vulkan SDK Cache
|
||||
uses: actions/cache@v5
|
||||
id: cache-sdk
|
||||
with:
|
||||
path: ./vulkan_sdk
|
||||
key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
|
||||
|
||||
- name: Setup Vulkan SDK
|
||||
if: steps.cache-sdk.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/linux-setup-vulkan
|
||||
id: setup
|
||||
uses: ./.github/actions/unarchive-tar
|
||||
with:
|
||||
url: https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkan_sdk.tar.xz
|
||||
path: ./vulkan_sdk
|
||||
version: ${{ env.VULKAN_SDK_VERSION }}
|
||||
strip: 1
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
@@ -133,3 +127,56 @@ jobs:
|
||||
# This is using llvmpipe and runs slower than other backends
|
||||
# test-backend-ops is too slow on llvmpipe, skip it
|
||||
ctest -L main -E test-backend-ops --verbose --timeout 900
|
||||
|
||||
windows:
|
||||
runs-on: windows-2025
|
||||
|
||||
env:
|
||||
VULKAN_VERSION: 1.4.357.0
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: cpu-windows-2025-x64-vulkan
|
||||
variant: ccache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install Vulkan SDK
|
||||
id: get_vulkan
|
||||
run: |
|
||||
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
|
||||
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
|
||||
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
|
||||
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
|
||||
|
||||
- name: Install Ninja
|
||||
id: install_ninja
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -S . -B build -G "Ninja Multi-Config" `
|
||||
-D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
|
||||
-DCMAKE_BUILD_TYPE=Release `
|
||||
-DGGML_NATIVE=OFF `
|
||||
-DLLAMA_BUILD_SERVER=ON `
|
||||
-DGGML_RPC=ON `
|
||||
-DGGML_BACKEND_DL=ON `
|
||||
-DGGML_CPU_ALL_VARIANTS=ON `
|
||||
-DGGML_VULKAN=ON `
|
||||
-DLLAMA_BUILD_BORINGSSL=ON
|
||||
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main -C Release --verbose --timeout 900
|
||||
|
||||
@@ -1439,7 +1439,9 @@ jobs:
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
# note: only macos and ios-device due to long build time
|
||||
# ref: https://github.com/ggml-org/llama.cpp/pull/27252
|
||||
./build-xcframework.sh macos ios-device
|
||||
|
||||
- name: Build Xcode project
|
||||
run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
@@ -1660,6 +1662,16 @@ jobs:
|
||||
run: |
|
||||
tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist .
|
||||
|
||||
- name: Create and push git tag
|
||||
run: |
|
||||
TAG="${{ steps.tag.outputs.name }}"
|
||||
if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null 2>&1; then
|
||||
echo "Tag ${TAG} already exists, skipping creation"
|
||||
else
|
||||
git tag "${TAG}"
|
||||
git push origin "${TAG}"
|
||||
fi
|
||||
|
||||
- name: Create release
|
||||
id: create_release
|
||||
uses: ggml-org/action-create-release@v1
|
||||
|
||||
@@ -2,6 +2,7 @@ You are a coding agent. Here are some very important rules that you must follow:
|
||||
|
||||
General:
|
||||
- Be very precise and concise when writing code, comments, explanations, etc.
|
||||
- If an inline comment exceeds 2 lines, replace it with: `// note: TODO LATER`
|
||||
- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
|
||||
- Don't try to build or run the code unless you are explicitly asked to do so
|
||||
- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
|
||||
|
||||
+4
-1
@@ -5,7 +5,7 @@ include(CheckIncludeFileCXX)
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 1)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_PATCH 1)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
# whether this is a development/nightly build
|
||||
@@ -224,6 +224,9 @@ add_subdirectory(src)
|
||||
# utils, programs, examples and tests
|
||||
#
|
||||
|
||||
# mtmd needs this even when common is not built
|
||||
add_subdirectory(vendor/hash)
|
||||
|
||||
if (LLAMA_BUILD_COMMON)
|
||||
add_subdirectory(common)
|
||||
add_subdirectory(vendor/cpp-httplib)
|
||||
|
||||
+209
-128
@@ -1,5 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# usage: ./build-xcframework.sh [BUILD ...] (default: all builds)
|
||||
# builds: ios-sim ios-device macos visionos visionos-sim tvos-sim tvos-device
|
||||
#
|
||||
# Options
|
||||
IOS_MIN_OS_VERSION=16.4
|
||||
MACOS_MIN_OS_VERSION=13.3
|
||||
@@ -19,6 +22,43 @@ GGML_METAL_EMBED_LIBRARY=ON
|
||||
GGML_BLAS_DEFAULT=ON
|
||||
GGML_OPENMP=OFF
|
||||
|
||||
# Max number of concurrent platform builds
|
||||
MAX_PARALLEL_BUILDS=1
|
||||
|
||||
# Split the available cores between the concurrent builds (min 1)
|
||||
JOBS_PER_BUILD=$(( $(sysctl -n hw.logicalcpu) / MAX_PARALLEL_BUILDS ))
|
||||
if [[ "$JOBS_PER_BUILD" -lt 1 ]]; then
|
||||
JOBS_PER_BUILD=1
|
||||
fi
|
||||
|
||||
# echo "build_fn build_dir release_dir platform is_simulator min_os" for a build name
|
||||
build_spec() {
|
||||
case "$1" in
|
||||
ios-sim) echo "build_ios_sim build-ios-sim Release-iphonesimulator ios true ${IOS_MIN_OS_VERSION}" ;;
|
||||
ios-device) echo "build_ios_device build-ios-device Release-iphoneos ios false ${IOS_MIN_OS_VERSION}" ;;
|
||||
macos) echo "build_macos build-macos Release macos false ${MACOS_MIN_OS_VERSION}" ;;
|
||||
visionos) echo "build_visionos build-visionos Release-xros visionos false ${VISIONOS_MIN_OS_VERSION}" ;;
|
||||
visionos-sim) echo "build_visionos_sim build-visionos-sim Release-xrsimulator visionos true ${VISIONOS_MIN_OS_VERSION}" ;;
|
||||
tvos-sim) echo "build_tvos_sim build-tvos-sim Release-appletvsimulator tvos true ${TVOS_MIN_OS_VERSION}" ;;
|
||||
tvos-device) echo "build_tvos_device build-tvos-device Release-appletvos tvos false ${TVOS_MIN_OS_VERSION}" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Default: build everything
|
||||
if [[ $# -eq 0 ]]; then
|
||||
BUILDS=(ios-sim ios-device macos visionos visionos-sim tvos-sim tvos-device)
|
||||
else
|
||||
BUILDS=("$@")
|
||||
fi
|
||||
for b in "${BUILDS[@]}"; do
|
||||
if ! build_spec "$b" >/dev/null; then
|
||||
echo "Error: unknown build '$b'" >&2
|
||||
echo "Valid builds: ios-sim ios-device macos visionos visionos-sim tvos-sim tvos-device" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
|
||||
COMMON_CXX_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
|
||||
|
||||
@@ -401,148 +441,189 @@ combine_static_libraries() {
|
||||
rm -rf "${temp_dir}"
|
||||
}
|
||||
|
||||
echo "Building for iOS simulator..."
|
||||
cmake -B build-ios-sim -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
|
||||
-DIOS=ON \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_SYSROOT=iphonesimulator \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-ios-sim --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_ios_sim() {
|
||||
echo "Building for iOS simulator..."
|
||||
cmake -B build-ios-sim -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
|
||||
-DIOS=ON \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_SYSROOT=iphonesimulator \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-ios-sim --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
echo "Building for iOS devices..."
|
||||
cmake -B build-ios-device -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_SYSROOT=iphoneos \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-ios-device --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_ios_device() {
|
||||
echo "Building for iOS devices..."
|
||||
cmake -B build-ios-device -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_SYSROOT=iphoneos \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-ios-device --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
echo "Building for macOS..."
|
||||
cmake -B build-macos -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-S .
|
||||
cmake --build build-macos --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_macos() {
|
||||
echo "Building for macOS..."
|
||||
cmake -B build-macos -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-S .
|
||||
cmake --build build-macos --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
echo "Building for visionOS..."
|
||||
cmake -B build-visionos -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_SYSROOT=xros \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-visionos --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_visionos() {
|
||||
echo "Building for visionOS..."
|
||||
cmake -B build-visionos -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_SYSROOT=xros \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-visionos --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
echo "Building for visionOS simulator..."
|
||||
cmake -B build-visionos-sim -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_SYSROOT=xrsimulator \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-visionos-sim --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_visionos_sim() {
|
||||
echo "Building for visionOS simulator..."
|
||||
cmake -B build-visionos-sim -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_SYSROOT=xrsimulator \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-visionos-sim --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
# Add tvOS builds (might need the same u_int definitions as watchOS and visionOS)
|
||||
echo "Building for tvOS simulator..."
|
||||
cmake -B build-tvos-sim -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_SYSROOT=appletvsimulator \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DGGML_METAL=ON \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-tvos-sim --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_tvos_sim() {
|
||||
echo "Building for tvOS simulator..."
|
||||
cmake -B build-tvos-sim -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_SYSROOT=appletvsimulator \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DGGML_METAL=ON \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-tvos-sim --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
echo "Building for tvOS devices..."
|
||||
cmake -B build-tvos-device -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_SYSROOT=appletvos \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DGGML_METAL=ON \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-tvos-device --config Release -j $(sysctl -n hw.logicalcpu) -- -quiet
|
||||
build_tvos_device() {
|
||||
echo "Building for tvOS devices..."
|
||||
cmake -B build-tvos-device -G Xcode \
|
||||
"${COMMON_CMAKE_ARGS[@]}" \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_SYSROOT=appletvos \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DGGML_METAL=ON \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DMTMD_VIDEO=OFF \
|
||||
-S .
|
||||
cmake --build build-tvos-device --config Release -j "${JOBS_PER_BUILD}" -- -quiet
|
||||
}
|
||||
|
||||
run_builds_parallel() {
|
||||
local -a pids=()
|
||||
local -a names=()
|
||||
local name i
|
||||
for name in "$@"; do
|
||||
# Wait for the oldest running build to free a slot
|
||||
if [[ "${#pids[@]}" -ge "$MAX_PARALLEL_BUILDS" ]]; then
|
||||
if ! wait "${pids[0]}"; then
|
||||
echo "ERROR: build '${names[0]}' failed, log follows (${names[0]}.log):" >&2
|
||||
kill "${pids[@]}" 2>/dev/null || true
|
||||
cat "${names[0]}.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
pids=("${pids[@]:1}")
|
||||
names=("${names[@]:1}")
|
||||
fi
|
||||
echo "Starting build: $name (log: ${name}.log, -j ${JOBS_PER_BUILD})"
|
||||
"$name" > "${name}.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
names+=("$name")
|
||||
done
|
||||
# Wait for the remaining builds
|
||||
for i in "${!pids[@]}"; do
|
||||
if ! wait "${pids[$i]}"; then
|
||||
echo "ERROR: build '${names[$i]}' failed, log follows (${names[$i]}.log):" >&2
|
||||
kill "${pids[@]}" 2>/dev/null || true
|
||||
cat "${names[$i]}.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
BUILD_FNS=()
|
||||
for b in "${BUILDS[@]}"; do
|
||||
read -r fn _ < <(build_spec "$b")
|
||||
BUILD_FNS+=("$fn")
|
||||
done
|
||||
echo "Building: ${BUILDS[*]} (max ${MAX_PARALLEL_BUILDS} at a time, -j ${JOBS_PER_BUILD} each)..."
|
||||
run_builds_parallel "${BUILD_FNS[@]}"
|
||||
|
||||
# Setup frameworks and copy binaries and headers
|
||||
echo "Setting up framework structures..."
|
||||
setup_framework_structure "build-ios-sim" ${IOS_MIN_OS_VERSION} "ios"
|
||||
setup_framework_structure "build-ios-device" ${IOS_MIN_OS_VERSION} "ios"
|
||||
setup_framework_structure "build-macos" ${MACOS_MIN_OS_VERSION} "macos"
|
||||
setup_framework_structure "build-visionos" ${VISIONOS_MIN_OS_VERSION} "visionos"
|
||||
setup_framework_structure "build-visionos-sim" ${VISIONOS_MIN_OS_VERSION} "visionos"
|
||||
setup_framework_structure "build-tvos-sim" ${TVOS_MIN_OS_VERSION} "tvos"
|
||||
setup_framework_structure "build-tvos-device" ${TVOS_MIN_OS_VERSION} "tvos"
|
||||
for b in "${BUILDS[@]}"; do
|
||||
read -r _ bdir _ platform _ min_os < <(build_spec "$b")
|
||||
setup_framework_structure "$bdir" "$min_os" "$platform"
|
||||
done
|
||||
|
||||
# Create dynamic libraries from static libraries
|
||||
echo "Creating dynamic libraries from static libraries..."
|
||||
combine_static_libraries "build-ios-sim" "Release-iphonesimulator" "ios" "true"
|
||||
combine_static_libraries "build-ios-device" "Release-iphoneos" "ios" "false"
|
||||
combine_static_libraries "build-macos" "Release" "macos" "false"
|
||||
combine_static_libraries "build-visionos" "Release-xros" "visionos" "false"
|
||||
combine_static_libraries "build-visionos-sim" "Release-xrsimulator" "visionos" "true"
|
||||
combine_static_libraries "build-tvos-sim" "Release-appletvsimulator" "tvos" "true"
|
||||
combine_static_libraries "build-tvos-device" "Release-appletvos" "tvos" "false"
|
||||
for b in "${BUILDS[@]}"; do
|
||||
read -r _ bdir rdir platform is_sim _ < <(build_spec "$b")
|
||||
combine_static_libraries "$bdir" "$rdir" "$platform" "$is_sim"
|
||||
done
|
||||
|
||||
# Create XCFramework with correct debug symbols paths
|
||||
echo "Creating XCFramework..."
|
||||
XCFW_ARGS=()
|
||||
for b in "${BUILDS[@]}"; do
|
||||
read -r _ bdir _ _ _ _ < <(build_spec "$b")
|
||||
XCFW_ARGS+=(-framework "$(pwd)/${bdir}/framework/llama.framework")
|
||||
XCFW_ARGS+=(-debug-symbols "$(pwd)/${bdir}/dSYMs/llama.dSYM")
|
||||
done
|
||||
xcrun xcodebuild -create-xcframework \
|
||||
-framework $(pwd)/build-ios-sim/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-ios-sim/dSYMs/llama.dSYM \
|
||||
-framework $(pwd)/build-ios-device/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-ios-device/dSYMs/llama.dSYM \
|
||||
-framework $(pwd)/build-macos/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-macos/dSYMs/llama.dSYM \
|
||||
-framework $(pwd)/build-visionos/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-visionos/dSYMs/llama.dSYM \
|
||||
-framework $(pwd)/build-visionos-sim/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-visionos-sim/dSYMs/llama.dSYM \
|
||||
-framework $(pwd)/build-tvos-device/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-tvos-device/dSYMs/llama.dSYM \
|
||||
-framework $(pwd)/build-tvos-sim/framework/llama.framework \
|
||||
-debug-symbols $(pwd)/build-tvos-sim/dSYMs/llama.dSYM \
|
||||
-output $(pwd)/build-apple/llama.xcframework
|
||||
"${XCFW_ARGS[@]}" \
|
||||
-output "$(pwd)/build-apple/llama.xcframework"
|
||||
|
||||
+1
-1
@@ -3362,7 +3362,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
{"--tools"}, "TOOL1,TOOL2,...",
|
||||
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
|
||||
"specify \"all\" to enable all tools\n"
|
||||
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
|
||||
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.server_tools = parse_csv_row(value);
|
||||
|
||||
+15
-8
@@ -926,6 +926,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
// draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
|
||||
const bool is_dspark;
|
||||
|
||||
// dspark speculators
|
||||
bool sample_from_anchor = true;
|
||||
|
||||
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
|
||||
uint32_t target_layer_ids_n = 0;
|
||||
|
||||
@@ -960,16 +963,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) {
|
||||
block_size = std::atoi(buf);
|
||||
}
|
||||
if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
|
||||
sample_from_anchor = std::strcmp(buf, "true") == 0;
|
||||
}
|
||||
}
|
||||
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
|
||||
|
||||
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
|
||||
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
|
||||
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);
|
||||
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,
|
||||
block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false");
|
||||
|
||||
// DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most
|
||||
// block_size-1 draft tokens, DSpark yield a full block_size draft tokens
|
||||
const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;
|
||||
// block_size-1 draft tokens, anchor-first DSpark yields a full block_size draft tokens
|
||||
const int32_t n_draft_max = is_dspark && sample_from_anchor ? block_size : block_size - 1;
|
||||
if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {
|
||||
LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",
|
||||
__func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);
|
||||
@@ -1175,7 +1182,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
|
||||
const int32_t n_draft = params.n_max;
|
||||
|
||||
const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);
|
||||
const int32_t n_block_tokens = n_draft + (is_dspark && sample_from_anchor ? 0 : 1);
|
||||
i_block_beg[seq_id] = batch.n_tokens;
|
||||
n_block [seq_id] = n_block_tokens;
|
||||
for (int32_t i = 0; i < n_block_tokens; ++i) {
|
||||
@@ -1208,11 +1215,11 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
auto & result = *dp.result;
|
||||
|
||||
if (is_dspark) {
|
||||
// DSpark predicts the next token from position 0 and optionally truncates
|
||||
// at the first position below the confidence threshold.
|
||||
// DSpark: read from the first draft slot, truncate below the confidence threshold
|
||||
const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
|
||||
|
||||
for (int32_t i = 0; i < n_block_tokens; ++i) {
|
||||
// bonus-anchor drafts read the mask positions only, like DFlash
|
||||
const int32_t i_draft_beg = sample_from_anchor ? 0 : 1;
|
||||
for (int32_t i = i_draft_beg; i < n_block_tokens; ++i) {
|
||||
const int32_t idx = beg + i;
|
||||
|
||||
if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {
|
||||
|
||||
@@ -55,6 +55,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"DeepseekV32ForCausalLM": "deepseek",
|
||||
"DFlashDraftModel": "qwen",
|
||||
"Qwen3DSparkModel": "qwen",
|
||||
"DSparkDraftModel": "qwen",
|
||||
"DSparkSpeculator": "qwen",
|
||||
"DeepseekV4ForCausalLM": "deepseek",
|
||||
"DeepseekV4DSparkModel": "deepseek",
|
||||
"DistilBertForMaskedLM": "bert",
|
||||
|
||||
+73
-12
@@ -4,12 +4,13 @@ import json
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf, logger
|
||||
from .base import LazyTorchTensor, ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("QWenLMHeadModel")
|
||||
@@ -708,22 +709,82 @@ class DFlashModel(Qwen3Model):
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3DSparkModel")
|
||||
@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator")
|
||||
@ModelBase.example("satgeze/Qwen3.6-27B-DSpark")
|
||||
class DSparkModel(DFlashModel):
|
||||
# DSpark = DFlash + a semi-autoregressive Markov head
|
||||
# DSpark = DFlash + a semi-autoregressive Markov head.
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# normalize the flat DeepSpec schema to DFlash's nested dflash_config
|
||||
self.hparams.setdefault("dflash_config", {
|
||||
k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams
|
||||
})
|
||||
def __init__(self, dir_model, *args, **kwargs):
|
||||
hparams = kwargs.pop("hparams", None)
|
||||
if hparams is None:
|
||||
hparams = ModelBase.load_hparams(dir_model, False)
|
||||
|
||||
# EAGLE3-style exports use the 1+N bonus-anchor block, DFlash-lineage exports sample from the anchor
|
||||
self._sample_from_anchor = hparams.get(
|
||||
"sample_from_anchor",
|
||||
"transformer_layer_config" not in hparams and "aux_hidden_state_layer_ids" not in hparams)
|
||||
if "transformer_layer_config" in hparams:
|
||||
hparams = {**hparams, **hparams["transformer_layer_config"]}
|
||||
|
||||
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
||||
|
||||
# normalize both schemas to DFlash's nested dflash_config
|
||||
if "aux_hidden_state_layer_ids" in self.hparams:
|
||||
self.hparams.setdefault("dflash_config", {
|
||||
"mask_token_id": self.hparams.get("mask_token_id"),
|
||||
"target_layer_ids": [i - 1 for i in self.hparams["aux_hidden_state_layer_ids"]],
|
||||
})
|
||||
else:
|
||||
self.hparams.setdefault("dflash_config", {
|
||||
k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams
|
||||
})
|
||||
|
||||
if (markov_head_type := self.hparams.get("markov_head_type", "vanilla")) != "vanilla":
|
||||
raise ValueError(f"unsupported markov_head_type {markov_head_type!r} (only 'vanilla' is supported)")
|
||||
|
||||
n_vocab = self.hparams["vocab_size"]
|
||||
self._n_vocab_draft = self.hparams.get("draft_vocab_size") or n_vocab
|
||||
if self._n_vocab_draft > n_vocab:
|
||||
raise ValueError(f"draft_vocab_size {self._n_vocab_draft} exceeds vocab_size {n_vocab}")
|
||||
self._d2t: Tensor | None = None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_sample_from_anchor(self._sample_from_anchor)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if name.endswith(("embed_tokens.weight", "lm_head.weight")):
|
||||
if item[0] == "t2d": # not used at runtime
|
||||
return None
|
||||
return super().filter_tensors((name, gen))
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name == "model.d2t":
|
||||
self._d2t = data_torch
|
||||
return
|
||||
|
||||
if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors()
|
||||
|
||||
n_vocab = self.hparams["vocab_size"]
|
||||
if self._n_vocab_draft < n_vocab and self._d2t is None:
|
||||
raise ValueError(f"draft_vocab_size {self._n_vocab_draft} < vocab_size {n_vocab} but no d2t table found")
|
||||
|
||||
# write d2t as absolute target token ids
|
||||
if self._d2t is not None:
|
||||
data = LazyTorchTensor.to_eager(self._d2t).to(torch.int64).cpu().numpy().reshape(-1)
|
||||
if data.size != self._n_vocab_draft:
|
||||
raise ValueError(f"d2t size {data.size} does not match draft_vocab_size {self._n_vocab_draft}")
|
||||
data = data + np.arange(data.size, dtype=np.int64)
|
||||
if np.any((data < 0) | (data >= n_vocab)):
|
||||
raise ValueError(f"d2t target ids out of range for target vocab size {n_vocab}")
|
||||
if np.unique(data).size != data.size:
|
||||
raise ValueError("d2t contains duplicate target ids")
|
||||
logger.info(f"{'d2t,':<30} --> I64, shape = {{{data.size}}}")
|
||||
self.gguf_writer.add_tensor("d2t", data, raw_dtype=gguf.GGMLQuantizationType.I64)
|
||||
|
||||
@@ -106,6 +106,10 @@ acceptance (from the draft's confidence head, if present) falls below `P` (defau
|
||||
Currently only drafts with a Qwen3 backbone are supported; support for other backbones
|
||||
(e.g. Gemma4) is planned.
|
||||
|
||||
DSpark drafts exported in the [speculators](https://github.com/vllm-project/speculators) format
|
||||
(for example [`RedHatAI/gemma-4-31B-it-speculator.dspark`](https://huggingface.co/RedHatAI/gemma-4-31B-it-speculator.dspark))
|
||||
convert the same way.
|
||||
|
||||
See:
|
||||
|
||||
- #25173
|
||||
|
||||
@@ -2,21 +2,5 @@ set(TARGET llama-gguf-hash)
|
||||
add_executable(${TARGET} gguf-hash.cpp)
|
||||
install(TARGETS ${TARGET} RUNTIME)
|
||||
|
||||
# clibs dependencies
|
||||
include_directories(deps/)
|
||||
|
||||
add_library(xxhash OBJECT deps/xxhash/xxhash.c deps/xxhash/xxhash.h)
|
||||
target_link_libraries(${TARGET} PRIVATE xxhash)
|
||||
|
||||
add_library(sha1 OBJECT deps/sha1/sha1.c deps/sha1/sha1.h)
|
||||
target_link_libraries(${TARGET} PRIVATE sha1)
|
||||
if (NOT MSVC)
|
||||
# disable warnings in 3rd party code
|
||||
target_compile_options(sha1 PRIVATE -w)
|
||||
endif()
|
||||
|
||||
add_library(sha256 OBJECT deps/sha256/sha256.c deps/sha256/sha256.h)
|
||||
target_link_libraries(${TARGET} PRIVATE sha256)
|
||||
|
||||
target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})
|
||||
target_link_libraries(${TARGET} PRIVATE vendor-hash ggml ${CMAKE_THREAD_LIBS_INIT})
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"name": "rotate-bits",
|
||||
"version": "0.1.1",
|
||||
"repo": "jb55/rotate-bits.h",
|
||||
"description": "rotate bits",
|
||||
"keywords": ["rotl", "rotr"],
|
||||
"src": ["rotate-bits.h"],
|
||||
"license": "Public Domain",
|
||||
"development": {
|
||||
"thlorenz/tap.c": "*"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "sha1",
|
||||
"version": "0.0.1",
|
||||
"repo": "clibs/sha1",
|
||||
"description": "sha1 hash algorithm",
|
||||
"keywords": ["sha1", "hash"],
|
||||
"license": "public domain",
|
||||
"src": ["sha1.c", "sha1.h"]
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"name": "sha256",
|
||||
"version": "0.0.2",
|
||||
"repo": "jb55/sha256.c",
|
||||
"description": "sha256 in c",
|
||||
"keywords": ["sha256", "sha2"],
|
||||
"src": ["sha256.c", "sha256.h"],
|
||||
"dependencies": {
|
||||
"jb55/rotate-bits.h": "0.1.1"
|
||||
},
|
||||
"development": {
|
||||
"thlorenz/tap.c": "*"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"name": "xxhash",
|
||||
"version": "0.8.2",
|
||||
"repo": "Cyan4973/xxhash",
|
||||
"description": "Extremely fast non-cryptographic hash algorithm",
|
||||
"keywords": ["xxhash", "hashing"],
|
||||
"license": "BSD-2-Clause",
|
||||
"src": [
|
||||
"xxhash.c",
|
||||
"xxhash.h"
|
||||
]
|
||||
}
|
||||
@@ -18,13 +18,16 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
#include "xxhash/xxhash.h"
|
||||
#include "sha1/sha1.h"
|
||||
#include "sha256/sha256.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py
|
||||
#include "sha1/sha1.h"
|
||||
using namespace vendor_hash;
|
||||
|
||||
|
||||
// uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp')
|
||||
#define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5"
|
||||
|
||||
@@ -4770,7 +4770,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
|
||||
}
|
||||
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/17368
|
||||
#if defined(__linux__)
|
||||
#if defined(__linux__) && !defined(GGML_USE_HIP)
|
||||
// Check if this is a UMA (Unified Memory Architecture) system
|
||||
cudaDeviceProp prop;
|
||||
CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device)));
|
||||
@@ -4790,7 +4790,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
|
||||
GGML_LOG_ERROR("%s: /proc/meminfo reading failed, using cudaMemGetInfo\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif // defined(__linux__)
|
||||
#endif // defined(__linux__) && !defined(GGML_USE_HIP)
|
||||
|
||||
// virtual devices sharing one physical GPU share its memory pool; split it between them
|
||||
const int share_count = ggml_cuda_physical_device_share_count(ctx->device);
|
||||
|
||||
@@ -162,6 +162,7 @@ class Keys:
|
||||
TARGET_LAYERS = "{arch}.target_layers"
|
||||
TARGET_HIDDEN_SIZE = "{arch}.target_hidden_size"
|
||||
BLOCK_SIZE = "{arch}.block_size"
|
||||
SAMPLE_FROM_ANCHOR = "{arch}.sample_from_anchor"
|
||||
NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"
|
||||
NORM_BEFORE_FC = "{arch}.norm_before_fc"
|
||||
|
||||
@@ -4819,6 +4820,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
],
|
||||
MODEL_ARCH.DFLASH: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
MODEL_TENSOR.ATTN_NORM,
|
||||
MODEL_TENSOR.ATTN_Q,
|
||||
@@ -4858,6 +4860,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_UP_SHEXP,
|
||||
MODEL_TENSOR.FC,
|
||||
MODEL_TENSOR.ENC_OUTPUT_NORM,
|
||||
MODEL_TENSOR.D2T,
|
||||
# optional DSpark heads
|
||||
MODEL_TENSOR.DSPARK_MARKOV_W1,
|
||||
MODEL_TENSOR.DSPARK_MARKOV_W2,
|
||||
|
||||
@@ -981,6 +981,9 @@ class GGUFWriter:
|
||||
def add_block_size(self, value: int) -> None:
|
||||
self.add_uint32(Keys.LLM.BLOCK_SIZE.format(arch=self.arch), value)
|
||||
|
||||
def add_sample_from_anchor(self, value: bool) -> None:
|
||||
self.add_bool(Keys.LLM.SAMPLE_FROM_ANCHOR.format(arch=self.arch), value)
|
||||
|
||||
def add_target_layers(self, value: Sequence[int]) -> None:
|
||||
self.add_array(Keys.LLM.TARGET_LAYERS.format(arch=self.arch), value)
|
||||
|
||||
|
||||
@@ -76,14 +76,14 @@ class TensorNameMap:
|
||||
# Output
|
||||
MODEL_TENSOR.OUTPUT: (
|
||||
"embed_out", # gptneox
|
||||
"lm_head", # gpt2 mpt falcon llama-hf baichuan qwen mamba dbrx jais nemotron exaone olmoe olmo2 phimoe plamo2
|
||||
"lm_head", # gpt2 mpt falcon llama-hf baichuan qwen mamba dbrx jais nemotron exaone olmoe olmo2 phimoe plamo2 llama4
|
||||
"output", # llama-pth bloom internlm2
|
||||
"word_embeddings_for_head", # persimmon
|
||||
"lm_head.linear", # phi2
|
||||
"output_layer", # chatglm
|
||||
"head", # rwkv
|
||||
"head.out", # wavtokenizer
|
||||
"lm_head", # llama4
|
||||
"model.lm_head", # dflash
|
||||
"model.transformer.ff_out", # llada
|
||||
"head.decoder", # modern-bert
|
||||
),
|
||||
|
||||
@@ -71,6 +71,26 @@ if git ls-remote --tags origin "${VERSION}" | grep -q "${VERSION}"; then
|
||||
fi
|
||||
echo "Tag ${VERSION} does not exist on remote - OK"
|
||||
|
||||
echo "Checking release.yml status for commit ${SHA}..."
|
||||
if [[ -z "${GITHUB_REPOSITORY:-}" ]]; then
|
||||
echo "Warning: GITHUB_REPOSITORY not set - skipping CI check (local run)"
|
||||
else
|
||||
RUNS=$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs?per_page=100" \
|
||||
--jq "[.workflow_runs[] | select(.head_sha == \"${SHA}\" and .conclusion == \"success\")] | length")
|
||||
if [[ "$RUNS" -eq 0 ]]; then
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo "Warning: no successful release.yml run found for HEAD (${SHA}) (dry run, continuing)."
|
||||
CHECKS_PASSED=false
|
||||
else
|
||||
echo "Error: no successful release.yml run found for HEAD (${SHA})"
|
||||
echo "The nightly build must complete successfully before making a release."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Found successful release.yml run for HEAD."
|
||||
fi
|
||||
fi
|
||||
|
||||
MAJOR=$(grep "set(GGML_VERSION_MAJOR" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+')
|
||||
MINOR=$(grep "set(GGML_VERSION_MINOR" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+')
|
||||
PATCH=$(grep "set(GGML_VERSION_PATCH" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+')
|
||||
|
||||
@@ -7,6 +7,12 @@ import subprocess
|
||||
|
||||
HTTPLIB_VERSION = "refs/tags/v0.53.1"
|
||||
|
||||
# used by examples/gguf-hash, these repos have no release tag, so we pin a commit
|
||||
XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68"
|
||||
SHA1_COMMIT = "e1e2536fcf6a8f9703be8c85d58724b408552287"
|
||||
SHA256_COMMIT = "5e637272c13f200872d55ff579f7e2ab6c3f252f"
|
||||
ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1"
|
||||
|
||||
vendor = {
|
||||
"https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp",
|
||||
"https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp",
|
||||
@@ -22,12 +28,95 @@ vendor = {
|
||||
f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE": "vendor/cpp-httplib/LICENSE",
|
||||
|
||||
"https://raw.githubusercontent.com/sheredom/subprocess.h/9ce0d701b6fb10f8f8c4445edd31e7c60a1237e3/subprocess.h": "vendor/sheredom/subprocess.h",
|
||||
|
||||
f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c": "vendor/hash/xxhash/xxhash.c",
|
||||
f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h": "vendor/hash/xxhash/xxhash.h",
|
||||
f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE": "vendor/hash/xxhash/LICENSE",
|
||||
|
||||
# clibs/sha1 ships no license file, the source header says public domain
|
||||
f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/hash/sha1/sha1.c",
|
||||
f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/hash/sha1/sha1.h",
|
||||
|
||||
f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/hash/sha256/sha256.c",
|
||||
f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/hash/sha256/sha256.h",
|
||||
f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE": "vendor/hash/sha256/LICENSE",
|
||||
|
||||
f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/hash/rotate-bits/rotate-bits.h",
|
||||
f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md": "vendor/hash/rotate-bits/LICENSE.md",
|
||||
}
|
||||
|
||||
# local changes kept on top of the upstream sources
|
||||
patches = {
|
||||
"vendor/hash/xxhash/xxhash.h": [(
|
||||
'#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n',
|
||||
'/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n'
|
||||
' before allowing the windows compiler to use the C11 form.\n'
|
||||
' Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n'
|
||||
'#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n'
|
||||
' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'
|
||||
)],
|
||||
|
||||
# sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time.
|
||||
# we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace.
|
||||
"vendor/hash/sha1/sha1.h": [
|
||||
(
|
||||
'#if defined(__cplusplus)\n'
|
||||
'extern "C" {\n'
|
||||
'#endif\n',
|
||||
|
||||
'namespace vendor_hash {\n'
|
||||
),
|
||||
(
|
||||
'#if defined(__cplusplus)\n'
|
||||
'}\n'
|
||||
'#endif\n',
|
||||
|
||||
'} // namespace vendor_hash\n'
|
||||
),
|
||||
],
|
||||
|
||||
"vendor/hash/sha1/sha1.c": [
|
||||
(
|
||||
'#include "sha1.h"\n',
|
||||
|
||||
'#include "sha1.h"\n'
|
||||
'\n'
|
||||
'namespace vendor_hash {\n'
|
||||
),
|
||||
(
|
||||
' SHA1Final((unsigned char *)hash_out, &ctx);\n'
|
||||
'}\n',
|
||||
|
||||
' SHA1Final((unsigned char *)hash_out, &ctx);\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'} // namespace vendor_hash\n'
|
||||
),
|
||||
],
|
||||
|
||||
# silence a maybe-uninitialized warning
|
||||
"vendor/hash/sha256/sha256.c": [(
|
||||
" uint32_t W[16];\n",
|
||||
" uint32_t W[16] = {0};\n"
|
||||
)],
|
||||
}
|
||||
|
||||
for url, filename in vendor.items():
|
||||
print(f"downloading {url} to {filename}") # noqa: NP100
|
||||
urllib.request.urlretrieve(url, filename)
|
||||
|
||||
for filename, replacements in patches.items():
|
||||
print(f"patching {filename}") # noqa: NP100
|
||||
with open(filename, "r", encoding="utf-8", newline="") as f:
|
||||
content = f.read()
|
||||
for old, new in replacements:
|
||||
if content.count(old) != 1:
|
||||
print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100
|
||||
sys.exit(1)
|
||||
content = content.replace(old, new)
|
||||
with open(filename, "w", encoding="utf-8", newline="") as f:
|
||||
f.write(content)
|
||||
|
||||
print("Splitting httplib.h...") # noqa: NP100
|
||||
try:
|
||||
subprocess.check_call([
|
||||
|
||||
+17
-4
@@ -2428,17 +2428,24 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|
||||
const uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx);
|
||||
|
||||
const float * scores = nullptr;
|
||||
const int * iscores = nullptr;
|
||||
const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());
|
||||
if (score_idx != -1) {
|
||||
if (gguf_get_kv_type(ctx, score_idx) != GGUF_TYPE_ARRAY ||
|
||||
gguf_get_arr_type(ctx, score_idx) != GGUF_TYPE_FLOAT32) {
|
||||
const gguf_type kv_type = gguf_get_kv_type(ctx, score_idx);
|
||||
const gguf_type arr_type = kv_type == GGUF_TYPE_ARRAY ? gguf_get_arr_type(ctx, score_idx) : GGUF_TYPE_COUNT;
|
||||
if (arr_type != GGUF_TYPE_INT32 &&
|
||||
arr_type != GGUF_TYPE_FLOAT32) {
|
||||
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_SCORES).c_str()));
|
||||
}
|
||||
const uint32_t n_scores = gguf_get_arr_n(ctx, score_idx);
|
||||
if (n_scores < n_tokens) {
|
||||
throw std::runtime_error("Index out of array bounds for scores (" + std::to_string(n_scores) + " < " + std::to_string(n_tokens) + ")\n");
|
||||
}
|
||||
scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
|
||||
if (arr_type == GGUF_TYPE_INT32) {
|
||||
iscores = (const int *) gguf_get_arr_data(ctx, score_idx);
|
||||
} else {
|
||||
scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
|
||||
}
|
||||
}
|
||||
|
||||
const int * toktypes = nullptr;
|
||||
@@ -2469,7 +2476,13 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|
||||
|
||||
auto & token_data = id_to_token[i];
|
||||
token_data.text = std::move(word);
|
||||
token_data.score = scores ? scores[i] : 0.0f;
|
||||
if (scores) {
|
||||
token_data.score = scores[i];
|
||||
} else if (iscores) {
|
||||
token_data.score = static_cast<float>(iscores[i]);
|
||||
} else {
|
||||
token_data.score = 0.0f;
|
||||
}
|
||||
token_data.attr = LLAMA_TOKEN_ATTR_NORMAL;
|
||||
|
||||
if (toktypes) { //TODO: remove, required until per token attributes are available from GGUF file
|
||||
|
||||
+52
-3
@@ -85,6 +85,16 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
|
||||
const int64_t n_embd_inp = hparams.n_embd_inp_enc();
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
|
||||
|
||||
// reduced draft vocab (optional): d2t maps draft rows to target token ids
|
||||
int64_t n_vocab_draft = n_vocab;
|
||||
const struct ggml_tensor * d2t_meta = ml->get_tensor_meta("d2t");
|
||||
if (d2t_meta) {
|
||||
n_vocab_draft = d2t_meta->ne[0];
|
||||
d2t = create_tensor(tn(LLM_TENSOR_D2T), { n_vocab_draft }, 0);
|
||||
LLAMA_LOG_INFO("%s: DFlash using d2t mapping (draft_vocab_size = %lld)\n", __func__, (long long) n_vocab_draft);
|
||||
}
|
||||
|
||||
// DSpark = DFlash + a semi-autoregressive Markov head and Confidence head
|
||||
//
|
||||
// TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4)
|
||||
@@ -94,7 +104,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
|
||||
const int64_t dspark_markov_rank = markov_meta->ne[0];
|
||||
|
||||
dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);
|
||||
dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab }, 0);
|
||||
dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab_draft }, 0);
|
||||
|
||||
dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0);
|
||||
dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);
|
||||
@@ -157,6 +167,9 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
|
||||
return;
|
||||
}
|
||||
|
||||
// optional: reduced-vocab drafts ship their own, full-vocab drafts share the target's via ctx_other
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab_draft }, TENSOR_NOT_REQUIRED);
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = layers[i];
|
||||
|
||||
@@ -242,6 +255,11 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &
|
||||
const int64_t block_size = std::stoi(it->second);
|
||||
GGML_ASSERT(block_size > 0);
|
||||
|
||||
// bonus anchor (SpecForge exports): slot 0 is a bonus token, not a prediction slot
|
||||
const auto it_anchor = model.gguf_kv.find("dflash.sample_from_anchor");
|
||||
const bool sample_from_anchor = it_anchor == model.gguf_kv.end() || it_anchor->second == "true";
|
||||
const int64_t i_draft_beg = sample_from_anchor ? 0 : 1;
|
||||
|
||||
const int64_t n_blocks = g.ubatch.n_seqs_unq;
|
||||
GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks");
|
||||
// runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size
|
||||
@@ -263,11 +281,26 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &
|
||||
ggml_tensor * cat = nullptr;
|
||||
ggml_tensor * cat_conf = nullptr;
|
||||
|
||||
if (!sample_from_anchor) {
|
||||
// bonus anchor slot: pass the logits through unbiased, pad the (unread) confidence column
|
||||
cat = ggml_cont(ctx0, ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, 0));
|
||||
cat_conf = ggml_sigmoid(ctx0, ggml_cont(ctx0, ggml_view_2d(ctx0, base, 1, n_blocks, base_stride, 0)));
|
||||
}
|
||||
|
||||
// TODO: the in-graph chain is greedy (argmax); sampling params affect only the final
|
||||
// token pick, not the Markov conditioning path
|
||||
for (int64_t i = 0; i < block_drafts; ++i) {
|
||||
for (int64_t i = i_draft_beg; i < block_drafts; ++i) {
|
||||
ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]
|
||||
ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab, n_blocks]
|
||||
ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab_draft, n_blocks]
|
||||
if (model.d2t) {
|
||||
// reduced draft vocab: scatter the bias to the target rows (base is -inf on the others)
|
||||
const int64_t n_draft_vocab = bias->ne[0];
|
||||
ggml_tensor * full = ggml_fill(ctx0, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, n_vocab, n_blocks), 0.0f);
|
||||
bias = ggml_set_rows(ctx0, full,
|
||||
ggml_reshape_3d(ctx0, bias, 1, n_draft_vocab, n_blocks),
|
||||
ggml_reshape_3d(ctx0, model.d2t, n_draft_vocab, 1, 1));
|
||||
bias = ggml_reshape_2d(ctx0, bias, n_vocab, n_blocks);
|
||||
}
|
||||
|
||||
// position i of every block: strided view [n_vocab, n_blocks]
|
||||
ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]);
|
||||
@@ -497,6 +530,22 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
}
|
||||
|
||||
cur = build_lora_mm(output, cur, output_s);
|
||||
|
||||
// reduced-draft-vocab exports: scatter the draft logits to the target vocabulary via d2t
|
||||
if (model.d2t) {
|
||||
const int64_t n_draft_vocab = cur->ne[0];
|
||||
const int64_t n_outputs = cur->ne[1];
|
||||
const int64_t n_vocab = (int64_t) model.vocab.n_tokens();
|
||||
|
||||
GGML_ASSERT(model.d2t->type == GGML_TYPE_I64);
|
||||
GGML_ASSERT(model.d2t->ne[0] == n_draft_vocab);
|
||||
|
||||
ggml_tensor * logits = ggml_fill(ctx0, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, n_vocab, n_outputs), -INFINITY);
|
||||
cur = ggml_set_rows(ctx0, logits,
|
||||
ggml_reshape_3d(ctx0, cur, 1, n_draft_vocab, n_outputs),
|
||||
ggml_reshape_3d(ctx0, model.d2t, n_draft_vocab, 1, 1));
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_outputs);
|
||||
}
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES
|
||||
)
|
||||
|
||||
target_link_libraries (mtmd PUBLIC ggml llama)
|
||||
target_link_libraries (mtmd PRIVATE Threads::Threads)
|
||||
target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash)
|
||||
target_include_directories(mtmd PUBLIC .)
|
||||
target_include_directories(mtmd PRIVATE ../..)
|
||||
target_include_directories(mtmd PRIVATE ../../vendor)
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
#include "mtmd-helper-common.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include "hash.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
#include <vector>
|
||||
@@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int
|
||||
|
||||
} // namespace audio_helpers
|
||||
|
||||
// Computes FNV-1a hash of the data
|
||||
static std::string fnv_hash(const uint8_t * data, size_t len) {
|
||||
const uint64_t fnv_prime = 0x100000001b3ULL;
|
||||
uint64_t hash = 0xcbf29ce484222325ULL;
|
||||
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
hash ^= data[i];
|
||||
hash *= fnv_prime;
|
||||
}
|
||||
return std::to_string(hash);
|
||||
}
|
||||
|
||||
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
|
||||
// calculate the hash if needed
|
||||
std::string id;
|
||||
mtmd_bitmap * result = nullptr;
|
||||
|
||||
if (!placeholder) {
|
||||
id = fnv_hash(buf, len);
|
||||
// use sha256 to prevent cache poisoning
|
||||
id = hash_sha256_hex(buf, len);
|
||||
}
|
||||
|
||||
if (audio_helpers::is_audio_file((const char *)buf, len)) {
|
||||
|
||||
@@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
|
||||
// note:
|
||||
// - for now, video input is only supported via C++ helper functions
|
||||
// - audio files will be auto-detected based on magic bytes
|
||||
// - output bitmap will have FNV hash as the ID
|
||||
// - output bitmap will have SHA-256 hash (hex string) as the ID
|
||||
// returns nullptr on failure
|
||||
// this function is thread-safe
|
||||
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);
|
||||
|
||||
@@ -999,6 +999,20 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi
|
||||
// mtmd_image_preprocessor_lfm2
|
||||
//
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) {
|
||||
auto const inst = get_slice_instructions(img.get_size());
|
||||
if (!inst.slices.empty()) {
|
||||
return mtmd_image_preprocessor_llava_uhd::preprocess(img);
|
||||
}
|
||||
|
||||
// single tile: no thumbnail
|
||||
// note: not using output.overview here because it will emit <|img_thumbnail|> token, which we don't want in this case
|
||||
auto sliced = slice_image(img, inst);
|
||||
mtmd_image_preproc_out output;
|
||||
output.append(hparams, sliced.overview, true);
|
||||
return output;
|
||||
}
|
||||
|
||||
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) {
|
||||
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
|
||||
const int align_size = hparams.patch_size * hparams.n_merge;
|
||||
@@ -1606,12 +1620,16 @@ mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_im
|
||||
|
||||
const clip_image_size orig_size = img.get_size();
|
||||
const int tile_size = hparams.image_size;
|
||||
GGML_ASSERT(tile_size > 0);
|
||||
|
||||
// llava-next always encodes an overview plus a grid of tiles, even for small images
|
||||
const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates);
|
||||
const int grid_x = refined_size.width / tile_size;
|
||||
const int grid_y = refined_size.height / tile_size;
|
||||
|
||||
// the tiles are stacked on the Y axis, a big grid overflows the stacked image height
|
||||
GGML_ASSERT(grid_x >= 0 && grid_x <= 1024 && grid_y >= 0 && grid_y <= 1024);
|
||||
|
||||
clip_image_u8 overview;
|
||||
img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov,
|
||||
hparams.image_pad_ov, hparams.image_pad_color_ov);
|
||||
|
||||
@@ -145,6 +145,7 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
|
||||
static constexpr int tile_size = 512;
|
||||
|
||||
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
|
||||
slice_instructions get_slice_instructions(const clip_image_size & original_size) override;
|
||||
|
||||
private:
|
||||
|
||||
+32
-14
@@ -2322,23 +2322,12 @@ void mtmd_input_chunk_free(mtmd_input_chunk * chunk) {
|
||||
}
|
||||
}
|
||||
|
||||
int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) {
|
||||
// returns 0 on success
|
||||
static int32_t mtmd_input_chunk_save_impl(const mtmd_input_chunk * chunk, std::vector<char> & out_buf) {
|
||||
try {
|
||||
mtmd_serialization ser(MTMD_SERIALIZATION_VERSION);
|
||||
chunk->serialize(ser);
|
||||
|
||||
if (expected_out_len) {
|
||||
*expected_out_len = ser.data.size();
|
||||
}
|
||||
if (!out_buf) {
|
||||
// caller is only querying the required size
|
||||
return 0;
|
||||
}
|
||||
if (out_len < ser.data.size()) {
|
||||
LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, ser.data.size(), out_len);
|
||||
return -1;
|
||||
}
|
||||
std::memcpy(out_buf, ser.data.data(), ser.data.size());
|
||||
out_buf = std::move(ser.data);
|
||||
return 0;
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: %s\n", __func__, e.what());
|
||||
@@ -2346,6 +2335,35 @@ int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, si
|
||||
}
|
||||
}
|
||||
|
||||
mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk) {
|
||||
// this is hacky, but still faster than copy the whole batch data
|
||||
std::vector<char> buf;
|
||||
if (mtmd_input_chunk_save_impl(chunk, buf) != 0) {
|
||||
return nullptr;
|
||||
}
|
||||
return mtmd_input_chunk_load(buf.data(), buf.size());
|
||||
}
|
||||
|
||||
int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) {
|
||||
std::vector<char> buf;
|
||||
if (mtmd_input_chunk_save_impl(chunk, buf) != 0) {
|
||||
return -1;
|
||||
}
|
||||
if (expected_out_len) {
|
||||
*expected_out_len = buf.size();
|
||||
}
|
||||
if (!out_buf) {
|
||||
// caller is only querying the required size
|
||||
return 0;
|
||||
}
|
||||
if (out_len < buf.size()) {
|
||||
LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, buf.size(), out_len);
|
||||
return -1;
|
||||
}
|
||||
std::memcpy(out_buf, buf.data(), buf.size());
|
||||
return 0;
|
||||
}
|
||||
|
||||
mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len) {
|
||||
try {
|
||||
mtmd_serialization ser(MTMD_SERIALIZATION_VERSION, buf, len);
|
||||
|
||||
@@ -233,6 +233,9 @@ MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd
|
||||
MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk);
|
||||
MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk);
|
||||
|
||||
// similar to mtmd_input_chunk_copy, but returns a placeholder chunk
|
||||
MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk);
|
||||
|
||||
// save/load an input chunk to/from a buffer (useful for KV save/load)
|
||||
// important: only chunk's metadata will be saved, the actual image/audio data will not be saved
|
||||
// the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode()
|
||||
|
||||
@@ -189,7 +189,7 @@ This endpoint is intended to be used internally by the Web UI and subject to cha
|
||||
Get a list of tools, each tool has these fields:
|
||||
- `tool` (string): the ID name of the tool, to be used in POST call. Example: `read_file`
|
||||
- `display_name` (string): the name to be displayed on UI. Example: `Read file`
|
||||
- `type` (string): `"builtin"` for a built-in tool, or `"mcp"` for a tool exposed by an MCP server
|
||||
- `type` (string): `"server"` for a server tool, or `"mcp"` for a tool exposed by an MCP server
|
||||
- `permissions` (object): a mapping string --> boolean that indicates the permission required by this tool. This is useful for the UI to ask the user before calling the tool. For now, the only permission supported is `"write"`
|
||||
- `definition` (object): the OAI-compat definition of this tool
|
||||
|
||||
|
||||
+58
-6
@@ -196,11 +196,11 @@ For the full list of features, please refer to [server's changelog](https://gith
|
||||
| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
|
||||
| `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
|
||||
| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
|
||||
| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
|
||||
| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
|
||||
| `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/> 'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/> 'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/> 'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
|
||||
| `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |
|
||||
| `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |
|
||||
| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
|
||||
| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
|
||||
| `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) |
|
||||
| `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) |
|
||||
| `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)<br/>(env: LLAMA_ARG_RERANKING) |
|
||||
@@ -337,12 +337,64 @@ It is currently available in the following endpoints:
|
||||
|
||||
For more details, please refer to [multimodal documentation](../../docs/multimodal.md)
|
||||
|
||||
### Built-in tools support
|
||||
### Server tools support
|
||||
|
||||
The server includes a set of built-in tools that enable the LLM to access the local file system directly from the Web UI.
|
||||
The server includes a set of server tools that enable the LLM to access the local file system directly from the Web UI.
|
||||
|
||||
To use this feature, start the server with `--tools all`. You can also enable only specific tools by passing a comma-separated list: `--tools name1,name2,...`. Run `--help` for the full list of available tool names.
|
||||
|
||||
### MCP servers
|
||||
|
||||
Besides the built-in tools, the server can expose tools coming from MCP servers, added in [#26062](https://github.com/ggml-org/llama.cpp/pull/26062). Only the stdio transport is supported: such a server is a child process reading JSON-RPC messages on its stdin and writing replies on its stdout, so nothing has to be started or maintained outside `llama-server`.
|
||||
|
||||
Servers are declared in a Cursor-compatible JSON file:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"example": { "command": "/path/to/server", "args": [] }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```sh
|
||||
llama-server -m model.gguf --mcp-servers-config mcp.json
|
||||
```
|
||||
|
||||
The same JSON can be passed inline with `--mcp-servers-json`. Each entry under `mcpServers` accepts:
|
||||
|
||||
| Key | Explanation |
|
||||
| --- | ----------- |
|
||||
| `command` | executable to spawn, required, entries without it are skipped |
|
||||
| `args` | array of arguments |
|
||||
| `env` | object merged over the parent environment |
|
||||
| `cwd` | working directory of the child process |
|
||||
| `timeout_ms` | per-tool-call timeout (default: 30000) |
|
||||
|
||||
Every server is spawned once at startup to list its tools, then stopped, and respawned on demand when one of its tools is called. Tools are exposed as `<server>_<tool>` alongside the built-in ones: they show up in the Web UI and in `GET /tools`, and the model calls them like any other tool. A name colliding with an already registered tool is skipped. This is independent of `--tools`, MCP servers can be the only tools available.
|
||||
|
||||
The child process runs with the same privileges as the server, so only declare commands you trust. As with `--tools`, `--cors-origins` then defaults to `localhost`.
|
||||
|
||||
Note: `--ui-mcp-proxy` is unrelated, it only lets the Web UI reach remote MCP servers from the browser.
|
||||
|
||||
Any server written against the [MCP specification](https://modelcontextprotocol.io) works as is, whether it uses an official SDK or not: the transport is one JSON-RPC message per line on stdio, so a script wrapping an existing program is a valid server too.
|
||||
|
||||
### CORS
|
||||
|
||||
By default the server reflects any `Origin` header back with credentials allowed. This matches the old, always-on `*` behavior and is fine as long as the server only exposes stateless, read-only endpoints.
|
||||
|
||||
Enabling `--tools` or `--agent` exposes file read/write over the API, so in that case `--cors-origins` defaults to `localhost` instead: only pages served from localhost can reach the server. Pass `--cors-origins` explicitly to override either default.
|
||||
|
||||
Recommended `--cors-origins` setting, depending on where the server runs:
|
||||
|
||||
| Deployment | Recommendation |
|
||||
| ---------- | --------------- |
|
||||
| Public | set an API key, put the server behind a reverse proxy, `--cors-origins` optional |
|
||||
| Local network | set `--cors-origins` to your frontend's origin |
|
||||
| Same machine | `--cors-origins localhost` (default once `--agent` is set) |
|
||||
|
||||
Related flags: `--cors-origins`, `--cors-methods`, `--cors-headers`, `--cors-credentials` / `--no-cors-credentials`. Background and rationale: [#25655](https://github.com/ggml-org/llama.cpp/pull/25655).
|
||||
|
||||
## Build
|
||||
|
||||
`llama-server` is built alongside everything else from the root of the project
|
||||
@@ -1579,9 +1631,9 @@ curl http://localhost:8080/v1/messages/count_tokens \
|
||||
{"input_tokens": 10}
|
||||
```
|
||||
|
||||
## Server built-in tools
|
||||
## Server tools
|
||||
|
||||
The server exposes a REST API under `/tools` that allows the Web UI to call built-in tools. This endpoint is intended to be used internally by the Web UI and subject to change or to be removed in the future.
|
||||
The server exposes a REST API under `/tools` that allows the Web UI to call server tools. This endpoint is intended to be used internally by the Web UI and subject to change or to be removed in the future.
|
||||
|
||||
**Please do NOT use this endpoint in a downstream application**
|
||||
|
||||
|
||||
@@ -507,6 +507,23 @@ void server_tokens::push_back(const mtmd_input_chunk * chunk) {
|
||||
}
|
||||
}
|
||||
|
||||
void server_tokens::push_back_placeholder(const mtmd_input_chunk * chunk) {
|
||||
auto type = mtmd_input_chunk_get_type(chunk);
|
||||
if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
|
||||
GGML_ASSERT(has_mtmd);
|
||||
mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_get_placeholder(chunk));
|
||||
GGML_ASSERT(new_chunk != nullptr && "failed to create placeholder chunk");
|
||||
const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);
|
||||
size_t start_idx = tokens.size();
|
||||
for (size_t i = 0; i < n_tokens; ++i) {
|
||||
tokens.emplace_back(LLAMA_TOKEN_NULL);
|
||||
}
|
||||
map_idx_to_media[start_idx] = std::move(new_chunk);
|
||||
} else {
|
||||
push_back(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
void server_tokens::push_back(server_tokens & tokens) {
|
||||
size_t start_idx = size();
|
||||
for (size_t i = 0; i < tokens.size(); i++) {
|
||||
|
||||
@@ -195,6 +195,10 @@ public:
|
||||
// will create a copy of the chunk if it contains non-text data
|
||||
void push_back(const mtmd_input_chunk * chunk);
|
||||
|
||||
// same as push_back, but media chunks are stored as placeholders (no image/audio data)
|
||||
// only use this if the chunk will never be encoded again (e.g. it is already in the KV cache)
|
||||
void push_back_placeholder(const mtmd_input_chunk * chunk);
|
||||
|
||||
// appends server tokens, updates the media map. copies media chunks.
|
||||
void push_back(server_tokens & tokens);
|
||||
|
||||
|
||||
@@ -3416,7 +3416,8 @@ private:
|
||||
// add the mtmd chunk to cache
|
||||
{
|
||||
const auto & chunk = input_tokens.find_chunk(cur_token_idx);
|
||||
slot.prompt.tokens.push_back(chunk.get()); // copy
|
||||
// the chunk is already in the KV cache at this point, so we don't need to keep its data around
|
||||
slot.prompt.tokens.push_back_placeholder(chunk.get());
|
||||
}
|
||||
|
||||
has_mtmd = true;
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
#include <regex>
|
||||
#include <thread>
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <atomic>
|
||||
#include <cstring>
|
||||
#include <cctype>
|
||||
@@ -1692,61 +1691,6 @@ private:
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// get_datetime: returns the current date and time
|
||||
//
|
||||
|
||||
struct server_tool_get_datetime : server_tool {
|
||||
server_tool_get_datetime() {
|
||||
name = "get_datetime";
|
||||
display_name = "Get Date & Time";
|
||||
permission_write = false;
|
||||
}
|
||||
|
||||
json get_definition() const override {
|
||||
return {
|
||||
{"type", "function"},
|
||||
{"function", {
|
||||
{"name", name},
|
||||
{"description", "Returns the current date and time in UTC"},
|
||||
{"parameters", {
|
||||
{"type", "object"},
|
||||
{"properties", {
|
||||
{"format", {
|
||||
{"type", "string"},
|
||||
{"description",
|
||||
"strftime()-style format string for the output (default: \"%Y-%m-%dT%H:%M:%SZ\", "
|
||||
"e.g. ISO 8601). Choose your own format if you need something else, "
|
||||
"e.g. \"%A, %B %d %Y\" for a human-readable date."},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
};
|
||||
}
|
||||
|
||||
json invoke(json params, server_tool::stream *) const override {
|
||||
std::string format = json_value(params, "format", std::string("%Y-%m-%dT%H:%M:%SZ"));
|
||||
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto time = std::chrono::system_clock::to_time_t(now);
|
||||
std::tm tm_utc;
|
||||
#ifdef _WIN32
|
||||
gmtime_s(&tm_utc, &time);
|
||||
#else
|
||||
gmtime_r(&time, &tm_utc);
|
||||
#endif
|
||||
|
||||
char buf[256];
|
||||
size_t len = std::strftime(buf, sizeof(buf), format.c_str(), &tm_utc);
|
||||
if (len == 0) {
|
||||
return {{"error", "invalid format string"}};
|
||||
}
|
||||
|
||||
return {{"result", std::string(buf, len)}};
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// get_info: returns runtime info (OS name/version and cwd)
|
||||
//
|
||||
@@ -2005,6 +1949,10 @@ static server_tool & find_tool(std::vector<std::unique_ptr<server_tool>> & tools
|
||||
//
|
||||
|
||||
static std::vector<std::unique_ptr<server_tool>> build_tools() {
|
||||
// IMPORTANT: for contributors, please keep this array of tools as minimal as possible
|
||||
// we only accept minimal i/o and shell command tools here
|
||||
// for example, do not add: web search, get date time, etc.
|
||||
// high-level functionality should be added either via MCP or web UI
|
||||
std::vector<std::unique_ptr<server_tool>> tools;
|
||||
tools.push_back(std::make_unique<server_tool_read_file>());
|
||||
tools.push_back(std::make_unique<server_tool_file_glob_search>());
|
||||
@@ -2012,7 +1960,6 @@ static std::vector<std::unique_ptr<server_tool>> build_tools() {
|
||||
tools.push_back(std::make_unique<server_tool_exec_shell_command>());
|
||||
tools.push_back(std::make_unique<server_tool_write_file>());
|
||||
tools.push_back(std::make_unique<server_tool_edit_file>());
|
||||
tools.push_back(std::make_unique<server_tool_get_datetime>());
|
||||
tools.push_back(std::make_unique<server_tool_get_info>());
|
||||
return tools;
|
||||
}
|
||||
@@ -2088,7 +2035,7 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools,
|
||||
}
|
||||
}
|
||||
|
||||
// append MCP tools, skipping any that collide with a built-in or another MCP tool of the same "<server>_<tool>" name
|
||||
// append MCP tools, skipping any that collide with a server tool or another MCP tool of the same "<server>_<tool>" name
|
||||
if (!mcp_mgr.empty()) {
|
||||
std::unordered_set<std::string> seen_names;
|
||||
for (auto & t : tools) {
|
||||
|
||||
@@ -18,7 +18,7 @@ struct server_tool {
|
||||
|
||||
virtual ~server_tool() = default;
|
||||
virtual json get_definition() const = 0;
|
||||
virtual std::string type() const { return "builtin"; }
|
||||
virtual std::string type() const { return "server"; }
|
||||
|
||||
struct stream {
|
||||
server_response & qr;
|
||||
|
||||
@@ -346,7 +346,7 @@ int llama_server(common_params & params, int argc, char ** argv) {
|
||||
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
|
||||
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
|
||||
if (!params.server_tools.empty()) {
|
||||
warn_names.push_back("built-in tools (experimental)");
|
||||
warn_names.push_back("server tools (experimental)");
|
||||
}
|
||||
if (!params.server_tools_runtime.empty()) {
|
||||
warn_names.push_back("tools runtime (experimental)");
|
||||
|
||||
@@ -259,6 +259,8 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
cpp += fmt("static const unsigned char asset_%d_data[] = {", i);
|
||||
append_bytes_hex(cpp, bytes);
|
||||
|
||||
// note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here
|
||||
const auto hash = fnv_hash(bytes.data(), bytes.size());
|
||||
|
||||
cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n",
|
||||
|
||||
@@ -61,6 +61,9 @@ export default ts.config(
|
||||
{ blankLine: 'always', next: ['return', 'throw', 'break', 'continue'], prev: '*' }
|
||||
],
|
||||
|
||||
// Alphabetical order for enum members
|
||||
'perfectionist/sort-enums': ['error', { type: 'natural' }],
|
||||
|
||||
'perfectionist/sort-objects': ['error', { type: 'natural' }],
|
||||
|
||||
// Alphabetical order for variable declarations and object keys
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@
|
||||
<span>
|
||||
Run llama-server with <code>{CLI_FLAGS.TOOLS}</code> flag to enable
|
||||
|
||||
<strong>Built-in Tools</strong>.
|
||||
<strong>Server Tools</strong>.
|
||||
</span>
|
||||
</span>
|
||||
|
||||
|
||||
+2
-2
@@ -62,7 +62,7 @@
|
||||
// it, the picker still opens for manual entry but explains why search is
|
||||
// unavailable instead of firing searches that would only fail. Browse is
|
||||
// hidden too: it resolves the picked folder name through the same tool.
|
||||
const fileSearchKey = $derived(toolsStore.getPermissionKey(BuiltInTool.FILE_GLOB_SEARCH));
|
||||
const fileSearchKey = $derived(toolsStore.getPermissionKey(BuiltInTool.SERVER_FILE_GLOB_SEARCH));
|
||||
const fileSearchEnabled = $derived(
|
||||
fileSearchKey !== null && toolsStore.isToolEnabled(fileSearchKey)
|
||||
);
|
||||
@@ -212,7 +212,7 @@
|
||||
// so the caller fails visibly instead of committing a bare leaf name.
|
||||
async function resolveNativeName(name: string): Promise<string | null> {
|
||||
try {
|
||||
const res = await ToolsService.executeToolRaw(BuiltInTool.FILE_GLOB_SEARCH, {
|
||||
const res = await ToolsService.executeToolRaw(BuiltInTool.SERVER_FILE_GLOB_SEARCH, {
|
||||
include: buildCaseInsensitiveGlob(name),
|
||||
limit: SEARCH.NATIVE_LIMIT,
|
||||
max_depth: SEARCH.NATIVE_MAX_DEPTH,
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@
|
||||
// When the server does not expose file_glob_search (started without
|
||||
// --tools) or the user disabled it, the picker still opens but explains
|
||||
// why instead of firing searches that would only fail.
|
||||
const fileSearchKey = $derived(toolsStore.getPermissionKey(BuiltInTool.FILE_GLOB_SEARCH));
|
||||
const fileSearchKey = $derived(toolsStore.getPermissionKey(BuiltInTool.SERVER_FILE_GLOB_SEARCH));
|
||||
const fileSearchEnabled = $derived(
|
||||
fileSearchKey !== null && toolsStore.isToolEnabled(fileSearchKey)
|
||||
);
|
||||
|
||||
+10
-10
@@ -35,19 +35,19 @@
|
||||
|
||||
{#if isSearchCall}
|
||||
<ChatMessageToolCallBlockSearchResults {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.GET_DATETIME}
|
||||
{:else if section.toolName === BuiltInTool.BROWSER_GET_DATETIME}
|
||||
<ChatMessageToolCallBlockGetDatetime {section} {isStreaming} />
|
||||
{:else if section.toolName === BuiltInTool.GET_INFO}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_GET_INFO}
|
||||
<ChatMessageToolCallBlockGetInfo {section} {isStreaming} />
|
||||
{:else if section.toolName === BuiltInTool.READ_FILE}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_READ_FILE}
|
||||
<ChatMessageToolCallBlockReadFile {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.READ_MEDIA}
|
||||
{:else if section.toolName === BuiltInTool.BROWSER_READ_MEDIA}
|
||||
<ChatMessageToolCallBlockReadMedia {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.EDIT_FILE}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_EDIT_FILE}
|
||||
<ChatMessageToolCallBlockEditFile {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.WRITE_FILE}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_WRITE_FILE}
|
||||
<ChatMessageToolCallBlockWriteFile {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.EXEC_SHELL_COMMAND}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND}
|
||||
<ChatMessageToolCallBlockExecShellCommand
|
||||
{section}
|
||||
{open}
|
||||
@@ -56,11 +56,11 @@
|
||||
{attachments}
|
||||
{onToggle}
|
||||
/>
|
||||
{:else if section.toolName === BuiltInTool.FILE_GLOB_SEARCH}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_FILE_GLOB_SEARCH}
|
||||
<ChatMessageToolCallBlockFileGlobSearch {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.GREP_SEARCH}
|
||||
{:else if section.toolName === BuiltInTool.SERVER_GREP_SEARCH}
|
||||
<ChatMessageToolCallBlockGrepSearch {section} {open} {isStreaming} {onToggle} />
|
||||
{:else if section.toolName === BuiltInTool.RUN_JAVASCRIPT}
|
||||
{:else if section.toolName === BuiltInTool.BROWSER_RUN_JAVASCRIPT}
|
||||
<ChatMessageToolCallBlockRunJavascript {section} {open} {isStreaming} {onToggle} />
|
||||
{:else}
|
||||
<ChatMessageToolCallBlockDefault {section} {open} {isStreaming} {attachments} {onToggle} />
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@
|
||||
import {
|
||||
classifyToolResult,
|
||||
formatJsonPretty,
|
||||
getBuiltinToolUi,
|
||||
getToolUi,
|
||||
parseToolResultWithMedia
|
||||
} from '$lib/utils';
|
||||
import { createBase64DataUrl } from '$lib/utils/data-url';
|
||||
@@ -27,7 +27,7 @@
|
||||
|
||||
let { attachments, isStreaming, onToggle, open, section }: Props = $props();
|
||||
|
||||
const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
|
||||
const title = $derived(getToolUi(section.toolName)?.label ?? section.toolName ?? '');
|
||||
const outputKind = $derived(classifyToolResult(section.toolResult));
|
||||
const parsedLines: ToolResultLine[] = $derived(
|
||||
section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@
|
||||
if (typeof obj.result === 'string') return { dateString: obj.result.trim() };
|
||||
}
|
||||
} catch {
|
||||
return { dateString: toolResultString.trim() };
|
||||
// not JSON - nothing to show
|
||||
}
|
||||
|
||||
return {};
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';
|
||||
import { FileTypeText } from '$lib/enums';
|
||||
import type { AgenticSection } from '$lib/types';
|
||||
import { getBuiltinToolUi } from '$lib/utils';
|
||||
import { getToolUi } from '$lib/utils';
|
||||
|
||||
interface Props {
|
||||
section: AgenticSection;
|
||||
@@ -18,7 +18,7 @@
|
||||
let { isStreaming, onToggle, open, section }: Props = $props();
|
||||
|
||||
const runJsMeta = $derived(parseRunJavascriptMeta(section));
|
||||
const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
|
||||
const title = $derived(getToolUi(section.toolName)?.label ?? section.toolName ?? '');
|
||||
</script>
|
||||
|
||||
<ToolCallBlock {section} {open} {isStreaming} meta={runJsMeta} {title} {onToggle}>
|
||||
|
||||
+3
-3
@@ -14,8 +14,8 @@
|
||||
import { ICON_CLASS_DEFAULT, ICON_CLASS_SPIN } from '$lib/constants';
|
||||
import { AgenticSectionType } from '$lib/enums';
|
||||
import { mcpStore } from '$lib/stores';
|
||||
import type { AgenticSection, BuiltinToolUiEntry } from '$lib/types';
|
||||
import { getBuiltinToolUi } from '$lib/utils';
|
||||
import type { AgenticSection, ToolUiEntry } from '$lib/types';
|
||||
import { getToolUi } from '$lib/utils';
|
||||
import type { Component, Snippet } from 'svelte';
|
||||
|
||||
type ToolCallBlockMetaWithError = TMeta & { errorMessage?: string };
|
||||
@@ -82,7 +82,7 @@
|
||||
const showSpinner = $derived(isPending || (isStreamingCall && isStreaming) || extraLiveStreaming);
|
||||
const isCodeStreaming = $derived(isStreaming && (isPending || isStreamingCall));
|
||||
|
||||
const toolUi: BuiltinToolUiEntry | null = $derived(getBuiltinToolUi(section.toolName));
|
||||
const toolUi: ToolUiEntry | null = $derived(getToolUi(section.toolName));
|
||||
const toolIcon: Component = $derived(
|
||||
spinIconWhenActive && showSpinner ? Loader2 : (toolUi?.icon ?? Wrench)
|
||||
);
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ export type EditFileMeta = {
|
||||
};
|
||||
|
||||
export function parseEditFileMeta(section: AgenticSection): EditFileMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.EDIT_FILE, section, { partial: true });
|
||||
const args = parseToolArgs(BuiltInTool.SERVER_EDIT_FILE, section, { partial: true });
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ export type ExecShellCommandMeta = {
|
||||
};
|
||||
|
||||
export function parseExecShellCommandMeta(section: AgenticSection): ExecShellCommandMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.EXEC_SHELL_COMMAND, section);
|
||||
const args = parseToolArgs(BuiltInTool.SERVER_EXEC_SHELL_COMMAND, section);
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ export type FileGlobSearchMeta = {
|
||||
};
|
||||
|
||||
export function parseFileGlobSearchMeta(section: AgenticSection): FileGlobSearchMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.FILE_GLOB_SEARCH, section);
|
||||
const args = parseToolArgs(BuiltInTool.SERVER_FILE_GLOB_SEARCH, section);
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ export type GrepSearchMeta = {
|
||||
};
|
||||
|
||||
export function parseGrepSearchMeta(section: AgenticSection): GrepSearchMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.GREP_SEARCH, section);
|
||||
const args = parseToolArgs(BuiltInTool.SERVER_GREP_SEARCH, section);
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ export type ReadFileMeta = {
|
||||
};
|
||||
|
||||
export function parseReadFileMeta(section: AgenticSection): ReadFileMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.READ_FILE, section, { partial: true });
|
||||
const args = parseToolArgs(BuiltInTool.SERVER_READ_FILE, section, { partial: true });
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ export type RunJavascriptMeta = {
|
||||
};
|
||||
|
||||
export function parseRunJavascriptMeta(section: AgenticSection): RunJavascriptMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.RUN_JAVASCRIPT, section);
|
||||
const args = parseToolArgs(BuiltInTool.BROWSER_RUN_JAVASCRIPT, section);
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ export type WriteFileMeta = {
|
||||
};
|
||||
|
||||
export function parseWriteFileMeta(section: AgenticSection): WriteFileMeta | null {
|
||||
const args = parseToolArgs(BuiltInTool.WRITE_FILE, section, { partial: true });
|
||||
const args = parseToolArgs(BuiltInTool.SERVER_WRITE_FILE, section, { partial: true });
|
||||
|
||||
if (!args) return null;
|
||||
|
||||
|
||||
+2
-2
@@ -61,8 +61,8 @@
|
||||
{:else}
|
||||
{@const source = toolsStore.getToolSource(toolName)}
|
||||
{@const providerName =
|
||||
source === ToolSource.BUILTIN
|
||||
? TOOL_SERVER_LABELS[ToolSource.BUILTIN]
|
||||
source === ToolSource.SERVER
|
||||
? TOOL_SERVER_LABELS[ToolSource.SERVER]
|
||||
: source === ToolSource.CUSTOM
|
||||
? TOOL_SERVER_LABELS[ToolSource.CUSTOM]
|
||||
: 'MCP Tools'}
|
||||
|
||||
@@ -278,7 +278,7 @@ export { default as ChatFormInput } from './ChatForm/ChatFormInput/ChatFormInput
|
||||
/**
|
||||
* Working directory selector for agent mode. Renders a chip below the chat
|
||||
* form; clicking it opens a popover with a directory picker backed by the
|
||||
* server's `file_glob_search` built-in tool (POST /tools). The picked
|
||||
* server's `file_glob_search` server tool (POST /tools). The picked
|
||||
* directory is exposed via `bind:directory`; changing it records a
|
||||
* synthetic "Set working directory to ..." user message into chat history
|
||||
* and is enforced on tool calls via the `x-tool-cwd` request header.
|
||||
@@ -380,7 +380,7 @@ export { default as ChatFormPickerListItemSkeleton } from './ChatForm/ChatFormPi
|
||||
|
||||
/**
|
||||
* `@`-triggered file/folder mention picker. Resolves `@<query>` in the chat
|
||||
* input to a filesystem match via the server's `file_glob_search` built-in
|
||||
* input to a filesystem match via the server's `file_glob_search` server tool
|
||||
* tool, scoped to the conversation cwd (or server home when unset).
|
||||
* Selection splices a `[name](file:///<abs path>)` link into the input.
|
||||
*/
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import { ICON_CLASS_DEFAULT } from '$lib/constants';
|
||||
import { ToolSource } from '$lib/enums/tools.enums';
|
||||
import { mcpStore, permissionsStore, toolsStore } from '$lib/stores';
|
||||
import { getBuiltinToolUi } from '$lib/utils';
|
||||
import { getToolUi } from '$lib/utils';
|
||||
import { SvelteSet } from 'svelte/reactivity';
|
||||
|
||||
let expandedGroups = new SvelteSet<string>();
|
||||
@@ -69,12 +69,12 @@
|
||||
|
||||
{#each group.tools as entry (entry.key)}
|
||||
{@const toolName = entry.definition.function.name}
|
||||
{@const builtinUi =
|
||||
entry.source === ToolSource.BUILTIN || entry.source === ToolSource.FRONTEND
|
||||
? getBuiltinToolUi(toolName)
|
||||
{@const toolUi =
|
||||
entry.source === ToolSource.SERVER || entry.source === ToolSource.BROWSER
|
||||
? getToolUi(toolName)
|
||||
: null}
|
||||
{@const displayLabel = builtinUi?.label ?? toolName}
|
||||
{@const IconComponent = builtinUi?.icon ?? null}
|
||||
{@const displayLabel = toolUi?.label ?? toolName}
|
||||
{@const IconComponent = toolUi?.icon ?? null}
|
||||
{@const isEnabled = toolsStore.isToolEnabled(entry.key)}
|
||||
{@const permissionKey = entry.key}
|
||||
{@const isAlwaysAllowed = permissionsStore.hasTool(permissionKey)}
|
||||
|
||||
@@ -69,7 +69,7 @@ export { default as SettingsChatFields } from './SettingsChat/SettingsChatFields
|
||||
/**
|
||||
* **SettingsChatToolsTab** - Tools configuration tab for chat settings
|
||||
*
|
||||
* Displays available tools grouped by source (built-in, MCP, custom) with
|
||||
* Displays available tools grouped by source (server, browser, MCP, custom) with
|
||||
* toggles to enable/disable individual tools and tool groups. Shows MCP
|
||||
* server favicons and permission management controls.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { CLI_FLAGS } from './cli-flags.constants';
|
||||
import { BuiltInTool, JsonSchemaType, ToolCallType } from '$lib/enums';
|
||||
import type { OpenAIToolDefinition } from '$lib/types';
|
||||
|
||||
// get_info is served by the server, but the browser falls back to this
|
||||
// implementation when the server does not provide it - same wire name.
|
||||
export const BROWSER_INFO_TOOL_NAME = BuiltInTool.SERVER_GET_INFO;
|
||||
|
||||
/** UA token to OS name, first match wins - Android and iOS UAs also carry the Linux / Mac OS X tokens */
|
||||
export const BROWSER_INFO_OS_UA_PATTERNS: readonly [RegExp, string][] = [
|
||||
[/Windows NT/, 'Windows'],
|
||||
[/Android/, 'Android'],
|
||||
[/iPhone|iPad|iPod/, 'iOS'],
|
||||
[/CrOS/, 'ChromeOS'],
|
||||
[/Mac OS X/, 'macOS'],
|
||||
[/Linux/, 'Linux']
|
||||
];
|
||||
|
||||
export const BROWSER_INFO_OS_UNKNOWN = 'unknown';
|
||||
|
||||
/** Sent to the model as the `note` field of the tool result, next to the OS name */
|
||||
export const BROWSER_INFO_NOTE = `This environment is browser-only, it cannot read or modify local files, and it cannot run shell commands. To get local file access, tell user to launch llama-server with the ${CLI_FLAGS.AGENT} argument.`;
|
||||
|
||||
export function buildBrowserInfoToolDefinition(): OpenAIToolDefinition {
|
||||
return {
|
||||
function: {
|
||||
description:
|
||||
'Get runtime info (OS name), may call when user asks about local files or shell commands',
|
||||
name: BROWSER_INFO_TOOL_NAME,
|
||||
parameters: {
|
||||
properties: {},
|
||||
required: [],
|
||||
type: JsonSchemaType.OBJECT
|
||||
}
|
||||
},
|
||||
type: ToolCallType.FUNCTION
|
||||
};
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
// Registry of built-in and frontend (browser) tools whose renderer
|
||||
// shows a recognizable icon and friendly label inline in the chat UI.
|
||||
//
|
||||
// To add a new built-in tool, add an entry to BUILTIN_TOOL_UI. To give a
|
||||
// tool a custom title or body renderer, add a dedicated component under
|
||||
// ChatMessageToolCall/ and route it in ChatMessageToolCallBlock.svelte
|
||||
// (see ChatMessageToolCallBlockGetDatetime and
|
||||
// ChatMessageToolCallBlockSearchResults for prior art).
|
||||
|
||||
import {
|
||||
Braces,
|
||||
Clock,
|
||||
Eye,
|
||||
FilePen,
|
||||
FilePlus,
|
||||
FileSearch,
|
||||
FileText,
|
||||
Info,
|
||||
SearchCode,
|
||||
Terminal
|
||||
} from '@lucide/svelte';
|
||||
import { BuiltInTool, ToolSource } from '$lib/enums';
|
||||
import type { BuiltinToolUiEntry } from '$lib/types';
|
||||
|
||||
export const BUILTIN_TOOL_UI: Readonly<Record<BuiltInTool, BuiltinToolUiEntry>> = {
|
||||
[BuiltInTool.EDIT_FILE]: { icon: FilePen, label: 'Edit file', source: ToolSource.BUILTIN },
|
||||
[BuiltInTool.EXEC_SHELL_COMMAND]: {
|
||||
icon: Terminal,
|
||||
label: 'Run command',
|
||||
source: ToolSource.BUILTIN
|
||||
},
|
||||
[BuiltInTool.FILE_GLOB_SEARCH]: {
|
||||
icon: FileSearch,
|
||||
label: 'Search files',
|
||||
source: ToolSource.BUILTIN
|
||||
},
|
||||
[BuiltInTool.GET_DATETIME]: { icon: Clock, label: 'Current time', source: ToolSource.BUILTIN },
|
||||
[BuiltInTool.GET_INFO]: { icon: Info, label: 'Runtime info', source: ToolSource.BUILTIN },
|
||||
[BuiltInTool.GREP_SEARCH]: {
|
||||
icon: SearchCode,
|
||||
label: 'Search in files',
|
||||
source: ToolSource.BUILTIN
|
||||
},
|
||||
[BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN },
|
||||
[BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND },
|
||||
[BuiltInTool.RUN_JAVASCRIPT]: {
|
||||
icon: Braces,
|
||||
label: 'Run JavaScript',
|
||||
source: ToolSource.FRONTEND
|
||||
},
|
||||
[BuiltInTool.WRITE_FILE]: { icon: FilePlus, label: 'Write file', source: ToolSource.BUILTIN }
|
||||
} as const;
|
||||
@@ -1,4 +1,5 @@
|
||||
export const CLI_FLAGS = {
|
||||
AGENT: '--agent',
|
||||
API_KEY: '--api-key',
|
||||
MCP_PROXY: '--ui-mcp-proxy',
|
||||
SLOTS: '--slots',
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { BuiltInTool, JsonSchemaType, ToolCallType } from '$lib/enums';
|
||||
import type { OpenAIToolDefinition } from '$lib/types';
|
||||
|
||||
export const GET_DATETIME_TOOL_NAME = BuiltInTool.BROWSER_GET_DATETIME;
|
||||
|
||||
export function buildGetDatetimeToolDefinition(): OpenAIToolDefinition {
|
||||
return {
|
||||
function: {
|
||||
description:
|
||||
'Returns the current local date and time in ISO 8601 format, with the IANA time zone name',
|
||||
name: GET_DATETIME_TOOL_NAME,
|
||||
parameters: {
|
||||
properties: {},
|
||||
required: [],
|
||||
type: JsonSchemaType.OBJECT
|
||||
}
|
||||
},
|
||||
type: ToolCallType.FUNCTION
|
||||
};
|
||||
}
|
||||
@@ -15,7 +15,7 @@ export * from './context-gauge-popup.constants';
|
||||
export * from './conversation-import.constants';
|
||||
export * from './binary-detection.constants';
|
||||
export * from './content-detection.constants';
|
||||
export * from './built-in-tools.constants';
|
||||
export * from './tool-ui.constants';
|
||||
export * from './cache.constants';
|
||||
export * from './chat-form.constants';
|
||||
export * from './cli-flags.constants';
|
||||
@@ -59,3 +59,5 @@ export * from './uri-template.constants';
|
||||
export * from './url.constants';
|
||||
export * from './working-directory.constants';
|
||||
export * from './read-media';
|
||||
export * from './get-datetime';
|
||||
export * from './browser-info';
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
} from '$lib/enums';
|
||||
import type { OpenAIToolDefinition } from '$lib/types';
|
||||
|
||||
export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA;
|
||||
export const READ_MEDIA_TOOL_NAME = BuiltInTool.BROWSER_READ_MEDIA;
|
||||
|
||||
// header lines of the tool result, parsed back by the read_media renderer
|
||||
export const PREFIX_FILE = 'File: ';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BuiltInTool } from '$lib/enums';
|
||||
|
||||
export const SANDBOX_TOOL_NAME = BuiltInTool.RUN_JAVASCRIPT;
|
||||
export const SANDBOX_TOOL_NAME = BuiltInTool.BROWSER_RUN_JAVASCRIPT;
|
||||
|
||||
export const SANDBOX_TIMEOUT_MS_DEFAULT = 10000;
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
// Registry of server and browser tools whose renderer
|
||||
// shows a recognizable icon and friendly label inline in the chat UI.
|
||||
//
|
||||
// To add a new tool, add an entry to TOOL_UI. To give a
|
||||
// tool a custom title or body renderer, add a dedicated component under
|
||||
// ChatMessageToolCall/ and route it in ChatMessageToolCallBlock.svelte
|
||||
// (see ChatMessageToolCallBlockGetDatetime and
|
||||
// ChatMessageToolCallBlockSearchResults for prior art).
|
||||
|
||||
import {
|
||||
Braces,
|
||||
Clock,
|
||||
Eye,
|
||||
FilePen,
|
||||
FilePlus,
|
||||
FileSearch,
|
||||
FileText,
|
||||
Info,
|
||||
SearchCode,
|
||||
Terminal
|
||||
} from '@lucide/svelte';
|
||||
import { BuiltInTool, ToolSource } from '$lib/enums';
|
||||
import type { ToolUiEntry } from '$lib/types';
|
||||
|
||||
export const TOOL_UI: Readonly<Record<BuiltInTool, ToolUiEntry>> = {
|
||||
[BuiltInTool.BROWSER_GET_DATETIME]: {
|
||||
icon: Clock,
|
||||
label: 'Current time',
|
||||
source: ToolSource.BROWSER
|
||||
},
|
||||
[BuiltInTool.BROWSER_READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.BROWSER },
|
||||
[BuiltInTool.BROWSER_RUN_JAVASCRIPT]: {
|
||||
icon: Braces,
|
||||
label: 'Run JavaScript',
|
||||
source: ToolSource.BROWSER
|
||||
},
|
||||
[BuiltInTool.SERVER_EDIT_FILE]: { icon: FilePen, label: 'Edit file', source: ToolSource.SERVER },
|
||||
[BuiltInTool.SERVER_EXEC_SHELL_COMMAND]: {
|
||||
icon: Terminal,
|
||||
label: 'Run command',
|
||||
source: ToolSource.SERVER
|
||||
},
|
||||
[BuiltInTool.SERVER_FILE_GLOB_SEARCH]: {
|
||||
icon: FileSearch,
|
||||
label: 'Search files',
|
||||
source: ToolSource.SERVER
|
||||
},
|
||||
[BuiltInTool.SERVER_GET_INFO]: { icon: Info, label: 'Runtime info', source: ToolSource.SERVER },
|
||||
[BuiltInTool.SERVER_GREP_SEARCH]: {
|
||||
icon: SearchCode,
|
||||
label: 'Search in files',
|
||||
source: ToolSource.SERVER
|
||||
},
|
||||
[BuiltInTool.SERVER_READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.SERVER },
|
||||
[BuiltInTool.SERVER_WRITE_FILE]: {
|
||||
icon: FilePlus,
|
||||
label: 'Write file',
|
||||
source: ToolSource.SERVER
|
||||
}
|
||||
} as const;
|
||||
@@ -18,15 +18,15 @@ export const UI_DATA_ATTRS = {
|
||||
} as const;
|
||||
|
||||
export const TOOL_GROUP_LABELS = {
|
||||
[ToolSource.BUILTIN]: 'Built-in',
|
||||
[ToolSource.BROWSER]: 'Browser',
|
||||
[ToolSource.CUSTOM]: 'JSON Schema',
|
||||
[ToolSource.FRONTEND]: 'Browser'
|
||||
[ToolSource.SERVER]: 'Server'
|
||||
} as const;
|
||||
|
||||
export const TOOL_SERVER_LABELS = {
|
||||
[ToolSource.BUILTIN]: 'Built-in Tools',
|
||||
[ToolSource.BROWSER]: 'Browser Tools',
|
||||
[ToolSource.CUSTOM]: 'Custom Tools',
|
||||
[ToolSource.FRONTEND]: 'Browser Tools'
|
||||
[ToolSource.SERVER]: 'Server Tools'
|
||||
} as const;
|
||||
|
||||
export const TOOLTIP_DELAY_DURATION = 500;
|
||||
|
||||
@@ -9,12 +9,12 @@ export enum ToolCallType {
|
||||
* Types of sections in agentic content display.
|
||||
*/
|
||||
export enum AgenticSectionType {
|
||||
REASONING = 'reasoning',
|
||||
REASONING_PENDING = 'reasoning_pending',
|
||||
TEXT = 'text',
|
||||
TOOL_CALL = 'tool_call',
|
||||
TOOL_CALL_PENDING = 'tool_call_pending',
|
||||
TOOL_CALL_STREAMING = 'tool_call_streaming',
|
||||
REASONING = 'reasoning',
|
||||
REASONING_PENDING = 'reasoning_pending'
|
||||
TOOL_CALL_STREAMING = 'tool_call_streaming'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -22,8 +22,8 @@ export enum AgenticSectionType {
|
||||
*/
|
||||
export enum ContinueIntentKind {
|
||||
APPEND_TEXT = 'append_text',
|
||||
RERUN_TURN = 'rerun_turn',
|
||||
NEXT_TURN = 'next_turn'
|
||||
NEXT_TURN = 'next_turn',
|
||||
RERUN_TURN = 'rerun_turn'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -39,7 +39,7 @@ export enum ToolResultKind {
|
||||
* Line classification for the unified-diff renderer of `edit_file` results.
|
||||
*/
|
||||
export enum DiffLineKind {
|
||||
CONTEXT = 'context',
|
||||
ADD = 'add',
|
||||
CONTEXT = 'context',
|
||||
REMOVE = 'remove'
|
||||
}
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
export enum AttachmentType {
|
||||
AUDIO = 'AUDIO',
|
||||
IMAGE = 'IMAGE',
|
||||
VIDEO = 'VIDEO',
|
||||
LEGACY_CONTEXT = 'context', // Legacy attachment type for backward compatibility
|
||||
MCP_PROMPT = 'MCP_PROMPT',
|
||||
MCP_RESOURCE = 'MCP_RESOURCE',
|
||||
PDF = 'PDF',
|
||||
TEXT = 'TEXT',
|
||||
LEGACY_CONTEXT = 'context' // Legacy attachment type for backward compatibility
|
||||
VIDEO = 'VIDEO'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -17,14 +17,14 @@ export enum AttachmentType {
|
||||
* Used to select which file upload or attachment action is triggered.
|
||||
*/
|
||||
export enum AttachmentMenuItemId {
|
||||
IMAGES = 'images',
|
||||
AUDIO = 'audio',
|
||||
VIDEO = 'video',
|
||||
TEXT = 'text',
|
||||
IMAGES = 'images',
|
||||
MCP_PROMPT = 'mcp-prompt',
|
||||
MCP_RESOURCES = 'mcp-resources',
|
||||
PDF = 'pdf',
|
||||
SYSTEM_MESSAGE = 'system-message',
|
||||
MCP_PROMPT = 'mcp-prompt',
|
||||
MCP_RESOURCES = 'mcp-resources'
|
||||
TEXT = 'text',
|
||||
VIDEO = 'video'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -32,9 +32,9 @@ export enum AttachmentMenuItemId {
|
||||
*/
|
||||
export enum AttachmentItemEnabledWhen {
|
||||
ALWAYS = 'always',
|
||||
HAS_VISION_MODALITY = 'hasVisionModality',
|
||||
HAS_AUDIO_MODALITY = 'hasAudioModality',
|
||||
HAS_VIDEO_MODALITY = 'hasVideoModality'
|
||||
HAS_VIDEO_MODALITY = 'hasVideoModality',
|
||||
HAS_VISION_MODALITY = 'hasVisionModality'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -42,9 +42,9 @@ export enum AttachmentItemEnabledWhen {
|
||||
*/
|
||||
export enum AttachmentAction {
|
||||
FILE_UPLOAD = 'onFileUpload',
|
||||
SYSTEM_PROMPT_CLICK = 'onSystemPromptClick',
|
||||
MCP_PROMPT_CLICK = 'onMcpPromptClick',
|
||||
MCP_RESOURCES_CLICK = 'onMcpResourcesClick'
|
||||
MCP_RESOURCES_CLICK = 'onMcpResourcesClick',
|
||||
SYSTEM_PROMPT_CLICK = 'onSystemPromptClick'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -52,9 +52,9 @@ export enum AttachmentAction {
|
||||
*/
|
||||
export enum AttachmentLabel {
|
||||
FILE = 'File',
|
||||
PDF_FILE = 'PDF File',
|
||||
MCP_PROMPT = 'MCP Prompt',
|
||||
MCP_RESOURCE = 'MCP Resource'
|
||||
MCP_RESOURCE = 'MCP Resource',
|
||||
PDF_FILE = 'PDF File'
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/** String representation of a boolean used in data attributes and persisted values. */
|
||||
export enum BooleanString {
|
||||
TRUE = 'true',
|
||||
FALSE = 'false'
|
||||
FALSE = 'false',
|
||||
TRUE = 'true'
|
||||
}
|
||||
|
||||
@@ -1,41 +1,41 @@
|
||||
export enum ChatMessageStatsView {
|
||||
GENERATION = 'generation',
|
||||
READING = 'reading',
|
||||
TOOLS = 'tools',
|
||||
SUMMARY = 'summary'
|
||||
SUMMARY = 'summary',
|
||||
TOOLS = 'tools'
|
||||
}
|
||||
|
||||
export enum ChatMessageStatisticsMode {
|
||||
SWITCHABLE = 'switchable',
|
||||
GENERATION = 'generation',
|
||||
READING = 'reading',
|
||||
GENERATION = 'generation'
|
||||
SWITCHABLE = 'switchable'
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection state of a streamed completion, drives the resume status indicator.
|
||||
*/
|
||||
export enum StreamConnectionState {
|
||||
STREAMING = 'streaming',
|
||||
LOST = 'lost',
|
||||
RESUMING = 'resuming',
|
||||
LOST = 'lost'
|
||||
STREAMING = 'streaming'
|
||||
}
|
||||
|
||||
/**
|
||||
* Reasoning format options for API requests.
|
||||
*/
|
||||
export enum ReasoningFormat {
|
||||
NONE = 'none',
|
||||
AUTO = 'auto'
|
||||
AUTO = 'auto',
|
||||
NONE = 'none'
|
||||
}
|
||||
|
||||
/**
|
||||
* Message roles for chat messages.
|
||||
*/
|
||||
export enum MessageRole {
|
||||
USER = 'user',
|
||||
ASSISTANT = 'assistant',
|
||||
SYSTEM = 'system',
|
||||
TOOL = 'tool'
|
||||
TOOL = 'tool',
|
||||
USER = 'user'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -43,27 +43,27 @@ export enum MessageRole {
|
||||
*/
|
||||
export enum MessageType {
|
||||
ROOT = 'root',
|
||||
SYSTEM = 'system',
|
||||
TEXT = 'text',
|
||||
THINK = 'think',
|
||||
SYSTEM = 'system'
|
||||
THINK = 'think'
|
||||
}
|
||||
|
||||
/**
|
||||
* Content part types for API chat message content.
|
||||
*/
|
||||
export enum ContentPartType {
|
||||
TEXT = 'text',
|
||||
IMAGE_URL = 'image_url',
|
||||
INPUT_AUDIO = 'input_audio',
|
||||
INPUT_VIDEO = 'input_video'
|
||||
INPUT_VIDEO = 'input_video',
|
||||
TEXT = 'text'
|
||||
}
|
||||
|
||||
/**
|
||||
* Error dialog types for displaying server/timeout errors.
|
||||
*/
|
||||
export enum ErrorDialogType {
|
||||
TIMEOUT = 'timeout',
|
||||
SERVER = 'server'
|
||||
SERVER = 'server',
|
||||
TIMEOUT = 'timeout'
|
||||
}
|
||||
|
||||
export enum ConversationSelectionMode {
|
||||
@@ -75,27 +75,27 @@ export enum ConversationSelectionMode {
|
||||
* PDF view mode options for previewing PDF attachments.
|
||||
*/
|
||||
export enum PdfViewMode {
|
||||
TEXT = 'text',
|
||||
PAGES = 'pages'
|
||||
PAGES = 'pages',
|
||||
TEXT = 'text'
|
||||
}
|
||||
|
||||
export enum ChatFormCommandAction {
|
||||
PROMPT = 'prompt',
|
||||
CWD = 'cwd',
|
||||
MODEL = 'model'
|
||||
MODEL = 'model',
|
||||
PROMPT = 'prompt'
|
||||
}
|
||||
|
||||
export enum FileMentionEntryType {
|
||||
FILE = 'file',
|
||||
DIRECTORY = 'directory'
|
||||
DIRECTORY = 'directory',
|
||||
FILE = 'file'
|
||||
}
|
||||
|
||||
/**
|
||||
* Kinds of tokens the chat-form-input-rich produces.
|
||||
*/
|
||||
export enum ChatFormInputRichTokenKind {
|
||||
TEXT = 'text',
|
||||
BADGE = 'badge',
|
||||
CODE_BLOCK = 'code_block',
|
||||
CODE_INLINE = 'code_inline',
|
||||
CODE_BLOCK = 'code_block'
|
||||
TEXT = 'text'
|
||||
}
|
||||
|
||||
@@ -4,6 +4,6 @@
|
||||
* message record belongs to it.
|
||||
*/
|
||||
export enum SessionRecordType {
|
||||
SESSION = 'session',
|
||||
MESSAGE = 'message'
|
||||
MESSAGE = 'message',
|
||||
SESSION = 'session'
|
||||
}
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
|
||||
// File type category enum
|
||||
export enum FileTypeCategory {
|
||||
IMAGE = 'image',
|
||||
AUDIO = 'audio',
|
||||
VIDEO = 'video',
|
||||
IMAGE = 'image',
|
||||
PDF = 'pdf',
|
||||
TEXT = 'text'
|
||||
TEXT = 'text',
|
||||
VIDEO = 'video'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -21,13 +21,13 @@ export enum SpecialFileType {
|
||||
|
||||
// Specific file type enums for each category
|
||||
export enum FileTypeImage {
|
||||
GIF = 'gif',
|
||||
HEIC = 'heic',
|
||||
HEIF = 'heif',
|
||||
JPEG = 'jpeg',
|
||||
PNG = 'png',
|
||||
GIF = 'gif',
|
||||
WEBP = 'webp',
|
||||
SVG = 'svg',
|
||||
HEIC = 'heic',
|
||||
HEIF = 'heif'
|
||||
WEBP = 'webp'
|
||||
}
|
||||
|
||||
export enum FileTypeAudio {
|
||||
@@ -46,55 +46,55 @@ export enum FileTypePdf {
|
||||
}
|
||||
|
||||
export enum FileTypeText {
|
||||
PLAIN_TEXT = 'plainText',
|
||||
MARKDOWN = 'md',
|
||||
ASCIIDOC = 'asciidoc',
|
||||
JAVASCRIPT = 'js',
|
||||
TYPESCRIPT = 'ts',
|
||||
JSX = 'jsx',
|
||||
TSX = 'tsx',
|
||||
CSS = 'css',
|
||||
HTML = 'html',
|
||||
JSON = 'json',
|
||||
XML = 'xml',
|
||||
YAML = 'yaml',
|
||||
CSV = 'csv',
|
||||
LOG = 'log',
|
||||
PYTHON = 'python',
|
||||
JAVA = 'java',
|
||||
BIBTEX = 'bibtex',
|
||||
CPP = 'cpp',
|
||||
PHP = 'php',
|
||||
RUBY = 'ruby',
|
||||
CSHARP = 'csharp',
|
||||
CSS = 'css',
|
||||
CSV = 'csv',
|
||||
CUDA = 'cuda',
|
||||
DART = 'dart',
|
||||
GO = 'go',
|
||||
HASKELL = 'haskell',
|
||||
HTML = 'html',
|
||||
JAVA = 'java',
|
||||
JAVASCRIPT = 'js',
|
||||
JSON = 'json',
|
||||
JSX = 'jsx',
|
||||
KOTLIN = 'kotlin',
|
||||
LATEX = 'latex',
|
||||
LOG = 'log',
|
||||
MARKDOWN = 'md',
|
||||
PHP = 'php',
|
||||
PLAIN_TEXT = 'plainText',
|
||||
PROPERTIES = 'properties',
|
||||
PYTHON = 'python',
|
||||
R = 'r',
|
||||
RUBY = 'ruby',
|
||||
RUST = 'rust',
|
||||
SCALA = 'scala',
|
||||
SHELL = 'shell',
|
||||
SQL = 'sql',
|
||||
R = 'r',
|
||||
SCALA = 'scala',
|
||||
KOTLIN = 'kotlin',
|
||||
SWIFT = 'swift',
|
||||
DART = 'dart',
|
||||
VUE = 'vue',
|
||||
SVELTE = 'svelte',
|
||||
LATEX = 'latex',
|
||||
BIBTEX = 'bibtex',
|
||||
CUDA = 'cuda',
|
||||
SWIFT = 'swift',
|
||||
TSX = 'tsx',
|
||||
TYPESCRIPT = 'ts',
|
||||
VUE = 'vue',
|
||||
VULKAN = 'vulkan',
|
||||
HASKELL = 'haskell',
|
||||
CSHARP = 'csharp',
|
||||
PROPERTIES = 'properties'
|
||||
XML = 'xml',
|
||||
YAML = 'yaml'
|
||||
}
|
||||
|
||||
// File extension enums
|
||||
export enum FileExtensionImage {
|
||||
JPG = '.jpg',
|
||||
JPEG = '.jpeg',
|
||||
PNG = '.png',
|
||||
GIF = '.gif',
|
||||
WEBP = '.webp',
|
||||
SVG = '.svg',
|
||||
HEIC = '.heic',
|
||||
HEIF = '.heif'
|
||||
HEIF = '.heif',
|
||||
JPEG = '.jpeg',
|
||||
JPG = '.jpg',
|
||||
PNG = '.png',
|
||||
SVG = '.svg',
|
||||
WEBP = '.webp'
|
||||
}
|
||||
|
||||
export enum FileExtensionAudio {
|
||||
@@ -112,64 +112,64 @@ export enum FileExtensionPdf {
|
||||
}
|
||||
|
||||
export enum FileExtensionText {
|
||||
TXT = '.txt',
|
||||
MD = '.md',
|
||||
ADOC = '.adoc',
|
||||
JS = '.js',
|
||||
TS = '.ts',
|
||||
JSX = '.jsx',
|
||||
TSX = '.tsx',
|
||||
BAT = '.bat',
|
||||
BIB = '.bib',
|
||||
C = '.c',
|
||||
COMP = '.comp',
|
||||
CPP = '.cpp',
|
||||
CS = '.cs',
|
||||
CSS = '.css',
|
||||
HTML = '.html',
|
||||
CSV = '.csv',
|
||||
CU = '.cu',
|
||||
CUH = '.cuh',
|
||||
DART = '.dart',
|
||||
GO = '.go',
|
||||
H = '.h',
|
||||
HPP = '.hpp',
|
||||
HS = '.hs',
|
||||
HTM = '.htm',
|
||||
HTML = '.html',
|
||||
JAVA = '.java',
|
||||
JS = '.js',
|
||||
JSON = '.json',
|
||||
JSONL = '.jsonl',
|
||||
ZIP = '.zip',
|
||||
JSX = '.jsx',
|
||||
KT = '.kt',
|
||||
LOG = '.log',
|
||||
MD = '.md',
|
||||
PHP = '.php',
|
||||
PROPERTIES = '.properties',
|
||||
PY = '.py',
|
||||
R = '.r',
|
||||
RB = '.rb',
|
||||
RS = '.rs',
|
||||
SCALA = '.scala',
|
||||
SH = '.sh',
|
||||
SQL = '.sql',
|
||||
SVELTE = '.svelte',
|
||||
SWIFT = '.swift',
|
||||
TEX = '.tex',
|
||||
TS = '.ts',
|
||||
TSX = '.tsx',
|
||||
TXT = '.txt',
|
||||
VUE = '.vue',
|
||||
XML = '.xml',
|
||||
YAML = '.yaml',
|
||||
YML = '.yml',
|
||||
CSV = '.csv',
|
||||
LOG = '.log',
|
||||
PY = '.py',
|
||||
JAVA = '.java',
|
||||
CPP = '.cpp',
|
||||
C = '.c',
|
||||
H = '.h',
|
||||
PHP = '.php',
|
||||
RB = '.rb',
|
||||
GO = '.go',
|
||||
RS = '.rs',
|
||||
SH = '.sh',
|
||||
BAT = '.bat',
|
||||
SQL = '.sql',
|
||||
R = '.r',
|
||||
SCALA = '.scala',
|
||||
KT = '.kt',
|
||||
SWIFT = '.swift',
|
||||
DART = '.dart',
|
||||
VUE = '.vue',
|
||||
SVELTE = '.svelte',
|
||||
TEX = '.tex',
|
||||
BIB = '.bib',
|
||||
CU = '.cu',
|
||||
CUH = '.cuh',
|
||||
COMP = '.comp',
|
||||
HPP = '.hpp',
|
||||
HS = '.hs',
|
||||
PROPERTIES = '.properties',
|
||||
CS = '.cs'
|
||||
ZIP = '.zip'
|
||||
}
|
||||
|
||||
// MIME type prefixes and includes for content detection
|
||||
export enum MimeTypePrefix {
|
||||
IMAGE = 'image/',
|
||||
AUDIO = 'audio/',
|
||||
IMAGE = 'image/',
|
||||
TEXT = 'text'
|
||||
}
|
||||
|
||||
export enum MimeTypeIncludes {
|
||||
JSON = 'json',
|
||||
JAVASCRIPT = 'javascript',
|
||||
JSON = 'json',
|
||||
TYPESCRIPT = 'typescript'
|
||||
}
|
||||
|
||||
@@ -182,23 +182,23 @@ export enum UriPattern {
|
||||
// MIME type enums
|
||||
export enum MimeTypeApplication {
|
||||
JSON = 'application/json',
|
||||
PDF = 'application/pdf',
|
||||
OCTET_STREAM = 'application/octet-stream',
|
||||
PDF = 'application/pdf',
|
||||
ZIP = 'application/zip'
|
||||
}
|
||||
|
||||
export enum MimeTypeAudio {
|
||||
MP3_MPEG = 'audio/mpeg',
|
||||
MP3 = 'audio/mp3',
|
||||
MP3_MPEG = 'audio/mpeg',
|
||||
MP4 = 'audio/mp4',
|
||||
VND_WAVE = 'audio/vnd.wave',
|
||||
WAV = 'audio/wav',
|
||||
WAVE = 'audio/wave',
|
||||
X_WAV = 'audio/x-wav',
|
||||
X_WAVE = 'audio/x-wave',
|
||||
VND_WAVE = 'audio/vnd.wave',
|
||||
X_PN_WAV = 'audio/x-pn-wav',
|
||||
WEBM = 'audio/webm',
|
||||
WEBM_OPUS = 'audio/webm;codecs=opus'
|
||||
WEBM_OPUS = 'audio/webm;codecs=opus',
|
||||
X_PN_WAV = 'audio/x-pn-wav',
|
||||
X_WAV = 'audio/x-wav',
|
||||
X_WAVE = 'audio/x-wave'
|
||||
}
|
||||
|
||||
export enum MimeTypeVideo {
|
||||
@@ -207,62 +207,62 @@ export enum MimeTypeVideo {
|
||||
}
|
||||
|
||||
export enum MimeTypeImage {
|
||||
GIF = 'image/gif',
|
||||
HEIC = 'image/heic',
|
||||
HEIF = 'image/heif',
|
||||
ICO = 'image/x-icon',
|
||||
ICO_MICROSOFT = 'image/vnd.microsoft.icon',
|
||||
JPEG = 'image/jpeg',
|
||||
JPG = 'image/jpg',
|
||||
PNG = 'image/png',
|
||||
GIF = 'image/gif',
|
||||
WEBP = 'image/webp',
|
||||
SVG = 'image/svg+xml',
|
||||
ICO = 'image/x-icon',
|
||||
ICO_MICROSOFT = 'image/vnd.microsoft.icon',
|
||||
HEIC = 'image/heic',
|
||||
HEIF = 'image/heif'
|
||||
WEBP = 'image/webp'
|
||||
}
|
||||
|
||||
export enum MimeTypeText {
|
||||
PLAIN = 'text/plain',
|
||||
MARKDOWN = 'text/markdown',
|
||||
ASCIIDOC = 'text/asciidoc',
|
||||
JAVASCRIPT = 'text/javascript',
|
||||
JAVASCRIPT_APP = 'application/javascript',
|
||||
TYPESCRIPT = 'text/typescript',
|
||||
JSX = 'text/jsx',
|
||||
TSX = 'text/tsx',
|
||||
CSS = 'text/css',
|
||||
HTML = 'text/html',
|
||||
JSON = 'application/json',
|
||||
JSONL = 'application/jsonl',
|
||||
XML_TEXT = 'text/xml',
|
||||
XML_APP = 'application/xml',
|
||||
YAML_TEXT = 'text/yaml',
|
||||
YAML_APP = 'application/yaml',
|
||||
CSV = 'text/csv',
|
||||
PYTHON = 'text/x-python',
|
||||
JAVA = 'text/x-java-source',
|
||||
BAT = 'application/x-bat',
|
||||
BIBTEX = 'text/x-bibtex',
|
||||
C_HDR = 'text/x-chdr',
|
||||
C_SRC = 'text/x-csrc',
|
||||
CPP_HDR = 'text/x-c++hdr',
|
||||
CPP_SRC = 'text/x-c++src',
|
||||
CSHARP = 'text/x-csharp',
|
||||
HASKELL = 'text/x-haskell',
|
||||
C_SRC = 'text/x-csrc',
|
||||
C_HDR = 'text/x-chdr',
|
||||
PHP = 'text/x-php',
|
||||
RUBY = 'text/x-ruby',
|
||||
GO = 'text/x-go',
|
||||
RUST = 'text/x-rust',
|
||||
SHELL = 'text/x-shellscript',
|
||||
BAT = 'application/x-bat',
|
||||
SQL = 'text/x-sql',
|
||||
R = 'text/x-r',
|
||||
SCALA = 'text/x-scala',
|
||||
KOTLIN = 'text/x-kotlin',
|
||||
SWIFT = 'text/x-swift',
|
||||
CSS = 'text/css',
|
||||
CSV = 'text/csv',
|
||||
CUDA = 'text/x-cuda',
|
||||
DART = 'text/x-dart',
|
||||
VUE = 'text/x-vue',
|
||||
GO = 'text/x-go',
|
||||
HASKELL = 'text/x-haskell',
|
||||
HTML = 'text/html',
|
||||
JAVA = 'text/x-java-source',
|
||||
JAVASCRIPT = 'text/javascript',
|
||||
JAVASCRIPT_APP = 'application/javascript',
|
||||
JSON = 'application/json',
|
||||
JSONL = 'application/jsonl',
|
||||
JSX = 'text/jsx',
|
||||
KOTLIN = 'text/x-kotlin',
|
||||
LATEX = 'application/x-latex',
|
||||
MARKDOWN = 'text/markdown',
|
||||
PHP = 'text/x-php',
|
||||
PLAIN = 'text/plain',
|
||||
PROPERTIES = 'text/properties',
|
||||
PYTHON = 'text/x-python',
|
||||
R = 'text/x-r',
|
||||
RUBY = 'text/x-ruby',
|
||||
RUST = 'text/x-rust',
|
||||
SCALA = 'text/x-scala',
|
||||
SHELL = 'text/x-shellscript',
|
||||
SQL = 'text/x-sql',
|
||||
SVELTE = 'text/x-svelte',
|
||||
SWIFT = 'text/x-swift',
|
||||
TEX = 'text/x-tex',
|
||||
TEX_APP = 'application/x-tex',
|
||||
LATEX = 'application/x-latex',
|
||||
BIBTEX = 'text/x-bibtex',
|
||||
CUDA = 'text/x-cuda',
|
||||
PROPERTIES = 'text/properties'
|
||||
TSX = 'text/tsx',
|
||||
TYPESCRIPT = 'text/typescript',
|
||||
VUE = 'text/x-vue',
|
||||
XML_APP = 'application/xml',
|
||||
XML_TEXT = 'text/xml',
|
||||
YAML_APP = 'application/yaml',
|
||||
YAML_TEXT = 'text/yaml'
|
||||
}
|
||||
|
||||
@@ -2,19 +2,19 @@
|
||||
* Keyboard key names for event handling
|
||||
*/
|
||||
export enum KeyboardKey {
|
||||
ENTER = 'Enter',
|
||||
ESCAPE = 'Escape',
|
||||
ARROW_UP = 'ArrowUp',
|
||||
ARROW_DOWN = 'ArrowDown',
|
||||
ARROW_LEFT = 'ArrowLeft',
|
||||
ARROW_RIGHT = 'ArrowRight',
|
||||
TAB = 'Tab',
|
||||
ARROW_UP = 'ArrowUp',
|
||||
B_LOWER = 'b',
|
||||
D_LOWER = 'd',
|
||||
D_UPPER = 'D',
|
||||
E_UPPER = 'E',
|
||||
ENTER = 'Enter',
|
||||
ESCAPE = 'Escape',
|
||||
K_LOWER = 'k',
|
||||
O_LOWER = 'o',
|
||||
O_UPPER = 'O',
|
||||
SPACE = ' '
|
||||
SPACE = ' ',
|
||||
TAB = 'Tab'
|
||||
}
|
||||
|
||||
@@ -2,61 +2,61 @@
|
||||
* Connection lifecycle phases for MCP protocol
|
||||
*/
|
||||
export enum MCPConnectionPhase {
|
||||
IDLE = 'idle',
|
||||
TRANSPORT_CREATING = 'transport_creating',
|
||||
TRANSPORT_READY = 'transport_ready',
|
||||
INITIALIZING = 'initializing',
|
||||
CAPABILITIES_EXCHANGED = 'capabilities_exchanged',
|
||||
LISTING_TOOLS = 'listing_tools',
|
||||
CONNECTED = 'connected',
|
||||
DISCONNECTED = 'disconnected',
|
||||
ERROR = 'error',
|
||||
DISCONNECTED = 'disconnected'
|
||||
IDLE = 'idle',
|
||||
INITIALIZING = 'initializing',
|
||||
LISTING_TOOLS = 'listing_tools',
|
||||
TRANSPORT_CREATING = 'transport_creating',
|
||||
TRANSPORT_READY = 'transport_ready'
|
||||
}
|
||||
|
||||
/**
|
||||
* Log level for connection events
|
||||
*/
|
||||
export enum MCPLogLevel {
|
||||
ERROR = 'error',
|
||||
INFO = 'info',
|
||||
WARN = 'warn',
|
||||
ERROR = 'error'
|
||||
WARN = 'warn'
|
||||
}
|
||||
|
||||
/**
|
||||
* Transport types for MCP connections
|
||||
*/
|
||||
export enum MCPTransportType {
|
||||
WEBSOCKET = 'websocket',
|
||||
SSE = 'sse',
|
||||
STREAMABLE_HTTP = 'streamable_http',
|
||||
SSE = 'sse'
|
||||
WEBSOCKET = 'websocket'
|
||||
}
|
||||
|
||||
/**
|
||||
* Health check status for MCP servers
|
||||
*/
|
||||
export enum HealthCheckStatus {
|
||||
IDLE = 'idle',
|
||||
CONNECTING = 'connecting',
|
||||
SUCCESS = 'success',
|
||||
ERROR = 'error'
|
||||
ERROR = 'error',
|
||||
IDLE = 'idle',
|
||||
SUCCESS = 'success'
|
||||
}
|
||||
|
||||
/**
|
||||
* Content types for MCP tool results
|
||||
*/
|
||||
export enum MCPContentType {
|
||||
TEXT = 'text',
|
||||
IMAGE = 'image',
|
||||
RESOURCE = 'resource'
|
||||
RESOURCE = 'resource',
|
||||
TEXT = 'text'
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON Schema types used in MCP tool definitions
|
||||
*/
|
||||
export enum JsonSchemaType {
|
||||
NUMBER = 'number',
|
||||
OBJECT = 'object',
|
||||
STRING = 'string',
|
||||
NUMBER = 'number'
|
||||
STRING = 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
export enum ModelModality {
|
||||
TEXT = 'TEXT',
|
||||
AUDIO = 'AUDIO',
|
||||
VISION = 'VISION',
|
||||
VIDEO = 'VIDEO'
|
||||
TEXT = 'TEXT',
|
||||
VIDEO = 'VIDEO',
|
||||
VISION = 'VISION'
|
||||
}
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
*/
|
||||
export enum ReasoningEffort {
|
||||
DEFAULT = 'default',
|
||||
OFF = 'off',
|
||||
LOW = 'low',
|
||||
MEDIUM = 'medium',
|
||||
HIGH = 'high',
|
||||
MAX = 'max'
|
||||
LOW = 'low',
|
||||
MAX = 'max',
|
||||
MEDIUM = 'medium',
|
||||
OFF = 'off'
|
||||
}
|
||||
|
||||
@@ -13,11 +13,11 @@ export enum ServerRole {
|
||||
* Used as the `value` field in the status object from /models endpoint
|
||||
*/
|
||||
export enum ServerModelStatus {
|
||||
UNLOADED = 'unloaded',
|
||||
LOADING = 'loading',
|
||||
FAILED = 'failed',
|
||||
LOADED = 'loaded',
|
||||
LOADING = 'loading',
|
||||
SLEEPING = 'sleeping',
|
||||
FAILED = 'failed'
|
||||
UNLOADED = 'unloaded'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -26,10 +26,10 @@ export enum ServerModelStatus {
|
||||
* tools/server/server-models.cpp from the C++ server.
|
||||
*/
|
||||
export enum ServerModelsSseEventType {
|
||||
STATUS_CHANGE = 'status_change',
|
||||
MODEL_STATUS = 'model_status',
|
||||
STATUS_UPDATE = 'status_update',
|
||||
MODELS_RELOAD = 'models_reload',
|
||||
DOWNLOAD_PROGRESS = 'download_progress',
|
||||
MODEL_REMOVE = 'model_remove',
|
||||
DOWNLOAD_PROGRESS = 'download_progress'
|
||||
MODEL_STATUS = 'model_status',
|
||||
MODELS_RELOAD = 'models_reload',
|
||||
STATUS_CHANGE = 'status_change',
|
||||
STATUS_UPDATE = 'status_update'
|
||||
}
|
||||
|
||||
@@ -2,26 +2,26 @@
|
||||
* Parameter source - indicates whether a parameter uses default or custom value
|
||||
*/
|
||||
export enum ParameterSource {
|
||||
DEFAULT = 'default',
|
||||
CUSTOM = 'custom'
|
||||
CUSTOM = 'custom',
|
||||
DEFAULT = 'default'
|
||||
}
|
||||
|
||||
/**
|
||||
* Syncable parameter type - data types for parameters that can be synced with server
|
||||
*/
|
||||
export enum SyncableParameterType {
|
||||
BOOLEAN = 'boolean',
|
||||
NUMBER = 'number',
|
||||
STRING = 'string',
|
||||
BOOLEAN = 'boolean'
|
||||
STRING = 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
* Settings field type - defines the input type for settings fields
|
||||
*/
|
||||
export enum SettingsFieldType {
|
||||
INPUT = 'input',
|
||||
TEXTAREA = 'textarea',
|
||||
CHECKBOX = 'checkbox',
|
||||
INPUT = 'input',
|
||||
RADIO = 'radio',
|
||||
SELECT = 'select',
|
||||
RADIO = 'radio'
|
||||
TEXTAREA = 'textarea'
|
||||
}
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
* Splash screen orientation for iOS apple-touch-startup-image
|
||||
*/
|
||||
export enum SplashOrientation {
|
||||
PORTRAIT = 'portrait',
|
||||
LANDSCAPE = 'landscape'
|
||||
LANDSCAPE = 'landscape',
|
||||
PORTRAIT = 'portrait'
|
||||
}
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
export enum ToolSource {
|
||||
BUILTIN = 'builtin',
|
||||
MCP = 'mcp',
|
||||
BROWSER = 'browser',
|
||||
CUSTOM = 'custom',
|
||||
FRONTEND = 'frontend'
|
||||
MCP = 'mcp',
|
||||
SERVER = 'server'
|
||||
}
|
||||
|
||||
export enum ToolPermissionDecision {
|
||||
ALWAYS = 'always',
|
||||
ALWAYS_SERVER = 'always_server',
|
||||
ONCE = 'once',
|
||||
DENY = 'deny'
|
||||
DENY = 'deny',
|
||||
ONCE = 'once'
|
||||
}
|
||||
|
||||
export enum ToolResponseField {
|
||||
PLAIN_TEXT = 'plain_text_response',
|
||||
ERROR = 'error'
|
||||
ERROR = 'error',
|
||||
PLAIN_TEXT = 'plain_text_response'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -22,28 +22,34 @@ export enum ToolResponseField {
|
||||
* Mirrors the server-side validation in server-tools.cpp.
|
||||
*/
|
||||
export enum GlobSearchType {
|
||||
FILE = 'file',
|
||||
ALL = 'all',
|
||||
DIR = 'dir',
|
||||
ALL = 'all'
|
||||
FILE = 'file'
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire-format identifiers for built-in and frontend tools. The string
|
||||
* Wire-format identifiers for server and browser tools. The string
|
||||
* value matches what the model emits in tool call names, so comparing
|
||||
* against `BuiltInTool.READ_FILE` is equivalent to comparing against the
|
||||
* raw `'read_file'` literal - the enum just keeps the two in lock-step
|
||||
* and gives TypeScript a single source of truth for autocomplete / rename
|
||||
* support.
|
||||
* against `BuiltInTool.SERVER_READ_FILE` is equivalent to comparing
|
||||
* against the raw `'read_file'` literal - the enum just keeps the two in
|
||||
* lock-step and gives TypeScript a single source of truth for autocomplete
|
||||
* / rename support.
|
||||
*
|
||||
* The `SERVER_` / `BROWSER_` prefixes mirror the tool's primary source
|
||||
* (llama-server vs llama-ui). `get_info` is the exception: it is served by
|
||||
* the server, but llama-ui falls back to a browser implementation when the
|
||||
* server does not provide it, so it can surface under both categories in
|
||||
* the UI while keeping a single wire name.
|
||||
*/
|
||||
export enum BuiltInTool {
|
||||
READ_FILE = 'read_file',
|
||||
READ_MEDIA = 'read_media',
|
||||
EDIT_FILE = 'edit_file',
|
||||
WRITE_FILE = 'write_file',
|
||||
GET_DATETIME = 'get_datetime',
|
||||
GET_INFO = 'get_info',
|
||||
FILE_GLOB_SEARCH = 'file_glob_search',
|
||||
GREP_SEARCH = 'grep_search',
|
||||
EXEC_SHELL_COMMAND = 'exec_shell_command',
|
||||
RUN_JAVASCRIPT = 'run_javascript'
|
||||
BROWSER_GET_DATETIME = 'get_datetime',
|
||||
BROWSER_READ_MEDIA = 'read_media',
|
||||
BROWSER_RUN_JAVASCRIPT = 'run_javascript',
|
||||
SERVER_EDIT_FILE = 'edit_file',
|
||||
SERVER_EXEC_SHELL_COMMAND = 'exec_shell_command',
|
||||
SERVER_FILE_GLOB_SEARCH = 'file_glob_search',
|
||||
SERVER_GET_INFO = 'get_info',
|
||||
SERVER_GREP_SEARCH = 'grep_search',
|
||||
SERVER_READ_FILE = 'read_file',
|
||||
SERVER_WRITE_FILE = 'write_file'
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
export enum ColorMode {
|
||||
LIGHT = 'light',
|
||||
DARK = 'dark',
|
||||
LIGHT = 'light',
|
||||
SYSTEM = 'system'
|
||||
}
|
||||
|
||||
export enum TooltipSide {
|
||||
TOP = 'top',
|
||||
RIGHT = 'right',
|
||||
BOTTOM = 'bottom',
|
||||
LEFT = 'left'
|
||||
LEFT = 'left',
|
||||
RIGHT = 'right',
|
||||
TOP = 'top'
|
||||
}
|
||||
|
||||
/**
|
||||
* MCP prompt display variant
|
||||
*/
|
||||
export enum McpPromptVariant {
|
||||
MESSAGE = 'message',
|
||||
ATTACHMENT = 'attachment'
|
||||
ATTACHMENT = 'attachment',
|
||||
MESSAGE = 'message'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -39,8 +39,8 @@ export enum HtmlInputType {
|
||||
* Alert level that drives the context gauge dial color.
|
||||
*/
|
||||
export enum ColorLevel {
|
||||
OK = 'ok',
|
||||
WARNING = 'warning',
|
||||
CRITICAL = 'critical',
|
||||
NEUTRAL = 'neutral'
|
||||
NEUTRAL = 'neutral',
|
||||
OK = 'ok',
|
||||
WARNING = 'warning'
|
||||
}
|
||||
|
||||
@@ -46,13 +46,13 @@ export function useToolsPanel(): UseToolsPanelReturn {
|
||||
|
||||
// Tools endpoint is unreachable (404) — server started without --tools
|
||||
if (toolsStore.isToolsEndpointUnreachable) {
|
||||
return `To enable Built-In Tools you need to run llama-server with ${CLI_FLAGS.TOOLS} all or ${CLI_FLAGS.TOOLS} <name> flag. To see MCP Tools you need to add / enable MCP Server(s).`;
|
||||
return `To enable Server Tools you need to run llama-server with ${CLI_FLAGS.TOOLS} all or ${CLI_FLAGS.TOOLS} <name> flag. To see MCP Tools you need to add / enable MCP Server(s).`;
|
||||
}
|
||||
|
||||
// Other errors — return null so UI shows "Failed to load tools"
|
||||
if (toolsStore.error) return null;
|
||||
|
||||
return `To enable Built-In Tools you need to run llama-server with ${CLI_FLAGS.TOOLS} all or ${CLI_FLAGS.TOOLS} <name> flag. To see MCP Tools you need to add / enable MCP Server(s).`;
|
||||
return `To enable Server Tools you need to run llama-server with ${CLI_FLAGS.TOOLS} all or ${CLI_FLAGS.TOOLS} <name> flag. To see MCP Tools you need to add / enable MCP Server(s).`;
|
||||
});
|
||||
|
||||
function isGroupChecked(group: ToolGroup): boolean {
|
||||
@@ -95,8 +95,8 @@ export function useToolsPanel(): UseToolsPanelReturn {
|
||||
}
|
||||
|
||||
function handleOpen(): void {
|
||||
if (toolsStore.builtinTools.length === 0 && !toolsStore.loading) {
|
||||
toolsStore.fetchBuiltinTools();
|
||||
if (toolsStore.serverTools.length === 0 && !toolsStore.loading) {
|
||||
toolsStore.fetchServerTools();
|
||||
}
|
||||
|
||||
mcpStore.runHealthChecksForServers(mcpStore.getServers().filter((s) => s.enabled));
|
||||
|
||||
@@ -262,9 +262,9 @@ export { ParameterSyncService } from './parameter-sync.service';
|
||||
export { MCPService } from './mcp.service';
|
||||
|
||||
/**
|
||||
* **SandboxService** - Frontend JavaScript execution in a browser sandbox
|
||||
* **SandboxService** - Browser JavaScript execution in a browser sandbox
|
||||
*
|
||||
* Stateless executor for the run_javascript frontend tool. Model generated
|
||||
* Stateless executor for the run_javascript browser tool. Model generated
|
||||
* code runs in a Web Worker spawned inside a sandboxed iframe with an opaque
|
||||
* origin: no access to the app origin, its storage or its API, and outgoing
|
||||
* requests carry a null origin. The code never touches a main thread, so the
|
||||
@@ -274,7 +274,7 @@ export { MCPService } from './mcp.service';
|
||||
* **Architecture & Relationships:**
|
||||
* - **SandboxService** (this class): Stateless sandbox execution
|
||||
* - **toolsStore**: Exposes the tool definition when the sandbox is enabled
|
||||
* - **agenticStore**: Dispatches ToolSource.FRONTEND calls here
|
||||
* - **agenticStore**: Dispatches ToolSource.BROWSER calls here
|
||||
*
|
||||
* @see buildSandboxToolDefinition in utils/sandbox-tool - tool schema sent to the LLM
|
||||
* @see agenticStore in stores/agentic.svelte.ts - tool dispatch
|
||||
|
||||
@@ -28,15 +28,15 @@ function fileExtension(path: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* **ReadMediaService** - frontend executor for the `read_media` tool
|
||||
* **ReadMediaService** - browser executor for the `read_media` tool
|
||||
*
|
||||
* The tool is synthetic: no such tool exists on the server. It reads the file
|
||||
* through the built-in `read_file` tool with the `base64` response type, then
|
||||
* through the server `read_file` tool with the `base64` response type, then
|
||||
* turns the bytes into a data URI line. The agentic store lifts that line into
|
||||
* an image or audio attachment on the tool result message, which is what makes
|
||||
* the model perceive the file instead of reading a wall of base64.
|
||||
*
|
||||
* Living in the frontend is what lets it exist only for models that can
|
||||
* Living in the browser is what lets it exist only for models that can
|
||||
* actually use the result - the server has no idea which model is selected.
|
||||
*
|
||||
* @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM
|
||||
@@ -82,7 +82,7 @@ export class ReadMediaService {
|
||||
}
|
||||
|
||||
const raw = await ToolsService.executeToolRaw(
|
||||
BuiltInTool.READ_FILE,
|
||||
BuiltInTool.SERVER_READ_FILE,
|
||||
{ path },
|
||||
signal,
|
||||
cwd,
|
||||
|
||||
@@ -68,7 +68,7 @@ function formatReply(reply: SandboxReply): ToolExecutionResult {
|
||||
|
||||
export class SandboxService {
|
||||
/**
|
||||
* Execute a frontend sandbox tool call and return its output.
|
||||
* Execute a browser sandbox tool call and return its output.
|
||||
* One disposable iframe per execution, removed on completion,
|
||||
* timeout or abort. Removing the iframe terminates the worker
|
||||
* at the browser level, so runaway code cannot outlive it.
|
||||
@@ -79,7 +79,7 @@ export class SandboxService {
|
||||
signal?: AbortSignal
|
||||
): Promise<ToolExecutionResult> {
|
||||
if (toolName !== SANDBOX_TOOL_NAME) {
|
||||
return { content: `Unknown frontend tool: ${toolName}`, isError: true };
|
||||
return { content: `Unknown browser tool: ${toolName}`, isError: true };
|
||||
}
|
||||
|
||||
const code = typeof params.code === 'string' ? params.code : '';
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
import { base } from '$app/paths';
|
||||
import { API_TOOLS, HEADERS } from '$lib/constants';
|
||||
import { ToolResponseField } from '$lib/enums';
|
||||
import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types';
|
||||
import type { ServerToolInfo, ToolExecutionResult } from '$lib/types';
|
||||
import { apiFetch } from '$lib/utils';
|
||||
import { getJsonHeaders } from '$lib/utils/api-headers';
|
||||
import { parseSseJsonStream, type SseJsonEvent } from '$lib/utils/sse';
|
||||
|
||||
export class ToolsService {
|
||||
/**
|
||||
* Fetch the list of built-in tools from the server.
|
||||
* Fetch the list of server tools from the server.
|
||||
*
|
||||
* @returns Array of tool definitions in OpenAI-compatible format
|
||||
*/
|
||||
static async list(): Promise<ServerBuiltinToolInfo[]> {
|
||||
return apiFetch<ServerBuiltinToolInfo[]>(API_TOOLS.LIST);
|
||||
static async list(): Promise<ServerToolInfo[]> {
|
||||
return apiFetch<ServerToolInfo[]>(API_TOOLS.LIST);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a built-in tool on the server.
|
||||
* Execute a server tool on the server.
|
||||
*
|
||||
* @param cwd - Working directory for the tool call, sent as the
|
||||
* x-tool-cwd request header. The server resolves relative paths
|
||||
@@ -48,7 +48,7 @@ export class ToolsService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a built-in tool and return the raw JSON response. Unlike
|
||||
* Execute a server tool and return the raw JSON response. Unlike
|
||||
* executeTool, this preserves structured fields (e.g. file_glob_search's
|
||||
* `entries` and `base`) that the flattened ToolExecutionResult drops.
|
||||
*
|
||||
@@ -77,7 +77,7 @@ export class ToolsService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a built-in tool's output chunks from the server. The server
|
||||
* Stream a server tool's output chunks from the server. The server
|
||||
* `POST /tools` endpoint with `{stream: true}` emits `data: {"chunk": "..."}`
|
||||
* events followed by a terminal `data: {"done": true}` (optionally with
|
||||
* `error`). Yields the chunk string for each partial event.
|
||||
|
||||
@@ -56,7 +56,8 @@ import type {
|
||||
AgenticSession,
|
||||
McpServerOverride,
|
||||
MCPToolCall,
|
||||
SettingsConfigType
|
||||
SettingsConfigType,
|
||||
ToolExecutionResult
|
||||
} from '$lib/types';
|
||||
import type {
|
||||
AgenticFlowCallbacks,
|
||||
@@ -83,7 +84,12 @@ import type {
|
||||
DatabaseMessageExtraAudioFile,
|
||||
DatabaseMessageExtraImageFile
|
||||
} from '$lib/types/database';
|
||||
import { getAudioInputFormat, isAbortError } from '$lib/utils';
|
||||
import {
|
||||
executeBrowserInfoTool,
|
||||
executeGetDatetimeTool,
|
||||
getAudioInputFormat,
|
||||
isAbortError
|
||||
} from '$lib/utils';
|
||||
import { SvelteMap } from 'svelte/reactivity';
|
||||
|
||||
function createDefaultSession(): AgenticSession {
|
||||
@@ -320,8 +326,8 @@ class AgenticStore {
|
||||
const maxTurns = Number(settings.agenticMaxTurns) || DEFAULT_AGENTIC_CONFIG.maxTurns;
|
||||
const hasTools =
|
||||
mcpStore.hasEnabledServers(perChatOverrides) ||
|
||||
toolsStore.builtinTools.length > 0 ||
|
||||
toolsStore.frontendTools.length > 0 ||
|
||||
toolsStore.serverTools.length > 0 ||
|
||||
toolsStore.browserTools.length > 0 ||
|
||||
toolsStore.customTools.length > 0;
|
||||
|
||||
return {
|
||||
@@ -449,9 +455,9 @@ class AgenticStore {
|
||||
this._continueResolvers.delete(conversationId);
|
||||
this._steeringMessages.delete(conversationId);
|
||||
|
||||
// Ensure built-in tools are fetched before checking if agentic is enabled
|
||||
if (toolsStore.builtinTools.length === 0 && !toolsStore.loading) {
|
||||
await toolsStore.fetchBuiltinTools();
|
||||
// Ensure server tools are fetched before checking if agentic is enabled
|
||||
if (toolsStore.serverTools.length === 0 && !toolsStore.loading) {
|
||||
await toolsStore.fetchServerTools();
|
||||
}
|
||||
|
||||
const agenticConfig = this.getConfig(settingsStore.config, perChatOverrides);
|
||||
@@ -900,8 +906,8 @@ class AgenticStore {
|
||||
} else {
|
||||
try {
|
||||
if (
|
||||
toolSource === ToolSource.BUILTIN &&
|
||||
toolName === BuiltInTool.EXEC_SHELL_COMMAND &&
|
||||
toolSource === ToolSource.SERVER &&
|
||||
toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND &&
|
||||
createToolResultMessage &&
|
||||
updateToolResultMessage
|
||||
) {
|
||||
@@ -932,7 +938,7 @@ class AgenticStore {
|
||||
}
|
||||
}
|
||||
result = accumulated;
|
||||
} else if (toolSource === ToolSource.BUILTIN) {
|
||||
} else if (toolSource === ToolSource.SERVER) {
|
||||
const args = this.parseToolArguments(toolCall.function.arguments);
|
||||
const cwd = conversationsStore.activeConversation?.cwd;
|
||||
const executionResult = await ToolsService.executeTool(toolName, args, signal, cwd);
|
||||
@@ -940,20 +946,28 @@ class AgenticStore {
|
||||
result = executionResult.content;
|
||||
|
||||
if (executionResult.isError) toolSuccess = false;
|
||||
} else if (toolSource === ToolSource.FRONTEND) {
|
||||
} else if (toolSource === ToolSource.BROWSER) {
|
||||
const args = this.parseToolArguments(toolCall.function.arguments);
|
||||
const executionResult =
|
||||
toolName === BuiltInTool.READ_MEDIA
|
||||
? await ReadMediaService.executeTool(
|
||||
args,
|
||||
{
|
||||
audio: modelsStore.modelSupportsAudio(effectiveModel),
|
||||
vision: modelsStore.modelSupportsVision(effectiveModel)
|
||||
},
|
||||
signal,
|
||||
conversationsStore.activeConversation?.cwd
|
||||
)
|
||||
: await SandboxService.executeTool(toolName, args, signal);
|
||||
|
||||
let executionResult: ToolExecutionResult;
|
||||
|
||||
if (toolName === BuiltInTool.BROWSER_GET_DATETIME) {
|
||||
executionResult = executeGetDatetimeTool();
|
||||
} else if (toolName === BuiltInTool.SERVER_GET_INFO) {
|
||||
executionResult = executeBrowserInfoTool();
|
||||
} else if (toolName === BuiltInTool.BROWSER_READ_MEDIA) {
|
||||
executionResult = await ReadMediaService.executeTool(
|
||||
args,
|
||||
{
|
||||
audio: modelsStore.modelSupportsAudio(effectiveModel),
|
||||
vision: modelsStore.modelSupportsVision(effectiveModel)
|
||||
},
|
||||
signal,
|
||||
conversationsStore.activeConversation?.cwd
|
||||
);
|
||||
} else {
|
||||
executionResult = await SandboxService.executeTool(toolName, args, signal);
|
||||
}
|
||||
|
||||
result = executionResult.content;
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { browser } from '$app/environment';
|
||||
import {
|
||||
buildBrowserInfoToolDefinition,
|
||||
buildGetDatetimeToolDefinition,
|
||||
buildReadMediaToolDefinition,
|
||||
DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
|
||||
HOME_TILDE,
|
||||
@@ -26,11 +28,11 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity';
|
||||
/** Stable selection identity for a tool, shared by the disabled set and the permission store */
|
||||
|
||||
class ToolsStore {
|
||||
private _builtinTools = $state<OpenAIToolDefinition[]>([]);
|
||||
private _serverTools = $state<OpenAIToolDefinition[]>([]);
|
||||
private _loading = $state(false);
|
||||
private _error = $state<string | null>(null);
|
||||
private _disabledTools = $state(new SvelteSet<string>());
|
||||
// builtin tools that resolve their paths against the working directory,
|
||||
// server tools that resolve their paths against the working directory,
|
||||
// as declared by the server in its `/tools` listing
|
||||
private _cwdAwareTools = $state(new SvelteSet<string>());
|
||||
private _toolsEndpointUnreachable = $state(false);
|
||||
@@ -56,7 +58,7 @@ class ToolsStore {
|
||||
console.error('[ToolsStore] Failed to load disabled tools from localStorage:', err);
|
||||
}
|
||||
|
||||
this.fetchBuiltinTools();
|
||||
this.fetchServerTools();
|
||||
}
|
||||
|
||||
private persistDisabledTools(): void {
|
||||
@@ -76,10 +78,10 @@ class ToolsStore {
|
||||
return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`;
|
||||
case ToolSource.CUSTOM:
|
||||
return `custom:${name}`;
|
||||
case ToolSource.FRONTEND:
|
||||
return `frontend:${name}`;
|
||||
case ToolSource.BROWSER:
|
||||
return `browser:${name}`;
|
||||
default:
|
||||
return `builtin:${name}`;
|
||||
return `server:${name}`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,8 +164,8 @@ class ToolsStore {
|
||||
};
|
||||
}
|
||||
|
||||
get builtinTools(): OpenAIToolDefinition[] {
|
||||
return this._builtinTools;
|
||||
get serverTools(): OpenAIToolDefinition[] {
|
||||
return this._serverTools;
|
||||
}
|
||||
|
||||
get serverHome(): string | null {
|
||||
@@ -174,8 +176,8 @@ class ToolsStore {
|
||||
return this.mcpEntries().map((e) => e.definition);
|
||||
}
|
||||
|
||||
get frontendTools(): OpenAIToolDefinition[] {
|
||||
const tools: OpenAIToolDefinition[] = [];
|
||||
get browserTools(): OpenAIToolDefinition[] {
|
||||
const tools: OpenAIToolDefinition[] = [buildGetDatetimeToolDefinition()];
|
||||
|
||||
if (settingsStore.config.jsSandboxEnabled) {
|
||||
tools.push(buildSandboxToolDefinition(!!settingsStore.config.symbolicMathEnabled));
|
||||
@@ -185,21 +187,26 @@ class ToolsStore {
|
||||
|
||||
if (readMedia) tools.push(readMedia);
|
||||
|
||||
// provide browser's get_info tool if server doesn't provide one
|
||||
if (!this.hasServerTool(BuiltInTool.SERVER_GET_INFO)) {
|
||||
tools.push(buildBrowserInfoToolDefinition());
|
||||
}
|
||||
|
||||
return tools;
|
||||
}
|
||||
|
||||
private hasServerTool(name: BuiltInTool): boolean {
|
||||
return this._serverTools.some((def) => def.function.name === name);
|
||||
}
|
||||
|
||||
/**
|
||||
* `read_media` runs in the frontend on top of the server's `read_file`, so it
|
||||
* `read_media` runs in the browser on top of the server's `read_file`, so it
|
||||
* exists only when that tool is served and the active model can perceive the
|
||||
* bytes. The server cannot make this call - it does not know which model the
|
||||
* conversation uses.
|
||||
*/
|
||||
private readMediaTool(): OpenAIToolDefinition | null {
|
||||
const hasReadFile = this._builtinTools.some(
|
||||
(def) => def.function.name === BuiltInTool.READ_FILE
|
||||
);
|
||||
|
||||
if (!hasReadFile) return null;
|
||||
if (!this.hasServerTool(BuiltInTool.SERVER_READ_FILE)) return null;
|
||||
|
||||
const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? '';
|
||||
|
||||
@@ -297,23 +304,23 @@ class ToolsStore {
|
||||
entries.push(entry);
|
||||
};
|
||||
|
||||
for (const def of this._builtinTools) {
|
||||
for (const def of this._serverTools) {
|
||||
const name = def.function.name;
|
||||
|
||||
push({
|
||||
definition: def,
|
||||
key: this.toolKey(ToolSource.BUILTIN, name),
|
||||
source: ToolSource.BUILTIN
|
||||
key: this.toolKey(ToolSource.SERVER, name),
|
||||
source: ToolSource.SERVER
|
||||
});
|
||||
}
|
||||
|
||||
for (const def of this.frontendTools) {
|
||||
for (const def of this.browserTools) {
|
||||
const name = def.function.name;
|
||||
|
||||
push({
|
||||
definition: def,
|
||||
key: this.toolKey(ToolSource.FRONTEND, name),
|
||||
source: ToolSource.FRONTEND
|
||||
key: this.toolKey(ToolSource.BROWSER, name),
|
||||
source: ToolSource.BROWSER
|
||||
});
|
||||
}
|
||||
|
||||
@@ -377,17 +384,17 @@ class ToolsStore {
|
||||
return entry.serverName ?? '';
|
||||
case ToolSource.CUSTOM:
|
||||
return TOOL_GROUP_LABELS[ToolSource.CUSTOM];
|
||||
case ToolSource.FRONTEND:
|
||||
return TOOL_GROUP_LABELS[ToolSource.FRONTEND];
|
||||
case ToolSource.BROWSER:
|
||||
return TOOL_GROUP_LABELS[ToolSource.BROWSER];
|
||||
default:
|
||||
return TOOL_GROUP_LABELS[ToolSource.BUILTIN];
|
||||
return TOOL_GROUP_LABELS[ToolSource.SERVER];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enabled tool definitions for sending to the LLM.
|
||||
* MCP tool schemas are normalized here so the wire payload is consistent
|
||||
* across all four sources (built-in, frontend/sandbox, MCP, custom JSON).
|
||||
* across all four sources (server, browser/sandbox, MCP, custom JSON).
|
||||
* The API identifies tools by name, so a name is sent at most once.
|
||||
*/
|
||||
getEnabledToolsForLLM(): OpenAIToolDefinition[] {
|
||||
@@ -410,8 +417,8 @@ class ToolsStore {
|
||||
result.push(def);
|
||||
};
|
||||
|
||||
for (const def of this._builtinTools) take(def);
|
||||
for (const def of this.frontendTools) take(def);
|
||||
for (const def of this._serverTools) take(def);
|
||||
for (const def of this.browserTools) take(def);
|
||||
// mcpEntries() over mcpStore directly so wire shape stays normalized and aligned with the tools UI.
|
||||
for (const entry of this.mcpEntries()) take(entry.definition);
|
||||
for (const def of this.customTools) take(def);
|
||||
@@ -535,11 +542,11 @@ class ToolsStore {
|
||||
|
||||
if (entry.serverName) return mcpStore.getServerDisplayName(entry.serverName);
|
||||
|
||||
if (entry.source === ToolSource.BUILTIN) return TOOL_SERVER_LABELS[ToolSource.BUILTIN];
|
||||
if (entry.source === ToolSource.SERVER) return TOOL_SERVER_LABELS[ToolSource.SERVER];
|
||||
|
||||
if (entry.source === ToolSource.CUSTOM) return TOOL_SERVER_LABELS[ToolSource.CUSTOM];
|
||||
|
||||
if (entry.source === ToolSource.FRONTEND) return TOOL_SERVER_LABELS[ToolSource.FRONTEND];
|
||||
if (entry.source === ToolSource.BROWSER) return TOOL_SERVER_LABELS[ToolSource.BROWSER];
|
||||
|
||||
return '';
|
||||
}
|
||||
@@ -549,27 +556,27 @@ class ToolsStore {
|
||||
return this.findEntryByName(toolName)?.key ?? null;
|
||||
}
|
||||
|
||||
/** Check if there are any enabled tools available (builtin, MCP, or custom) */
|
||||
/** Check if there are any enabled tools available (server, MCP, or custom) */
|
||||
get hasEnabledTools(): boolean {
|
||||
return this.getEnabledToolsForLLM().length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a working directory is worth setting: at least one builtin tool
|
||||
* Check if a working directory is worth setting: at least one server tool
|
||||
* that reads it is both served and left enabled by the user.
|
||||
*/
|
||||
get hasEnabledCwdTools(): boolean {
|
||||
return this._builtinTools.some((def) => {
|
||||
return this._serverTools.some((def) => {
|
||||
const name = def.function.name;
|
||||
|
||||
return (
|
||||
this._cwdAwareTools.has(name) &&
|
||||
!this._disabledTools.has(this.toolKey(ToolSource.BUILTIN, name))
|
||||
!this._disabledTools.has(this.toolKey(ToolSource.SERVER, name))
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async fetchBuiltinTools(): Promise<void> {
|
||||
async fetchServerTools(): Promise<void> {
|
||||
if (this._loading) return;
|
||||
|
||||
this._loading = true;
|
||||
@@ -579,7 +586,7 @@ class ToolsStore {
|
||||
try {
|
||||
const toolInfos = await ToolsService.list();
|
||||
|
||||
this._builtinTools = toolInfos.map((info) => info.definition);
|
||||
this._serverTools = toolInfos.map((info) => info.definition);
|
||||
this._cwdAwareTools = new SvelteSet(
|
||||
toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool)
|
||||
);
|
||||
@@ -592,9 +599,9 @@ class ToolsStore {
|
||||
// TODO: check status code instead of relying on message
|
||||
if (errorMessage.includes('this feature is disabled')) {
|
||||
this._toolsEndpointUnreachable = true;
|
||||
console.info('[ToolsStore] Built-in tools are disabled on the server');
|
||||
console.info('[ToolsStore] Server tools are disabled on the server');
|
||||
} else {
|
||||
console.error('[ToolsStore] Failed to fetch built-in tools:', err);
|
||||
console.error('[ToolsStore] Failed to fetch server tools:', err);
|
||||
}
|
||||
} finally {
|
||||
this._loading = false;
|
||||
@@ -611,7 +618,7 @@ class ToolsStore {
|
||||
if (this._serverHome !== undefined) return this._serverHome;
|
||||
|
||||
try {
|
||||
const res = await ToolsService.executeToolRaw(BuiltInTool.FILE_GLOB_SEARCH, {
|
||||
const res = await ToolsService.executeToolRaw(BuiltInTool.SERVER_FILE_GLOB_SEARCH, {
|
||||
limit: 1,
|
||||
max_depth: 1,
|
||||
path: HOME_TILDE,
|
||||
|
||||
@@ -147,7 +147,7 @@ export type {
|
||||
ServerStatus,
|
||||
ToolCallParams,
|
||||
ToolExecutionResult,
|
||||
ServerBuiltinToolInfo,
|
||||
ServerToolInfo,
|
||||
Tool,
|
||||
Prompt,
|
||||
GetPromptResult,
|
||||
@@ -208,7 +208,7 @@ export type {
|
||||
export type { DesktopIconStripItem } from './navigation';
|
||||
|
||||
// Tools types
|
||||
export type { ToolEntry, ToolGroup, BuiltinToolUiEntry } from './tools';
|
||||
export type { ToolEntry, ToolGroup, ToolUiEntry } from './tools';
|
||||
|
||||
// Reasoning
|
||||
export type { ReasoningEffortLevel } from './reasoning';
|
||||
|
||||
Vendored
+2
-2
@@ -285,10 +285,10 @@ export interface ToolExecutionResult {
|
||||
isError: boolean;
|
||||
}
|
||||
|
||||
export interface ServerBuiltinToolInfo {
|
||||
export interface ServerToolInfo {
|
||||
display_name: string;
|
||||
tool: string;
|
||||
type: ToolSource.BUILTIN;
|
||||
type: ToolSource.SERVER;
|
||||
permissions: {
|
||||
write: boolean;
|
||||
};
|
||||
|
||||
Vendored
+4
-4
@@ -3,12 +3,12 @@ import type { ToolSource } from '$lib/enums';
|
||||
import type { Component } from 'svelte';
|
||||
|
||||
/**
|
||||
* UI metadata for a built-in or frontend tool, keyed by its `BuiltInTool` id.
|
||||
* UI metadata for a server or browser tool, keyed by its `BuiltInTool` id.
|
||||
*/
|
||||
export interface BuiltinToolUiEntry {
|
||||
export interface ToolUiEntry {
|
||||
icon: Component;
|
||||
label: string;
|
||||
source: ToolSource.BUILTIN | ToolSource.FRONTEND;
|
||||
source: ToolSource.SERVER | ToolSource.BROWSER;
|
||||
}
|
||||
|
||||
export interface ToolEntry {
|
||||
@@ -17,7 +17,7 @@ export interface ToolEntry {
|
||||
serverName?: string;
|
||||
/** For MCP tools, the server ID (used for permission keys) */
|
||||
serverId?: string;
|
||||
/** Stable selection identity: builtin:name, mcp-<serverId>:name, mcp:name, custom:name */
|
||||
/** Stable selection identity: server:name, mcp-<serverId>:name, mcp:name, custom:name */
|
||||
key: string;
|
||||
definition: OpenAIToolDefinition;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Browser fallback for the server's `get_info` tool, offered only when the
|
||||
* server does not serve one (llama-server without --agent). It tells the model
|
||||
* which OS the browser runs on and that there is no local file or shell access,
|
||||
* so it does not plan around tools that are not there.
|
||||
*
|
||||
* @see server_tool_get_info in tools/server/server-tools.cpp - the served variant
|
||||
* @see buildBrowserInfoToolDefinition in constants/browser-info.ts - tool schema sent to the LLM
|
||||
*/
|
||||
|
||||
import { browser } from '$app/environment';
|
||||
import {
|
||||
BROWSER_INFO_NOTE,
|
||||
BROWSER_INFO_OS_UA_PATTERNS,
|
||||
BROWSER_INFO_OS_UNKNOWN
|
||||
} from '$lib/constants';
|
||||
import type { ToolExecutionResult } from '$lib/types';
|
||||
|
||||
function detectOs(userAgent: string): string {
|
||||
for (const [pattern, os] of BROWSER_INFO_OS_UA_PATTERNS) {
|
||||
if (pattern.test(userAgent)) return os;
|
||||
}
|
||||
|
||||
return BROWSER_INFO_OS_UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result shape mirrors the server tool's JSON so the `get_info` renderer reads
|
||||
* `os` the same way, minus `cwd` - there is no working directory to report.
|
||||
*/
|
||||
export function executeBrowserInfoTool(): ToolExecutionResult {
|
||||
return {
|
||||
content: JSON.stringify({
|
||||
note: BROWSER_INFO_NOTE,
|
||||
os: browser ? detectOs(navigator.userAgent) : BROWSER_INFO_OS_UNKNOWN
|
||||
}),
|
||||
isError: false
|
||||
};
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { BUILTIN_TOOL_UI } from '$lib/constants';
|
||||
import type { BuiltinToolUiEntry } from '$lib/types';
|
||||
|
||||
/**
|
||||
* Resolve the UI metadata (label + icon) for a built-in tool by its name.
|
||||
* Falls back to null for unknown or non-built-in tools so callers can render
|
||||
* a generic chrome instead.
|
||||
*/
|
||||
export function getBuiltinToolUi(toolName: string | undefined): BuiltinToolUiEntry | null {
|
||||
if (!toolName) return null;
|
||||
|
||||
return (BUILTIN_TOOL_UI as Record<string, BuiltinToolUiEntry>)[toolName] ?? null;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user