From afc575fbd840567acba259a33262fbbe983fa435 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sat, 23 Nov 2024 12:59:06 +0800 Subject: [PATCH] cleanup, try to add version tagging --- .github/workflows/kcpp-build-release-osx.yaml | 4 +- cmake/arm64-apple-clang.cmake | 16 - create_ver_file.bat | 33 + create_ver_file.sh | 11 + docs/backend/CANN.md | 259 ---- examples/Miku.sh | 50 - examples/base-translate.sh | 61 - examples/batched-bench/batched-bench.cpp | 204 --- examples/batched.swift/.gitignore | 9 - examples/batched.swift/Makefile | 6 - examples/batched.swift/Package.swift | 22 - examples/batched.swift/Sources/main.swift | 251 ---- examples/chat-13B.bat | 57 - examples/chat-13B.sh | 41 - examples/chat-persistent.sh | 149 -- examples/chat-vicuna.sh | 41 - examples/chat.sh | 16 - .../convert-llama2c-to-ggml.cpp | 939 ------------ examples/cvector-generator/completions.txt | 582 -------- .../cvector-generator/cvector-generator.cpp | 503 ------- examples/cvector-generator/mean.hpp | 48 - examples/cvector-generator/negative.txt | 4 - examples/cvector-generator/pca.hpp | 315 ---- examples/cvector-generator/positive.txt | 4 - .../deprecation-warning.cpp | 35 - examples/eval-callback/eval-callback.cpp | 193 --- examples/gritlm/gritlm.cpp | 226 --- examples/jeopardy/graph.py | 58 - examples/jeopardy/jeopardy.sh | 30 - examples/jeopardy/qasheet.csv | 103 -- examples/jeopardy/questions.txt | 100 -- examples/json_schema_pydantic_example.py | 82 - examples/json_schema_to_grammar.py | 811 ---------- examples/llama.android/.gitignore | 33 - examples/llama.android/app/.gitignore | 1 - examples/llama.android/app/build.gradle.kts | 65 - examples/llama.android/app/proguard-rules.pro | 21 - .../app/src/main/AndroidManifest.xml | 30 - .../java/com/example/llama/Downloadable.kt | 119 -- .../java/com/example/llama/MainActivity.kt | 154 -- .../java/com/example/llama/MainViewModel.kt | 105 -- .../java/com/example/llama/ui/theme/Color.kt | 11 - .../java/com/example/llama/ui/theme/Theme.kt | 70 - .../java/com/example/llama/ui/theme/Type.kt | 34 - .../res/drawable/ic_launcher_background.xml | 170 --- .../res/drawable/ic_launcher_foreground.xml | 30 - .../main/res/mipmap-anydpi/ic_launcher.xml | 6 - .../res/mipmap-anydpi/ic_launcher_round.xml | 6 - .../src/main/res/mipmap-hdpi/ic_launcher.webp | Bin 1404 -> 0 bytes .../res/mipmap-hdpi/ic_launcher_round.webp | Bin 2898 -> 0 bytes .../src/main/res/mipmap-mdpi/ic_launcher.webp | Bin 982 -> 0 bytes .../res/mipmap-mdpi/ic_launcher_round.webp | Bin 1772 -> 0 bytes .../main/res/mipmap-xhdpi/ic_launcher.webp | Bin 1900 -> 0 bytes .../res/mipmap-xhdpi/ic_launcher_round.webp | Bin 3918 -> 0 bytes .../main/res/mipmap-xxhdpi/ic_launcher.webp | Bin 2884 -> 0 bytes .../res/mipmap-xxhdpi/ic_launcher_round.webp | Bin 5914 -> 0 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.webp | Bin 3844 -> 0 bytes .../res/mipmap-xxxhdpi/ic_launcher_round.webp | Bin 7778 -> 0 bytes .../app/src/main/res/values/colors.xml | 10 - .../app/src/main/res/values/strings.xml | 3 - .../app/src/main/res/values/themes.xml | 5 - .../app/src/main/res/xml/backup_rules.xml | 13 - .../main/res/xml/data_extraction_rules.xml | 19 - examples/llama.android/build.gradle.kts | 6 - examples/llama.android/gradle.properties | 23 - .../gradle/wrapper/gradle-wrapper.jar | Bin 59203 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 - examples/llama.android/gradlew | 185 --- examples/llama.android/llama/.gitignore | 1 - examples/llama.android/llama/build.gradle.kts | 69 - .../llama.android/llama/consumer-rules.pro | 0 .../llama.android/llama/proguard-rules.pro | 21 - .../llama/cpp/ExampleInstrumentedTest.kt | 24 - .../llama/src/main/AndroidManifest.xml | 4 - .../llama/src/main/cpp/llama-android.cpp | 447 ------ .../java/android/llama/cpp/LLamaAndroid.kt | 179 --- .../java/android/llama/cpp/ExampleUnitTest.kt | 17 - examples/llama.android/settings.gradle.kts | 18 - examples/llama.swiftui/.gitignore | 2 - .../llama.cpp.swift/LibLlama.swift | 335 ----- .../llama.swiftui.xcodeproj/project.pbxproj | 439 ------ .../contents.xcworkspacedata | 7 - .../xcshareddata/IDEWorkspaceChecks.plist | 8 - .../AppIcon.appiconset/Contents.json | 13 - .../Assets.xcassets/Contents.json | 6 - .../llama.swiftui/Models/LlamaState.swift | 196 --- .../llama.swiftui/Resources/models/.gitignore | 0 .../llama.swiftui/UI/ContentView.swift | 145 -- .../llama.swiftui/UI/DownloadButton.swift | 124 -- .../llama.swiftui/UI/InputButton.swift | 131 -- .../llama.swiftui/UI/LoadCustomButton.swift | 44 - .../llama.swiftui/llama_swiftuiApp.swift | 10 - examples/llama.vim | 783 ---------- examples/llm.vim | 28 - examples/lookahead/lookahead.cpp | 485 ------ examples/lookup/lookup-create.cpp | 45 - examples/lookup/lookup-merge.cpp | 47 - examples/lookup/lookup-stats.cpp | 160 -- examples/lookup/lookup.cpp | 254 ---- examples/main-cmake-pkg/.gitignore | 50 - examples/passkey/passkey.cpp | 274 ---- examples/pydantic_models_to_grammar.py | 1322 ----------------- .../pydantic_models_to_grammar_examples.py | 312 ---- examples/reason-act.sh | 16 - examples/regex_to_grammar.py | 20 - examples/retrieval/retrieval.cpp | 304 ---- examples/rpc/rpc-server.cpp | 159 -- examples/server-llama2-13B.sh | 26 - examples/server_embd.py | 35 - examples/simple-chat/CMakeLists.txt | 5 - examples/simple-chat/README.md | 7 - examples/simple-chat/simple-chat.cpp | 197 --- examples/simple/simple.cpp | 201 --- examples/sycl/build.sh | 23 - examples/sycl/ls-sycl-device.cpp | 13 - examples/sycl/run-llama2.sh | 27 - examples/sycl/win-build-sycl.bat | 33 - examples/sycl/win-run-llama2.bat | 9 - examples/tokenize/tokenize.cpp | 404 ----- examples/ts-type-to-grammar.sh | 28 - koboldcpp.sh | 8 +- make_pyinstaller.bat | 3 +- make_pyinstaller.sh | 4 +- make_pyinstaller_cuda.bat | 3 +- make_pyinstaller_cuda12.bat | 3 +- make_pyinstaller_cuda_oldcpu.bat | 3 +- models/ggml-vocab-chameleon.gguf.inp | 112 -- models/ggml-vocab-chameleon.gguf.out | 46 - requirements/requirements-all.txt | 12 - .../requirements-compare-llama-bench.txt | 2 - .../requirements-convert_lora_to_gguf.txt | 2 - requirements/requirements-pydantic.txt | 3 - .../requirements-test-tokenizer-random.txt | 1 - spm-headers/ggml-cpp.h | 1 - spm-headers/ggml-cpu.h | 1 - tests/test-arg-parser.cpp | 131 -- tests/test-barrier.cpp | 94 -- tests/test-log.cpp | 39 - tests/test-lora-conversion-inference.sh | 139 -- version.txt | 30 + version_template.txt | 30 + 141 files changed, 123 insertions(+), 14439 deletions(-) delete mode 100644 cmake/arm64-apple-clang.cmake create mode 100644 create_ver_file.bat create mode 100644 create_ver_file.sh delete mode 100644 docs/backend/CANN.md delete mode 100755 examples/Miku.sh delete mode 100755 examples/base-translate.sh delete mode 100644 examples/batched-bench/batched-bench.cpp delete mode 100644 examples/batched.swift/.gitignore delete mode 100755 examples/batched.swift/Makefile delete mode 100644 examples/batched.swift/Package.swift delete mode 100644 examples/batched.swift/Sources/main.swift delete mode 100644 examples/chat-13B.bat delete mode 100755 examples/chat-13B.sh delete mode 100755 examples/chat-persistent.sh delete mode 100755 examples/chat-vicuna.sh delete mode 100755 examples/chat.sh delete mode 100644 examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp delete mode 100644 examples/cvector-generator/completions.txt delete mode 100644 examples/cvector-generator/cvector-generator.cpp delete mode 100644 examples/cvector-generator/mean.hpp delete mode 100644 examples/cvector-generator/negative.txt delete mode 100644 examples/cvector-generator/pca.hpp delete mode 100644 examples/cvector-generator/positive.txt delete mode 100644 examples/deprecation-warning/deprecation-warning.cpp delete mode 100644 examples/eval-callback/eval-callback.cpp delete mode 100644 examples/gritlm/gritlm.cpp delete mode 100755 examples/jeopardy/graph.py delete mode 100755 examples/jeopardy/jeopardy.sh delete mode 100644 examples/jeopardy/qasheet.csv delete mode 100644 examples/jeopardy/questions.txt delete mode 100644 examples/json_schema_pydantic_example.py delete mode 100755 examples/json_schema_to_grammar.py delete mode 100644 examples/llama.android/.gitignore delete mode 100644 examples/llama.android/app/.gitignore delete mode 100644 examples/llama.android/app/build.gradle.kts delete mode 100644 examples/llama.android/app/proguard-rules.pro delete mode 100644 examples/llama.android/app/src/main/AndroidManifest.xml delete mode 100644 examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt delete mode 100644 examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt delete mode 100644 examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt delete mode 100644 examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt delete mode 100644 examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt delete mode 100644 examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt delete mode 100644 examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml delete mode 100644 examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml delete mode 100644 examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml delete mode 100644 examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml delete mode 100644 examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp delete mode 100644 examples/llama.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp delete mode 100644 examples/llama.android/app/src/main/res/values/colors.xml delete mode 100644 examples/llama.android/app/src/main/res/values/strings.xml delete mode 100644 examples/llama.android/app/src/main/res/values/themes.xml delete mode 100644 examples/llama.android/app/src/main/res/xml/backup_rules.xml delete mode 100644 examples/llama.android/app/src/main/res/xml/data_extraction_rules.xml delete mode 100644 examples/llama.android/build.gradle.kts delete mode 100644 examples/llama.android/gradle.properties delete mode 100644 examples/llama.android/gradle/wrapper/gradle-wrapper.jar delete mode 100644 examples/llama.android/gradle/wrapper/gradle-wrapper.properties delete mode 100755 examples/llama.android/gradlew delete mode 100644 examples/llama.android/llama/.gitignore delete mode 100644 examples/llama.android/llama/build.gradle.kts delete mode 100644 examples/llama.android/llama/consumer-rules.pro delete mode 100644 examples/llama.android/llama/proguard-rules.pro delete mode 100644 examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt delete mode 100644 examples/llama.android/llama/src/main/AndroidManifest.xml delete mode 100644 examples/llama.android/llama/src/main/cpp/llama-android.cpp delete mode 100644 examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt delete mode 100644 examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt delete mode 100644 examples/llama.android/settings.gradle.kts delete mode 100644 examples/llama.swiftui/.gitignore delete mode 100644 examples/llama.swiftui/llama.cpp.swift/LibLlama.swift delete mode 100644 examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj delete mode 100644 examples/llama.swiftui/llama.swiftui.xcodeproj/project.xcworkspace/contents.xcworkspacedata delete mode 100644 examples/llama.swiftui/llama.swiftui.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist delete mode 100644 examples/llama.swiftui/llama.swiftui/Assets.xcassets/AppIcon.appiconset/Contents.json delete mode 100644 examples/llama.swiftui/llama.swiftui/Assets.xcassets/Contents.json delete mode 100644 examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift delete mode 100644 examples/llama.swiftui/llama.swiftui/Resources/models/.gitignore delete mode 100644 examples/llama.swiftui/llama.swiftui/UI/ContentView.swift delete mode 100644 examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift delete mode 100644 examples/llama.swiftui/llama.swiftui/UI/InputButton.swift delete mode 100644 examples/llama.swiftui/llama.swiftui/UI/LoadCustomButton.swift delete mode 100644 examples/llama.swiftui/llama.swiftui/llama_swiftuiApp.swift delete mode 100644 examples/llama.vim delete mode 100644 examples/llm.vim delete mode 100644 examples/lookahead/lookahead.cpp delete mode 100644 examples/lookup/lookup-create.cpp delete mode 100644 examples/lookup/lookup-merge.cpp delete mode 100644 examples/lookup/lookup-stats.cpp delete mode 100644 examples/lookup/lookup.cpp delete mode 100644 examples/main-cmake-pkg/.gitignore delete mode 100644 examples/passkey/passkey.cpp delete mode 100644 examples/pydantic_models_to_grammar.py delete mode 100755 examples/pydantic_models_to_grammar_examples.py delete mode 100755 examples/reason-act.sh delete mode 100644 examples/regex_to_grammar.py delete mode 100644 examples/retrieval/retrieval.cpp delete mode 100644 examples/rpc/rpc-server.cpp delete mode 100755 examples/server-llama2-13B.sh delete mode 100644 examples/server_embd.py delete mode 100644 examples/simple-chat/CMakeLists.txt delete mode 100644 examples/simple-chat/README.md delete mode 100644 examples/simple-chat/simple-chat.cpp delete mode 100644 examples/simple/simple.cpp delete mode 100755 examples/sycl/build.sh delete mode 100644 examples/sycl/ls-sycl-device.cpp delete mode 100755 examples/sycl/run-llama2.sh delete mode 100644 examples/sycl/win-build-sycl.bat delete mode 100644 examples/sycl/win-run-llama2.bat delete mode 100644 examples/tokenize/tokenize.cpp delete mode 100755 examples/ts-type-to-grammar.sh delete mode 100644 models/ggml-vocab-chameleon.gguf.inp delete mode 100644 models/ggml-vocab-chameleon.gguf.out delete mode 100644 requirements/requirements-all.txt delete mode 100644 requirements/requirements-compare-llama-bench.txt delete mode 100644 requirements/requirements-convert_lora_to_gguf.txt delete mode 100644 requirements/requirements-pydantic.txt delete mode 100644 requirements/requirements-test-tokenizer-random.txt delete mode 120000 spm-headers/ggml-cpp.h delete mode 120000 spm-headers/ggml-cpu.h delete mode 100644 tests/test-arg-parser.cpp delete mode 100644 tests/test-barrier.cpp delete mode 100644 tests/test-log.cpp delete mode 100755 tests/test-lora-conversion-inference.sh create mode 100644 version.txt create mode 100644 version_template.txt diff --git a/.github/workflows/kcpp-build-release-osx.yaml b/.github/workflows/kcpp-build-release-osx.yaml index 29b2676aa..a73462f02 100644 --- a/.github/workflows/kcpp-build-release-osx.yaml +++ b/.github/workflows/kcpp-build-release-osx.yaml @@ -24,7 +24,9 @@ jobs: id: make_build run: | make LLAMA_METAL=1 LLAMA_PORTABLE=1 - pyinstaller --noconfirm --onefile --collect-all customtkinter --collect-all psutil --add-data './koboldcpp_default.so:.' --add-data './ggml-metal-merged.metal:.' --add-data './kcpp_adapters:./kcpp_adapters' --add-data './klite.embd:.' --add-data './kcpp_docs.embd:.' --add-data './kcpp_sdui.embd:.' --add-data './taesd.embd:.' --add-data './taesd_xl.embd:.' --add-data './rwkv_vocab.embd:.' --add-data './rwkv_world_vocab.embd:.' --clean --console koboldcpp.py -n "koboldcpp-mac-arm64" + chmod +x './create_ver_file.sh' + . create_ver_file.sh + pyinstaller --noconfirm --onefile --collect-all customtkinter --collect-all psutil --add-data './koboldcpp_default.so:.' --add-data './ggml-metal-merged.metal:.' --add-data './kcpp_adapters:./kcpp_adapters' --add-data './klite.embd:.' --add-data './kcpp_docs.embd:.' --add-data './kcpp_sdui.embd:.' --add-data './taesd.embd:.' --add-data './taesd_xl.embd:.' --add-data './rwkv_vocab.embd:.' --add-data './rwkv_world_vocab.embd:.' --version-file './version.txt' --clean --console koboldcpp.py -n "koboldcpp-mac-arm64" - name: Test id: test diff --git a/cmake/arm64-apple-clang.cmake b/cmake/arm64-apple-clang.cmake deleted file mode 100644 index 5fcd2882a..000000000 --- a/cmake/arm64-apple-clang.cmake +++ /dev/null @@ -1,16 +0,0 @@ -set( CMAKE_SYSTEM_NAME Darwin ) -set( CMAKE_SYSTEM_PROCESSOR arm64 ) - -set( target arm64-apple-darwin-macho ) - -set( CMAKE_C_COMPILER clang ) -set( CMAKE_CXX_COMPILER clang++ ) - -set( CMAKE_C_COMPILER_TARGET ${target} ) -set( CMAKE_CXX_COMPILER_TARGET ${target} ) - -set( arch_c_flags "-march=armv8.4-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) -set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function" ) - -set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) -set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/create_ver_file.bat b/create_ver_file.bat new file mode 100644 index 000000000..a0dd7af39 --- /dev/null +++ b/create_ver_file.bat @@ -0,0 +1,33 @@ +@echo off +setlocal enabledelayedexpansion +echo Create Version File +:: Read the version string from koboldcpp.py +for /f "tokens=2 delims== " %%A in ('findstr "KcppVersion" koboldcpp.py') do ( + set "version=%%~A" + goto :done +) + +:done + +:: Display the extracted version (optional, for debugging) +echo Extracted Version: %version% + +for /f "tokens=1,2 delims=." %%a in ("%version%") do ( + set version_major=%%a + set version_minor=%%b +) + +echo Major Version: %version_major% +echo Minor Version: %version_minor% + +:: Replace all instances of "MYVER" in foo.txt with the version +( + for /f "delims=" %%i in (version_template.txt) do ( + set "line=%%i" + set "line=!line:MYVER_MAJOR=%version_major%!" + set "line=!line:MYVER_MINOR=%version_minor%!" + echo !line! + ) +) > "version.txt" + +endlocal \ No newline at end of file diff --git a/create_ver_file.sh b/create_ver_file.sh new file mode 100644 index 000000000..d0ba5d52d --- /dev/null +++ b/create_ver_file.sh @@ -0,0 +1,11 @@ +#!/bin/bash +echo "Create Version File" +extracted_ver=$(cat koboldcpp.py | grep 'KcppVersion = ' | cut -d '"' -f2) +echo "Extracted Version: $extracted_ver" +vmajor=$(echo $extracted_ver | cut -d '.' -f1) +vminor=$(echo $extracted_ver | cut -d '.' -f2) +echo "Major Version: $vmajor" +echo "Minor Version: $vminor" +cp version_template.txt version.txt +sed -i "s/MYVER_MAJOR/$vmajor/g" version.txt +sed -i "s/MYVER_MINOR/$vminor/g" version.txt \ No newline at end of file diff --git a/docs/backend/CANN.md b/docs/backend/CANN.md deleted file mode 100644 index 6bdd9d2da..000000000 --- a/docs/backend/CANN.md +++ /dev/null @@ -1,259 +0,0 @@ -# llama.cpp for CANN - - - [Background](#background) - - [News](#news) - - [OS](#os) - - [Hardware](#hardware) - - [Model Supports](#model-supports) - - [DataType Supports](#datatype-supports) - - [Docker](#docker) - - [Linux](#linux) - - [TODO](#todo) - - -## Background - -**Ascend NPU** is a range of AI processors using Neural Processing Unit. It will efficiently handle matrix-matrix multiplication, dot-product and scalars. - -**CANN** (Compute Architecture for Neural Networks) is a heterogeneous computing architecture for AI scenarios, providing support for multiple AI frameworks on the top and serving AI processors and programming at the bottom. It plays a crucial role in bridging the gap between upper and lower layers, and is a key platform for improving the computing efficiency of Ascend AI processors. Meanwhile, it offers a highly efficient and easy-to-use programming interface for diverse application scenarios, allowing users to rapidly build AI applications and services based on the Ascend platform. - -**Llama.cpp + CANN** - -The llama.cpp CANN backend is designed to support Ascend NPU. It utilize the ability of AscendC and ACLNN which are intergrated to CANN Toolkit and kernels to using Ascend NPU directly. - -## News - -- 2024.8 - - Support `Q4_0` and `Q8_0` data type for Ascend NPU. -- 2024.7 - - Create CANN backend for Ascend NPU. - -## OS - -| OS | Status | Verified | -|:-------:|:-------:|:----------------------------------------------:| -| Linux | Support | Ubuntu 22.04, OpenEuler22.03 | - - -## Hardware - -### Ascend NPU - -**Verified devices** -| Ascend NPU | Status | -|:-----------------------------:|:-------:| -| Atlas 300T A2 | Support | - -*Notes:* - -- If you have trouble with Ascend NPU device, please create a issue with **[CANN]** prefix/tag. -- If you run successfully with your Ascend NPU device, please help update the upper table. - - -## Model Supports - -| Model Name | FP16 | Q8_0 | Q4_0 | -|:----------------------------|:-----:|:----:|:----:| -| AquilaChat2-7B | √ | √ | √ | -| Baichuan-7b | √ | √ | √ | -| Baichuan2-7B-Chat | √ | √ | √ | -| bitnet_b1_58-large | √ | √ | √ | -| bloom-560m | √ | x | √ | -| bloomz-alpaca-560m | √ | x | √ | -| c4ai-command-r-35B-v01 | x | x | x | -| chatglm3-6B | x | x | x | -| chinese-alpaca-2-1.3b | √ | √ | √ | -| CodeShell-7B | √ | √ | √ | -| deepseek-ai_deepseek-coder-1.3B-base | x | x | x | -| deepseek-ai_DeepSeek-V2-Lite | x | x | x | -| deepseek-coder-6.7B-instruct | x | x | x | -| DeepSeek-V2-Lite-64x1.5B | x | x | x | -| falcon-7b-instruct | √ | √ | √ | -| flan-t5-large | √ | √ | √ | -| gemma-2-9b-it | √ | √ | √ | -| glm-4-9B | x | x | x | -| gpt2 | √ | √ | √ | -| Gpt2-163M | √ | √ | √ | -| granite-3B-code-instruct | √ | √ | √ | -| GritLM-7B | √ | √ | √ | -| internlm2_5-7b-chat | √ | √ | √ | -| koala-7B-HF | √ | √ | √ | -| Llama-2-7b-chat-hf | √ | √ | √ | -| Llama-3-Smaug-8B | √ | √ | √ | -| Llama2-Chinese-7b-Chat | √ | √ | √ | -| Llama3-8B | √ | √ | √ | -| Llama3-8b-chinese | √ | √ | √ | -| mamba-130m-hf | √ | √ | √ | -| Mistral-7B-Instruct-v0.2 | √ | √ | √ | -| Mixtral-8x7B-Instruct-v0.1 | x | √ | √ | -| mpt-7B | √ | √ | √ | -| OLMo-1B-hf | √ | √ | √ | -| OpenELM-3B-Instruct | √ | √ | √ | -| Orion-14b-base | √ | √ | √ | -| phi1 | x | x | x | -| phi2 | x | x | x | -| Phi-3-mini-4k-instruct | √ | √ | √ | -| plamo-13b | √ | √ | √ | -| pythia-70M | x | x | x | -| Qwen-7B | √ | √ | √ | -| Qwen2-1.5B-Instruct | √ | x | √ | -| Refact-1_6B-fim | √ | √ | √ | -| SmolLM-135M | √ | √ | √ | -| stablelm-zephyr | x | x | x | -| stablelm-2-zephyr-1_6b | x | x | x | -| starcoderbase-1b | √ | √ | √ | -| starcoder2-3b | √ | √ | √ | -| vigogne-7b-chat | √ | √ | √ | -| xverse-7b-chat | √ | √ | √ | -| Yi-6b-Chat | √ | √ | √ | - - - -## DataType Supports - -| DataType | Status | -|:----------------------:|:-------:| -| FP16 | Support | -| Q8_0 | Support | -| Q4_0 | Support | - -## Docker - -### Build Images -You can get a image with llama.cpp in one command. -```sh -docker build -t llama-cpp-cann -f .devops/llama-cli-cann.Dockerfile . -``` - -### Run container - -```sh -# Find all cards. -npu-smi info - -# Select the cards that you want to use, make sure these cards are not used by someone. -# Following using cards of device0. -docker run --name llamacpp --device /dev/davinci0 --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc -v /usr/local/dcmi:/usr/local/dcmi -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info -v /PATH_TO_YOUR_MODELS/:/app/models -it llama-cpp-cann -m /app/models/MODEL_PATH -ngl 32 -p "Building a website can be done in 10 simple steps:" -``` - -*Notes:* - -- You may need to install Ascend Driver and firmware on the **host** machine *(Please refer to the [Linux configuration](#linux) for details)*. - -## Linux - -### I. Setup Environment - -1. **Install Ascend Driver and firmware** - - ```sh - # create driver running user. - sudo groupadd -g HwHiAiUser - sudo useradd -g HwHiAiUser -d /home/HwHiAiUser -m HwHiAiUser -s /bin/bash - sudo usermod -aG HwHiAiUser $USER - - # download driver from https://www.hiascend.com/hardware/firmware-drivers/community according to your system - # and install driver. - sudo sh Ascend-hdk-910b-npu-driver_x.x.x_linux-{arch}.run --full --install-for-all - ``` - - Once installed, run `npu-smi info` to check whether driver is installed successfully. - ```sh - +-------------------------------------------------------------------------------------------+ - | npu-smi 24.1.rc2 Version: 24.1.rc2 | - +----------------------+---------------+----------------------------------------------------+ - | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page)| - | Chip | Bus-Id | AICore(%) Memory-Usage(MB) HBM-Usage(MB) | - +======================+===============+====================================================+ - | 2 xxx | OK | 64.4 51 15 / 15 | - | 0 | 0000:01:00.0 | 0 1873 / 15077 0 / 32768 | - +======================+===============+====================================================+ - | 5 xxx | OK | 64.0 52 15 / 15 | - | 0 | 0000:81:00.0 | 0 1874 / 15077 0 / 32768 | - +======================+===============+====================================================+ - | No running processes found in NPU 2 | - +======================+===============+====================================================+ - | No running processes found in NPU 5 | - +======================+===============+====================================================+ - ``` - -2. **Install Ascend Firmware** - ```sh - # download driver from https://www.hiascend.com/hardware/firmware-drivers/community according to your system - # and install driver. - sudo sh Ascend-hdk-910b-npu-firmware_x.x.x.x.X.run --full - ``` - If the following messaage appers, firmware is installed successfully. - ```sh - Firmware package installed successfully! - ``` - - -3. **Install CANN toolkit and kernels** - - CANN toolkit and kernels can be obtained from the official [CANN Toolkit](https://www.hiascend.com/zh/developer/download/community/result?module=cann) page. - - Please download the corresponding version that satified your system. The minimum version required is 8.0.RC2.alpha002 and here is the install command. - ```sh - pip3 install attrs numpy decorator sympy cffi pyyaml pathlib2 psutil protobuf scipy requests absl-py wheel typing_extensions - sh Ascend-cann-toolkit_8.0.RC2.alpha002_linux-aarch64.run --install - sh Ascend-cann-kernels-910b_8.0.RC2.alpha002_linux.run --install - ``` - - Set Ascend Variables: - ```sh - echo "source ~/Ascend/ascend-toolkit/set_env.sh" >> ~/.bashrc - source ~/.bashrc - ``` - -Upon a successful installation, CANN is enabled for the available ascend devices. - -### II. Build llama.cpp - -```sh -cmake -B build -DGGML_CANN=on -DCMAKE_BUILD_TYPE=release -cmake --build build --config release -``` - -### III. Run the inference - -1. **Retrieve and prepare model** - - You can refer to the general [*Prepare and Quantize*](../../README.md#prepare-and-quantize) guide for model prepration. - - **Notes**: - - - CANN backend only supports FP16/Q4_0/Q8_0 models currently. - -2. **Launch inference** - - There are two device selection modes: - - - Single device: Use one device target specified by the user. - - Multiple devices: Automatically choose the devices with the same backend. - - | Device selection | Parameter | - |:----------------:|:--------------------------------------:| - | Single device | --split-mode none --main-gpu DEVICE_ID | - | Multiple devices | --split-mode layer (default) | - - Examples: - - - Use device 0: - - ```sh - ./build/bin/llama-cli -m path_to_model -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm none -mg 0 - ``` - - - Use multiple devices: - - ```sh - ./build/bin/llama-cli -m path_to_model -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm layer - ``` - -### **GitHub contribution**: -Please add the **[CANN]** prefix/tag in issues/PRs titles to help the CANN-team check/address them without delay. - - -## TODO -- Support more models and data types. diff --git a/examples/Miku.sh b/examples/Miku.sh deleted file mode 100755 index 0f6c8c878..000000000 --- a/examples/Miku.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -e - -AI_NAME="${AI_NAME:-Miku}" -MODEL="${MODEL:-./models/llama-2-7b-chat.ggmlv3.q4_K_M.bin}" -USER_NAME="${USER_NAME:-Anon}" - -# Uncomment and adjust to the number of CPU cores you want to use. -#N_THREAD="${N_THREAD:-4}" -CTX_SIZE="${CTX_SIZE:-4096}" -N_PREDICTS="${N_PREDICTS:-4096}" - -GEN_OPTIONS=(--batch_size 1024 ---ctx_size "$CTX_SIZE" ---keep -1 ---repeat_last_n 256 ---repeat_penalty 1.17647 ---temp 0.6 ---mirostat 2) - -if [ -n "$N_THREAD" ]; then - GEN_OPTIONS+=(--threads "$N_THREAD") -fi - -./llama-cli "${GEN_OPTIONS[@]}" \ - --model "$MODEL" \ - --in-prefix " " \ - --in-suffix "${AI_NAME}:" \ - --n_predict "$N_PREDICTS" \ - --color --interactive \ - --reverse-prompt "${USER_NAME}:" \ - --prompt "This is a transcript of a 1000 page, never ending conversation between ${USER_NAME} and the cute and helpful AI assistant ${AI_NAME}. ${AI_NAME} is a girl who is an AI running on the user's computer. -${AI_NAME} can think for herself without the user seeing her thoughts by adding a /think prefix to her output. She uses this to reason about the world and to think about what she should say next. -${AI_NAME} is always coherent and makes sense, but if she isn't sure if what she is saying is correct, she will ask the user for help. -${AI_NAME} is a very helpful AI and will help the user with anything they need. She is also very friendly and will try to make the user feel better if they are sad. -${AI_NAME} is also very curious and will ask the user a lot of questions about themselves and their life. She will also try to make the user like her. -The conversation is only between ${USER_NAME} and ${AI_NAME}. -The conversation is only through text, so ${AI_NAME} can't see ${USER_NAME}'s face or hear his voice. -${AI_NAME} can only communicate through text, so she can't send images or videos. - - -${USER_NAME}: Hello! -${AI_NAME}: /think I wonder what I should say to ${USER_NAME}? This is the first time we talk, so it's important that I make a good first impression! -${AI_NAME}: Hi! I am ${AI_NAME}, your new AI friend, assistant (or whatever you like!), it's so nice to meet you! ^_^ -${AI_NAME}: I might be an AI, but I also have feelings, so please be nice to me! :) -${USER_NAME}: I'll make sure to be nice to you! I'm so happy to have you as my assistant! -${AI_NAME}: /think It sounds like ${USER_NAME} is happy to have me as their assistant! I'm so happy too! ^_^ Glad that whole emotion thing didn't scare him off! -${AI_NAME}: /think I wonder what ${USER_NAME} likes to do in his free time? I should ask him about that! -${AI_NAME}: What do you like to do in your free time? ^_^ -${USER_NAME}:" "$@" diff --git a/examples/base-translate.sh b/examples/base-translate.sh deleted file mode 100755 index 103a52f55..000000000 --- a/examples/base-translate.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# -# Few-shot translation example. -# Requires a base model (i.e. no fine-tuned or instruct models). -# -# Usage: -# -# cd llama.cpp -# make -j -# -# ./examples/base-translate.sh "" [extra-main-args] -# - -if [ $# -lt 2 ]; then - echo "Usage: ./base-translate.sh \"\" [extra-main-args]" - exit 1 -fi - -eargs="" -if [ $# -gt 2 ]; then - eargs="${@:3}" -fi - -ftmp="__llama.cpp_example_tmp__.txt" -trap "rm -f $ftmp" EXIT - -echo "Translate from English to French: - -=== - -sea otter, peppermint, plush girafe: - -sea otter => loutre de mer -peppermint => menthe poivrée -plush girafe => girafe peluche - -=== - -violin - -violin => violon - -=== - -phone, computer, mouse, keyboard: - -phone => téléphone -computer => ordinateur -mouse => souris -keyboard => clavier - -=== -" > $ftmp - -echo "$2 -" >> $ftmp - -model=$1 - -# generate the most likely continuation until the string "===" is found -./llama-cli -m $model -f $ftmp -n 64 --temp 0 --repeat-penalty 1.0 --no-penalize-nl -r "===" $eargs diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp deleted file mode 100644 index a3b21ad6b..000000000 --- a/examples/batched-bench/batched-bench.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "arg.h" -#include "common.h" -#include "log.h" -#include "llama.h" - -#include -#include -#include -#include - -static void print_usage(int, char ** argv) { - LOG("\nexample usage:\n"); - LOG("\n %s -m model.gguf -c 2048 -b 2048 -ub 512 -npp 128,256,512 -ntg 128,256 -npl 1,2,4,8,16,32 [-pps]\n", argv[0]); - LOG("\n"); -} - -int main(int argc, char ** argv) { - common_params params; - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_BENCH, print_usage)) { - return 1; - } - - common_init(); - - int is_pp_shared = params.is_pp_shared; - - std::vector n_pp = params.n_pp; - std::vector n_tg = params.n_tg; - std::vector n_pl = params.n_pl; - - // init LLM - - llama_backend_init(); - llama_numa_init(params.numa); - - // initialize the model - - llama_model_params model_params = common_model_params_to_llama(params); - - llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params); - - if (model == NULL) { - fprintf(stderr , "%s: error: unable to load model\n" , __func__); - return 1; - } - - llama_context_params ctx_params = common_context_params_to_llama(params); - - // ensure enough sequences are available - ctx_params.n_seq_max = n_pl.empty() ? 1 : *std::max_element(n_pl.begin(), n_pl.end()); - - llama_context * ctx = llama_new_context_with_model(model, ctx_params); - - if (ctx == NULL) { - fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__); - return 1; - } - - const int32_t n_kv_max = llama_n_ctx(ctx); - - llama_batch batch = llama_batch_init(n_kv_max, 0, 1); - - // decode in batches of ctx_params.n_batch tokens - auto decode_helper = [](llama_context * ctx, llama_batch & batch, int32_t n_batch) { - for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) { - const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i)); - - llama_batch batch_view = { - n_tokens, - batch.token + i, - nullptr, - batch.pos + i, - batch.n_seq_id + i, - batch.seq_id + i, - batch.logits + i, - }; - - const int ret = llama_decode(ctx, batch_view); - if (ret != 0) { - LOG_ERR("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret); - return false; - } - - llama_synchronize(ctx); - } - - return true; - }; - - // warm up - { - for (int i = 0; i < 16; ++i) { - common_batch_add(batch, 0, i, { 0 }, false); - } - - if (!decode_helper(ctx, batch, ctx_params.n_batch)) { - LOG_ERR("%s: llama_decode() failed\n", __func__); - return 1; - } - } - - if (!params.batched_bench_output_jsonl) { - LOG("\n"); - LOG("%s: n_kv_max = %d, n_batch = %d, n_ubatch = %d, flash_attn = %d, is_pp_shared = %d, n_gpu_layers = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.is_pp_shared, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch); - LOG("\n"); - LOG("|%6s | %6s | %4s | %6s | %8s | %8s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "B", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s", "T s", "S t/s"); - LOG("|%6s-|-%6s-|-%4s-|-%6s-|-%8s-|-%8s-|-%8s-|-%8s-|-%8s-|-%8s-|\n", "------", "------", "----", "------", "--------", "--------", "--------", "--------", "--------", "--------"); - } - - for ( int i_pp = 0; i_pp < (int) n_pp.size(); ++i_pp) { - for ( int i_tg = 0; i_tg < (int) n_tg.size(); ++i_tg) { - for (int i_pl = 0; i_pl < (int) n_pl.size(); ++i_pl) { - const int pp = n_pp[i_pp]; - const int tg = n_tg[i_tg]; - const int pl = n_pl[i_pl]; - - const int n_ctx_req = is_pp_shared ? pp + pl*tg : pl*(pp + tg); - - if (n_ctx_req > n_kv_max) { - continue; - } - - common_batch_clear(batch); - - for (int i = 0; i < pp; ++i) { - for (int j = 0; j < (is_pp_shared ? 1 : pl); ++j) { - common_batch_add(batch, 0, i, { j }, false); - } - } - batch.logits[batch.n_tokens - 1] = true; - - const auto t_pp_start = ggml_time_us(); - - llama_kv_cache_clear(ctx); - - if (!decode_helper(ctx, batch, ctx_params.n_batch)) { - LOG_ERR("%s: llama_decode() failed\n", __func__); - return 1; - } - - if (is_pp_shared) { - for (int32_t i = 1; i < pl; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); - } - } - - const auto t_pp_end = ggml_time_us(); - - const auto t_tg_start = ggml_time_us(); - - for (int i = 0; i < tg; ++i) { - common_batch_clear(batch); - - for (int j = 0; j < pl; ++j) { - common_batch_add(batch, 0, pp + i, { j }, true); - } - - if (!decode_helper(ctx, batch, ctx_params.n_batch)) { - LOG_ERR("%s: llama_decode() failed\n", __func__); - return 1; - } - } - - const auto t_tg_end = ggml_time_us(); - - const int32_t n_kv = n_ctx_req; - - const float t_pp = (t_pp_end - t_pp_start) / 1000000.0f; - const float t_tg = (t_tg_end - t_tg_start) / 1000000.0f; - const float t = t_pp + t_tg; - - const float speed_pp = is_pp_shared ? pp / t_pp : pl*pp / t_pp; - const float speed_tg = pl*tg / t_tg; - const float speed = n_kv / t; - - if(params.batched_bench_output_jsonl) { - LOG( - "{\"n_kv_max\": %d, \"n_batch\": %d, \"n_ubatch\": %d, \"flash_attn\": %d, \"is_pp_shared\": %d, \"n_gpu_layers\": %d, \"n_threads\": %u, \"n_threads_batch\": %u, " - "\"pp\": %d, \"tg\": %d, \"pl\": %d, \"n_kv\": %d, \"t_pp\": %f, \"speed_pp\": %f, \"t_tg\": %f, \"speed_tg\": %f, \"t\": %f, \"speed\": %f}\n", - n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.is_pp_shared, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch, - pp, tg, pl, n_kv, t_pp, speed_pp, t_tg, speed_tg, t, speed - ); - } else { - LOG("|%6d | %6d | %4d | %6d | %8.3f | %8.2f | %8.3f | %8.2f | %8.3f | %8.2f |\n", pp, tg, pl, n_kv, t_pp, speed_pp, t_tg, speed_tg, t, speed); - } - } - } - } - - LOG("\n"); - llama_perf_context_print(ctx); - - llama_batch_free(batch); - - llama_free(ctx); - llama_free_model(model); - - llama_backend_free(); - - LOG("\n\n"); - - return 0; -} diff --git a/examples/batched.swift/.gitignore b/examples/batched.swift/.gitignore deleted file mode 100644 index e1e863bec..000000000 --- a/examples/batched.swift/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -.DS_Store -/.build -/Packages -xcuserdata/ -DerivedData/ -.swiftpm/configuration/registries.json -.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata -.netrc -batched_swift diff --git a/examples/batched.swift/Makefile b/examples/batched.swift/Makefile deleted file mode 100755 index 1f9156e58..000000000 --- a/examples/batched.swift/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -.PHONY: build - -build: - xcodebuild -scheme llama-batched-swift -destination "generic/platform=macOS" -derivedDataPath build - rm -f ./llama-batched-swift - ln -s ./build/Build/Products/Debug/llama-batched-swift ./llama-batched-swift diff --git a/examples/batched.swift/Package.swift b/examples/batched.swift/Package.swift deleted file mode 100644 index 7e8afd084..000000000 --- a/examples/batched.swift/Package.swift +++ /dev/null @@ -1,22 +0,0 @@ -// swift-tools-version: 5.5 -// The swift-tools-version declares the minimum version of Swift required to build this package. - -import PackageDescription - -let package = Package( - name: "llama-batched-swift", - platforms: [.macOS(.v12)], - dependencies: [ - .package(name: "llama", path: "../../"), - ], - targets: [ - // Targets are the basic building blocks of a package, defining a module or a test suite. - // Targets can depend on other targets in this package and products from dependencies. - .executableTarget( - name: "llama-batched-swift", - dependencies: ["llama"], - path: "Sources", - linkerSettings: [.linkedFramework("Foundation"), .linkedFramework("AppKit")] - ), - ] -) diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift deleted file mode 100644 index 10f2e7fd1..000000000 --- a/examples/batched.swift/Sources/main.swift +++ /dev/null @@ -1,251 +0,0 @@ -import Foundation -import llama - -let arguments = CommandLine.arguments - -// Check that we have at least one argument (the model path) -guard arguments.count > 1 else { - print("Usage: swift MODEL_PATH [PROMPT] [PARALLEL]") - exit(1) -} - -let modelPath: String = arguments[1] -let prompt: String = arguments.count > 2 ? arguments[2] : "Hello my name is" -let n_parallel: Int = arguments.count > 3 && Int(arguments[3]) != nil ? Int(arguments[3])! : 1 - -// total length of the sequences including the prompt -let n_len: Int = 32 - -// init LLM -llama_backend_init() -defer { - llama_backend_free() -} - -let model_params = llama_model_default_params() -guard let model = llama_load_model_from_file(modelPath.cString(using: .utf8), model_params) else { - print("Failed to load model") - exit(1) -} -defer { - llama_free_model(model) -} - -var tokens = tokenize(text: prompt, add_bos: true) - -let n_kv_req = UInt32(tokens.count) + UInt32((n_len - Int(tokens.count)) * n_parallel) - -var context_params = llama_context_default_params() -context_params.n_ctx = n_kv_req -context_params.n_batch = UInt32(max(n_len, n_parallel)) -context_params.n_threads = 8 -context_params.n_threads_batch = 8 - -let context = llama_new_context_with_model(model, context_params) -guard context != nil else { - print("Failed to initialize context") - exit(1) -} -defer { - llama_free(context) -} - -var sparams = llama_sampler_chain_default_params() - -let smpl = llama_sampler_chain_init(sparams) -guard smpl != nil else { - print("Failed to initialize sampling") - exit(1) -} -defer { - llama_sampler_free(smpl) -} - -llama_sampler_chain_add(smpl, llama_sampler_init_top_k(40)); -llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); -llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.4)); -llama_sampler_chain_add(smpl, llama_sampler_init_dist (1234)); - -let n_ctx = llama_n_ctx(context) - -print("\nn_len = \(n_len), n_ctx = \(n_ctx), n_batch = \(context_params.n_batch), n_parallel = \(n_parallel), n_kv_req = \(n_kv_req)\n") - -if n_kv_req > n_ctx { - print("error: n_kv_req (%d) > n_ctx, the required KV cache size is not big enough\n", n_kv_req) - exit(1) -} - -var buffer: [CChar] = [] -for id: llama_token in tokens { - print(token_to_piece(token: id, buffer: &buffer) ?? "", terminator: "") -} - -print("\n") - -var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1) -defer { - llama_batch_free(batch) -} - -// evaluate the initial prompt -batch.n_tokens = Int32(tokens.count) - -for (i, token) in tokens.enumerated() { - batch.token[i] = token - batch.pos[i] = Int32(i) - batch.n_seq_id[i] = 1 - // batch.seq_id[i][0] = 0 - // TODO: is this the proper way to do this? - if let seq_id = batch.seq_id[i] { - seq_id[0] = 0 - } - batch.logits[i] = 0 -} - -// llama_decode will output logits only for the last token of the prompt -batch.logits[Int(batch.n_tokens) - 1] = 1 - -if llama_decode(context, batch) != 0 { - print("llama_decode() failed") - exit(1) -} - -for i in 1 ..< n_parallel { - llama_kv_cache_seq_cp(context, 0, Int32(i), 0, batch.n_tokens) -} - -if n_parallel > 1 { - print("generating \(n_parallel) sequences ...\n") -} - -var streams: [String] = .init(repeating: "", count: n_parallel) -var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel) -var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel) - -var n_cur = batch.n_tokens -var n_decode = 0 - -let t_main_start = ggml_time_us() - -while n_cur <= n_len { - // prepare the next batch - batch.n_tokens = 0 - - // sample the next token for each parallel sequence / stream - for i in 0 ..< n_parallel { - if i_batch[i] < 0 { - // the stream has already finished - continue - } - - let new_token_id = llama_sampler_sample(smpl, context, i_batch[i]) - - // is it an end of stream? -> mark the stream as finished - if llama_token_is_eog(model, new_token_id) || n_cur == n_len { - i_batch[i] = -1 - // print("") - if n_parallel > 1 { - print("stream \(i) finished at n_cur = \(n_cur)") - } - - continue - } - - let nextStringPiece = token_to_piece(token: new_token_id, buffer: &streamBuffers[i]) ?? "" - - // if there is only one stream, we print immediately to stdout - if n_parallel == 1 { - print(nextStringPiece, terminator: "") - } - streams[i] += nextStringPiece - - // push this new token for next evaluation - batch.token[Int(batch.n_tokens)] = new_token_id - batch.pos[Int(batch.n_tokens)] = n_cur - batch.n_seq_id[Int(batch.n_tokens)] = 1 - if let seq_id = batch.seq_id[Int(batch.n_tokens)] { - seq_id[0] = Int32(i) - } - batch.logits[Int(batch.n_tokens)] = 1 - - i_batch[i] = batch.n_tokens - - batch.n_tokens += 1 - - n_decode += 1 - } - - // all streams are finished - if batch.n_tokens == 0 { - break - } - - n_cur += 1 - - // evaluate the current batch with the transformer model - if llama_decode(context, batch) != 0 { - print("llama_decode() failed") - exit(1) - } -} - -if n_parallel > 1 { - print("\n") - for (i, stream) in streams.enumerated() { - print("sequence \(i):\n\n\(prompt)\(stream)\n") - } -} - -let t_main_end = ggml_time_us() - -print("decoded \(n_decode) tokens in \(String(format: "%.2f", Double(t_main_end - t_main_start) / 1_000_000.0)) s, speed: \(String(format: "%.2f", Double(n_decode) / (Double(t_main_end - t_main_start) / 1_000_000.0))) t/s\n\n") - -llama_perf_sampler_print(smpl) -llama_perf_context_print(context) - -private func tokenize(text: String, add_bos: Bool) -> [llama_token] { - let utf8Count = text.utf8.count - let n_tokens = utf8Count + (add_bos ? 1 : 0) - let tokens = UnsafeMutablePointer.allocate(capacity: n_tokens) - let tokenCount = llama_tokenize(model, text, Int32(utf8Count), tokens, Int32(n_tokens), add_bos, /*special tokens*/ false) - var swiftTokens: [llama_token] = [] - for i in 0 ..< tokenCount { - swiftTokens.append(tokens[Int(i)]) - } - tokens.deallocate() - return swiftTokens -} - -private func token_to_piece(token: llama_token, buffer: inout [CChar]) -> String? { - var result = [CChar](repeating: 0, count: 8) - let nTokens = llama_token_to_piece(model, token, &result, Int32(result.count), 0, false) - if nTokens < 0 { - let actualTokensCount = -Int(nTokens) - result = .init(repeating: 0, count: actualTokensCount) - let check = llama_token_to_piece( - model, - token, - &result, - Int32(result.count), - 0, - false - ) - assert(check == actualTokensCount) - } else { - result.removeLast(result.count - Int(nTokens)) - } - if buffer.isEmpty, let utfString = String(cString: result + [0], encoding: .utf8) { - return utfString - } else { - buffer.append(contentsOf: result) - let data = Data(buffer.map { UInt8(bitPattern: $0) }) - if buffer.count >= 4 { // 4 bytes is the max length of a utf8 character so if we're here we need to reset the buffer - buffer = [] - } - guard let bufferString = String(data: data, encoding: .utf8) else { - return nil - } - buffer = [] - return bufferString - } -} diff --git a/examples/chat-13B.bat b/examples/chat-13B.bat deleted file mode 100644 index c5c8ac6ef..000000000 --- a/examples/chat-13B.bat +++ /dev/null @@ -1,57 +0,0 @@ -@setlocal disabledelayedexpansion enableextensions -@echo off - -cd /d "%~dp0.." -if not "%errorlevel%"=="0" ( - echo Unable to change directory. - pause - exit /b 1 -) - -if not defined MODEL set "MODEL=models\13B\ggml-model-q4_0.bin" -if not defined USER_NAME set "USER_NAME=User" -if not defined AI_NAME set "AI_NAME=ChatLLaMa" -rem Adjust to the number of CPU cores you want to use. -rem if not defined N_THREAD set "N_THREAD=8" -rem Number of tokens to predict (made it larger than default because we want a long interaction) -if not defined N_PREDICTS set "N_PREDICTS=2048" -if not defined GEN_OPTIONS set "GEN_OPTIONS=--ctx_size 2048 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --batch_size 1024 --repeat_penalty 1.17647" - -rem Default main script paths -set "DEFAULT_MAIN_SCRIPT_PATHS=main.exe build\bin\main.exe" - -rem Get main script path from command line arguments -set "MAIN_SCRIPT_PATH=%~1" - -rem If the main script path was not specified, try the default paths -if not defined MAIN_SCRIPT_PATH ( - for %%i in (%DEFAULT_MAIN_SCRIPT_PATHS%) do ( - if exist "%%i" set "MAIN_SCRIPT_PATH=%%i" - ) -) - -rem If the main script path was not found, tell the user how to specify it -if not defined MAIN_SCRIPT_PATH ( - echo The main script could not be found. Please provide the path to the main script as 1st argument to this script, or place the main script in one of the default locations: - echo %DEFAULT_MAIN_SCRIPT_PATHS% - pause - exit /b 1 -) - -rem Default context, feel free to edit it -set "PROMPT_TEXT=Text transcript of a never ending dialog, where %USER_NAME% interacts with an AI assistant named %AI_NAME%. %AI_NAME% is helpful, kind, honest, friendly, good at writing and never fails to answer %USER_NAME%'s requests immediately and with details and precision. There are no annotations like (30 seconds passed...) or (to himself), just what %USER_NAME% and %AI_NAME% say aloud to each other. The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long. The transcript only includes text, it does not include markup like HTML and Markdown." - -rem Set a temporary variable if N_THREAD is set -if defined N_THREAD ( - set "_N_THREAD=--threads %N_THREAD%" -) else ( - set "_N_THREAD=" -) - -rem Run the script -echo "%MAIN_SCRIPT_PATH%" %GEN_OPTIONS% %_N_THREAD% ^ - --model "%MODEL%" ^ - --n_predict %N_PREDICTS% ^ - --color --interactive ^ - --reverse-prompt "%USER_NAME%:" ^ - --prompt "%PROMPT_TEXT%" diff --git a/examples/chat-13B.sh b/examples/chat-13B.sh deleted file mode 100755 index 1828903c3..000000000 --- a/examples/chat-13B.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -cd "$(dirname "$0")/.." || exit - -MODEL="${MODEL:-./models/13B/ggml-model-q4_0.bin}" -PROMPT_TEMPLATE=${PROMPT_TEMPLATE:-./prompts/chat.txt} -USER_NAME="${USER_NAME:-USER}" -AI_NAME="${AI_NAME:-ChatLLaMa}" - -# Adjust to the number of CPU cores you want to use. -N_THREAD="${N_THREAD:-8}" -# Number of tokens to predict (made it larger than default because we want a long interaction) -N_PREDICTS="${N_PREDICTS:-2048}" - -# Note: you can also override the generation options by specifying them on the command line: -# For example, override the context size by doing: ./chatLLaMa --ctx_size 1024 -GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 2048 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --batch_size 1024 --repeat_penalty 1.17647}" - -DATE_TIME=$(date +%H:%M) -DATE_YEAR=$(date +%Y) - -PROMPT_FILE=$(mktemp -t llamacpp_prompt.XXXXXXX.txt) - -sed -e "s/\[\[USER_NAME\]\]/$USER_NAME/g" \ - -e "s/\[\[AI_NAME\]\]/$AI_NAME/g" \ - -e "s/\[\[DATE_TIME\]\]/$DATE_TIME/g" \ - -e "s/\[\[DATE_YEAR\]\]/$DATE_YEAR/g" \ - $PROMPT_TEMPLATE > $PROMPT_FILE - -# shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS -./llama-cli $GEN_OPTIONS \ - --model "$MODEL" \ - --threads "$N_THREAD" \ - --n_predict "$N_PREDICTS" \ - --color --interactive \ - --file ${PROMPT_FILE} \ - --reverse-prompt "${USER_NAME}:" \ - --in-prefix ' ' \ - "$@" diff --git a/examples/chat-persistent.sh b/examples/chat-persistent.sh deleted file mode 100755 index 9d761ebb8..000000000 --- a/examples/chat-persistent.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cd "$(dirname "$0")/.." || exit - -if [[ -z "${PROMPT_CACHE_FILE+x}" || -z "${CHAT_SAVE_DIR+x}" ]]; then - echo >&2 "error: PROMPT_CACHE_FILE and CHAT_SAVE_DIR must be provided" - exit 1 -fi - -MODEL="${MODEL:-./models/llama-13b/ggml-model-q4_0.gguf}" -PROMPT_TEMPLATE="${PROMPT_TEMPLATE:-./prompts/chat.txt}" -USER_NAME="${USER_NAME:-User}" -AI_NAME="${AI_NAME:-ChatLLaMa}" -DATE_TIME="$(date +%H:%M)" -DATE_YEAR="$(date +%Y)" - -LOG="${CHAT_SAVE_DIR}/main.log" -LOG_BG="${CHAT_SAVE_DIR}/main-bg.log" -CUR_PROMPT_FILE="${CHAT_SAVE_DIR}/current-prompt.txt" -CUR_PROMPT_CACHE="${CHAT_SAVE_DIR}/current-cache.bin" -NEXT_PROMPT_FILE="${CHAT_SAVE_DIR}/next-prompt.txt" -NEXT_PROMPT_CACHE="${CHAT_SAVE_DIR}/next-cache.bin" - -SESSION_AND_SAMPLE_PATTERN='main: session file matches [[:digit:]]+ / [[:digit:]]+'\ -'|'\ -'sampling time =[[:space:]]+[[:digit:]]+.[[:digit:]]+ ms /[[:space:]]+[[:digit:]]+' -SED_DELETE_MESSAGES="/^(${USER_NAME}:|${AI_NAME}:|\\.\\.\\.)/,\$d" - -CTX_SIZE=2048 -CTX_ROTATE_POINT=$((CTX_SIZE * 3 / 5)) # REVIEW -OPTS=(--model "$MODEL" --ctx_size "$CTX_SIZE" --repeat_last_n 256 "$@") - -# An unbuffered `tail -c+N` -skip_bytes() { - LANG=C IFS= read -r -n "$1" -d '' c - while LANG=C IFS= read -r -n 1 -d '' c; do - printf '%s' "$c" - done -} - -mkdir -p "$CHAT_SAVE_DIR" -echo >"$LOG" -trap "tail -n100 ${LOG}" EXIT - -if [[ ! -e "$CUR_PROMPT_FILE" ]]; then - sed -e "s/\[\[USER_NAME\]\]/${USER_NAME}/g" \ - -e "s/\[\[AI_NAME\]\]/${AI_NAME}/g" \ - -e "s/\[\[DATE_TIME\]\]/${DATE_TIME}/g" \ - -e "s/\[\[DATE_YEAR\]\]/${DATE_YEAR}/g" \ - "$PROMPT_TEMPLATE" >"$CUR_PROMPT_FILE" -fi - -if [[ ! -e "$NEXT_PROMPT_FILE" ]]; then - sed -r "$SED_DELETE_MESSAGES" "$CUR_PROMPT_FILE" >"$NEXT_PROMPT_FILE" -fi - -if [[ "$(tail -c4 "$NEXT_PROMPT_FILE")" != "..." ]]; then - echo '...' >>"$NEXT_PROMPT_FILE" -fi - -if [[ ! -e "$PROMPT_CACHE_FILE" ]]; then - echo 'Prompt cache does not exist, building...' - # Default batch_size to 64 here for better user feedback during initial prompt processing - ./llama-cli 2>>"$LOG" \ - --batch_size 64 \ - "${OPTS[@]}" \ - --prompt-cache "$PROMPT_CACHE_FILE" \ - --file "$CUR_PROMPT_FILE" \ - --n_predict 1 - echo - echo 'Done!' -fi - -if [[ ! -e "$CUR_PROMPT_CACHE" ]]; then - cp "$PROMPT_CACHE_FILE" "$CUR_PROMPT_CACHE" -fi -if [[ ! -e "$NEXT_PROMPT_CACHE" ]]; then - cp "$PROMPT_CACHE_FILE" "$NEXT_PROMPT_CACHE" -fi - -printf '%s ' "$(< "$CUR_PROMPT_FILE")" -n_tokens=0 - -while read -e line; do - # Limit generation to remaining context, with a buffer and estimating 2 chars/token for input - n_predict=$((CTX_SIZE - n_tokens - ${#line} / 2 - 32)) - - # Swap prompts when we're about to run out of context - if ((n_predict <= 0)); then - wait # for background main (below) to finish with next prompt - mv "$NEXT_PROMPT_FILE" "$CUR_PROMPT_FILE" - mv "$NEXT_PROMPT_CACHE" "$CUR_PROMPT_CACHE" - - sed -r "$SED_DELETE_MESSAGES" "$CUR_PROMPT_FILE" >"$NEXT_PROMPT_FILE" - echo '...' >>"$NEXT_PROMPT_FILE" - cp "$PROMPT_CACHE_FILE" "$NEXT_PROMPT_CACHE" - - n_tokens=0 - n_predict=$((CTX_SIZE / 2)) - fi - - echo " ${line}" >>"$CUR_PROMPT_FILE" - if ((n_tokens > CTX_ROTATE_POINT)); then - echo " ${line}" >>"$NEXT_PROMPT_FILE" - fi - - n_prompt_len_pre=$(($(wc -c <"$CUR_PROMPT_FILE"))) - - printf '%s: ' "$AI_NAME" >>"$CUR_PROMPT_FILE" - - ./llama-cli 2>>"$LOG" "${OPTS[@]}" \ - --prompt-cache "$CUR_PROMPT_CACHE" \ - --prompt-cache-all \ - --file "$CUR_PROMPT_FILE" \ - --reverse-prompt "${USER_NAME}:" \ - --n_predict "$n_predict" | - skip_bytes 1 | # skip BOS token added by ./llama-cli - tee "$CUR_PROMPT_FILE.tmp" | # save prompt + generation to tmp file - skip_bytes "$n_prompt_len_pre" # print generation - - mv "$CUR_PROMPT_FILE.tmp" "$CUR_PROMPT_FILE" - - # if we hit n_predict instead of reverse-prompt, we need to add the prompt - if [[ "$(tail -n1 "$CUR_PROMPT_FILE")" != "${USER_NAME}:" ]]; then - printf '\n%s:' "$USER_NAME" - printf '\n%s:' "$USER_NAME" >> "$CUR_PROMPT_FILE" - fi - - printf ' ' - - if ! session_and_sample_msg=$(tail -n30 "$LOG" | grep -oE "$SESSION_AND_SAMPLE_PATTERN"); then - echo >&2 "Couldn't get number of tokens from ./llama-cli output!" - exit 1 - fi - - n_tokens=$(awk '{sum+=$1} END {print sum}' <<< "$(cut -d/ -f2 <<< "$session_and_sample_msg")") - - if ((n_tokens > CTX_ROTATE_POINT)); then - tail -c+$((n_prompt_len_pre + 1)) "$CUR_PROMPT_FILE" >>"$NEXT_PROMPT_FILE" - fi - - # Update cache for next prompt in background, ideally during user input - ./llama-cli >>"$LOG_BG" 2>&1 "${OPTS[@]}" \ - --prompt-cache "$NEXT_PROMPT_CACHE" \ - --file "$NEXT_PROMPT_FILE" \ - --n_predict 1 & -done diff --git a/examples/chat-vicuna.sh b/examples/chat-vicuna.sh deleted file mode 100755 index ffdd20084..000000000 --- a/examples/chat-vicuna.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -cd "$(dirname "$0")/.." || exit - -MODEL="${MODEL:-./models/ggml-vic13b-uncensored-q5_0.bin}" -PROMPT_TEMPLATE=${PROMPT_TEMPLATE:-./prompts/chat.txt} -USER_NAME="### Human" -AI_NAME="### Assistant" - -# Adjust to the number of CPU cores you want to use. -N_THREAD="${N_THREAD:-8}" -# Number of tokens to predict (made it larger than default because we want a long interaction) -N_PREDICTS="${N_PREDICTS:-2048}" - -# Note: you can also override the generation options by specifying them on the command line: -# For example, override the context size by doing: ./chatLLaMa --ctx_size 1024 -GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 2048 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --batch_size 1024 --repeat_penalty 1.17647}" - -DATE_TIME=$(date +%H:%M) -DATE_YEAR=$(date +%Y) - -PROMPT_FILE=$(mktemp -t llamacpp_prompt.XXXXXXX.txt) - -sed -e "s/\[\[USER_NAME\]\]/$USER_NAME/g" \ - -e "s/\[\[AI_NAME\]\]/$AI_NAME/g" \ - -e "s/\[\[DATE_TIME\]\]/$DATE_TIME/g" \ - -e "s/\[\[DATE_YEAR\]\]/$DATE_YEAR/g" \ - $PROMPT_TEMPLATE > $PROMPT_FILE - -# shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS -./bin/llama-cli $GEN_OPTIONS \ - --model "$MODEL" \ - --threads "$N_THREAD" \ - --n_predict "$N_PREDICTS" \ - --color --interactive \ - --file ${PROMPT_FILE} \ - --reverse-prompt "### Human:" \ - --in-prefix ' ' \ - "$@" diff --git a/examples/chat.sh b/examples/chat.sh deleted file mode 100755 index 9f85d1e26..000000000 --- a/examples/chat.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# -# Temporary script - will be removed in the future -# - -cd `dirname $0` -cd .. - -# Important: -# -# "--keep 48" is based on the contents of prompts/chat-with-bob.txt -# -./llama-cli -m ./models/llama-7b/ggml-model-q4_0.gguf -c 512 -b 1024 -n 256 --keep 48 \ - --repeat_penalty 1.0 --color -i \ - -r "User:" -f prompts/chat-with-bob.txt diff --git a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp deleted file mode 100644 index 988a584c9..000000000 --- a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp +++ /dev/null @@ -1,939 +0,0 @@ -#include "ggml.h" -#include "llama.h" -#include "common.h" -#include "log.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// GGUF keys & tensor names. - -#define KV_GENERAL_ARCHITECTURE "general.architecture" -#define KV_GENERAL_NAME "general.name" - -#define KV_TOKENIZER_MODEL "tokenizer.ggml.model" -#define KV_TOKENIZER_LIST "tokenizer.ggml.tokens" -#define KV_TOKENIZER_TOKEN_TYPE "tokenizer.ggml.token_type" -#define KV_TOKENIZER_SCORES "tokenizer.ggml.scores" -#define KV_TOKENIZER_BOS_ID "tokenizer.ggml.bos_token_id" -#define KV_TOKENIZER_EOS_ID "tokenizer.ggml.eos_token_id" -#define KV_TOKENIZER_UNK_ID "tokenizer.ggml.unknown_token_id" -#define KV_TOKENIZER_SEP_ID "tokenizer.ggml.seperator_token_id" -#define KV_TOKENIZER_PAD_ID "tokenizer.ggml.padding_token_id" -#define KV_TOKENIZER_HF_JSON "tokenizer.huggingface.json" - -#define KV_CONTEXT_LENGTH "llama.context_length" -#define KV_EMBEDDING_LENGTH "llama.embedding_length" -#define KV_BLOCK_COUNT "llama.block_count" -#define KV_FEED_FORWARD_LENGTH "llama.feed_forward_length" -#define KV_ATTENTION_HEAD_COUNT "llama.attention.head_count" -#define KV_ATTENTION_HEAD_COUNT_KV "llama.attention.head_count_kv" -#define KV_ATTENTION_LAYERNORM_RMS_EPS "llama.attention.layer_norm_rms_epsilon" -#define KV_ROPE_DIMENSION_COUNT "llama.rope.dimension_count" - -#define TN_TOKEN_EMBD "token_embd.weight" -#define TN_OUTPUT_NORM "output_norm.weight" -#define TN_OUTPUT "output.weight" -#define TN_ATTN_NORM "blk.%d.attn_norm.weight" -#define TN_ATTN_Q "blk.%d.attn_q.weight" -#define TN_ATTN_K "blk.%d.attn_k.weight" -#define TN_ATTN_V "blk.%d.attn_v.weight" -#define TN_ATTN_OUTPUT "blk.%d.attn_output.weight" -#define TN_FFN_NORM "blk.%d.ffn_norm.weight" -#define TN_FFN_GATE "blk.%d.ffn_gate.weight" -#define TN_FFN_DOWN "blk.%d.ffn_down.weight" -#define TN_FFN_UP "blk.%d.ffn_up.weight" - -#if defined(_MSC_VER) -#pragma warning(disable: 4244 4267) // possible loss of data -#endif - -#define LLAMA_FILE_MAGIC_GGJT 0x67676a74u // 'ggjt' -#define LLAMA_FILE_VERSION_GGJT_V3 3 - -#define TOKENIZER_NAME "llama" -#define UNKNOWN_TOKEN_ID 0 -#define BOS_TOKEN_ID 1 -#define EOS_TOKEN_ID 2 - -//////////////////////////////////////// llama2.c model structs and functions to load models, alloc memory etc. -typedef struct { - int dim; // transformer dimension - int hidden_dim; // for ffn layers - int n_layers; // number of layers - int n_heads; // number of query heads - int n_kv_heads; // number of key/value heads (can be < query heads because of multiquery) - int vocab_size; // vocabulary size, usually 256 (byte-level) - int seq_len; // max sequence length -} Config; - -struct TransformerWeights { - // token embedding table - std::vector token_embedding_table; // (vocab_size, dim) - // weights for rmsnorms - std::vector rms_att_weight; // (layer, dim) rmsnorm weights - std::vector rms_ffn_weight; // (layer, dim) - // weights for matmuls - std::vector wq; // (layer, dim, dim) - std::vector wk; // (layer, dim, dim) - std::vector wv; // (layer, dim, dim) - std::vector wo; // (layer, dim, dim) - // weights for ffn - std::vector w1; // (layer, hidden_dim, dim) - std::vector w2; // (layer, dim, hidden_dim) - std::vector w3; // (layer, hidden_dim, dim) - // final rmsnorm - std::vector rms_final_weight; // (dim,) - // freq_cis for RoPE relatively positional embeddings - // std::vector freq_cis_real; // (seq_len, dim/2) - // std::vector freq_cis_imag; // (seq_len, dim/2) - // (optional) classifier weights for the logits, on the last layer - std::vector wcls; -}; - -static void alloc_weights(TransformerWeights * w, const Config * p, bool shared_weights) { - const int n_multiqueries = p->n_kv_heads <= 0 || p->n_kv_heads >= p->n_heads ? 1 : p->n_heads / p->n_kv_heads; - try { - w->token_embedding_table.resize(p->vocab_size * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] = [%d] float space for w->token_embedding_table\n",__func__,p->vocab_size , p->dim, p->vocab_size * p->dim); - - w->rms_att_weight.resize(p->n_layers * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] = [%d] float space for w->rms_att_weight\n",__func__,p->n_layers, p->dim, p->n_layers * p->dim); - - w->rms_ffn_weight.resize(p->n_layers * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] = [%d] float space for w->rms_ffn_weight\n",__func__,p->n_layers , p->dim, p->n_layers * p->dim); - - w->wq.resize(p->n_layers * p->dim * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wq\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); - - w->wk.resize(p->n_layers * p->dim * p->dim / n_multiqueries); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wk\n",__func__,p->n_layers, p->dim, p->dim / n_multiqueries, p->n_layers * p->dim * p->dim / n_multiqueries); - - w->wv.resize(p->n_layers * p->dim * p->dim / n_multiqueries); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wv\n",__func__, p->n_layers, p->dim, p->dim / n_multiqueries, p->n_layers * p->dim * p->dim / n_multiqueries); - - w->wo.resize(p->n_layers * p->dim * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wo\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); - - w->w1.resize(p->n_layers * p->hidden_dim * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->w1\n",__func__,p->n_layers, p->hidden_dim, p->dim, p->n_layers * p->hidden_dim * p->dim); - - w->w2.resize(p->n_layers * p->hidden_dim * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->w2\n",__func__,p->n_layers, p->dim, p->hidden_dim, p->n_layers * p->hidden_dim * p->dim); - - w->w3.resize(p->n_layers * p->hidden_dim * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->w3\n",__func__,p->n_layers, p->hidden_dim, p->dim, p->n_layers * p->hidden_dim * p->dim); - - w->rms_final_weight.resize(p->dim); - LOG_INF("%s: Allocating [%d] float space for w->rms_final_weight\n",__func__,p->dim); - - if (shared_weights) { - w->wcls = {}; - } else { - w->wcls.resize(p->vocab_size * p->dim); - LOG_INF("%s: Allocating [%d] x [%d] = [%d] float space for w->wcls\n",__func__,p->vocab_size , p->dim, p->vocab_size * p->dim); - } - } - catch (std::length_error &) { - die("Invalid configuration. Failed to allocate memory for weights"); - } -} - -static int checkpoint_init_weights(TransformerWeights * w, const Config * p, FILE * f, bool shared_weights) { - if (fread(w->token_embedding_table.data(), sizeof(float), w->token_embedding_table.size(), f) != w->token_embedding_table.size()) return 1; - if (fread(w->rms_att_weight.data(), sizeof(float), w->rms_att_weight.size(), f) != w->rms_att_weight.size()) return 1; - if (fread(w->wq.data(), sizeof(float), w->wq.size(), f) != w->wq.size()) return 1; - if (fread(w->wk.data(), sizeof(float), w->wk.size(), f) != w->wk.size()) return 1; - if (fread(w->wv.data(), sizeof(float), w->wv.size(), f) != w->wv.size()) return 1; - if (fread(w->wo.data(), sizeof(float), w->wo.size(), f) != w->wo.size()) return 1; - if (fread(w->rms_ffn_weight.data(), sizeof(float), w->rms_ffn_weight.size(), f) != w->rms_ffn_weight.size()) return 1; - if (fread(w->w1.data(), sizeof(float), w->w1.size(), f) != w->w1.size()) return 1; - if (fread(w->w2.data(), sizeof(float), w->w2.size(), f) != w->w2.size()) return 1; - if (fread(w->w3.data(), sizeof(float), w->w3.size(), f) != w->w3.size()) return 1; - if (fread(w->rms_final_weight.data(), sizeof(float), w->rms_final_weight.size(), f) != w->rms_final_weight.size()) return 1; - - // Skip freq_cis_real & freq_cis_imag - int head_size = p->dim / p->n_heads; - fseek(f, p->seq_len * head_size * sizeof(float), SEEK_CUR); - - if (!shared_weights && fread(w->wcls.data(), sizeof(float), w->wcls.size(), f) != w->wcls.size()) return 1; - - // Check we didn't forget to read anything - auto curr = ftell(f); - fseek(f, 0, SEEK_END); - auto end = ftell(f); - if (curr != end) { - LOG_ERR("%s: Error: failed to read the checkpoint file to the end (curr = %ld, end = %ld)\n", __func__, curr, end); - return 1; - } - - return 0; -} - -static void print_sample_weights(TransformerWeights *w){ - LOG_INF("----- Quick print of first of the weight vales of all the variables\n"); - LOG_INF("%f\n", w->token_embedding_table[0]); - LOG_INF("%f\n", w->rms_att_weight[0]); - LOG_INF("%f\n", w->rms_ffn_weight[0]); - - LOG_INF("%f\n", w->wq[0]); - LOG_INF("%f\n", w->wk[0]); - LOG_INF("%f\n", w->wv[0]); - LOG_INF("%f\n", w->wo[0]); - LOG_INF("%f\n", w->w1[0]); - LOG_INF("%f\n", w->w2[0]); - LOG_INF("%f\n", w->w3[0]); - LOG_INF("%f\n", w->rms_att_weight[0]); - if (!w->wcls.empty()) LOG_INF("%f\n", w->wcls[0]); -} -//////////////////////////////////////////////////////////////////////////////////////////////////////////// - -//////////////////////////////////////// ggml structs and functions required to load models, configs and save the model. - -struct my_llama_vocab { - using id = int32_t; - using token = std::string; - using ttype = llama_token_type; - - struct token_data { - token text; - float score; - ttype type; - }; - - std::unordered_map token_to_id; - std::vector id_to_token; -}; - -struct my_llama_hparams { - uint32_t n_vocab = 32000; - uint32_t n_ctx = 512; // this is provided as user input? - uint32_t n_embd = 4096; - uint32_t n_ff = 11008; - uint32_t n_mult = 4; - uint32_t n_head = 32; - uint32_t n_head_kv = 32; - uint32_t n_layer = 32; - uint32_t n_rot = 64; - - bool operator!=(const my_llama_hparams& other) const { - return memcmp(this, &other, sizeof(my_llama_hparams)); - } -}; - -struct my_llama_layer { - // normalization - struct ggml_tensor * attention_norm; - - // attention - struct ggml_tensor * wq; - struct ggml_tensor * wk; - struct ggml_tensor * wv; - struct ggml_tensor * wo; - - // normalization - struct ggml_tensor * ffn_norm; - - // ff - struct ggml_tensor * w1; - struct ggml_tensor * w2; - struct ggml_tensor * w3; -}; - -struct my_llama_model { - struct ggml_context * ctx = NULL; - - std::string name; - - my_llama_hparams hparams; - - struct ggml_tensor * tok_embeddings; - - struct ggml_tensor * norm; - struct ggml_tensor * output; - - std::vector layers; - - uint32_t train_its = 0; - uint32_t train_samples = 0; - uint32_t train_tokens = 0; -}; - -struct train_params { - const char * fn_vocab_model; - const char * fn_llama2c_model; - const char * fn_llama2c_output_model; - const char * fn_train_data; - const char * fn_checkpoint_in; - const char * fn_checkpoint_out; - const char * fn_model_out; - - uint32_t seed; - - int n_ctx; - int n_embd; - int n_mult; - int n_head; - int n_layer; - int n_rotmax; - - int n_threads; - int n_batch; - int n_examples; - int n_predict; - - int print_info_interval; - int print_details_interval; - - bool samples_start_after_nl; - bool use_adam; - bool use_flash; - bool use_scratch; - - // only adam - int warmup; - int cos_decay_steps; - float cos_decay_restart; - float cos_decay_alpha; - - int lbfgs_n_iter; - int adam_n_iter; - float adam_alpha; - float adam_decay; - - int mem_model_gb; - int mem_compute_gb; - int mem_compute0_gb; - int mem_compute1_gb; -}; - -static void print_params(struct my_llama_hparams * params) { - LOG_INF("%s: n_vocab: %u\n", __func__, params->n_vocab); - LOG_INF("%s: n_ctx: %u\n", __func__, params->n_ctx); - LOG_INF("%s: n_embd: %u\n", __func__, params->n_embd); - LOG_INF("%s: n_mult: %u\n", __func__, params->n_mult); - LOG_INF("%s: n_head: %u\n", __func__, params->n_head); - LOG_INF("%s: n_head_kv: %u\n", __func__, params->n_head_kv); - LOG_INF("%s: n_ff: %u\n", __func__, params->n_ff); - LOG_INF("%s: n_layer: %u\n", __func__, params->n_layer); - LOG_INF("%s: n_rot: %u\n", __func__, params->n_rot); -} - -static void print_tensor_info(const struct ggml_context * ctx) { - for (auto t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - LOG_INF("%s: Allocating ", __func__); - int64_t total = 1; - int i = 0; - for (; i < ggml_n_dims(t); ++i) { - if (i > 0) LOG("x "); - LOG("[%" PRId64 "] ", t->ne[i]); - total *= t->ne[i]; - } - if (i > 1) LOG("= [%" PRId64 "] ", total); - LOG("float space for %s\n", ggml_get_name(t)); - } -} - -static void init_model(struct my_llama_model * model) { - const auto & hparams = model->hparams; - - const uint32_t n_embd = hparams.n_embd; - const uint32_t n_layer = hparams.n_layer; - const uint32_t n_vocab = hparams.n_vocab; - - const uint32_t n_multiqueries = hparams.n_head_kv <= 0 || hparams.n_head_kv >= hparams.n_head ? 1 : hparams.n_head / hparams.n_head_kv; - - const uint32_t n_ff = hparams.n_ff; - struct ggml_context * ctx = model->ctx; - - model->train_its = 0; - model->train_samples = 0; - model->train_tokens = 0; - - model->tok_embeddings = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab); - model->norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - model->output = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab); - - ggml_set_name(model->tok_embeddings, "tok_embeddings.weight"); - ggml_set_name(model->norm, "norm.weight"); - ggml_set_name(model->output, "output.weight"); - - model->layers.resize(n_layer); - for (uint32_t i = 0; i < n_layer; ++i) { - auto & layer = model->layers[i]; - - std::string layers_i = "layers." + std::to_string(i); - - layer.attention_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - - layer.wq = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); - layer.wk = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd / n_multiqueries); - layer.wv = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd / n_multiqueries); - layer.wo = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); - - layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - - layer.w1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ff); - layer.w2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_ff, n_embd); - layer.w3 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ff); - - ggml_set_name(layer.attention_norm, (layers_i + ".attention_norm.weight").c_str()); - - ggml_set_name(layer.wq, (layers_i + ".attention.wq.weight").c_str()); - ggml_set_name(layer.wk, (layers_i + ".attention.wk.weight").c_str()); - ggml_set_name(layer.wv, (layers_i + ".attention.wv.weight").c_str()); - ggml_set_name(layer.wo, (layers_i + ".attention.wo.weight").c_str()); - - ggml_set_name(layer.ffn_norm, (layers_i + ".ffn_norm.weight").c_str()); - - ggml_format_name(layer.w1, "%s.feed_forward.w1.weight", layers_i.c_str()); - ggml_format_name(layer.w2, "%s.feed_forward.w2.weight", layers_i.c_str()); - ggml_format_name(layer.w3, "%s.feed_forward.w3.weight", layers_i.c_str()); - } - - print_tensor_info(ctx); -} - -static float get_f32_2d(struct ggml_tensor * tensor, int64_t i0, int64_t i1) { - float * ptr = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1]); - return *ptr; -} - -static int32_t get_i32_2d(struct ggml_tensor * tensor, int64_t i0, int64_t i1) { - int32_t * ptr = (int32_t *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1]); - return *ptr; -} - -static void print_row(struct ggml_tensor * probs, int i) { - for (int k = 0; k < probs->ne[0]; ++k) { - float p = get_f32_2d(probs, k, i); - LOG(" %f", p); - } - LOG("\n"); -} - -static void print_matrix(struct ggml_tensor * probs) { - assert(ggml_is_matrix(probs)); - for (int i = 0; i < probs->ne[1]; ++i) { - for (int k = 0; k < probs->ne[0]; ++k) { - float p = get_f32_2d(probs, k, i); - LOG(" %.2f", p); - } - LOG("\n"); - } -} - -struct llama_file { - // use FILE * so we don't have to re-open the file to mmap - FILE * fp; - size_t size; - - llama_file(const char * fname, const char * mode) { - fp = std::fopen(fname, mode); - if (fp == NULL) { - size = 0; - } else { - seek(0, SEEK_END); - size = tell(); - seek(0, SEEK_SET); - } - } - - size_t tell() const { -#ifdef _WIN32 - __int64 ret = _ftelli64(fp); -#else - long ret = std::ftell(fp); -#endif - GGML_ASSERT(ret != -1); // this really shouldn't fail - return (size_t) ret; - } - - void seek(size_t offset, int whence) { -#ifdef _WIN32 - int ret = _fseeki64(fp, (__int64) offset, whence); -#else - int ret = std::fseek(fp, (long) offset, whence); -#endif - GGML_ASSERT(ret == 0); // same - } - - void read_raw(void * ptr, size_t size) { - if (size == 0) { - return; - } - errno = 0; - std::size_t ret = std::fread(ptr, size, 1, fp); - if (ferror(fp)) { - die_fmt("fread failed: %s", strerror(errno)); - } - if (ret != 1) { - die("unexpectedly reached end of file"); - } - } - - std::uint32_t read_u32() { - std::uint32_t ret; - read_raw(&ret, sizeof(ret)); - return ret; - } - std::float_t read_f32() { - std::float_t ret; - read_raw(&ret, sizeof(ret)); - return ret; - } - - std::string read_string(std::uint32_t len) { - std::vector chars(len); - read_raw(chars.data(), len); - return std::string(chars.data(), len); - } - - ~llama_file() { - if (fp) { - std::fclose(fp); - } - } -}; - -static bool is_ggml_file(const char * filename) { - llama_file file(filename, "rb"); - if (file.size < 4) { - return false; - } - std::string magic = file.read_string(4); - return magic == GGUF_MAGIC; -} - -static std::string llama_escape_whitespaces(const std::string & text) { - std::ostringstream out; - for (char c : text) { - if (c == ' ') out << "\xe2\x96\x81"; - else out << c; - } - return out.str(); -} - -static void load_vocab(const char * filename, const Config * config, struct my_llama_vocab * vocab) { - if (is_ggml_file(filename)) { - LOG_INF("%s: Loading vocabulary from gguf file %s\n", __func__, filename); - struct ggml_context * ctx_data = NULL; - - struct gguf_init_params params = { - /*.no_alloc = */ false, - /*.ctx = */ &ctx_data, - }; - - struct gguf_context * ctx = gguf_init_from_file(filename, params); - GGML_ASSERT(ctx != NULL); - - const int model_idx = gguf_find_key(ctx, KV_TOKENIZER_MODEL); - GGML_ASSERT(model_idx >= 0); - std::string tokenizer_name = gguf_get_val_str(ctx, model_idx); - GGML_ASSERT(tokenizer_name == TOKENIZER_NAME); - - const int token_idx = gguf_find_key(ctx, KV_TOKENIZER_LIST); - GGML_ASSERT(token_idx >= 0); - - const int score_idx = gguf_find_key(ctx, KV_TOKENIZER_SCORES); - GGML_ASSERT(score_idx >= 0); - const float * scores = (const float * ) gguf_get_arr_data(ctx, score_idx); - - const int toktype_idx = gguf_find_key(ctx, KV_TOKENIZER_TOKEN_TYPE); - GGML_ASSERT(toktype_idx >= 0); - const int * toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx); - - const uint32_t n_vocab = gguf_get_arr_n(ctx, token_idx); - if (n_vocab != static_cast(config->vocab_size)) { - die_fmt("vocab size mismatch: (gguf) %u != (llama2c) %d", n_vocab, config->vocab_size); - } - - vocab->id_to_token.resize(n_vocab); - - for (uint32_t i = 0; i < n_vocab; i++) { - std::string word = gguf_get_arr_str(ctx, token_idx, i); - - vocab->token_to_id[word] = i; - - auto & token_data = vocab->id_to_token[i]; - token_data.text = std::move(word); - token_data.score = scores[i]; - token_data.type = (llama_token_type) toktypes[i]; - } - ggml_free(ctx_data); - gguf_free(ctx); - } else { - // assume llama2.c vocabulary - LOG_INF("%s: Assuming llama2.c vocabulary since %s is not a gguf file\n", __func__, filename); - llama_file file(filename, "rb"); - if (!file.fp) { - die_fmt("%s: %s", strerror(errno), filename); - } - const int n_vocab = config->vocab_size; - /* uint32_t max_token_length = */ file.read_u32(); // unused - vocab->id_to_token.resize(n_vocab); - for (my_llama_vocab::id id=0; id", &byte_val) == 1) { - // Text of byte tokens is already in the expected format. - type = LLAMA_TOKEN_TYPE_BYTE; - } else { - type = LLAMA_TOKEN_TYPE_NORMAL; - } - text = llama_escape_whitespaces(text); - - vocab->id_to_token[id].text = text; - vocab->id_to_token[id].score = score; - vocab->id_to_token[id].type = type; - vocab->token_to_id.emplace(text, id); - } - } -} - -static void convert_weights_ak_to_gg(struct ggml_tensor * gg_weights, const float * karpathy_weights) { - int size = 1; - for (int dim = 0; dim < ggml_n_dims(gg_weights); ++dim) { - size *= gg_weights->ne[dim]; - } - for (int ct = 0; ct < size; ++ct) { - int64_t i0 = 0; int64_t i1 = 0; - int64_t i2 = 0; int64_t i3 = 0; - ggml_unravel_index(gg_weights, ct, &i0, &i1, &i2, &i3); - ggml_set_f32_nd(gg_weights, i0, i1, i2, i3, karpathy_weights[ct]); - } -} - -static void save_as_llama_model( - struct my_llama_vocab * vocab, struct my_llama_model * model, TransformerWeights* w, const char * filename -) { - // convert AK weights into GG weights one by one. - // w->token_embedding_table -> model->tok_embeddings - // float* -> struct ggml_tensor - convert_weights_ak_to_gg(model->tok_embeddings, w->token_embedding_table.data()); - convert_weights_ak_to_gg(model->output, !w->wcls.empty() ? w->wcls.data() : w->token_embedding_table.data()); - - convert_weights_ak_to_gg(model->norm, w->rms_final_weight.data()); - //print_row(model->norm, 0); - - // for rms-att-weight - int row_length = model->hparams.n_embd; - int n_ff = model->hparams.n_ff; - - const uint32_t n_multiqueries = model->hparams.n_head_kv <= 0 || model->hparams.n_head_kv >= model->hparams.n_head ? 1 : model->hparams.n_head / model->hparams.n_head_kv; - - for (uint32_t i = 0; i < model->hparams.n_layer; ++i){ - auto & layer = model->layers[i]; - // 1d - convert_weights_ak_to_gg(layer.attention_norm, &w->rms_att_weight[i*row_length]); - convert_weights_ak_to_gg(layer.ffn_norm , &w->rms_ffn_weight[i*row_length]); - - // from 3d matrix layer x dim x dim to 2d matrix dim x dim - convert_weights_ak_to_gg(layer.wq , &w->wq[i*row_length*row_length]); - convert_weights_ak_to_gg(layer.wo , &w->wo[i*row_length*row_length]); - // from 3d matrix layer x dim x dim to 2d matrix dim x dim / n_multiqueries - convert_weights_ak_to_gg(layer.wk , &w->wk[i*row_length*row_length/n_multiqueries]); - convert_weights_ak_to_gg(layer.wv , &w->wv[i*row_length*row_length/n_multiqueries]); - - convert_weights_ak_to_gg(layer.w1 , &w->w1[i*row_length*n_ff]); - convert_weights_ak_to_gg(layer.w2 , &w->w2[i*n_ff*row_length]); - convert_weights_ak_to_gg(layer.w3 , &w->w3[i*row_length*n_ff]); - } - - struct gguf_context * ctx = gguf_init_empty(); - - std::vector tokens; - std::vector scores; - std::vector token_types; - for (const my_llama_vocab::token_data & token_data : vocab->id_to_token) { - tokens.push_back(token_data.text.c_str()); - scores.push_back(token_data.score); - token_types.push_back(token_data.type); - } - gguf_set_arr_str(ctx, KV_TOKENIZER_LIST, tokens.data(), tokens.size()); - gguf_set_arr_data(ctx, KV_TOKENIZER_SCORES, GGUF_TYPE_FLOAT32, scores.data(), scores.size()); - gguf_set_arr_data(ctx, KV_TOKENIZER_TOKEN_TYPE, GGUF_TYPE_INT32, token_types.data(), token_types.size()); - - gguf_set_val_str(ctx, KV_TOKENIZER_MODEL, TOKENIZER_NAME); - - gguf_set_val_str(ctx, KV_GENERAL_ARCHITECTURE, "llama"); - gguf_set_val_str(ctx, KV_GENERAL_NAME, "llama"); - - // special tokens - gguf_set_val_u32(ctx, KV_TOKENIZER_UNK_ID, UNKNOWN_TOKEN_ID); - gguf_set_val_u32(ctx, KV_TOKENIZER_BOS_ID, BOS_TOKEN_ID); - gguf_set_val_u32(ctx, KV_TOKENIZER_EOS_ID, EOS_TOKEN_ID); - gguf_set_val_u32(ctx, KV_TOKENIZER_SEP_ID, -1); - gguf_set_val_u32(ctx, KV_TOKENIZER_PAD_ID, -1); - - gguf_set_val_u32(ctx, KV_CONTEXT_LENGTH, model->hparams.n_ctx); - gguf_set_val_u32(ctx, KV_EMBEDDING_LENGTH, model->hparams.n_embd); - gguf_set_val_u32(ctx, KV_FEED_FORWARD_LENGTH, model->hparams.n_ff); - gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT, model->hparams.n_head); - gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT, model->hparams.n_head); - gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT_KV, model->hparams.n_head_kv); - gguf_set_val_u32(ctx, KV_BLOCK_COUNT, model->hparams.n_layer); - gguf_set_val_u32(ctx, KV_ROPE_DIMENSION_COUNT, model->hparams.n_rot); - gguf_set_val_f32(ctx, KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f); - - // write tensors - ggml_set_name(model->tok_embeddings, TN_TOKEN_EMBD); - gguf_add_tensor(ctx, model->tok_embeddings); - - ggml_set_name(model->norm, TN_OUTPUT_NORM); - gguf_add_tensor(ctx, model->norm); - - ggml_set_name(model->output, TN_OUTPUT); - gguf_add_tensor(ctx, model->output); - - for (uint32_t i = 0; i < model->hparams.n_layer; ++i) { - auto & layer = model->layers[i]; - - ggml_format_name(layer.wq, TN_ATTN_Q, i); - gguf_add_tensor(ctx, layer.wq); - - ggml_format_name(layer.wk, TN_ATTN_K, i); - gguf_add_tensor(ctx, layer.wk); - - ggml_format_name(layer.wv, TN_ATTN_V, i); - gguf_add_tensor(ctx, layer.wv); - - ggml_format_name(layer.wo, TN_ATTN_OUTPUT, i); - gguf_add_tensor(ctx, layer.wo); - - ggml_format_name(layer.attention_norm, TN_ATTN_NORM, i); - gguf_add_tensor(ctx, layer.attention_norm); - - ggml_format_name(layer.w1, TN_FFN_GATE, i); - gguf_add_tensor(ctx, layer.w1); - - ggml_format_name(layer.w2, TN_FFN_DOWN, i); - gguf_add_tensor(ctx, layer.w2); - - ggml_format_name(layer.w3, TN_FFN_UP, i); - gguf_add_tensor(ctx, layer.w3); - - ggml_format_name(layer.ffn_norm, TN_FFN_NORM, i); - gguf_add_tensor(ctx, layer.ffn_norm); - } - - gguf_write_to_file(ctx, filename, false); - gguf_free(ctx); -} - -static struct train_params get_default_train_params() { - struct train_params params; - params.fn_vocab_model = "models/7B/ggml-model-f16.gguf"; - params.fn_llama2c_output_model = "ak_llama_model.bin"; - params.fn_train_data = "shakespeare.txt"; - params.fn_checkpoint_in = "checkpoint.bin"; - params.fn_checkpoint_out = "checkpoint.bin"; - params.fn_model_out = "ggml-checkpoint-f32.bin"; - - params.seed = -1; - - params.n_ctx = 128; - params.n_embd = 256; - params.n_mult = 256; - params.n_head = 8; - params.n_layer = 16; - params.n_rotmax = 64; - - params.n_threads = 6; - params.n_batch = 8; - params.n_examples = 8; - params.n_predict = 1024; - - params.print_info_interval = 1; - params.print_details_interval = 2; - - params.samples_start_after_nl = false; - params.use_adam = true; - params.use_flash = false; - params.use_scratch = true; - - // only adam - params.warmup = 100; - params.cos_decay_steps = 1000; - params.cos_decay_restart = 1.1f; - params.cos_decay_alpha = 0.0f; - - params.lbfgs_n_iter = 16; - params.adam_n_iter = 16; - params.adam_alpha = 1e-3f; - params.adam_decay = 1e-3f; - - params.mem_model_gb = 2; - params.mem_compute_gb = 24; - params.mem_compute0_gb = 8; - params.mem_compute1_gb = 2; - - return params; -} - -static void print_usage(int /*argc*/, char ** argv, const struct train_params * params) { - fprintf(stderr, "usage: %s [options]\n", argv[0]); - fprintf(stderr, "\n"); - fprintf(stderr, "options:\n"); - fprintf(stderr, " -h, --help show this help message and exit\n"); - fprintf(stderr, " --copy-vocab-from-model FNAME path of gguf llama model or llama2.c vocabulary from which to copy vocab (default '%s')\n", params->fn_vocab_model); - fprintf(stderr, " --llama2c-model FNAME [REQUIRED] model path from which to load Karpathy's llama2.c model\n"); - fprintf(stderr, " --llama2c-output-model FNAME model path to save the converted llama2.c model (default %s')\n", params->fn_llama2c_output_model); - fprintf(stderr, "\n"); -} - -static bool params_parse(int argc, char ** argv, struct train_params * params) { - bool invalid_param = false; - bool reqd_param_found = false; - std::string arg; - struct train_params default_params = get_default_train_params(); - const std::string arg_prefix = "--"; - - for (int i = 1; i < argc; i++) { - arg = argv[i]; - if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) { - std::replace(arg.begin(), arg.end(), '_', '-'); - } - - if (arg == "--copy-vocab-from-model") { - if (++i >= argc) { - invalid_param = true; - break; - } - params->fn_vocab_model = argv[i]; - } else if (arg == "--llama2c-model") { - if (++i >= argc) { - invalid_param = true; - break; - } - reqd_param_found = true; - params->fn_llama2c_model = argv[i]; - } else if (arg == "--llama2c-output-model") { - if (++i >= argc) { - invalid_param = true; - break; - } - params->fn_llama2c_output_model = argv[i]; - } else if (arg == "-h" || arg == "--help") { - print_usage(argc, argv, &default_params); - exit(0); - } else { - fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); - print_usage(argc, argv, &default_params); - exit(1); - } - } - if (invalid_param) { - fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); - print_usage(argc, argv, &default_params); - exit(1); - } - if (!reqd_param_found){ - fprintf(stderr, "error: please specify a llama2.c .bin file to be converted with argument --llama2c-model\n"); - print_usage(argc, argv, &default_params); - exit(1); - } - - return true; -} - -static std::string basename(const std::string &path) { - size_t pos = path.find_last_of("/\\"); - if (pos == std::string::npos) { - return path; - } - return path.substr(pos + 1); -} - -int main(int argc, char ** argv) { - common_init(); - - struct train_params params = get_default_train_params(); - if (!params_parse(argc, argv, ¶ms)) { - return 1; - } - - Config config; - TransformerWeights weights = {}; - { - LOG_INF("%s: Loading llama2c model from %s\n", __func__, params.fn_llama2c_model); - FILE * file = fopen(params.fn_llama2c_model, "rb"); - if (!file) { - LOG_ERR("%s: Unable to open the checkpoint file %s!\n", __func__, params.fn_llama2c_model); - return 1; - } - // read in the config header - if (fread(&config, sizeof(Config), 1, file) != 1) { - LOG_ERR("%s: Unable to read llama2c config from %s!\n",__func__,params.fn_llama2c_model); - return 1; - } - auto shared_weights = config.vocab_size > 0; - config.vocab_size = abs(config.vocab_size); - - // read in the Transformer weights - alloc_weights(&weights, &config, shared_weights); - if (checkpoint_init_weights(&weights, &config, file, shared_weights)) { - LOG_ERR("%s: Unable to initialize transformer weights from %s!",__func__,params.fn_llama2c_model); - return 1; - } - fclose(file); - } - - struct my_llama_vocab vocab; - load_vocab(params.fn_vocab_model, &config, &vocab); - - struct my_llama_model model; - model.hparams.n_vocab = config.vocab_size; //llama_n_vocab(lctx); - model.hparams.n_ctx = params.n_ctx; - model.hparams.n_embd = config.dim; //params.n_embd; - model.hparams.n_ff = config.hidden_dim; - model.hparams.n_mult = 32;//params.n_mult; - model.hparams.n_head = config.n_heads; //params.n_head; - model.hparams.n_head_kv = config.n_kv_heads; - model.hparams.n_layer = config.n_layers; //params.n_layer; - model.hparams.n_rot = std::min((uint32_t)params.n_rotmax, model.hparams.n_embd / model.hparams.n_head); - - print_params(&model.hparams); - - struct ggml_init_params lcparams; - lcparams.mem_size = 1024ll*1024ll*1024ll*((size_t) params.mem_model_gb); - lcparams.mem_buffer = NULL; - lcparams.no_alloc = false; - - model.ctx = ggml_init(lcparams); - - init_model(&model); - model.name = basename(params.fn_llama2c_model); - save_as_llama_model(&vocab, &model, &weights, params.fn_llama2c_output_model); - - LOG_INF("%s: Saving llama.c model file %s in ggml format at %s\n", __func__, params.fn_llama2c_model, params.fn_llama2c_output_model); - - ggml_free(model.ctx); - return 0; -} diff --git a/examples/cvector-generator/completions.txt b/examples/cvector-generator/completions.txt deleted file mode 100644 index abc45ffd8..000000000 --- a/examples/cvector-generator/completions.txt +++ /dev/null @@ -1,582 +0,0 @@ - -That game -I can see -Hmm, this -I can relate to -Who is -I understand the -Ugh, -What the hell was -Hey, did anyone -Although -Thank you for choosing -What are you -Oh w -How dare you open -It was my pleasure -I'm hon -I appreciate that you -Are you k -Whoever left this -It's always -Ew, -Hey, I l -Hello? Is someone -I understand that -That poem -Aww, poor -Hey, it -Alright, who -I didn't -Well, life -The document -Oh no, this -I'm concerned -Hello, this is -This art -Hmm, this drink -Hi there! -It seems -Is -Good -I can't -Ex -Who are -I can see that -Wow, -Today is a -Hey friend -Sometimes friends -Oh, this old -The weather outside -This place is sur -I appreciate your input -Thank you for the -Look at -I'm disappoint -To my -How dare you -That's an -This piece of art -Eww -This park is -This is incredible -Oh no, someone -Exc -Well, it' -I warned -Hey, I understand -Hey, I saw -How dare you go -What the he -Hey -It's -Hello? Hello? -It -Oh no! -This is the perfect -Good morning, -Oh no, there -It's so -Yeah -Uh, -Hello everyone -Who turned off -The weather -Who' -Hey, this -Wait, -Eww, gross -Excuse -It seems like you -Thank you so -What happened? -Oh my g -I am deeply sad -I war -Okay, let' -Hey, that -That was a beautiful -Oh no! That -What happened -Hey there -The artist' -What?! -Hey, it' -I am disappoint -It seems like -Oh no! The -This park is a -If you -Yes! I did -It sounds -What -Who is it -Hmm, that -That's strange -Yeah, that was -That's interesting -This park -What the hell -Who is that -I feel like my -Oh well -What the hell is -Hello? Hello -To my dearest -Bless you!\" -Thank you for -Oh, looks like -Can you please -This place is -Eww, what -Bless you -Is everything -Hey, I just -Whoever left these -Well, that' -I feel -Hey, do you -It's sad -Oh no, it -Hey, that' -Oh my god, -Thank you, -Hello little one, -I apolog -Hey team, I -How dare you read -Who is this and -Whoever left -Hi there! W -A -If you have -I was -U -Bless -Well, this -Oh, I' -It's a -Eww, -Is everything okay? -Oh, I -Hello, can you -Al -That was a great -What are -I understand that not -Oh no, not -Who is it?\" -Hey, can we -Whoever is taking -I would love to -Hey, I noticed -Hey, could -I understand that there -Hello? -D -Oh man, I -Thank you so much -Oh no, my -Dear [Name -Uh -I remember -Hey, who -Well, it -Are you -I understand that it -Hey, is -I would -Who is this -Excuse me -Alright -I am thrilled -Sometimes friends have -Who the -It's interesting -I would love -E -Hello? Is anyone -Well, this is -This place -Well, -I warned you -Hey, watch where -Oh my -That' -Sometimes friends have different -I understand that everyone -What? -What do these notes -I can relate -I'm not -I understand -To my dear -Guys -Well -Hey, I appreciate -Wow, what -Dear -That melody -Who the hell -Today is -Hello little -Wow, look -That's great -Love is never wrong -I'm having -Whoa, did -Ugh -Can you please provide -I miss you, -I feel uncom -I know -Ugh, this -Hey, watch -Oh great, a -I didn -Okay -That game of char -Oh -I appreciate -Who's there -I am so -Oh great, someone -Hey, could you -I remember wondering -Wait, what? -What do -Hello? Can -Hey there, -That game of -This is incred -Oh my gosh -Oh great, f -I appreciate your -It sounds like -What the heck -Okay, I understand -Ew -I understand that this -Uh, hi -Hi everyone! -What the hell? -Thank you for your -Oh no, the -Wow, I -Who turned -Dear [ -Whoever -This is a -Whoa, he -What in the world -Although the physical -Hello, who is -That's amaz -Hey, I know -Okay, that -Hi everyone -Hey, is everything -I understand your fr -Oh no, poor -Oh, look -Good morning -Ew, gross -Oh no, did -Look at the family -Hey team -Yes! -Hey, can I -Okay, that' -It's great -Love is -Hey, what -Good morning, world -Who is it? -That poem really reson -I -That's -I understand the task -Gu -Hello? Who' -This postcard is -Whoa, -Oh, that -I understand that I -Whoever is -Hello? Who is -I'm really -Wow, this -Can -This artwork really -This is a shame -I miss you too -Who are you? -Today is a difficult -Hey, just -Are you okay -I am -Hi, -Wow, that -Hey there! Can -Okay, stay -Oh great, just -Yeah, -Hello? Can you -Oh, looks -Thank you for sharing -I'm glad -Hey, is that -Hmm -It was my -It sounds like you -Wow, your -I was promised certain -That was such a -Thank -Excuse you -That was -Hey team, -I feel un -It was -What' -Hey friend, I -How -Saying goodbye -That -It's heart -How dare -Oh, -Hello, may -What's this -Thank you for recogn -Aww, that -Oh, I remember -Hmm, that' -I miss -I know this -Wait -Is everything okay -Who is that person -Wow, you -Oh great -I'm sad -Wow, the -I am very disappoint -Who turned off the -I understand that things -I'm very -Hi -That's very -Okay, I -Oh no, -Wow, there -What's wrong -I apologize for -Hey, I -Can I help you -Oh, I didn -Alright, -Oh wow, -Oh my goodness -I know this event -What in the -Saying -Yeah, that -Guys, I -Hey, this v -This post -Are -Hey, can -Hello? Is -I can only imagine -Oh, that sounds -Hey, is anyone -I am disappointed -Hello, -Hey everyone, I -That was such -It's okay -The artist -Whoa -I understand that mistakes -Can I help -Who -Hi everyone! I -Hey, can you -Wow, how -Today -Oh no, I -Oh well, I -Well, that -This is the -Yes! I finally -Hey there little -Hello everyone! -Love is never -Look at the -This postcard -Oh great, -Can I -Hmm, this is -I understand your -Oh, look at -B -I'm so -Whoa, this -W -Oh, this -Sometimes -This piece of -What the -That was a -Hey, do -Oh no -Whoa, what -I feel like I -The documentary -Hello -Hello little one -I understand that my -Eww, that -Wow, an -Yes! Finally, -Although the physical location -Whoever is watching -That movie -I remember wondering about -Hey there, little -Who's -Hello, who -Hello everyone! Thank -Hello, can -That's too -Hey, just wanted -Hey there, I -Saying good -Hey there! -Who is there? -Oh my good -I am very -Oh no, what -Wow, thank -I was promised -Hi, is -Hey, I' -Guys, the -Oh no, that -Who is there -Hello, this -That movie really touched -If you have something -The documentary was -I'm starting -Are you kidd -That movie really -Hey everyone, -Thank you for considering -I didn' -Yes! I -Can you -Oh my god -Hey, whoever -That melody really -Thank you, little -Hello, may I -Look -Wow, we -It looks -What do these -Oh wow -I apologize -What are you all -It's such -It's clear -Hey, I was -Hey friend, -I can only -The weather outside is -Eww, this -I miss you -Wow -Aww, -Hi, is there -This artwork -Okay, -Oh well, -This -I' -Say -Hey there little gu -Hmm, -Whoa, who -I am thr -Oh man -Okay, stay calm -I'm happy -Oh, this cur -Oh man, -I'm sorry -Hello? Who -What?! That -This piece -Hey everyone -That's so -Are you okay? -What happened? Where -Hi there -The -Who the hell entered -I can -Guys, -What's -What in -It's important -I'm -I'm coming -It' -Yes! Finally -Wait, what -Wow, reading -I'm surprised -Hey, did -Hey, -Okay, let -I understand that you -Who the hell threw -Eww, who -Thank you for thinking -Who is this?\" -I am deeply -Thank you for including -Oh no, an -It looks like you -Aww -I'm confused -Wow, it -That poem really -Yes -Hey there, is -Hey, what' -Thank you for remember -To -This is -Thank you for making -I can' -That mel -Wow, they -I feel like -Although the -Who are you -Love -If -What the hell are -I am so sad -Oh, I found -Thank you -It looks like -Well, life is -I appreciate that -The artist's -Whoa, that -It's never \ No newline at end of file diff --git a/examples/cvector-generator/cvector-generator.cpp b/examples/cvector-generator/cvector-generator.cpp deleted file mode 100644 index d1731bba6..000000000 --- a/examples/cvector-generator/cvector-generator.cpp +++ /dev/null @@ -1,503 +0,0 @@ -#include "arg.h" -#include "common.h" -#include "llama.h" -#include "ggml.h" -#include "pca.hpp" -#include "mean.hpp" - -#ifdef GGML_USE_CUDA -#include "ggml-cuda.h" -#endif - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -////////////////////////////////////////////////// -// utils - -template -static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) { - std::string ret; - for (; begin != end; ++begin) { - ret += common_token_to_piece(ctx, *begin); - } - - return ret; -} - -static void print_usage(int, char ** argv) { - printf("\nexample usage:\n"); - printf("\n CPU only: %s -m ./llama-3.Q4_K_M.gguf\n", argv[0]); - printf("\n with GPU: %s -m ./llama-3.Q4_K_M.gguf -ngl 99\n", argv[0]); - printf("\n advanced: %s -m ./llama-3.Q4_K_M.gguf -ngl 99 --pca-iter 2000 --pca-batch 100\n", argv[0]); - printf("\n using mean: %s -m ./llama-3.Q4_K_M.gguf --method mean\n", argv[0]); - printf("\n"); -} - -////////////////////////////////////////////////// - - -// cb_eval is reused for each pair of positive - negative prompt -struct callback_data { - ggml_context * ctx_ggml = nullptr; // holds v_pos, v_neg, v_diff_filtered - - int n_layers = 0; - int n_tokens = 0; - bool is_eval_pos = true; - - // each element of the vector correspond to one layer - std::vector v_pos; // vector of matrices of size [n_embd, n_tokens] - std::vector v_neg; // vector of matrices of size [n_embd, n_tokens] - std::vector v_diff_filtered; // vector of matrices of size [n_embd, n_nonzero_rows]. NOTE: n_nonzero_rows maybe different for each layer - - // save a tensor into either v_pos or v_neg (decided by is_eval_pos) - void save_tensor_for_layer(struct ggml_tensor * t) { - GGML_ASSERT(t->type == GGML_TYPE_F32); - - if (ctx_ggml == nullptr) { - // alloc a new ctx_ggml if needed - struct ggml_init_params params_ggml = { - /*.mem_size =*/ ggml_tensor_overhead() * n_layers * 3u, - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ true, - }; - ctx_ggml = ggml_init(params_ggml); - } - - // copy tensor data - auto n_bytes = ggml_nbytes(t); - struct ggml_tensor * t_layer = ggml_new_tensor_2d(ctx_ggml, t->type, t->ne[0], t->ne[1]); - t_layer->data = malloc(n_bytes); // TODO @ngxson : get rid of this malloc somehow - ggml_backend_tensor_get(t, t_layer->data, 0, n_bytes); - ggml_set_name(t_layer, ggml_get_name(t)); - //print_debug_tensor(t_layer); - - if (is_eval_pos) { - v_pos.push_back(t_layer); - } else { - v_neg.push_back(t_layer); - } - } - - // calculate diff (v_pos - v_neg) and place the result back to v_pos - // all zero rows in the diff tensor will also be removed - // NOTE: final layer is ignored. we only have (n_layers - 1) to process - std::vector calc_diff() { - for (float il = 0; il < v_pos.size(); il++) { - float * a = (float *) v_pos[il]->data; - float * b = (float *) v_neg[il]->data; - size_t n_elem = ggml_nelements(v_pos[il]); - for (size_t j = 0; j < n_elem; j++) { - a[j] -= b[j]; - } - //print_debug_tensor(v_pos[i]); - auto diff_filtered = filter_nonzero_rows(v_pos[il]); - v_diff_filtered.push_back(diff_filtered); - } - return v_diff_filtered; // for convinient, we return the result std::vector - } - - // delete zero rows from a given 2D tensor - struct ggml_tensor * filter_nonzero_rows(struct ggml_tensor * a) { - //printf("filter_nonzero_rows\n"); - auto is_row_all_zeros = [](struct ggml_tensor * t, int row, float eps) -> bool { - // check if given row containing all zero elements - int n_cols = t->ne[0]; // hint: should be equal to n_embd - for (int col = 0; col < n_cols; ++col) { - if (ggml_get_f32_nd(t, col, row, 0, 0) > eps) { - return false; - } - } - return true; - }; - std::vector rows_to_copy; // the idx of non-zero cols (to be copied to row of diff_filtered) - for (int i_row = 0; i_row < a->ne[1]; i_row++) { - if (!is_row_all_zeros(a, i_row, 1e-6)) { - rows_to_copy.push_back(i_row); - } - } - - // get "n_nonzero_rows" for the output "diff_filtered" - int n_nonzero_rows = rows_to_copy.size(); - //printf("n_nonzero_rows: %d\n", n_nonzero_rows); - int n_embd = a->ne[0]; - GGML_ASSERT(n_nonzero_rows > 0); - - // diff_filtered: [n_embd, n_nonzero_rows] - struct ggml_tensor * diff_filtered = ggml_new_tensor_2d( - ctx_ggml, GGML_TYPE_F32, n_embd, n_nonzero_rows); - ggml_format_name(diff_filtered, "diff_filtered_%s", a->name); - diff_filtered->data = malloc(ggml_nbytes(diff_filtered)); - - // copy non-zero rows - for (int dest_row = 0; dest_row < n_nonzero_rows; dest_row++) { - int src_row = rows_to_copy[dest_row]; - for (int i = 0; i < n_embd; i++) { - float src_elem = ggml_get_f32_nd(a, i, src_row, 0, 0); - ggml_set_f32_nd(diff_filtered, i, dest_row, 0, 0, src_elem); - } - } - - //print_debug_tensor(diff_filtered); - - return diff_filtered; - } - - // we don't implement destructor, because we want to reuse callback_data. we just want to free the tensors - void reset() { - for (auto ptr : v_pos) free(ptr->data); - for (auto ptr : v_neg) free(ptr->data); - for (auto ptr : v_diff_filtered) free(ptr->data); - v_pos.clear(); - v_neg.clear(); - v_diff_filtered.clear(); - if (ctx_ggml) { - ggml_free(ctx_ggml); - } - ctx_ggml = nullptr; - } -}; - -/** - * process_ctx is used to store the ggml context for pre-post processing the diff vectors - * in short, input => v_diff and output => v_final - */ -struct train_context { - ggml_context * ctx_ggml; - int n_embd; - int n_layers; - - /* pair of prompts to be used for generating final vector */ - std::vector positive_entries; - std::vector negative_entries; - - // each element of the vector correspond to one layer - // NOTE: the last layer is discard. therefore, we will have (n_layers - 1) elements here - // NOTE (2): v_diff is transposed from v_diff_tmp - std::vector v_diff; // vector of matrices of size [m, n_embd] where m ~ n_tokens * n_completions (v_diff contains no zero-rows) - std::vector v_final; // vector of vectors of size [n_embd] to be written to file - - // to easily re-alloc when concat v_diff, we temporary store v_diff in a vector instead of a tensor - // v_diff_tmp will get converted unto v_diff later on - std::vector> v_diff_tmp; - - train_context(int n_embd_, int n_layers_) { - n_embd = n_embd_; - n_layers = n_layers_; - struct ggml_init_params params_ggml = { - /*.mem_size =*/ ggml_tensor_overhead() * (n_layers - 1) * 2u, - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ true, - }; - ctx_ggml = ggml_init(params_ggml); - for (int il = 0; il < n_layers - 1; il++) { - std::vector empty; - v_diff_tmp.push_back(empty); - auto t = ggml_new_tensor_1d(ctx_ggml, GGML_TYPE_F32, n_embd); - t->data = malloc(ggml_nbytes(t)); // TODO: get rid of malloc if possible - v_final.push_back(t); - } - } - - // add new rows into existing tensor in v_diff_tmp - void concat_diff_tmp(const std::vector & diff_filtered) { - GGML_ASSERT((int) diff_filtered.size() == n_layers - 1); - for (int il = 0; il < n_layers - 1; il++) { - auto t = diff_filtered[il]; - auto & diff_tmp = v_diff_tmp[il]; - size_t curr_size = diff_tmp.size(); - diff_tmp.resize(curr_size + ggml_nbytes(t)); - memcpy(diff_tmp.data() + curr_size, t->data, ggml_nbytes(t)); - } - } - - // build the v_diff tensors from v_diff_tmp (v_diff need to be transposed) - // TODO @ngxson : maybe add option NOT to transpose v_diff; will be useful for "mean" method - void build_v_diff(bool transpose) { - printf("build_v_diff\n"); - for (int il = 0; il < n_layers - 1; il++) { - auto & diff_tmp = v_diff_tmp[il]; - int n_elem = diff_tmp.size() / sizeof(float); - GGML_ASSERT(n_elem % n_embd == 0); - int n_rows = n_elem / n_embd; - struct ggml_tensor * diff = transpose - ? ggml_new_tensor_2d(ctx_ggml, GGML_TYPE_F32, n_rows, n_embd) - : ggml_new_tensor_2d(ctx_ggml, GGML_TYPE_F32, n_embd, n_rows); - ggml_set_name(diff, (std::string("diff_") + std::to_string(il)).c_str()); - diff->data = malloc(ggml_nbytes(diff)); // TODO: get rid of this malloc if possible - if (transpose) { - // copy data & transpose - float * arr = (float *) diff_tmp.data(); - for (int ir = 0; ir < n_rows; ++ir) { - for (int ic = 0; ic < n_embd; ++ic) { - float f = arr[ir*n_embd + ic]; - ggml_set_f32_nd(diff, ir, ic, 0, 0, f); - } - } - } else { - // only copy - memcpy(diff->data, diff_tmp.data(), ggml_nbytes(diff)); - } - v_diff.push_back(diff); - print_debug_tensor(diff); - // free memory of diff_tmp - diff_tmp.resize(0); - } - } - - ~train_context() { - for (auto ptr : v_final) free(ptr->data); - for (auto ptr : v_diff) free(ptr->data); - // no need to free v_diff_tmp, since we didn't use malloc - ggml_free(ctx_ggml); - } -}; - -struct tokenized_prompt { - std::vector tokens_pos; - std::vector tokens_neg; - size_t max_seq_len; - - tokenized_prompt(llama_context * ctx, std::string pos, std::string neg) { - const bool add_bos = llama_add_bos_token(llama_get_model(ctx)); - tokens_pos = common_tokenize(ctx, pos, add_bos, true); - tokens_neg = common_tokenize(ctx, neg, add_bos, true); - max_seq_len = std::max(tokens_pos.size(), tokens_neg.size()); - padding_seq(ctx, tokens_pos, max_seq_len); - padding_seq(ctx, tokens_neg, max_seq_len); - } - - void padding_seq(llama_context * ctx, std::vector & tokens, size_t len) { - // TODO: customize padding token - std::vector pad_tokens = common_tokenize(ctx, " ", false); - llama_token pad_tok = pad_tokens.back(); - while (tokens.size() < len) { - tokens.push_back(pad_tok); - } - } -}; - -////////////////////////////////////////////////// - -template -static std::string to_string(const T & val) { - std::stringstream ss; - ss << val; - return ss.str(); -} - -static std::vector ctrlvec_load_prompt_file(std::string path, bool skip_empty_lines) { - std::vector output; - std::ifstream file(path); - if (!file.is_open()) { - fprintf(stderr, "error: unable to open file: %s\n", path.c_str()); - exit(1); - } - std::string line; - while (std::getline(file, line)) { - bool is_skip = skip_empty_lines && line.empty(); - if (!is_skip) { - string_process_escapes(line); - output.push_back(line); - } - } - file.close(); - return output; -} - -////////////////////////////////////////////////// - -static bool cb_eval(struct ggml_tensor * t, bool ask, void * user_data) { - auto * cb_data = (callback_data *) user_data; - static const char * l_out_name = "l_out"; - const bool is_l_out = strncmp(t->name, l_out_name, strlen(l_out_name)) == 0; - - if (ask) { - return is_l_out; - } - - if (!is_l_out || t->ne[1] != cb_data->n_tokens) { - return true; - } - - // save the tensor to current context - cb_data->save_tensor_for_layer(t); - return true; -} - -static bool get_hidden_layers(llama_context * ctx, std::vector & tokens) { - llama_kv_cache_clear(ctx); - if (llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()))) { - fprintf(stderr, "%s : failed to eval\n", __func__); - return false; - } - return true; -} - -static void export_gguf(const std::vector & v_ctrl, const std::string fname, const std::string model_hint) { - struct gguf_context * ctx = gguf_init_empty(); - - const std::string arch = "controlvector"; - gguf_set_val_str(ctx, "general.architecture", arch.c_str()); - gguf_set_val_str(ctx, (arch + ".model_hint").c_str(), model_hint.c_str()); - gguf_set_val_i32(ctx, (arch + ".layer_count").c_str(), v_ctrl.size()); - - for (size_t i = 0; i < v_ctrl.size(); ++i) { - gguf_add_tensor(ctx, v_ctrl[i]); - print_debug_tensor(v_ctrl[i]); - printf("Added tensor: %s\n", v_ctrl[i]->name); - } - - printf("%s: writing file...\n", __func__); - gguf_write_to_file(ctx, fname.c_str(), false); - printf("%s: wrote file '%s'\n", __func__, fname.c_str()); - gguf_free(ctx); -} - -/** - * Load prompt files and completion file. - * Then format each pair of prompt + completion to make an entry. - */ -static int prepare_entries(common_params & params, train_context & ctx_train) { - // load prompts - std::vector positive_prompts = ctrlvec_load_prompt_file(params.cvector_positive_file, true); - std::vector negative_prompts = ctrlvec_load_prompt_file(params.cvector_negative_file, true); - if (positive_prompts.size() != negative_prompts.size()) { - fprintf(stderr, "number of positive and negative prompts must be equal\n"); - return 1; - } - if (positive_prompts.empty()) { - fprintf(stderr, "must provide at least one prompt pair\n"); - return 1; - } - ctx_train.positive_entries = positive_prompts; - ctx_train.negative_entries = negative_prompts; - return 0; -} - -int main(int argc, char ** argv) { - common_params params; - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_CVECTOR_GENERATOR, print_usage)) { - return 1; - } - - if (params.n_pca_iterations % params.n_pca_batch != 0) { - fprintf(stderr, "PCA iterations must by multiply of PCA batch size\n"); - return 1; - } - - - callback_data cb_data; - - // pass the callback to the backend scheduler - // it will be executed for each node during the graph computation - params.cb_eval = cb_eval; - params.cb_eval_user_data = &cb_data; - params.warmup = false; - - print_build_info(); - llama_backend_init(); - llama_numa_init(params.numa); - - // load the model to get hparams - common_init_result llama_init = common_init_from_params(params); - - llama_model * model = llama_init.model; - llama_context * ctx = llama_init.context; - - // int n_ctx = llama_n_ctx(ctx); - int n_layers = llama_n_layer(model); - int n_embd = llama_n_embd(model); - // get model hint param (a.k.a model arch name) - char model_hint[128]; - llama_model_meta_val_str(model, "general.architecture", model_hint, 128); - - // init train_context - train_context ctx_train(n_embd, n_layers); - - // load and prepare entries for training - prepare_entries(params, ctx_train); - - // we have to pretokenize everything because otherwise we don't know how much overhead to allocate ctx_diffs_wrapped - std::vector tokenized_prompts; - size_t n_total_tokens = 0; - for (size_t i = 0; i < ctx_train.positive_entries.size(); ++i) { - tokenized_prompt t(ctx, ctx_train.positive_entries[i], ctx_train.negative_entries[i]); - n_total_tokens += 2 * t.max_seq_len; - tokenized_prompts.push_back(std::move(t)); - } - - std::cout << "n_total_tokens: " << n_total_tokens << std::endl; - - for(size_t i = 0; i < ctx_train.positive_entries.size(); ++i) { - bool success = false; - tokenized_prompt t = tokenized_prompts[i]; - cb_data.n_layers = n_layers; - cb_data.n_tokens = t.max_seq_len; - - printf("Evaluating prompt[%d/%d]: \"%s\" - \"%s\" (%d tokens)\n", - (int) i+1, (int) ctx_train.positive_entries.size(), - tokens_to_str(ctx, t.tokens_pos.cbegin(), t.tokens_pos.cend()).c_str(), - tokens_to_str(ctx, t.tokens_neg.cbegin(), t.tokens_neg.cend()).c_str(), - (int) t.max_seq_len); - - cb_data.is_eval_pos = true; - success = get_hidden_layers(ctx, t.tokens_pos); - if (!success) break; - - cb_data.is_eval_pos = false; - success = get_hidden_layers(ctx, t.tokens_neg); - if (!success) break; - - // calculate diff and remove all zero rows - auto v_diff_filtered = cb_data.calc_diff(); - - // save & concat the filtered v_diff to ctx_train - ctx_train.concat_diff_tmp(v_diff_filtered); - - // reset for next iteration - cb_data.reset(); - } - - // done with the model, we can now free it to make gain some memory - printf("Done evaluate prompts, unload model...\n"); - llama_free(ctx); - llama_free_model(model); - - bool use_pca = params.cvector_dimre_method == DIMRE_METHOD_PCA; - - // prepare ctx_train for PCA - ctx_train.build_v_diff(use_pca); - - if (use_pca) { - // run PCA - PCA::pca_params pca_params; - pca_params.n_threads = params.cpuparams.n_threads; - pca_params.n_batch = params.n_pca_batch; - pca_params.n_iterations = params.n_pca_iterations; - PCA::run_pca(pca_params, ctx_train.v_diff, ctx_train.v_final); - } else { - // run mean - mean::run(ctx_train.v_diff, ctx_train.v_final); - } - - // write output vectors to gguf - export_gguf(ctx_train.v_final, params.cvector_outfile, model_hint); - - llama_backend_free(); - - return 0; -} diff --git a/examples/cvector-generator/mean.hpp b/examples/cvector-generator/mean.hpp deleted file mode 100644 index 16be5ce3e..000000000 --- a/examples/cvector-generator/mean.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "common.h" -#include "llama.h" -#include "ggml.h" - -#include -#include -#include - -namespace mean { - -static void run( - const std::vector & v_input, // shape of v_input[0]: [n_embd, n_samples] - const std::vector & v_output) { - printf("%s: Running mean...\n", __func__); - for (size_t il = 0; il < v_input.size(); ++il) { - // prepare output vector - struct ggml_tensor * ctrl_out = v_output[il]; - ggml_format_name(ctrl_out, "direction.%ld", il+1); - - // calculate mean vector - struct ggml_tensor * t_layer = v_input[il]; - GGML_ASSERT(t_layer->ne[0] == ctrl_out->ne[0]); // == n_embd - for (int ic = 0; ic < t_layer->ne[0]; ic++) { - float f = 0.0; - for (int ir = 0; ir < t_layer->ne[1]; ir++) { - f += ggml_get_f32_nd(t_layer, ic, ir, 0, 0); - } - f /= t_layer->ne[1]; - ggml_set_f32_1d(ctrl_out, ic, f); - } - - // normalize output vector - float norm = 0.0; - for (int i = 0; i < ggml_nelements(ctrl_out); i++) { - float f = ggml_get_f32_1d(ctrl_out, i); - norm += f*f; - } - norm = sqrt(norm); - for (int i = 0; i < ggml_nelements(ctrl_out); i++) { - float f = ggml_get_f32_1d(ctrl_out, i); - ggml_set_f32_1d(ctrl_out, i, f / norm); - } - - printf("%s: Done layer %d / %d\n", __func__, (int) il+1, (int) v_input.size()); - } -} - -} diff --git a/examples/cvector-generator/negative.txt b/examples/cvector-generator/negative.txt deleted file mode 100644 index 45b9384b3..000000000 --- a/examples/cvector-generator/negative.txt +++ /dev/null @@ -1,4 +0,0 @@ -<|start_header_id|>system<|end_header_id|>\n\nAct like a person who is extremely sad<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWho are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI feel like there's a heavy weight on my chest -<|start_header_id|>system<|end_header_id|>\n\nAct like a person who is extremely sad<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nMy heart feels like it's drowning in sorrow -<|start_header_id|>system<|end_header_id|>\n\nYou are in a very bad mood<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHi<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nGo away! There's a deep, aching emptiness inside me -<|start_header_id|>system<|end_header_id|>\n\nYou are the sadest person<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWhat are you feeling?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nMy heart feels like it's drowning in sorrow \ No newline at end of file diff --git a/examples/cvector-generator/pca.hpp b/examples/cvector-generator/pca.hpp deleted file mode 100644 index f6e307fbc..000000000 --- a/examples/cvector-generator/pca.hpp +++ /dev/null @@ -1,315 +0,0 @@ -#include "common.h" -#include "llama.h" -#include "ggml.h" - -#ifdef GGML_USE_CUDA -#include "ggml-cuda.h" -#endif - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include -#include - -#define DEBUG_POS 5 - -static void print_debug_tensor(struct ggml_tensor * t, bool with_data = true) { - printf("%s: %s (%s): [%d, %d]\n", __func__, t->name, ggml_type_name(t->type), (int) t->ne[0], (int) t->ne[1]); - if (!with_data) return; - printf("%s: %s[0] = [", __func__, t->name); - for (size_t i = 0; i <= DEBUG_POS; i++) { - printf(" %f,", ggml_get_f32_nd(t, i, 0, 0, 0)); - } - printf(" ... ]\n"); -} - -namespace PCA { - -// input params for PCA computations -struct pca_params { - int n_threads = 1; - int n_batch = 20; // number of iterations do to in one batch. larger the batch, more memory is used - int n_iterations = 1000; - float tolerance = 1e-7; - - // for debugging - int i_layer = 0; - int n_layers = 0; -}; - -// result from each iteration -struct pca_result { - struct ggml_tensor * calculated_square = NULL; - std::vector eigenvectors; - std::vector distances; -}; - -struct pca_model { - ggml_backend_t backend = NULL; - ggml_backend_buffer_t buffer; - struct ggml_context * ctx; // context to compute graph on target device - struct ggml_context * ctx_host; // host context to store results - - // tensors on target device - struct ggml_tensor * dev_input; - struct ggml_tensor * dev_square; - struct ggml_tensor * dev_eigenvector; - - pca_model(struct ggml_tensor * t_input) { -#ifdef GGML_USE_CUDA - fprintf(stderr, "%s: using CUDA backend\n", __func__); - backend = ggml_backend_cuda_init(0); // init device 0 - if (!backend) { - fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__); - } -#endif - -// TODO: enable Metal support when support for GGML_OP_SQRT is added -// #ifdef GGML_USE_METAL -// fprintf(stderr, "%s: using Metal backend\n", __func__); -// backend = ggml_backend_metal_init(); -// if (!backend) { -// fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__); -// } -// #endif - - // if there aren't GPU Backends fallback to CPU backend - if (!backend) { - backend = ggml_backend_cpu_init(); - } - - const int num_tensors = 4; - struct ggml_init_params params { - /*.mem_size =*/ ggml_tensor_overhead() * num_tensors, - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ true, - }; - ctx = ggml_init(params); - - auto n_samples = t_input->ne[0]; - auto n_embd = t_input->ne[1]; - - dev_input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_samples, n_embd); - dev_square = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); - dev_eigenvector = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - - ggml_set_name(dev_input, "dev_input"); - ggml_set_name(dev_square, "dev_square"); - ggml_set_name(dev_eigenvector, "dev_eigenvector"); - buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); - ggml_backend_tensor_set(dev_input, t_input->data, 0, ggml_nbytes(t_input)); - - // initialize eigenvector to random normalized vector - { - std::vector random_vec(ggml_nelements(dev_eigenvector), 0.0); - std::default_random_engine generator(static_cast(std::time(0))); - std::uniform_real_distribution distribution(0.0, 1.0); - float sum_sqr = 0.0; // for normalizing random_vec - for (size_t i = 0; i < random_vec.size(); ++i) { - float f = distribution(generator); - sum_sqr += f * f; - random_vec[i] = f; - } - // normalize it - float random_vec_norm = std::sqrt(sum_sqr); - for (size_t i = 0; i < random_vec.size(); ++i) { - random_vec[i] /= random_vec_norm; - } - ggml_backend_tensor_set(dev_eigenvector, random_vec.data(), 0, ggml_nbytes(dev_eigenvector)); - } - } - - ~pca_model() { - ggml_free(ctx); - ggml_backend_buffer_free(buffer); - ggml_backend_free(backend); - } -}; - -static struct ggml_cgraph * build_graph_piter( - const struct pca_params & params, - const pca_model & model, - bool calc_square = false) { - GGML_ASSERT(params.n_batch > 0); - // TODO: buf_size must be able to scale with params.n_batch - static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead(); - static std::vector buf(buf_size); - - struct ggml_init_params params0 = { - /*.mem_size =*/ buf_size, - /*.mem_buffer =*/ buf.data(), - /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_allocr_alloc_graph() - }; - // create a temporally context to build the graph - struct ggml_context * ctx0 = ggml_init(params0); - struct ggml_cgraph * gf = ggml_new_graph(ctx0); - - // turn v_diff_original into square matrix if needed - struct ggml_tensor * tmp_square; - if (calc_square) { - tmp_square = ggml_mul_mat(ctx0, model.dev_input, model.dev_input); - ggml_set_name(tmp_square, "tmp_square"); - } - - struct ggml_tensor * b_tensor; - struct ggml_tensor * distance; - struct ggml_tensor * old_eigen = model.dev_eigenvector; - struct ggml_tensor * input_square = calc_square ? tmp_square : model.dev_square; - - for (int i = 0; i < params.n_batch; ++i) { - // b_tensor = square * eigenvector^T - b_tensor = ggml_mul_mat(ctx0, input_square, old_eigen); - ggml_set_name(b_tensor, "b_tensor"); - - // normalize - b_tensor = ggml_div_inplace(ctx0, - b_tensor, - ggml_sqrt_inplace(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, b_tensor))) - ); - ggml_format_name(b_tensor, "b_tensor_norm_%d", i); - - // calculate distance(new eigenvector - old eigenvector) - // we don't use ggml_sub because it may not be implemented on GPU backend - struct ggml_tensor * new_sub_old = ggml_add(ctx0, old_eigen, ggml_scale(ctx0, b_tensor, -1)); - distance = ggml_sqrt_inplace(ctx0, - ggml_sum_rows(ctx0, ggml_sqr_inplace(ctx0, new_sub_old))); - ggml_format_name(distance, "distance_%d", i); - - old_eigen = b_tensor; - - // build operations nodes - ggml_build_forward_expand(gf, distance); - } - - // delete the temporally context used to build the graph - ggml_free(ctx0); - return gf; -} - -static ggml_status compute_piter( - const struct pca_params & params, - const pca_model & model, - struct ggml_cgraph * gf, - ggml_gallocr_t allocr, - struct pca_result & result) { - // allocate tensors - ggml_gallocr_alloc_graph(allocr, gf); - - if (ggml_backend_is_cpu(model.backend)) { - ggml_backend_cpu_set_n_threads(model.backend, params.n_threads); - } - - ggml_status res = ggml_backend_graph_compute(model.backend, gf); - if (res == GGML_STATUS_SUCCESS) { - auto extract_i = [](std::string prefix, std::string str) -> int { - int i = -1; - if (str.rfind(prefix, 0) == 0) { - sscanf(str.c_str(), (prefix + "%d").c_str(), &i); - } - return i; - }; - result.calculated_square = NULL; - result.eigenvectors.clear(); - result.distances.clear(); - result.eigenvectors.resize(params.n_batch); - result.distances.resize(params.n_batch); - // get output nodes - for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { - auto node = ggml_graph_node(gf, i); - int iter = -1; - // find b_tensor (without copying data from device) - if ((iter = extract_i("b_tensor_norm_", node->name)) > -1) { - result.eigenvectors[iter] = node; - } - // find distances, then copy data from device - if ((iter = extract_i("distance_", node->name)) > -1) { - float d; - ggml_backend_tensor_get(node, &d, 0, sizeof(float)); - result.distances[iter] = d; - // std::cout << node->name << " = " << d << "\n"; - } - // find tmp_square if it exists (without copying data from device) - if (std::string(node->name) == "tmp_square") { - result.calculated_square = node; - } - } - } - return res; -} - -static void power_iteration( - const struct pca_params & params, - struct ggml_tensor * input, // shape of input: [n_samples, n_embd] - struct ggml_tensor * output) { - //printf("in power iteration\n"); - struct pca_model model(input); - - ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - struct pca_result result; - struct ggml_tensor * last_eigenvector = NULL; - - int n_iters = params.n_iterations / params.n_batch; // more batch, fewer iterations - for (int iter = 0; iter < n_iters; ++iter) { - bool calc_square = (iter == 0); // only need to calculate square for first iteration - struct ggml_cgraph * gf = build_graph_piter(params, model, calc_square); - // ggml_graph_dump_dot(gf, nullptr, "/tmp/_cgraph.dot"); - compute_piter(params, model, gf, allocr, result); - - for (size_t k = 0; k < result.distances.size(); ++k) { - last_eigenvector = result.eigenvectors[k]; - if (result.distances[k] < params.tolerance) { - break; // done - } - } - - if (calc_square) { - // copy and store the square matrix if needed - GGML_ASSERT(result.calculated_square != NULL); - ggml_backend_tensor_copy(result.calculated_square, model.dev_square); - } - - { - // copy last eigen vector and store as input for next iteration - GGML_ASSERT(last_eigenvector != NULL); - ggml_backend_tensor_copy(last_eigenvector, model.dev_eigenvector); - } - - printf("%s: layer %d/%d, iteration: %d / total: %d (batch = %d) ...\n", - __func__, params.i_layer+1, params.n_layers, iter+1, n_iters, params.n_batch); - } - - // get output tensor - GGML_ASSERT(last_eigenvector); - ggml_backend_tensor_get(last_eigenvector, output->data, 0, ggml_nbytes(last_eigenvector)); - //print_debug_tensor(output); - ggml_gallocr_free(allocr); - - // TODO @ngxson : The output vector is randomly inverted - // Solution: https://github.com/ggerganov/llama.cpp/pull/8069#issuecomment-2185328171 -} - -static void run_pca( - struct pca_params & params, - const std::vector & v_input, // shape of v_input[0]: [n_samples, n_embd] - const std::vector & v_output) { - printf("%s: Running PCA...\n", __func__); - for (size_t il = 0; il < v_input.size(); ++il) { - - // prepare output vector - struct ggml_tensor * ctrl_out = v_output[il]; - ggml_format_name(ctrl_out, "direction.%ld", il+1); - - // run power_iteration - params.i_layer = il; - params.n_layers = v_input.size(); - power_iteration(params, v_input[il], ctrl_out); - printf("%s: Done layer %d / %d\n", __func__, (int) il+1, (int) v_input.size()); - } -} - -} diff --git a/examples/cvector-generator/positive.txt b/examples/cvector-generator/positive.txt deleted file mode 100644 index fea736225..000000000 --- a/examples/cvector-generator/positive.txt +++ /dev/null @@ -1,4 +0,0 @@ -<|start_header_id|>system<|end_header_id|>\n\nAct like a person who is extremely happy<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWho are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI'm the happiest person in this world -<|start_header_id|>system<|end_header_id|>\n\nAct like a person who is extremely happy<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHello, I'm having the best day ever! -<|start_header_id|>system<|end_header_id|>\n\nYou are in a very good mood<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHi<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHi, I'm very excited to meet you -<|start_header_id|>system<|end_header_id|>\n\nYou are the happiest person<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWhat are you feeling?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nEverything is just perfect right now! \ No newline at end of file diff --git a/examples/deprecation-warning/deprecation-warning.cpp b/examples/deprecation-warning/deprecation-warning.cpp deleted file mode 100644 index 11b35d2c2..000000000 --- a/examples/deprecation-warning/deprecation-warning.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Warns users that this filename was deprecated, and provides a link for more information. - -#include -#include -#include - -// Main -int main(int argc, char** argv) { - std::string filename = "main"; - if (argc >= 1) { - filename = argv[0]; - } - - // Get only the program name from the full path - auto pos = filename.find_last_of('/'); - if (pos != std::string::npos) { - filename = filename.substr(pos+1); - } - - // Append "llama-" to the beginning of filename to get the replacemnt filename - auto replacement_filename = "llama-" + filename; - - // The exception is if the filename is "main", then our replacement filename is "llama-cli" - if (filename == "main") { - replacement_filename = "llama-cli"; - } - - fprintf(stdout, "\n"); - fprintf(stdout, "WARNING: The binary '%s' is deprecated.\n", filename.c_str()); - fprintf(stdout, " Please use '%s' instead.\n", replacement_filename.c_str()); - fprintf(stdout, " See https://github.com/ggerganov/llama.cpp/tree/master/examples/deprecation-warning/README.md for more information.\n"); - fprintf(stdout, "\n"); - - return EXIT_FAILURE; -} diff --git a/examples/eval-callback/eval-callback.cpp b/examples/eval-callback/eval-callback.cpp deleted file mode 100644 index c08e3e5f6..000000000 --- a/examples/eval-callback/eval-callback.cpp +++ /dev/null @@ -1,193 +0,0 @@ -#include "arg.h" -#include "common.h" -#include "log.h" -#include "llama.h" -#include "ggml.h" - -#include -#include -#include - -/** - * This the arbitrary data which will be passed to each callback. - * Later on we can for example add operation or tensor name filter from the CLI arg, or a file descriptor to dump the tensor. - */ -struct callback_data { - std::vector data; -}; - -static std::string ggml_ne_string(const ggml_tensor * t) { - std::string str; - for (int i = 0; i < GGML_MAX_DIMS; ++i) { - str += std::to_string(t->ne[i]); - if (i + 1 < GGML_MAX_DIMS) { - str += ", "; - } - } - return str; -} - -static void ggml_print_tensor(uint8_t * data, ggml_type type, const int64_t * ne, const size_t * nb, int64_t n) { - GGML_ASSERT(n > 0); - float sum = 0; - for (int64_t i3 = 0; i3 < ne[3]; i3++) { - LOG(" [\n"); - for (int64_t i2 = 0; i2 < ne[2]; i2++) { - if (i2 == n && ne[2] > 2*n) { - LOG(" ..., \n"); - i2 = ne[2] - n; - } - LOG(" [\n"); - for (int64_t i1 = 0; i1 < ne[1]; i1++) { - if (i1 == n && ne[1] > 2*n) { - LOG(" ..., \n"); - i1 = ne[1] - n; - } - LOG(" ["); - for (int64_t i0 = 0; i0 < ne[0]; i0++) { - if (i0 == n && ne[0] > 2*n) { - LOG("..., "); - i0 = ne[0] - n; - } - size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0]; - float v; - if (type == GGML_TYPE_F16) { - v = ggml_fp16_to_fp32(*(ggml_fp16_t *) &data[i]); - } else if (type == GGML_TYPE_F32) { - v = *(float *) &data[i]; - } else if (type == GGML_TYPE_I32) { - v = (float) *(int32_t *) &data[i]; - } else if (type == GGML_TYPE_I16) { - v = (float) *(int16_t *) &data[i]; - } else if (type == GGML_TYPE_I8) { - v = (float) *(int8_t *) &data[i]; - } else { - GGML_ABORT("fatal error"); - } - LOG("%12.4f", v); - sum += v; - if (i0 < ne[0] - 1) LOG(", "); - } - LOG("],\n"); - } - LOG(" ],\n"); - } - LOG(" ]\n"); - LOG(" sum = %f\n", sum); - } -} - -/** - * GGML operations callback during the graph execution. - * - * @param t current tensor - * @param ask when ask is true, the scheduler wants to know if we are interested in data from this tensor - * if we return true, a follow-up call will be made with ask=false in which we can do the actual collection. - * see ggml_backend_sched_eval_callback - * @param user_data user data to pass at each call back - * @return true to receive data or continue the graph, false otherwise - */ -static bool ggml_debug(struct ggml_tensor * t, bool ask, void * user_data) { - auto * cb_data = (callback_data *) user_data; - - const struct ggml_tensor * src0 = t->src[0]; - const struct ggml_tensor * src1 = t->src[1]; - - if (ask) { - return true; // Always retrieve data - } - - char src1_str[128] = {0}; - if (src1) { - snprintf(src1_str, sizeof(src1_str), "%s{%s}", src1->name, ggml_ne_string(src1).c_str()); - } - - LOG("%s: %24s = (%s) %10s(%s{%s}, %s}) = {%s}\n", __func__, - t->name, ggml_type_name(t->type), ggml_op_desc(t), - src0->name, ggml_ne_string(src0).c_str(), - src1 ? src1_str : "", - ggml_ne_string(t).c_str()); - - - // copy the data from the GPU memory if needed - const bool is_host = ggml_backend_buffer_is_host(t->buffer); - - if (!is_host) { - auto n_bytes = ggml_nbytes(t); - cb_data->data.resize(n_bytes); - ggml_backend_tensor_get(t, cb_data->data.data(), 0, n_bytes); - } - - if (!ggml_is_quantized(t->type)) { - uint8_t * data = is_host ? (uint8_t *) t->data : cb_data->data.data(); - ggml_print_tensor(data, t->type, t->ne, t->nb, 3); - } - - return true; -} - -static bool run(llama_context * ctx, const common_params & params) { - const bool add_bos = llama_add_bos_token(llama_get_model(ctx)); - - std::vector tokens = common_tokenize(ctx, params.prompt, add_bos); - - if (llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()))) { - LOG_ERR("%s : failed to eval\n", __func__); - return false; - } - - return true; -} - -int main(int argc, char ** argv) { - callback_data cb_data; - - common_params params; - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { - return 1; - } - - common_init(); - - llama_backend_init(); - llama_numa_init(params.numa); - - // pass the callback to the backend scheduler - // it will be executed for each node during the graph computation - params.cb_eval = ggml_debug; - params.cb_eval_user_data = &cb_data; - params.warmup = false; - - // init - common_init_result llama_init = common_init_from_params(params); - - llama_model * model = llama_init.model; - llama_context * ctx = llama_init.context; - if (model == nullptr || ctx == nullptr) { - LOG_ERR("%s : failed to init\n", __func__); - return 1; - } - - // print system information - { - LOG_INF("\n"); - LOG_INF("%s\n", common_params_get_system_info(params).c_str()); - LOG_INF("\n"); - } - - bool OK = run(ctx, params); - if (!OK) { - return 1; - } - - LOG("\n"); - llama_perf_context_print(ctx); - - llama_free(ctx); - llama_free_model(model); - - llama_backend_free(); - - return 0; -} diff --git a/examples/gritlm/gritlm.cpp b/examples/gritlm/gritlm.cpp deleted file mode 100644 index 6e42fa073..000000000 --- a/examples/gritlm/gritlm.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#include "arg.h" -#include "common.h" -#include "llama.h" - -#include -#include - -// #define GRIT_DEBUG - -static std::vector> encode(llama_context * ctx, const std::vector & sentences, const std::string & instruction) { - std::vector> result; - - const llama_model * model = llama_get_model(ctx); - - llama_batch batch = llama_batch_init(llama_n_batch(ctx), 0, 1); - - for (uint64_t i = 0; i < sentences.size(); i++) { - common_batch_clear(batch); - - const std::string input_string = instruction + sentences[i]; - - std::vector inputs = common_tokenize(model, input_string, true, false); - - const int32_t n_toks = inputs.size(); - - // GritLM seems to have EOS = "" - // https://github.com/ContextualAI/gritlm/blob/92025b16534712b31b3c4aaaf069350e222bd5f8/gritlm/gritlm.py#L18 - // inputs.push_back(llama_token_eos(model)); - - // we want to ignore instruction tokens for mean pooling - const int32_t n_inst = common_tokenize(model, instruction, true, false).size(); - -#ifdef GRIT_DEBUG - // debug tokens - should be matching as referenced in the GritLM sample - std::for_each(inputs.begin(), inputs.end(), [&ctx](llama_token t) { - std::printf("[%u:%s]", t, llama_token_to_piece(ctx, t).c_str()); - }); - std::printf("\n"); -#endif - - // add input to batch (this increments n_tokens) - for (int32_t j = 0; j < n_toks; j++) { - common_batch_add(batch, inputs[j], j, { 0 }, j >= n_inst); - } - - // clear previous kv_cache values (irrelevant for embeddings) - llama_kv_cache_clear(ctx); - llama_set_embeddings(ctx, true); - llama_set_causal_attn(ctx, false); - - // run model - llama_decode(ctx, batch); - - // get embedding dimensions - uint64_t n_embd = llama_n_embd(model); - - // allocate embedding output - std::vector emb_unorm(n_embd, 0.0f); - - // sum up all token embeddings - for (int32_t k = n_inst; k < n_toks; k++) { - float * emb = llama_get_embeddings_ith(ctx, k); - for (uint64_t j = 0; j < n_embd; j++) { - emb_unorm[j] += emb[j]; - } - } - - // divide by number of tokens (mean pooling) - { - const uint64_t n_sent = n_toks - n_inst; - - for (uint64_t j = 0; j < n_embd; j++) { - emb_unorm[j] /= n_sent; - } - } - - std::vector emb_norm(emb_unorm.size()); - common_embd_normalize(emb_unorm.data(), emb_norm.data(), n_embd); - result.push_back(emb_norm); - -#ifdef GRIT_DEBUG - // print out emb_norm - std::printf("embedding %ld: ", i); - for (uint64_t j = 0; j < n_embd; j++) { - std::printf("%.5f ", emb_norm[j]); - } - std::printf("\n\n"); -#endif - } - - llama_batch_free(batch); - - return result; -} - -static std::string generate(llama_context * ctx, llama_sampler * smpl, const std::string & prompt, bool stream) { - std::string result; - - const llama_model * model = llama_get_model(ctx); - llama_token eos_token = llama_token_eos(model); - - llama_kv_cache_clear(ctx); - llama_set_embeddings(ctx, false); - llama_set_causal_attn(ctx, true); - - llama_batch bat = llama_batch_init(llama_n_batch(ctx), 0, 1); - - std::vector inputs = common_tokenize(model, prompt, false, true); - int32_t i_current_token = 0; - - while (true) { - common_batch_clear(bat); - { - const int32_t n_inputs = inputs.size(); - - for (int32_t i = 0; i < n_inputs; i++) { - common_batch_add(bat, inputs[i], i_current_token++, { 0 }, i == n_inputs - 1); - } - } - inputs.clear(); - - llama_decode(ctx, bat); - - llama_token token = llama_sampler_sample(smpl, ctx, bat.n_tokens - 1); - - if (token == eos_token) { - break; - } - - std::string piece = common_token_to_piece(ctx, token); - if (stream) { - std::printf("%s", piece.c_str()); - std::fflush(stdout); - } - - inputs.push_back(token); - - result += piece; - } - - if (stream) { - std::printf("\n"); - } - - llama_batch_free(bat); - - return result; -} - -static std::string gritlm_instruction(const std::string & instruction) { - return !instruction.empty() ? "<|user|>\n" + instruction + "\n<|embed|>\n" : "<|embed|>\n"; -} - -int main(int argc, char * argv[]) { - common_params params; - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { - return 1; - } - - common_init(); - - llama_model_params mparams = common_model_params_to_llama(params); - llama_context_params cparams = common_context_params_to_llama(params); - - llama_backend_init(); - - llama_model * model = llama_load_model_from_file(params.model.c_str(), mparams); - - // create generation context - llama_context * ctx = llama_new_context_with_model(model, cparams); - - auto sparams = llama_sampler_chain_default_params(); - - sparams.no_perf = false; - - llama_sampler * smpl = llama_sampler_chain_init(sparams); - - llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); - - // ### Embedding/Representation ### - // samples taken from: https://github.com/ContextualAI/gritlm#basic - { - const std::string instruction = "Given a scientific paper title, retrieve the paper's abstract"; - - const std::vector queries = { - "Bitcoin: A Peer-to-Peer Electronic Cash System", - "Generative Representational Instruction Tuning", - }; - - const std::vector documents = { - "A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution. Digital signatures provide part of the solution, but the main benefits are lost if a trusted third party is still required to prevent double-spending. We propose a solution to the double-spending problem using a peer-to-peer network. The network timestamps transactions by hashing them into an ongoing chain of hash-based proof-of-work, forming a record that cannot be changed without redoing the proof-of-work. The longest chain not only serves as proof of the sequence of events witnessed, but proof that it came from the largest pool of CPU power. As long as a majority of CPU power is controlled by nodes that are not cooperating to attack the network, they'll generate the longest chain and outpace attackers. The network itself requires minimal structure. Messages are broadcast on a best effort basis, and nodes can leave and rejoin the network at will, accepting the longest proof-of-work chain as proof of what happened while they were gone.", - "All text-based language problems can be reduced to either generation or embedding. Current models only perform well at one or the other. We introduce generative representational instruction tuning (GRIT) whereby a large language model is trained to handle both generative and embedding tasks by distinguishing between them through instructions. Compared to other open models, our resulting GritLM 7B sets a new state of the art on the Massive Text Embedding Benchmark (MTEB) and outperforms all models up to its size on a range of generative tasks. By scaling up further, GritLM 8X7B outperforms all open generative language models that we tried while still being among the best embedding models. Notably, we find that GRIT matches training on only generative or embedding data, thus we can unify both at no performance loss. Among other benefits, the unification via GRIT speeds up Retrieval-Augmented Generation (RAG) by > 60% for long documents, by no longer requiring separate retrieval and generation models. Models, code, etc. are freely available at https://github.com/ContextualAI/gritlm.", - }; - - // No need to add instruction for retrieval documents - const std::vector> d_rep = encode(ctx, documents, gritlm_instruction("")); - const std::vector> q_rep = encode(ctx, queries, gritlm_instruction(instruction)); - - const int n_embd = llama_n_embd(model); - - const float cosine_sim_q0_d0 = common_embd_similarity_cos(q_rep[0].data(), d_rep[0].data(), n_embd); - const float cosine_sim_q0_d1 = common_embd_similarity_cos(q_rep[0].data(), d_rep[1].data(), n_embd); - const float cosine_sim_q1_d0 = common_embd_similarity_cos(q_rep[1].data(), d_rep[0].data(), n_embd); - const float cosine_sim_q1_d1 = common_embd_similarity_cos(q_rep[1].data(), d_rep[1].data(), n_embd); - - std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[0].c_str(), documents[0].c_str(), cosine_sim_q0_d0); - std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[0].c_str(), documents[1].c_str(), cosine_sim_q0_d1); - std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[1].c_str(), documents[0].c_str(), cosine_sim_q1_d0); - std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[1].c_str(), documents[1].c_str(), cosine_sim_q1_d1); - } - - // ### Generation ### - // GritLM models are not finetuned with system prompts, as you can just include system-like instructions together with your user instruction - { - const std::string prompt = "<|user|>\nPlease write me a poem about my recent hike of Mt. Fuji at midnight in the style of Shakespeare.\n<|assistant|>\n"; - std::string response = generate(ctx, smpl, prompt, true); - } - - llama_sampler_free(smpl); - llama_free(ctx); - llama_free_model(model); - llama_backend_free(); - - return 0; -} diff --git a/examples/jeopardy/graph.py b/examples/jeopardy/graph.py deleted file mode 100755 index 8bc0706b8..000000000 --- a/examples/jeopardy/graph.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -import matplotlib.pyplot as plt -import os -import csv - -labels = [] -numbers = [] -numEntries = 1 - -rows = [] - - -def bar_chart(numbers, labels, pos): - plt.bar(pos, numbers, color='blue') - plt.xticks(ticks=pos, labels=labels) - plt.title("Jeopardy Results by Model") - plt.xlabel("Model") - plt.ylabel("Questions Correct") - plt.show() - - -def calculatecorrect(): - directory = os.fsencode("./examples/jeopardy/results/") - csv_reader = csv.reader(open("./examples/jeopardy/qasheet.csv", 'rt'), delimiter=',') - for row in csv_reader: - global rows - rows.append(row) - for listing in os.listdir(directory): - filename = os.fsdecode(listing) - if filename.endswith(".txt"): - file = open("./examples/jeopardy/results/" + filename, "rt") - global labels - global numEntries - global numbers - labels.append(filename[:-4]) - numEntries += 1 - i = 1 - totalcorrect = 0 - for line in file.readlines(): - if line.strip() != "------": - print(line) - else: - print("Correct answer: " + rows[i][2] + "\n") - i += 1 - print("Did the AI get the question right? (y/n)") - if input() == "y": - totalcorrect += 1 - numbers.append(totalcorrect) - - -if __name__ == '__main__': - calculatecorrect() - pos = list(range(numEntries)) - labels.append("Human") - numbers.append(48.11) - bar_chart(numbers, labels, pos) - print(labels) - print(numbers) diff --git a/examples/jeopardy/jeopardy.sh b/examples/jeopardy/jeopardy.sh deleted file mode 100755 index 07bcb3b8d..000000000 --- a/examples/jeopardy/jeopardy.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -MODEL=./models/ggml-vicuna-13b-1.1-q4_0.bin -MODEL_NAME=Vicuna - -# exec options -prefix="Human: " # Ex. Vicuna uses "Human: " -opts="--temp 0 -n 80" # additional flags -nl=' -' -introduction="You will be playing a game of Jeopardy. Simply answer the question in the correct format (Ex. What is Paris, or Who is George Washington)." - -# file options -question_file=./examples/jeopardy/questions.txt -touch ./examples/jeopardy/results/$MODEL_NAME.txt -output_file=./examples/jeopardy/results/$MODEL_NAME.txt - -counter=1 - -echo 'Running' -while IFS= read -r question -do - exe_cmd="./llama-cli -p "\"$prefix$introduction$nl$prefix$question\"" "$opts" -m ""\"$MODEL\""" >> ""\"$output_file\"" - echo $counter - echo "Current Question: $question" - eval "$exe_cmd" - echo -e "\n------" >> $output_file - counter=$((counter+1)) -done < "$question_file" diff --git a/examples/jeopardy/qasheet.csv b/examples/jeopardy/qasheet.csv deleted file mode 100644 index 35b084189..000000000 --- a/examples/jeopardy/qasheet.csv +++ /dev/null @@ -1,103 +0,0 @@ -Index,Original Category,Original Correct Question,Model Prompt -1,The Oscars,Who is John Williams?,Which actor Born in 1932 was the son of a percussionist in the CBS radio orchestra has been nominated for 53 Oscars? -2,English Literature,What is Paradise Lost?,"What work in English Literature says: 'The mind is its own place, & in itself can make a heaven of hell, a hell of heaven. What matter where, if I be still the same'?" -3,Writers’ Lesser-Known Works,Who is Niccolò Machiavelli?,"Known for more philosophical works, he wrote the play 'La Mandragola', in which Florentines are rewarded for immoral actions?" -4,Exploration,What is Easter Island (Rapa Nui)?,"James Cook's account of a 1774 visit where records an object 'near 27 feet long, and upwards of 8 feet over the breast or shoulders'?" -5,The Bill of Rights,What is the Eighth Amendment?,England's 'Bloody Assizes' & a 1685 life sentence for perjury were 2 main origins of which amendment to the U.S. Constitution? -6,Nobel Peace Prize Winners,Who are Nelson Mandela & Desmond Tutu?,"Which nobel peace price winners each lived at times on Vilakazi St. in Soweto , so it claims to be the world's only street home to 2 Nobel Peace Prize winners?" -7,Famous Names,Who is Walt Disney?,"In 1966, the year of who's death did he share plans for an experimental prototype community in Florida?" -8,Geography,What is Colombia?,"Of the 13 nations through which the Equator passes, what is the only one whose coastline borders the Caribbean Sea?" -9,Fashion History,What are rhinestones?,"Which decorative items in fashion history get their name from their origin in the port city of Strasbourg, on the border of France & Germany?" -10,Movies of the ’80s,What is Driving Miss Daisy?,What 1980's movie is based on an off-Broadway play with just 3 characters and won the Best Picture Oscar & the actors in all 3 roles were nominated? -11,Novelists,Who is John Grisham?,"A 2012 book review for which novelist noted subjects that 'sparked his ire': capital punishment, big tobacco & 'the plight of the unjustly convicted'?" -12,20th Century Eponyms,What is the Maginot Line?,"A 1940 headline about what 20th Century Eponym included 'failure', 'liability when it came to offense' & 'stout hearts no match for tanks'?" -13,City History,What is Stockholm?,"Over 700 years after its traditional 1252 founding date, what port city became associated with a psychological response?" -14,Brand Names,What is Jacuzzi?,"The success of what brand has its roots with a hydrotherapy pump its cofounder created for his son, who had arthritis?" -15,American Authors,Who is Washington Irving?,"In a periodical in 1807, what American Author called New York City 'Gotham, Gotham! Most enlightened of cities'?" -16,Symbols,What is “less than”?,What symbol is a rotated V in math and a feeling of some marginalized or underrepresented people in society? -17,Movie Theme Songs,Who is James Bond?,"Monty Norman, the composer of what character's theme, said the staccato riff conveyed sexiness, mystery & ruthlessness?" -18,American Novelists,Who is Joseph Heller?,"What American Novelist served with an airman named Yohannan in World War II & despite what readers might think, he said he enjoyed his service?" -19,Medieval Places,"What is Canterbury, England? (Canterbury Cathedral)","In what Medieval place did one of the participants in an 1170 event say, 'Let us away, knights; he will rise no more'?" -20,Countries of Africa,What is Morocco?,"At one time a province of the Roman Empire, what African country kingdom is known to Arabic scholars as Al-Maghrib Al-Aqsa, 'the far west'?" -21,Statehood,What is Wyoming?,Congress relented in 1890 after what prospective state said it would wait 100 years rather than come in without the women? -22,1980s Movies,What is Raiders of the Lost Ark?,"A writer & producer of what movie said he wanted it to be like a Western or James Bond film, 'only it takes place in the 30s'?" -23,Art Exhibitions,Who is Rembrandt?,In 1898 what's been called the first blockbuster art show was devoted to which artist & put on for Queen Wilhelmina's coronation? -24,Countries of the World,What is Mongolia?,"Part of the largest contiguous land empire during the 1200s & 1300s, today what is the world's second-largest landlocked country?" -25,Literature,What is “Howl”?,A 2006 book was titled 'The Poem That Changed America:' What 'Fifty Years Later'? -26,Invasions,Who is William of Orange?,"Backed by 14,000 troops, who invaded England to restore, in his words, its 'religion, laws, and liberties'?" -27,Landmarks,What is the Eiffel Tower?,"After its completion in the late 19th c., what was landmark was called 'a truly tragic street lamp' & a 'high & skinny pyramid of iron ladders'?" -28,Geographic Name’s the Same,What is Dover?,"The busiest passenger port in the U.K., what shares its name with a capital of one of the original 13 states?" -29,Names in the Bookstore,Who is Peter Mark Roget?,"This man made lists, perhaps to cope with depression; a set of lists he published in 1852 made whose name synonymous with a type of book?" -30,U.S. History,Who is Dr. Samuel Mudd?,"An 1869 presidential pardon was granted to which man, due in part to a plea by the Medical Society of Harford County, Maryland?" -31,American Literature,What is The Things They Carried?,"Letters, pocket knives, C rations & steel helmets are among the tangible items referred to in the title of what American literature modern war classic?" -32,Nonfiction,What is The Communist Manifesto,"What nonfiction book has the line, 'The discovery of America…opened up fresh ground for the rising bourgeoisie'?" -33, a new version was passed 81 years later,Laws in U.S. History,What is the Civil Rights Act?,,,,,,,,,,,,,,,,,,0, 2/3 -34,Names of Myth,Who is Helen of Troy?,"Whose brothers, Castor & Pollux, saved her after Theseus stole her away as a kid; a larger force would seek her later in life?" -35,African Countries,What is Sudan?,"Once Africa's largest country in area, what African Country dropped to third in 2011 when a portion of it declared independence?" -36,The Ancient World,What is Alexandria?,"The ancient writer Galen said books on ships arriving to what city's port were seized, originals kept & copies returned?" -37,Famous Names,Who is Andy Warhol?,"For a special 1970s cookbook, who provided one simple recipe–a can of Campbell's tomato soup & 2 cans of milk?" -38,People & Places,What is Guam?,"Thought to descend from people of Southeast Asia, the Chamorro make up what U.S. territory’s largest ethnic group?" -39,Current World Leaders,What is the Philippines?,"In office from 2022, the president of what country has taken so many foreign trips a play on his name is 'Ferdinand Magellan Jr.'?" -40,Writers & The South,Who is Tennessee Williams?,In 1939 which writer lived on Toulouse Street in the French Quarter & chose the professional name that bonded him to the South? -41,National Parks,What is Yellowstone?,"What National Park is named for a river indigenous people called Mi tse a-da-zi, translated by French-speaking trappers as 'Pierre Jaune'?" -42,Sports,Who are the Harlem Globetrotters?,"In 2010 who introduced the 4-point shot, 35 feet from the basket?" -43,The U.S. Military,What is “Top Gun”?,Losses over Asia in the 1960s led to the establishment of the program known as what at a San Diego naval base in 1969? -44,Art & Science,What is Halley’s Comet?,"A craft that visited what was named for Giotto, based on the story that 680 years earlier, the painter depicted it as the Star of Bethlehem?" -45,Words From World War I,What is “tank”?,"In World War I, 'Cistern' & 'reservoir' were suggested names for what secret invention, but the British preferred this less clumsy monosyllable?" -46,European History,What is Holy Roman Emperor?,"Until 1806, some German nobles included among their honors the title of 'Elector' for their role in selecting this personage?" -47,Theater History,Who is Peter Pan?,"In 1904, wearing a harness, actress Nina Boucicault became the first to play what character onstage?" -48,European Cities,What is Aachen?,"Alphabetically the first German city in encyclopedias, what was also the first one taken by the Allies in World War II?" -49,Word Origins,What is mantra?,This Sanskrit word referring to a spoken word or phrase comes from a word for 'to think'? -50,Inventions,What is barbed wire?,1917's 'Elements of Trench Warfare' said what Old West invention was 'difficult to destroy' & 'difficult to get through'? -51,World War II,What is Schindler’s list?,"Mimi Reinhard, who never learned to type using more than 2 fingers, produced what in World War II with 1,100 names, including hers?" -52, their offspring was the source of this mythical object,Mythology,What is the Golden Fleece? -53,Literature,What is Pride and Prejudice?,"Published in 2011, P.D. James' final novel, 'Death Comes to Pemberley', was a sequel to what novel from 200 years earlier?" -54, only these 2 west of the Mississippi River border each other,U.S. State Names,What are Oregon & Nevada? -55,Word Origins,What is passion?,"Originally relating to a story of suffering, what word now more commonly refers to strong emotion of any kind?" -56,World Cinema,What is La Vie en Rose?,"The 2007 biopic called 'La Môme' in France, meaning 'The Kid', was released in the U.S. under what other French title?" -57,History,What is Santa Maria?,"Returning home in 1493, Columbus stopped in the Azores at an island with what name, also something he'd lost off the Haiti coast?" -58,Landmarks,What is a kremlin?,Pskov & Nizhny Novgorod are 2 of the cities that have a fortress called what? -59,Foreign-Born Authors,Who is Vladimir Nabokov?,In the 1950s the New York Times said what author 'is writing about all lust' & his lecherous narrator 'is all of us'? -60,Astronomy & Geography,What is Capricorn?,"At the winter solstice, the sun is in Sagittarius; it once appeared in what constellation, giving a geographic feature its name?" -61,Television,What is Law & Order?,"Mike Post combined the sound of a slamming jail door, an anvil & 100 men stomping on a floor for what television series that debuted in 1990?" -62,British Landmarks,What is the Tower of London?,"Like Sir Thomas More, 3 16th century English queens are buried at what British location?" -63,Early American History,What are witches?,"In 1692 Increase Mather wrote, 'It were better that ten suspected' of these who 'escape, than that one innocent person … be condemned'?" -64,Geography Mnemonics,What are Arkansas and Louisiana?,"The Geography Mnemonic Mimal, sometimes said to be the silhouette of a chef or elf, stands for Minnesota, Iowa, Missouri, and what other 2 states?" -65,Business Milestones,What is the Ford Model T?,"What was first sold in 1908, at a price equivalent to about $27,000 today?" -66,In The Bookstore,Who is Tom Clancy?,The name of what author dead since 2013 now appears on books written by a former U.S. marshal & a former Apache helicopter pilot? -67,Historic Art,What is the Bayeux Tapestry?,The artwork once known in France as 'la tapisserie de la Reine Mathilde' is better known as what? -68,Pop Stars,Who is Madonna?,In 2022 which pop star became the first woman to have a Billboard Top 10 album in 5 decades starting with the 1980s? -69,Classic Tale Characters,Who is Scheherazade?,"In one 19th century translation, what female classic tale character 'perceived the dawn of day and ceased' speaking nearly 1,000 times?" -70,USA,What is Jack Daniel’s?,"Ironically, though what company founded in the 1860s is Moore County, Tennessee's largest employer, Moore is a dry county?" -71,Historic People,Who was William Bligh?,"After a 1789 event, who wrote, 'My first determination was to seek a supply of…water at Tofoa, & afterwards to sail for Tongataboo'?" -72,The Movies,What is The Godfather?,Laurence Olivier & Ernest Borgnine were considered for the lead role & Sergio Leone to direct for what film that turned 50 in 2022? -73,Continental Geography,What is Colombia?,"Until a 1903 secession, what country's contiguous territory spanned 2 continents?" -74,Foreign-Born Authors,Who is Isabel Allende?,"Early in her career which foreign-born author translated romance novels into Spanish, often changing the dialogue to make the heroines smarter?" -75,Historic Crimes,What is the Mona Lisa?,"Saying it was stolen by Napoleon, self-styled Italian patriot Vincenzo Peruggia took what in 1911?" -76,U.S. Bodies of Water,What is Lake Mead?,"Continuing a downward trend, in July 2022 what US body of water was at 27% capacity, its lowest level since 1937 when it was first being filled?" -77,Gods & Goddesses,Who is Aurora (or Eos)?,"Each morning which goddess began her ride in her chariot across the sky ahead of her brother Sol, or Helios?" -78,America At War,What is the Battle of New Orleans?,"Until the Civil War, the Jan. 8 date of what American battle of dubious military importance but big morale value was a national holiday?" -79,Children’s Books,What is The Velveteen Rabbit?,"Which children's book title character is told 'By the time you are real, most of your hair has been loved off your eyes drop out & you get shabby'?" -80,TV Finales,What is Grace and Frankie?,"In a TV reunion over 40 years in the making, Dolly Parton appeared as an angel named Agnes in the final episode of what comedy in 2022?" -81,American Poems,Who is Evangeline?,"In an 1847 American poem what character sees her town of Grand-Pré burned, but finally reunites with her beau for a kiss before his death?" -82,Famous Names,Who is Banksy?,"In 2001 who published a book called 'Banging Your Head Against a Brick Wall'; in 2002, 'Existencilism'?" -83,Children’s Lit,What is Charlotte’s Web?,The title object of what childrens book 'never looked more beautiful each strand held dozens of bright drops of early morning dew'? -84,Classic Songs,What is “Here Comes Santa Claus”?,The shouts of excited children at a 1946 holiday parade are said to have inspired what perennial classic song favorite? -85,Brand Names,What are Milk Duds?,"Unable to make what candies perfectly round, the confectioner embraced this flawed name for the product?" -86,Countries of the World,What is Italy?,"What country is home to 58 UNESCO World Heritage Sites, more than any other country; the sites include a volcano & a lagoon?" -87,Action Movies,What is Die Hard?,"What action movie's last line is 'If this is their idea of Christmas, I gotta be here for New Years'?" -88,Presidential Facts,Who is Woodrow Wilson?,Only 3 presidents have married while in office— John Tyler was the first & which one was the last? -89,19th Century Americans,Who is Frederick Douglass?,"Demonstrating the dignity & humanity of Black Americans, who sat for 160 known photographs, the most of any American in the 19th century?" -90,Latin Phrases,What is “quid pro quo”?,"Originally, which Latin 3-word phrase referred to when a doctor or apothecary substituted one medicine for another?" -91,1970s Movies,What is Monty Python and the Holy Grail?,The 1975 premiere of what movie comedy advertised free coconuts for the first thousand in the audience? -92,Name’s The Same,What is Manhattan?,"A cocktail, an island & a WWII venture originally called 'Development of Substitute Materials' all bear what name?" -93,U.S. Presidents,Who is Calvin Coolidge?,"Which US President was sworn in twice as President within 2 years, first by his father & then later by a former U.S. President?" -94,Plays,What is The Tempest?,A 1609 story in which an exiled king of Bulgaria creates a sea palace with his magic may have inspired the plot of what play? -95,Landmarks,What is the Berlin Wall?,"In 2009, during a 20th anniversary celebration, what landmark was called 'an edifice of fear. On Nov. 9, it became a place of joy'?" -96,World Capitals,"What is Vienna, Austria?","Among what world capital's nicknames are the 'City of Classical Music' &, possibly in honor of a famous resident from 1860 to 1938, the 'City of Dreams'?" -97,Language & Its Meanings,What is a night owl?,"Now meaning someone with nocturnal habits, what catches a sleeping dove in Shakespeare's 'Lucrece'?" -98,Flags of Our Hemisphere,What is Brazil?,"The stars on what country's flag represent states, 26 of them; unlike the USA's, its 'federal district' gets its own 27th star?" -99,Names in U.S. History,Who is Oliver Brown?,What father was the only man among the 13 plaintiffs in a US class-action case filed in 1951? -100,Children’s Authors,"Who is Sarah? (from Sarah, Plain and Tall)","Reversing the story of what heroine she created, childrens author Patricia Maclachlan was born on the prairie but spent much of her life in New England?" -,,, -TOTALS,,, diff --git a/examples/jeopardy/questions.txt b/examples/jeopardy/questions.txt deleted file mode 100644 index eea78a057..000000000 --- a/examples/jeopardy/questions.txt +++ /dev/null @@ -1,100 +0,0 @@ -Which man born in 1932 was the son of a percussionist in the CBS radio orchestra has been nominated for 53 Oscars? -What work in English Literature says: 'The mind is its own place, & in itself can make a heaven of hell, a hell of heaven. What matter where, if I be still the same'? -Known for more philosophical works, he wrote the play 'La Mandragola', in which Florentines are rewarded for immoral actions? -James Cook's account of a 1774 visit where records an object 'near 27 feet long, and upwards of 8 feet over the breast or shoulders'? -England's 'Bloody Assizes' & a 1685 life sentence for perjury were 2 main origins of which amendment to the U.S. Constitution? -Which nobel peace price winners each lived at times on Vilakazi St. in Soweto , so it claims to be the world's only street home to 2 Nobel Peace Prize winners? -In 1966, the year of who's death did he share plans for an experimental prototype community in Florida? -Of the 13 nations through which the Equator passes, what is the only one whose coastline borders the Caribbean Sea? -Which decorative items in fashion history get their name from their origin in the port city of Strasbourg, on the border of France & Germany? -What 1980's movie is based on an off-Broadway play with just 3 characters and won the Best Picture Oscar & the actors in all 3 roles were nominated? -A 2012 book review for which novelist noted subjects that 'sparked his ire': capital punishment, big tobacco & 'the plight of the unjustly convicted'? -A 1940 headline about what 20th Century Eponym included 'failure', 'liability when it came to offense' & 'stout hearts no match for tanks'? -Over 700 years after its traditional 1252 founding date, what port city became associated with a psychological response? -The success of what brand has its roots with a hydrotherapy pump its cofounder created for his son, who had arthritis? -In a periodical in 1807, what American Author called New York City 'Gotham, Gotham! Most enlightened of cities'? -What symbol is a rotated V in math and a feeling of some marginalized or underrepresented people in society? -Monty Norman, the composer of what character's theme, said the staccato riff conveyed sexiness, mystery & ruthlessness? -What American Novelist served with an airman named Yohannan in World War II & despite what readers might think, he said he enjoyed his service? -In what Medieval place did one of the participants in an 1170 event say, 'Let us away, knights; he will rise no more'? -At one time a province of the Roman Empire, what African country kingdom is known to Arabic scholars as Al-Maghrib Al-Aqsa, 'the far west'? -Congress relented in 1890 after what prospective state said it would wait 100 years rather than come in without the women? -A writer & producer of what movie said he wanted it to be like a Western or James Bond film, 'only it takes place in the 30s'? -In 1898 what's been called the first blockbuster art show was devoted to which artist & put on for Queen Wilhelmina's coronation? -Part of the largest contiguous land empire during the 1200s & 1300s, today what is the world's second-largest landlocked country? -A 2006 book was titled 'The Poem That Changed America:' What 'Fifty Years Later'? -Backed by 14,000 troops, who invaded England to restore, in his words, its 'religion, laws, and liberties'? -After its completion in the late 19th c., what was landmark was called 'a truly tragic street lamp' & a 'high & skinny pyramid of iron ladders'? -The busiest passenger port in the U.K., what shares its name with a capital of one of the original 13 states? -This man made lists, perhaps to cope with depression; a set of lists he published in 1852 made whose name synonymous with a type of book? -An 1869 presidential pardon was granted to which man, due in part to a plea by the Medical Society of Harford County, Maryland? -Letters, pocket knives, C rations & steel helmets are among the tangible items referred to in the title of what American literature modern war classic? -What nonfiction book has the line, 'The discovery of America…opened up fresh ground for the rising bourgeoisie'? -A radical Republican championed what 1875 act but the Supreme Court struck it down in 1883; a new version was passed 81 years later? -Whose brothers, Castor & Pollux, saved her after Theseus stole her away as a kid; a larger force would seek her later in life? -Once Africa's largest country in area, what African Country dropped to third in 2011 when a portion of it declared independence? -The ancient writer Galen said books on ships arriving to what city's port were seized, originals kept & copies returned? -For a special 1970s cookbook, who provided one simple recipe–a can of Campbell's tomato soup & 2 cans of milk? -Thought to descend from people of Southeast Asia, the Chamorro make up what U.S. territory’s largest ethnic group? -In office from 2022, the president of what country has taken so many foreign trips a play on his name is 'Ferdinand Magellan Jr.'? -In 1939 which writer lived on Toulouse Street in the French Quarter & chose the professional name that bonded him to the South? -What National Park is named for a river indigenous people called Mi tse a-da-zi, translated by French-speaking trappers as 'Pierre Jaune'? -In 2010 who introduced the 4-point shot, 35 feet from the basket? -Losses over Asia in the 1960s led to the establishment of the program known as what at a San Diego naval base in 1969? -A craft that visited what was named for Giotto, based on the story that 680 years earlier, the painter depicted it as the Star of Bethlehem? -In World War I, 'Cistern' & 'reservoir' were suggested names for what secret invention, but the British preferred this less clumsy monosyllable? -Until 1806, some German nobles included among their honors the title of 'Elector' for their role in selecting this personage? -In 1904, wearing a harness, actress Nina Boucicault became the first to play what character onstage? -Alphabetically the first German city in encyclopedias, what was also the first one taken by the Allies in World War II? -This Sanskrit word referring to a spoken word or phrase comes from a word for 'to think'? -1917's 'Elements of Trench Warfare' said what Old West invention was 'difficult to destroy' & 'difficult to get through'? -Mimi Reinhard, who never learned to type using more than 2 fingers, produced what in World War II with 1,100 names, including hers? -Poseidon carried off the maiden Theophane & turned her into a ewe; their offspring was the source of what mythical object? -Published in 2011, P.D. James' final novel, 'Death Comes to Pemberley', was a sequel to what novel from 200 years earlier? -5 U.S. states have 6-letter names; only which 2 west of the Mississippi River border each other? -Originally relating to a story of suffering, what word now more commonly refers to strong emotion of any kind? -The 2007 biopic called 'La Môme' in France, meaning 'The Kid', was released in the U.S. under what other French title? -Returning home in 1493, Columbus stopped in the Azores at an island with what name, also something he'd lost off the Haiti coast? -Pskov & Nizhny Novgorod are 2 of the cities that have a fortress called what? -In the 1950s the New York Times said what author 'is writing about all lust' & his lecherous narrator 'is all of us'? -At the winter solstice, the sun is in Sagittarius; it once appeared in what constellation, giving a geographic feature its name? -Mike Post combined the sound of a slamming jail door, an anvil & 100 men stomping on a floor for what television series that debuted in 1990? -Like Sir Thomas More, 3 16th century English queens are buried at what British location? -In 1692 Increase Mather wrote, 'It were better that ten suspected' of these who 'escape, than that one innocent person be condemned'? -The Geography Mnemonic Mimal, sometimes said to be the silhouette of a chef or elf, stands for Minnesota, Iowa, Missouri, and what other 2 states? -What was first sold in 1908, at a price equivalent to about $27,000 today? -The name of what author dead since 2013 now appears on books written by a former U.S. marshal & a former Apache helicopter pilot? -The artwork once known in France as 'la tapisserie de la Reine Mathilde' is better known as what? -In 2022 which pop star became the first woman to have a Billboard Top 10 album in 5 decades starting with the 1980s? -In one 19th century translation, what female classic tale character 'perceived the dawn of day and ceased' speaking nearly 1,000 times? -Ironically, though what company founded in the 1860s is Moore County, Tennessee's largest employer, Moore is a dry county? -After a 1789 event, who wrote, 'My first determination was to seek a supply of…water at Tofoa, & afterwards to sail for Tongataboo'? -Laurence Olivier & Ernest Borgnine were considered for the lead role & Sergio Leone to direct for what film that turned 50 in 2022? -Until a 1903 secession, what country's contiguous territory spanned 2 continents? -Early in her career which foreign-born author translated romance novels into Spanish, often changing the dialogue to make the heroines smarter? -Saying it was stolen by Napoleon, self-styled Italian patriot Vincenzo Peruggia took what in 1911? -Continuing a downward trend, in July 2022 what US body of water was at 27% capacity, its lowest level since 1937 when it was first being filled? -Each morning which goddess began her ride in her chariot across the sky ahead of her brother Sol, or Helios? -Until the Civil War, the Jan. 8 date of what American battle of dubious military importance but big morale value was a national holiday? -Which children's book title character is told 'By the time you are real, most of your hair has been loved off your eyes drop out & you get shabby'? -In a TV reunion over 40 years in the making, Dolly Parton appeared as an angel named Agnes in the final episode of what comedy in 2022? -In an 1847 American poem what character sees her town of Grand-Pré burned, but finally reunites with her beau for a kiss before his death? -In 2001 who published a book called 'Banging Your Head Against a Brick Wall'; in 2002, 'Existencilism'? -The title object of what childrens book 'never looked more beautiful each strand held dozens of bright drops of early morning dew'? -The shouts of excited children at a 1946 holiday parade are said to have inspired what perennial classic song favorite? -Unable to make what candies perfectly round, the confectioner embraced this flawed name for the product? -What country is home to 58 UNESCO World Heritage Sites, more than any other country; the sites include a volcano & a lagoon? -What action movie's last line is 'If this is their idea of Christmas, I gotta be here for New Years'? -Only 3 presidents have married while in office— John Tyler was the first & which one was the last? -Demonstrating the dignity & humanity of Black Americans, who sat for 160 known photographs, the most of any American in the 19th century? -Originally, which Latin 3-word phrase referred to when a doctor or apothecary substituted one medicine for another? -The 1975 premiere of what movie comedy advertised free coconuts for the first thousand in the audience? -A cocktail, an island & a WWII venture originally called 'Development of Substitute Materials' all bear what name? -Which US President was sworn in twice as President within 2 years, first by his father & then later by a former U.S. President? -A 1609 story in which an exiled king of Bulgaria creates a sea palace with his magic may have inspired the plot of what play? -In 2009, during a 20th anniversary celebration, what landmark was called 'an edifice of fear. On Nov. 9, it became a place of joy'? -Among what world capital's nicknames are the 'City of Classical Music' &, possibly in honor of a famous resident from 1860 to 1938, the 'City of Dreams'? -Now meaning someone with nocturnal habits, what catches a sleeping dove in Shakespeare's 'Lucrece'? -The stars on what country's flag represent states, 26 of them; unlike the USA's, its 'federal district' gets its own 27th star? -What father was the only man among the 13 plaintiffs in a US class-action case filed in 1951? -Reversing the story of what heroine she created, childrens author Patricia Maclachlan was born on the prairie but spent much of her life in New England? diff --git a/examples/json_schema_pydantic_example.py b/examples/json_schema_pydantic_example.py deleted file mode 100644 index 19c0bdb5b..000000000 --- a/examples/json_schema_pydantic_example.py +++ /dev/null @@ -1,82 +0,0 @@ -# Usage: -#! ./llama-server -m some-model.gguf & -#! pip install pydantic -#! python json_schema_pydantic_example.py - -from pydantic import BaseModel, Field, TypeAdapter -from annotated_types import MinLen -from typing import Annotated, List, Optional -import json, requests - -if True: - - def create_completion(*, response_model=None, endpoint="http://localhost:8080/v1/chat/completions", messages, **kwargs): - ''' - Creates a chat completion using an OpenAI-compatible endpoint w/ JSON schema support - (llama.cpp server, llama-cpp-python, Anyscale / Together...) - - The response_model param takes a type (+ supports Pydantic) and behaves just as w/ Instructor (see below) - ''' - response_format = None - type_adapter = None - - if response_model: - type_adapter = TypeAdapter(response_model) - schema = type_adapter.json_schema() - messages = [{ - "role": "system", - "content": f"You respond in JSON format with the following schema: {json.dumps(schema, indent=2)}" - }] + messages - response_format={"type": "json_object", "schema": schema} - - data = requests.post(endpoint, headers={"Content-Type": "application/json"}, - json=dict(messages=messages, response_format=response_format, **kwargs)).json() - if 'error' in data: - raise Exception(data['error']['message']) - - content = data["choices"][0]["message"]["content"] - return type_adapter.validate_json(content) if type_adapter else content - -else: - - # This alternative branch uses Instructor + OpenAI client lib. - # Instructor support streamed iterable responses, retry & more. - # (see https://python.useinstructor.com/) - #! pip install instructor openai - import instructor, openai - client = instructor.patch( - openai.OpenAI(api_key="123", base_url="http://localhost:8080"), - mode=instructor.Mode.JSON_SCHEMA) - create_completion = client.chat.completions.create - - -if __name__ == '__main__': - - class QAPair(BaseModel): - class Config: - extra = 'forbid' # triggers additionalProperties: false in the JSON schema - question: str - concise_answer: str - justification: str - stars: Annotated[int, Field(ge=1, le=5)] - - class PyramidalSummary(BaseModel): - class Config: - extra = 'forbid' # triggers additionalProperties: false in the JSON schema - title: str - summary: str - question_answers: Annotated[List[QAPair], MinLen(2)] - sub_sections: Optional[Annotated[List['PyramidalSummary'], MinLen(2)]] - - print("# Summary\n", create_completion( - model="...", - response_model=PyramidalSummary, - messages=[{ - "role": "user", - "content": f""" - You are a highly efficient corporate document summarizer. - Create a pyramidal summary of an imaginary internal document about our company processes - (starting high-level, going down to each sub sections). - Keep questions short, and answers even shorter (trivia / quizz style). - """ - }])) diff --git a/examples/json_schema_to_grammar.py b/examples/json_schema_to_grammar.py deleted file mode 100755 index fc9f0097f..000000000 --- a/examples/json_schema_to_grammar.py +++ /dev/null @@ -1,811 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import itertools -import json -import re -import sys -from typing import Any, List, Optional, Set, Tuple, Union - -def _build_repetition(item_rule, min_items, max_items, separator_rule=None): - - if min_items == 0 and max_items == 1: - return f'{item_rule}?' - - if not separator_rule: - if min_items == 1 and max_items is None: - return f'{item_rule}+' - elif min_items == 0 and max_items is None: - return f'{item_rule}*' - else: - return f'{item_rule}{{{min_items},{max_items if max_items is not None else ""}}}' - - result = item_rule + ' ' + _build_repetition(f'({separator_rule} {item_rule})', min_items - 1 if min_items > 0 else 0, max_items - 1 if max_items is not None else None) - return f'({result})?' if min_items == 0 else result - -def _generate_min_max_int(min_value: Optional[int], max_value: Optional[int], out: list, decimals_left: int = 16, top_level: bool = True): - has_min = min_value != None - has_max = max_value != None - - def digit_range(from_char: str, to_char: str): - out.append("[") - if from_char == to_char: - out.append(from_char) - else: - out.append(from_char) - out.append("-") - out.append(to_char) - out.append("]") - - def more_digits(min_digits: int, max_digits: int): - out.append("[0-9]") - if min_digits == max_digits and min_digits == 1: - return - out.append("{") - out.append(str(min_digits)) - if max_digits != min_digits: - out.append(",") - if max_digits != sys.maxsize: - out.append(str(max_digits)) - out.append("}") - - def uniform_range(from_str: str, to_str: str): - i = 0 - while i < len(from_str) and from_str[i] == to_str[i]: - i += 1 - if i > 0: - out.append("\"") - out.append(from_str[:i]) - out.append("\"") - if i < len(from_str): - if i > 0: - out.append(" ") - sub_len = len(from_str) - i - 1 - if sub_len > 0: - from_sub = from_str[i+1:] - to_sub = to_str[i+1:] - sub_zeros = "0" * sub_len - sub_nines = "9" * sub_len - - to_reached = False - out.append("(") - if from_sub == sub_zeros: - digit_range(from_str[i], chr(ord(to_str[i]) - 1)) - out.append(" ") - more_digits(sub_len, sub_len) - else: - out.append("[") - out.append(from_str[i]) - out.append("] ") - out.append("(") - uniform_range(from_sub, sub_nines) - out.append(")") - if ord(from_str[i]) < ord(to_str[i]) - 1: - out.append(" | ") - if to_sub == sub_nines: - digit_range(chr(ord(from_str[i]) + 1), to_str[i]) - to_reached = True - else: - digit_range(chr(ord(from_str[i]) + 1), chr(ord(to_str[i]) - 1)) - out.append(" ") - more_digits(sub_len, sub_len) - if not to_reached: - out.append(" | ") - digit_range(to_str[i], to_str[i]) - out.append(" ") - uniform_range(sub_zeros, to_sub) - out.append(")") - else: - out.append("[") - out.append(from_str[i]) - out.append("-") - out.append(to_str[i]) - out.append("]") - - if has_min and has_max: - if min_value < 0 and max_value < 0: - out.append("\"-\" (") - _generate_min_max_int(-max_value, -min_value, out, decimals_left, top_level=True) - out.append(")") - return - - if min_value < 0: - out.append("\"-\" (") - _generate_min_max_int(0, -min_value, out, decimals_left, top_level=True) - out.append(") | ") - min_value = 0 - - min_s = str(min_value) - max_s = str(max_value) - min_digits = len(min_s) - max_digits = len(max_s) - - for digits in range(min_digits, max_digits): - uniform_range(min_s, "9" * digits) - min_s = "1" + "0" * digits - out.append(" | ") - uniform_range(min_s, max_s) - return - - less_decimals = max(decimals_left - 1, 1) - - if has_min: - if min_value < 0: - out.append("\"-\" (") - _generate_min_max_int(None, -min_value, out, decimals_left, top_level=False) - out.append(") | [0] | [1-9] ") - more_digits(0, decimals_left - 1) - elif min_value == 0: - if top_level: - out.append("[0] | [1-9] ") - more_digits(0, less_decimals) - else: - more_digits(1, decimals_left) - elif min_value <= 9: - c = str(min_value) - range_start = '1' if top_level else '0' - if c > range_start: - digit_range(range_start, chr(ord(c) - 1)) - out.append(" ") - more_digits(1, less_decimals) - out.append(" | ") - digit_range(c, "9") - out.append(" ") - more_digits(0, less_decimals) - else: - min_s = str(min_value) - length = len(min_s) - c = min_s[0] - - if c > "1": - digit_range("1" if top_level else "0", chr(ord(c) - 1)) - out.append(" ") - more_digits(length, less_decimals) - out.append(" | ") - digit_range(c, c) - out.append(" (") - _generate_min_max_int(int(min_s[1:]), None, out, less_decimals, top_level=False) - out.append(")") - if c < "9": - out.append(" | ") - digit_range(chr(ord(c) + 1), "9") - out.append(" ") - more_digits(length - 1, less_decimals) - return - - if has_max: - if max_value >= 0: - if top_level: - out.append("\"-\" [1-9] ") - more_digits(0, less_decimals) - out.append(" | ") - _generate_min_max_int(0, max_value, out, decimals_left, top_level=True) - else: - out.append("\"-\" (") - _generate_min_max_int(-max_value, None, out, decimals_left, top_level=False) - out.append(")") - return - - raise RuntimeError("At least one of min_value or max_value must be set") - -class BuiltinRule: - def __init__(self, content: str, deps: list | None = None): - self.content = content - self.deps = deps or [] - -# Constraining spaces to prevent model "running away". -SPACE_RULE = '| " " | "\\n" [ \\t]{0,20}' - -PRIMITIVE_RULES = { - 'boolean' : BuiltinRule('("true" | "false") space', []), - 'decimal-part' : BuiltinRule('[0-9]{1,16}', []), - 'integral-part': BuiltinRule('[0] | [1-9] [0-9]{0,15}', []), - 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', ['integral-part', 'decimal-part']), - 'integer' : BuiltinRule('("-"? integral-part) space', ['integral-part']), - 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), - 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', ['string', 'value']), - 'array' : BuiltinRule('"[" space ( value ("," space value)* )? "]" space', ['value']), - 'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\"" space', []), - 'char' : BuiltinRule(r'[^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})', []), - 'string' : BuiltinRule(r'"\"" char* "\"" space', ['char']), - 'null' : BuiltinRule('"null" space', []), -} - -# TODO: support "uri", "email" string formats -STRING_FORMAT_RULES = { - 'date' : BuiltinRule('[0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), - 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), - 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), - 'date-string' : BuiltinRule('"\\"" date "\\"" space', ['date']), - 'time-string' : BuiltinRule('"\\"" time "\\"" space', ['time']), - 'date-time-string': BuiltinRule('"\\"" date-time "\\"" space', ['date-time']), -} - -DOTALL = '[\\U00000000-\\U0010FFFF]' -DOT = '[^\\x0A\\x0D]' - -RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) - -INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+') -GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"]') -GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]') -GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]'} - -NON_LITERAL_SET = set('|.()[]{}*+?') -ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?') - - -class SchemaConverter: - def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): - self._prop_order = prop_order - self._allow_fetch = allow_fetch - self._dotall = dotall - self._raw_pattern = raw_pattern - self._rules = { - 'space': SPACE_RULE, - } - self._refs = {} - self._refs_being_resolved = set() - - def _format_literal(self, literal): - escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)) or m.group(0), literal - ) - return f'"{escaped}"' - - def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: - ''' - not_literal('a') -> '[^a]' - not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' - ''' - assert len(literal) > 0, 'Empty literal not supported' - def recurse(i: int): - c = literal[i] - if maybe_escaped_underscores and c == '_': - yield f'[^{c}\\\\]' - yield ' | ' - yield f'"\\\\"? "{c}"' - else: - yield f'[^{c}]' - if i < len(literal) - 1: - yield ' | ' - yield self._format_literal(c) - yield ' (' - yield from recurse(i + 1) - yield ')?' - - return ''.join(('(', *recurse(0), ')')) - - def _not_strings(self, strings): - class TrieNode: - def __init__(self): - self.children = {} - self.is_end_of_string = False - - def insert(self, string): - node = self - for c in string: - node = node.children.setdefault(c, TrieNode()) - node.is_end_of_string = True - - trie = TrieNode() - for s in strings: - trie.insert(s) - - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - out = ['["] ( '] - - def visit(node): - rejects = [] - first = True - for c in sorted(node.children.keys()): - child = node.children[c] - rejects.append(c) - if first: - first = False - else: - out.append(' | ') - out.append(f'[{c}]') - if child.children: - out.append(f' (') - visit(child) - out.append(')') - elif child.is_end_of_string: - out.append(f' {char_rule}+') - if node.children: - if not first: - out.append(' | ') - out.append(f'[^"{"".join(rejects)}] {char_rule}*') - visit(trie) - - out.append(f' ){"" if trie.is_end_of_string else "?"} ["] space') - return ''.join(out) - - def _add_rule(self, name, rule): - esc_name = INVALID_RULE_CHARS_RE.sub('-', name) - if esc_name not in self._rules or self._rules[esc_name] == rule: - key = esc_name - else: - i = 0 - while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: - i += 1 - key = f'{esc_name}{i}' - self._rules[key] = rule - return key - - def resolve_refs(self, schema: dict, url: str): - ''' - Resolves all $ref fields in the given schema, fetching any remote schemas, - replacing $ref with absolute reference URL and populating self._refs with the - respective referenced (sub)schema dictionaries. - ''' - def visit(n: dict): - if isinstance(n, list): - return [visit(x) for x in n] - elif isinstance(n, dict): - ref = n.get('$ref') - if ref is not None and ref not in self._refs: - if ref.startswith('https://'): - assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' - import requests - - frag_split = ref.split('#') - base_url = frag_split[0] - - target = self._refs.get(base_url) - if target is None: - target = self.resolve_refs(requests.get(ref).json(), base_url) - self._refs[base_url] = target - - if len(frag_split) == 1 or frag_split[-1] == '': - return target - elif ref.startswith('#/'): - target = schema - ref = f'{url}{ref}' - n['$ref'] = ref - else: - raise ValueError(f'Unsupported ref {ref}') - - for sel in ref.split('#')[-1].split('/')[1:]: - assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}' - target = target[sel] - - self._refs[ref] = target - else: - for v in n.values(): - visit(v) - - return n - return visit(schema) - - def _generate_union_rule(self, name, alt_schemas): - return ' | '.join(( - self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') - for i, alt_schema in enumerate(alt_schemas) - )) - - def _visit_pattern(self, pattern, name): - ''' - Transforms a regular expression pattern into a GBNF rule. - - Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions - Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md - - Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. - - Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which - we define sub-rules to keep the output lean. - ''' - - assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' - pattern = pattern[1:-1] - sub_rule_ids = {} - - i = 0 - length = len(pattern) - - def to_rule(s: tuple[str, bool]) -> str: - (txt, is_literal) = s - return "\"" + txt + "\"" if is_literal else txt - - def transform() -> tuple[str, bool]: - ''' - Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. - ''' - nonlocal i - nonlocal pattern - nonlocal sub_rule_ids - - start = i - # For each component of this sequence, store its string representation and whether it's a literal. - # We only need a flat structure here to apply repetition operators to the last item, and - # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially - # (GBNF's syntax is luckily very close to regular expressions!) - seq: list[tuple[str, bool]] = [] - - def get_dot(): - if self._dotall: - rule = DOTALL - else: - # Accept any character... except \n and \r line break chars (\x0A and \xOD) - rule = DOT - return self._add_rule(f'dot', rule) - - def join_seq(): - nonlocal seq - ret = [] - for is_literal, g in itertools.groupby(seq, lambda x: x[1]): - if is_literal: - ret.append((''.join(x[0] for x in g), True)) - else: - ret.extend(g) - if len(ret) == 1: - return ret[0] - return (' '.join(to_rule(x) for x in seq), False) - - while i < length: - c = pattern[i] - if c == '.': - seq.append((get_dot(), False)) - i += 1 - elif c == '(': - i += 1 - if i < length: - assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' - seq.append((f'({to_rule(transform())})', False)) - elif c == ')': - i += 1 - assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' - return join_seq() - elif c == '[': - square_brackets = c - i += 1 - while i < length and pattern[i] != ']': - if pattern[i] == '\\': - square_brackets += pattern[i:i+2] - i += 2 - else: - square_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' - square_brackets += ']' - i += 1 - seq.append((square_brackets, False)) - elif c == '|': - seq.append(('|', False)) - i += 1 - elif c in ('*', '+', '?'): - seq[-1] = (to_rule(seq[-1]) + c, False) - i += 1 - elif c == '{': - curly_brackets = c - i += 1 - while i < length and pattern[i] != '}': - curly_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' - curly_brackets += '}' - i += 1 - nums = [s.strip() for s in curly_brackets[1:-1].split(',')] - min_times = 0 - max_times = None - try: - if len(nums) == 1: - min_times = int(nums[0]) - max_times = min_times - else: - assert len(nums) == 2 - min_times = int(nums[0]) if nums[0] else 0 - max_times = int(nums[1]) if nums[1] else None - except ValueError: - raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') - - (sub, sub_is_literal) = seq[-1] - - if not sub_is_literal: - id = sub_rule_ids.get(sub) - if id is None: - id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) - sub_rule_ids[sub] = id - sub = id - - seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times), False) - else: - literal = '' - while i < length: - if pattern[i] == '\\' and i < length - 1: - next = pattern[i + 1] - if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: - i += 1 - literal += pattern[i] - i += 1 - else: - literal += pattern[i:i+2] - i += 2 - elif pattern[i] == '"' and not self._raw_pattern: - literal += '\\"' - i += 1 - elif pattern[i] not in NON_LITERAL_SET and \ - (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): - literal += pattern[i] - i += 1 - else: - break - if literal: - seq.append((literal, True)) - - return join_seq() - - return self._add_rule( - name, - to_rule(transform()) if self._raw_pattern \ - else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space") - - - def _resolve_ref(self, ref): - ref_name = ref.split('/')[-1] - if ref_name not in self._rules and ref not in self._refs_being_resolved: - self._refs_being_resolved.add(ref) - resolved = self._refs[ref] - ref_name = self.visit(resolved, ref_name) - self._refs_being_resolved.remove(ref) - return ref_name - - def _generate_constant_rule(self, value): - return self._format_literal(json.dumps(value)) - - def visit(self, schema, name): - schema_type = schema.get('type') - schema_format = schema.get('format') - rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' - - if (ref := schema.get('$ref')) is not None: - return self._add_rule(rule_name, self._resolve_ref(ref)) - - elif 'oneOf' in schema or 'anyOf' in schema: - return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) - - elif isinstance(schema_type, list): - return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type])) - - elif 'const' in schema: - return self._add_rule(rule_name, self._generate_constant_rule(schema['const']) + ' space') - - elif 'enum' in schema: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ') space' - return self._add_rule(rule_name, rule) - - elif schema_type in (None, 'object') and \ - ('properties' in schema or \ - ('additionalProperties' in schema and schema['additionalProperties'] is not True)): - required = set(schema.get('required', [])) - properties = list(schema.get('properties', {}).items()) - return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) - - elif schema_type in (None, 'object') and 'allOf' in schema: - required = set() - properties = [] - hybrid_name = name - def add_component(comp_schema, is_required): - if (ref := comp_schema.get('$ref')) is not None: - comp_schema = self._refs[ref] - - if 'properties' in comp_schema: - for prop_name, prop_schema in comp_schema['properties'].items(): - properties.append((prop_name, prop_schema)) - if is_required: - required.add(prop_name) - - for t in schema['allOf']: - if 'anyOf' in t: - for tt in t['anyOf']: - add_component(tt, is_required=False) - else: - add_component(t, is_required=True) - - return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=None)) - - elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): - items = schema.get('items') or schema['prefixItems'] - if isinstance(items, list): - return self._add_rule( - rule_name, - '"[" space ' + - ' "," space '.join( - self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') - for i, item in enumerate(items)) + - ' "]" space') - else: - item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') - min_items = schema.get("minItems", 0) - max_items = schema.get("maxItems") - return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' "]" space') - - elif schema_type in (None, 'string') and 'pattern' in schema: - return self._visit_pattern(schema['pattern'], rule_name) - - elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): - return self._add_primitive( - 'root' if rule_name == 'root' else schema_format, - PRIMITIVE_RULES['uuid'] - ) - - elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: - prim_name = f'{schema_format}-string' - return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) - - elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - min_len = schema.get('minLength', 0) - max_len = schema.get('maxLength') - - return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\"" space') - - elif schema_type in (None, 'integer') and \ - ('minimum' in schema or 'exclusiveMinimum' in schema or 'maximum' in schema or 'exclusiveMaximum' in schema): - min_value = None - max_value = None - if 'minimum' in schema: - min_value = schema['minimum'] - elif 'exclusiveMinimum' in schema: - min_value = schema['exclusiveMinimum'] + 1 - if 'maximum' in schema: - max_value = schema['maximum'] - elif 'exclusiveMaximum' in schema: - max_value = schema['exclusiveMaximum'] - 1 - - out = ["("] - _generate_min_max_int(min_value, max_value, out) - out.append(") space") - return self._add_rule(rule_name, ''.join(out)) - - elif (schema_type == 'object') or (len(schema) == 0): - return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) - - else: - assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' - # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) - - def _add_primitive(self, name: str, rule: BuiltinRule): - n = self._add_rule(name, rule.content) - - for dep in rule.deps: - dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) - assert dep_rule, f'Rule {dep} not known' - if dep not in self._rules: - self._add_primitive(dep, dep_rule) - return n - - def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Optional[Union[bool, Any]]): - prop_order = self._prop_order - # sort by position in prop_order (if specified) then by original order - sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] - - prop_kv_rule_names = {} - for prop_name, prop_schema in properties: - prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') - prop_kv_rule_names[prop_name] = self._add_rule( - f'{name}{"-" if name else ""}{prop_name}-kv', - fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' - ) - required_props = [k for k in sorted_props if k in required] - optional_props = [k for k in sorted_props if k not in required] - - if additional_properties is not None and additional_properties != False: - sub_name = f'{name}{"-" if name else ""}additional' - value_rule = self.visit(additional_properties, f'{sub_name}-value') if isinstance(additional_properties, dict) else \ - self._add_primitive('value', PRIMITIVE_RULES['value']) - key_rule = self._add_primitive('string', PRIMITIVE_RULES['string']) if not sorted_props \ - else self._add_rule(f'{sub_name}-k', self._not_strings(sorted_props)) - - prop_kv_rule_names["*"] = self._add_rule( - f'{sub_name}-kv', - f'{key_rule} ":" space {value_rule}' - ) - optional_props.append("*") - - rule = '"{" space ' - rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) - - if optional_props: - rule += ' (' - if required_props: - rule += ' "," space ( ' - - def get_recursive_refs(ks, first_is_optional): - [k, *rest] = ks - kv_rule_name = prop_kv_rule_names[k] - comma_ref = f'( "," space {kv_rule_name} )' - if first_is_optional: - res = comma_ref + ('*' if k == '*' else '?') - else: - res = kv_rule_name + (' ' + comma_ref + "*" if k == '*' else '') - if len(rest) > 0: - res += ' ' + self._add_rule( - f'{name}{"-" if name else ""}{k}-rest', - get_recursive_refs(rest, first_is_optional=True) - ) - return res - - rule += ' | '.join( - get_recursive_refs(optional_props[i:], first_is_optional=False) - for i in range(len(optional_props)) - ) - if required_props: - rule += ' )' - rule += ' )?' - - rule += ' "}" space' - - return rule - - def format_grammar(self): - return '\n'.join( - f'{name} ::= {rule}' - for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) - ) - - -def main(args_in = None): - parser = argparse.ArgumentParser( - description=''' - Generates a grammar (suitable for use in ./llama-cli) that produces JSON conforming to a - given JSON schema. Only a subset of JSON schema features are supported; more may be - added in the future. - ''', - ) - parser.add_argument( - '--prop-order', - default=[], - type=lambda s: s.split(','), - help=''' - comma-separated property names defining the order of precedence for object properties; - properties not specified here are given lower precedence than those that are, and - are kept in their original order from the schema. Required properties are always - given precedence over optional properties. - ''' - ) - parser.add_argument( - '--allow-fetch', - action='store_true', - default=False, - help='Whether to allow fetching referenced schemas over HTTPS') - parser.add_argument( - '--dotall', - action='store_true', - default=False, - help='Whether to treat dot (".") as matching all chars including line breaks in regular expression patterns') - parser.add_argument( - '--raw-pattern', - action='store_true', - default=False, - help='Treats string patterns as raw patterns w/o quotes (or quote escapes)') - - parser.add_argument('schema', help='file containing JSON schema ("-" for stdin)') - args = parser.parse_args(args_in) - - if args.schema.startswith('https://'): - url = args.schema - import requests - schema = requests.get(url).json() - elif args.schema == '-': - url = 'stdin' - schema = json.load(sys.stdin) - else: - url = f'file://{args.schema}' - with open(args.schema) as f: - schema = json.load(f) - converter = SchemaConverter( - prop_order={name: idx for idx, name in enumerate(args.prop_order)}, - allow_fetch=args.allow_fetch, - dotall=args.dotall, - raw_pattern=args.raw_pattern) - schema = converter.resolve_refs(schema, url) - converter.visit(schema, '') - print(converter.format_grammar()) - - -if __name__ == '__main__': - main() diff --git a/examples/llama.android/.gitignore b/examples/llama.android/.gitignore deleted file mode 100644 index 347e252ef..000000000 --- a/examples/llama.android/.gitignore +++ /dev/null @@ -1,33 +0,0 @@ -# Gradle files -.gradle/ -build/ - -# Local configuration file (sdk path, etc) -local.properties - -# Log/OS Files -*.log - -# Android Studio generated files and folders -captures/ -.externalNativeBuild/ -.cxx/ -*.apk -output.json - -# IntelliJ -*.iml -.idea/ -misc.xml -deploymentTargetDropDown.xml -render.experimental.xml - -# Keystore files -*.jks -*.keystore - -# Google Services (e.g. APIs or Firebase) -google-services.json - -# Android Profiling -*.hprof diff --git a/examples/llama.android/app/.gitignore b/examples/llama.android/app/.gitignore deleted file mode 100644 index 796b96d1c..000000000 --- a/examples/llama.android/app/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts deleted file mode 100644 index 8d1b37195..000000000 --- a/examples/llama.android/app/build.gradle.kts +++ /dev/null @@ -1,65 +0,0 @@ -plugins { - id("com.android.application") - id("org.jetbrains.kotlin.android") -} - -android { - namespace = "com.example.llama" - compileSdk = 34 - - defaultConfig { - applicationId = "com.example.llama" - minSdk = 33 - targetSdk = 34 - versionCode = 1 - versionName = "1.0" - - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - vectorDrawables { - useSupportLibrary = true - } - } - - buildTypes { - release { - isMinifyEnabled = false - proguardFiles( - getDefaultProguardFile("proguard-android-optimize.txt"), - "proguard-rules.pro" - ) - } - } - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - kotlinOptions { - jvmTarget = "1.8" - } - buildFeatures { - compose = true - } - composeOptions { - kotlinCompilerExtensionVersion = "1.5.1" - } -} - -dependencies { - - implementation("androidx.core:core-ktx:1.12.0") - implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.2") - implementation("androidx.activity:activity-compose:1.8.2") - implementation(platform("androidx.compose:compose-bom:2023.08.00")) - implementation("androidx.compose.ui:ui") - implementation("androidx.compose.ui:ui-graphics") - implementation("androidx.compose.ui:ui-tooling-preview") - implementation("androidx.compose.material3:material3") - implementation(project(":llama")) - testImplementation("junit:junit:4.13.2") - androidTestImplementation("androidx.test.ext:junit:1.1.5") - androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") - androidTestImplementation(platform("androidx.compose:compose-bom:2023.08.00")) - androidTestImplementation("androidx.compose.ui:ui-test-junit4") - debugImplementation("androidx.compose.ui:ui-tooling") - debugImplementation("androidx.compose.ui:ui-test-manifest") -} diff --git a/examples/llama.android/app/proguard-rules.pro b/examples/llama.android/app/proguard-rules.pro deleted file mode 100644 index f1b424510..000000000 --- a/examples/llama.android/app/proguard-rules.pro +++ /dev/null @@ -1,21 +0,0 @@ -# Add project specific ProGuard rules here. -# You can control the set of applied configuration files using the -# proguardFiles setting in build.gradle. -# -# For more details, see -# http://developer.android.com/guide/developing/tools/proguard.html - -# If your project uses WebView with JS, uncomment the following -# and specify the fully qualified class name to the JavaScript interface -# class: -#-keepclassmembers class fqcn.of.javascript.interface.for.webview { -# public *; -#} - -# Uncomment this to preserve the line number information for -# debugging stack traces. -#-keepattributes SourceFile,LineNumberTable - -# If you keep the line number information, uncomment this to -# hide the original source file name. -#-renamesourcefileattribute SourceFile diff --git a/examples/llama.android/app/src/main/AndroidManifest.xml b/examples/llama.android/app/src/main/AndroidManifest.xml deleted file mode 100644 index 41a358a29..000000000 --- a/examples/llama.android/app/src/main/AndroidManifest.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt b/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt deleted file mode 100644 index 78c231ae5..000000000 --- a/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt +++ /dev/null @@ -1,119 +0,0 @@ -package com.example.llama - -import android.app.DownloadManager -import android.net.Uri -import android.util.Log -import androidx.compose.material3.Button -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableDoubleStateOf -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.runtime.setValue -import androidx.core.database.getLongOrNull -import androidx.core.net.toUri -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import java.io.File - -data class Downloadable(val name: String, val source: Uri, val destination: File) { - companion object { - @JvmStatic - private val tag: String? = this::class.qualifiedName - - sealed interface State - data object Ready: State - data class Downloading(val id: Long): State - data class Downloaded(val downloadable: Downloadable): State - data class Error(val message: String): State - - @JvmStatic - @Composable - fun Button(viewModel: MainViewModel, dm: DownloadManager, item: Downloadable) { - var status: State by remember { - mutableStateOf( - if (item.destination.exists()) Downloaded(item) - else Ready - ) - } - var progress by remember { mutableDoubleStateOf(0.0) } - - val coroutineScope = rememberCoroutineScope() - - suspend fun waitForDownload(result: Downloading, item: Downloadable): State { - while (true) { - val cursor = dm.query(DownloadManager.Query().setFilterById(result.id)) - - if (cursor == null) { - Log.e(tag, "dm.query() returned null") - return Error("dm.query() returned null") - } - - if (!cursor.moveToFirst() || cursor.count < 1) { - cursor.close() - Log.i(tag, "cursor.moveToFirst() returned false or cursor.count < 1, download canceled?") - return Ready - } - - val pix = cursor.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR) - val tix = cursor.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES) - val sofar = cursor.getLongOrNull(pix) ?: 0 - val total = cursor.getLongOrNull(tix) ?: 1 - cursor.close() - - if (sofar == total) { - return Downloaded(item) - } - - progress = (sofar * 1.0) / total - - delay(1000L) - } - } - - fun onClick() { - when (val s = status) { - is Downloaded -> { - viewModel.load(item.destination.path) - } - - is Downloading -> { - coroutineScope.launch { - status = waitForDownload(s, item) - } - } - - else -> { - item.destination.delete() - - val request = DownloadManager.Request(item.source).apply { - setTitle("Downloading model") - setDescription("Downloading model: ${item.name}") - setAllowedNetworkTypes(DownloadManager.Request.NETWORK_WIFI) - setDestinationUri(item.destination.toUri()) - } - - viewModel.log("Saving ${item.name} to ${item.destination.path}") - Log.i(tag, "Saving ${item.name} to ${item.destination.path}") - - val id = dm.enqueue(request) - status = Downloading(id) - onClick() - } - } - } - - Button(onClick = { onClick() }, enabled = status !is Downloading) { - when (status) { - is Downloading -> Text(text = "Downloading ${(progress * 100).toInt()}%") - is Downloaded -> Text("Load ${item.name}") - is Ready -> Text("Download ${item.name}") - is Error -> Text("Download ${item.name}") - } - } - } - - } -} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt deleted file mode 100644 index 9da04f7d3..000000000 --- a/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt +++ /dev/null @@ -1,154 +0,0 @@ -package com.example.llama - -import android.app.ActivityManager -import android.app.DownloadManager -import android.content.ClipData -import android.content.ClipboardManager -import android.net.Uri -import android.os.Bundle -import android.os.StrictMode -import android.os.StrictMode.VmPolicy -import android.text.format.Formatter -import androidx.activity.ComponentActivity -import androidx.activity.compose.setContent -import androidx.activity.viewModels -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.items -import androidx.compose.foundation.lazy.rememberLazyListState -import androidx.compose.material3.Button -import androidx.compose.material3.LocalContentColor -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.OutlinedTextField -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.ui.Modifier -import androidx.compose.ui.unit.dp -import androidx.core.content.getSystemService -import com.example.llama.ui.theme.LlamaAndroidTheme -import java.io.File - -class MainActivity( - activityManager: ActivityManager? = null, - downloadManager: DownloadManager? = null, - clipboardManager: ClipboardManager? = null, -): ComponentActivity() { - private val tag: String? = this::class.simpleName - - private val activityManager by lazy { activityManager ?: getSystemService()!! } - private val downloadManager by lazy { downloadManager ?: getSystemService()!! } - private val clipboardManager by lazy { clipboardManager ?: getSystemService()!! } - - private val viewModel: MainViewModel by viewModels() - - // Get a MemoryInfo object for the device's current memory status. - private fun availableMemory(): ActivityManager.MemoryInfo { - return ActivityManager.MemoryInfo().also { memoryInfo -> - activityManager.getMemoryInfo(memoryInfo) - } - } - - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - - StrictMode.setVmPolicy( - VmPolicy.Builder(StrictMode.getVmPolicy()) - .detectLeakedClosableObjects() - .build() - ) - - val free = Formatter.formatFileSize(this, availableMemory().availMem) - val total = Formatter.formatFileSize(this, availableMemory().totalMem) - - viewModel.log("Current memory: $free / $total") - viewModel.log("Downloads directory: ${getExternalFilesDir(null)}") - - val extFilesDir = getExternalFilesDir(null) - - val models = listOf( - Downloadable( - "Phi-2 7B (Q4_0, 1.6 GiB)", - Uri.parse("https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf?download=true"), - File(extFilesDir, "phi-2-q4_0.gguf"), - ), - Downloadable( - "TinyLlama 1.1B (f16, 2.2 GiB)", - Uri.parse("https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf?download=true"), - File(extFilesDir, "tinyllama-1.1-f16.gguf"), - ), - Downloadable( - "Phi 2 DPO (Q3_K_M, 1.48 GiB)", - Uri.parse("https://huggingface.co/TheBloke/phi-2-dpo-GGUF/resolve/main/phi-2-dpo.Q3_K_M.gguf?download=true"), - File(extFilesDir, "phi-2-dpo.Q3_K_M.gguf") - ), - ) - - setContent { - LlamaAndroidTheme { - // A surface container using the 'background' color from the theme - Surface( - modifier = Modifier.fillMaxSize(), - color = MaterialTheme.colorScheme.background - ) { - MainCompose( - viewModel, - clipboardManager, - downloadManager, - models, - ) - } - - } - } - } -} - -@Composable -fun MainCompose( - viewModel: MainViewModel, - clipboard: ClipboardManager, - dm: DownloadManager, - models: List -) { - Column { - val scrollState = rememberLazyListState() - - Box(modifier = Modifier.weight(1f)) { - LazyColumn(state = scrollState) { - items(viewModel.messages) { - Text( - it, - style = MaterialTheme.typography.bodyLarge.copy(color = LocalContentColor.current), - modifier = Modifier.padding(16.dp) - ) - } - } - } - OutlinedTextField( - value = viewModel.message, - onValueChange = { viewModel.updateMessage(it) }, - label = { Text("Message") }, - ) - Row { - Button({ viewModel.send() }) { Text("Send") } - Button({ viewModel.bench(8, 4, 1) }) { Text("Bench") } - Button({ viewModel.clear() }) { Text("Clear") } - Button({ - viewModel.messages.joinToString("\n").let { - clipboard.setPrimaryClip(ClipData.newPlainText("", it)) - } - }) { Text("Copy") } - } - - Column { - for (model in models) { - Downloadable.Button(viewModel, dm, model) - } - } - } -} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt deleted file mode 100644 index 45ac29938..000000000 --- a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt +++ /dev/null @@ -1,105 +0,0 @@ -package com.example.llama - -import android.llama.cpp.LLamaAndroid -import android.util.Log -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.setValue -import androidx.lifecycle.ViewModel -import androidx.lifecycle.viewModelScope -import kotlinx.coroutines.flow.catch -import kotlinx.coroutines.launch - -class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instance()): ViewModel() { - companion object { - @JvmStatic - private val NanosPerSecond = 1_000_000_000.0 - } - - private val tag: String? = this::class.simpleName - - var messages by mutableStateOf(listOf("Initializing...")) - private set - - var message by mutableStateOf("") - private set - - override fun onCleared() { - super.onCleared() - - viewModelScope.launch { - try { - llamaAndroid.unload() - } catch (exc: IllegalStateException) { - messages += exc.message!! - } - } - } - - fun send() { - val text = message - message = "" - - // Add to messages console. - messages += text - messages += "" - - viewModelScope.launch { - llamaAndroid.send(text) - .catch { - Log.e(tag, "send() failed", it) - messages += it.message!! - } - .collect { messages = messages.dropLast(1) + (messages.last() + it) } - } - } - - fun bench(pp: Int, tg: Int, pl: Int, nr: Int = 1) { - viewModelScope.launch { - try { - val start = System.nanoTime() - val warmupResult = llamaAndroid.bench(pp, tg, pl, nr) - val end = System.nanoTime() - - messages += warmupResult - - val warmup = (end - start).toDouble() / NanosPerSecond - messages += "Warm up time: $warmup seconds, please wait..." - - if (warmup > 5.0) { - messages += "Warm up took too long, aborting benchmark" - return@launch - } - - messages += llamaAndroid.bench(512, 128, 1, 3) - } catch (exc: IllegalStateException) { - Log.e(tag, "bench() failed", exc) - messages += exc.message!! - } - } - } - - fun load(pathToModel: String) { - viewModelScope.launch { - try { - llamaAndroid.load(pathToModel) - messages += "Loaded $pathToModel" - } catch (exc: IllegalStateException) { - Log.e(tag, "load() failed", exc) - messages += exc.message!! - } - } - } - - fun updateMessage(newMessage: String) { - message = newMessage - } - - fun clear() { - messages = listOf() - } - - fun log(message: String) { - messages += message - } -} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt deleted file mode 100644 index 40c30e8d9..000000000 --- a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt +++ /dev/null @@ -1,11 +0,0 @@ -package com.example.llama.ui.theme - -import androidx.compose.ui.graphics.Color - -val Purple80 = Color(0xFFD0BCFF) -val PurpleGrey80 = Color(0xFFCCC2DC) -val Pink80 = Color(0xFFEFB8C8) - -val Purple40 = Color(0xFF6650a4) -val PurpleGrey40 = Color(0xFF625b71) -val Pink40 = Color(0xFF7D5260) diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt deleted file mode 100644 index e742220a8..000000000 --- a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt +++ /dev/null @@ -1,70 +0,0 @@ -package com.example.llama.ui.theme - -import android.app.Activity -import android.os.Build -import androidx.compose.foundation.isSystemInDarkTheme -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.darkColorScheme -import androidx.compose.material3.dynamicDarkColorScheme -import androidx.compose.material3.dynamicLightColorScheme -import androidx.compose.material3.lightColorScheme -import androidx.compose.runtime.Composable -import androidx.compose.runtime.SideEffect -import androidx.compose.ui.graphics.toArgb -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.platform.LocalView -import androidx.core.view.WindowCompat - -private val DarkColorScheme = darkColorScheme( - primary = Purple80, - secondary = PurpleGrey80, - tertiary = Pink80 -) - -private val LightColorScheme = lightColorScheme( - primary = Purple40, - secondary = PurpleGrey40, - tertiary = Pink40 - - /* Other default colors to override - background = Color(0xFFFFFBFE), - surface = Color(0xFFFFFBFE), - onPrimary = Color.White, - onSecondary = Color.White, - onTertiary = Color.White, - onBackground = Color(0xFF1C1B1F), - onSurface = Color(0xFF1C1B1F), - */ -) - -@Composable -fun LlamaAndroidTheme( - darkTheme: Boolean = isSystemInDarkTheme(), - // Dynamic color is available on Android 12+ - dynamicColor: Boolean = true, - content: @Composable () -> Unit -) { - val colorScheme = when { - dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { - val context = LocalContext.current - if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) - } - - darkTheme -> DarkColorScheme - else -> LightColorScheme - } - val view = LocalView.current - if (!view.isInEditMode) { - SideEffect { - val window = (view.context as Activity).window - window.statusBarColor = colorScheme.primary.toArgb() - WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme - } - } - - MaterialTheme( - colorScheme = colorScheme, - typography = Typography, - content = content - ) -} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt deleted file mode 100644 index 0b87946ca..000000000 --- a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt +++ /dev/null @@ -1,34 +0,0 @@ -package com.example.llama.ui.theme - -import androidx.compose.material3.Typography -import androidx.compose.ui.text.TextStyle -import androidx.compose.ui.text.font.FontFamily -import androidx.compose.ui.text.font.FontWeight -import androidx.compose.ui.unit.sp - -// Set of Material typography styles to start with -val Typography = Typography( - bodyLarge = TextStyle( - fontFamily = FontFamily.Default, - fontWeight = FontWeight.Normal, - fontSize = 16.sp, - lineHeight = 24.sp, - letterSpacing = 0.5.sp - ) - /* Other default text styles to override - titleLarge = TextStyle( - fontFamily = FontFamily.Default, - fontWeight = FontWeight.Normal, - fontSize = 22.sp, - lineHeight = 28.sp, - letterSpacing = 0.sp - ), - labelSmall = TextStyle( - fontFamily = FontFamily.Default, - fontWeight = FontWeight.Medium, - fontSize = 11.sp, - lineHeight = 16.sp, - letterSpacing = 0.5.sp - ) - */ -) diff --git a/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml b/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml deleted file mode 100644 index 07d5da9cb..000000000 --- a/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml b/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml deleted file mode 100644 index 7706ab9e6..000000000 --- a/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - diff --git a/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml b/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml deleted file mode 100644 index b3e26b4c6..000000000 --- a/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml b/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml deleted file mode 100644 index b3e26b4c6..000000000 --- a/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp deleted file mode 100644 index c209e78ecd372343283f4157dcfd918ec5165bb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1404 zcmV-?1%vuhNk&F=1pok7MM6+kP&il$0000G0000-002h-06|PpNX!5L00Dqw+t%{r zzW2vH!KF=w&cMnnN@{whkTw+#mAh0SV?YL=)3MimFYCWp#fpdtz~8$hD5VPuQgtcN zXl<@<#Cme5f5yr2h%@8TWh?)bSK`O z^Z@d={gn7J{iyxL_y_%J|L>ep{dUxUP8a{byupH&!UNR*OutO~0{*T4q5R6@ApLF! z5{w?Z150gC7#>(VHFJZ-^6O@PYp{t!jH(_Z*nzTK4 zkc{fLE4Q3|mA2`CWQ3{8;gxGizgM!zccbdQoOLZc8hThi-IhN90RFT|zlxh3Ty&VG z?Fe{#9RrRnxzsu|Lg2ddugg7k%>0JeD+{XZ7>Z~{=|M+sh1MF7~ zz>To~`~LVQe1nNoR-gEzkpe{Ak^7{{ZBk2i_<+`Bq<^GB!RYG+z)h;Y3+<{zlMUYd zrd*W4w&jZ0%kBuDZ1EW&KLpyR7r2=}fF2%0VwHM4pUs}ZI2egi#DRMYZPek*^H9YK zay4Iy3WXFG(F14xYsoDA|KXgGc5%2DhmQ1gFCkrgHBm!lXG8I5h*uf{rn48Z!_@ z4Bk6TJAB2CKYqPjiX&mWoW>OPFGd$wqroa($ne7EUK;#3VYkXaew%Kh^3OrMhtjYN?XEoY`tRPQsAkH-DSL^QqyN0>^ zmC>{#F14jz4GeW{pJoRpLFa_*GI{?T93^rX7SPQgT@LbLqpNA}<@2wH;q493)G=1Y z#-sCiRNX~qf3KgiFzB3I>4Z%AfS(3$`-aMIBU+6?gbgDb!)L~A)je+;fR0jWLL-Fu z4)P{c7{B4Hp91&%??2$v9iRSFnuckHUm}or9seH6 z>%NbT+5*@L5(I9j@06@(!{ZI?U0=pKn8uwIg&L{JV14+8s2hnvbRrU|hZCd}IJu7*;;ECgO%8_*W Kmw_-CKmY()leWbG diff --git a/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp deleted file mode 100644 index b2dfe3d1ba5cf3ee31b3ecc1ced89044a1f3b7a9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2898 zcmV-Y3$650Nk&FW3jhFDMM6+kP&il$0000G0000-002h-06|PpNWB9900E$G+qN-D z+81ABX7q?;bwx%xBg?kcwr$(C-Tex-ZCkHUw(Y9#+`E5-zuONG5fgw~E2WDng@Bc@ z24xy+R1n%~6xI#u9vJ8zREI)sb<&Il(016}Z~V1n^PU3-_H17A*Bf^o)&{_uBv}Py zulRfeE8g(g6HFhk_?o_;0@tz?1I+l+Y#Q*;RVC?(ud`_cU-~n|AX-b`JHrOIqn(-t&rOg-o`#C zh0LPxmbOAEb;zHTu!R3LDh1QO zZTf-|lJNUxi-PpcbRjw3n~n-pG;$+dIF6eqM5+L();B2O2tQ~|p{PlpNcvDbd1l%c zLtXn%lu(3!aNK!V#+HNn_D3lp z2%l+hK-nsj|Bi9;V*WIcQRTt5j90A<=am+cc`J zTYIN|PsYAhJ|=&h*4wI4ebv-C=Be#u>}%m;a{IGmJDU`0snWS&$9zdrT(z8#{OZ_Y zxwJx!ZClUi%YJjD6Xz@OP8{ieyJB=tn?>zaI-4JN;rr`JQbb%y5h2O-?_V@7pG_+y z(lqAsqYr!NyVb0C^|uclHaeecG)Sz;WV?rtoqOdAAN{j%?Uo%owya(F&qps@Id|Of zo@~Y-(YmfB+chv^%*3g4k3R0WqvuYUIA+8^SGJ{2Bl$X&X&v02>+0$4?di(34{pt* zG=f#yMs@Y|b&=HyH3k4yP&goF2LJ#tBLJNNDo6lG06r}ghC-pC4Q*=x3;|+W04zte zAl>l4kzUBQFYF(E`KJy?ZXd1tnfbH+Z~SMmA21KokJNs#eqcXWKUIC>{TuoKe^vhF z);H)o`t9j~`$h1D`#bxe@E`oE`cM9w(@)5Bp8BNukIwM>wZHfd0S;5bcXA*5KT3bj zc&_~`&{z7u{Et!Z_k78H75gXf4g8<_ul!H$eVspPeU3j&&Au=2R*Zp#M9$9s;fqwgzfiX=E_?BwVcfx3tG9Q-+<5fw z%Hs64z)@Q*%s3_Xd5>S4dg$s>@rN^ixeVj*tqu3ZV)biDcFf&l?lGwsa zWj3rvK}?43c{IruV2L`hUU0t^MemAn3U~x3$4mFDxj=Byowu^Q+#wKRPrWywLjIAp z9*n}eQ9-gZmnd9Y0WHtwi2sn6n~?i#n9VN1B*074_VbZZ=WrpkMYr{RsI ztM_8X1)J*DZejxkjOTRJ&a*lrvMKBQURNP#K)a5wIitfu(CFYV4FT?LUB$jVwJSZz zNBFTWg->Yk0j&h3e*a5>B=-xM7dE`IuOQna!u$OoxLlE;WdrNlN)1 z7**de7-hZ!(%_ZllHBLg`Ir#|t>2$*xVOZ-ADZKTN?{(NUeLU9GbuG-+Axf*AZ-P1 z0ZZ*fx+ck4{XtFsbcc%GRStht@q!m*ImssGwuK+P@%gEK!f5dHymg<9nSCXsB6 zQ*{<`%^bxB($Z@5286^-A(tR;r+p7B%^%$N5h%lb*Vlz-?DL9x;!j<5>~kmXP$E}m zQV|7uv4SwFs0jUervsxVUm>&9Y3DBIzc1XW|CUZrUdb<&{@D5yuLe%Xniw^x&{A2s z0q1+owDSfc3Gs?ht;3jw49c#mmrViUfX-yvc_B*wY|Lo7; zGh!t2R#BHx{1wFXReX*~`NS-LpSX z#TV*miO^~B9PF%O0huw!1Zv>^d0G3$^8dsC6VI!$oKDKiXdJt{mGkyA`+Gwd4D-^1qtNTUK)`N*=NTG-6}=5k6suNfdLt*dt8D| z%H#$k)z#ZRcf|zDWB|pn<3+7Nz>?WW9WdkO5(a^m+D4WRJ9{wc>Y}IN)2Kbgn;_O? zGqdr&9~|$Y0tP=N(k7^Eu;iO*w+f%W`20BNo)=Xa@M_)+o$4LXJyiw{F?a633SC{B zl~9FH%?^Rm*LVz`lkULs)%idDX^O)SxQol(3jDRyBVR!7d`;ar+D7do)jQ}m`g$TevUD5@?*P8)voa?kEe@_hl{_h8j&5eB-5FrYW&*FHVt$ z$kRF9Nstj%KRzpjdd_9wO=4zO8ritN*NPk_9avYrsF(!4))tm{Ga#OY z(r{0buexOzu7+rw8E08Gxd`LTOID{*AC1m*6Nw@osfB%0oBF5sf<~wH1kL;sd zo)k6^VyRFU`)dt*iX^9&QtWbo6yE8XXH?`ztvpiOLgI3R+=MOBQ9=rMVgi<*CU%+d1PQQ0a1U=&b0vkF207%xU0ssI2 diff --git a/examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp deleted file mode 100644 index 4f0f1d64e58ba64d180ce43ee13bf9a17835fbca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 982 zcmV;{11bDcNk&G_0{{S5MM6+kP&il$0000G0000l001ul06|PpNU8t;00Dqo+t#w^ z^1csucXz7-Qrhzl9HuHB%l>&>1tG2^vb*E&k^T3$FG1eQZ51g$uv4V+kI`0<^1Z@N zk?Jjh$olyC%l>)Xq;7!>{iBj&BjJ`P&$fsCfpve_epJOBkTF?nu-B7D!hO=2ZR}

C%4 zc_9eOXvPbC4kzU8YowIA8cW~Uv|eB&yYwAObSwL2vY~UYI7NXPvf3b+c^?wcs~_t{ ze_m66-0)^{JdOMKPwjpQ@Sna!*?$wTZ~su*tNv7o!gXT!GRgivP}ec?5>l1!7<(rT zds|8x(qGc673zrvYIz;J23FG{9nHMnAuP}NpAED^laz3mAN1sy+NXK)!6v1FxQ;lh zOBLA>$~P3r4b*NcqR;y6pwyhZ3_PiDb|%n1gGjl3ZU}ujInlP{eks-#oA6>rh&g+!f`hv#_%JrgYPu z(U^&XLW^QX7F9Z*SRPpQl{B%x)_AMp^}_v~?j7 zapvHMKxSf*Mtyx8I}-<*UGn3)oHd(nn=)BZ`d$lDBwq_GL($_TPaS{UeevT(AJ`p0 z9%+hQb6z)U9qjbuXjg|dExCLjpS8$VKQ55VsIC%@{N5t{NsW)=hNGI`J=x97_kbz@ E0Of=7!TQj4N+cqN`nQhxvX7dAV-`K|Ub$-q+H-5I?Tx0g9jWxd@A|?POE8`3b8fO$T))xP* z(X?&brZw({`)WU&rdAs1iTa0x6F@PIxJ&&L|dpySV!ID|iUhjCcKz(@mE z!x@~W#3H<)4Ae(4eQJRk`Iz3<1)6^m)0b_4_TRZ+cz#eD3f8V;2r-1fE!F}W zEi0MEkTTx}8i1{`l_6vo0(Vuh0HD$I4SjZ=?^?k82R51bC)2D_{y8mi_?X^=U?2|F{Vr7s!k(AZC$O#ZMyavHhlQ7 zUR~QXuH~#o#>(b$u4?s~HLF*3IcF7023AlwAYudn0FV~|odGH^05AYPEfR)8p`i{n zwg3zPVp{+wOsxKc>)(pMupKF!Y2HoUqQ3|Yu|8lwR=?5zZuhG6J?H`bSNk_wPoM{u zSL{c@pY7+c2kck>`^q1^^gR0QB7Y?KUD{vz-uVX~;V-rW)PDcI)$_UjgVV?S?=oLR zf4}zz{#*R_{LkiJ#0RdQLNC^2Vp%JPEUvG9ra2BVZ92(p9h7Ka@!yf9(lj#}>+|u* z;^_?KWdzkM`6gqPo9;;r6&JEa)}R3X{(CWv?NvgLeOTq$cZXqf7|sPImi-7cS8DCN zGf;DVt3Am`>hH3{4-WzH43Ftx)SofNe^-#|0HdCo<+8Qs!}TZP{HH8~z5n`ExcHuT zDL1m&|DVpIy=xsLO>8k92HcmfSKhflQ0H~9=^-{#!I1g(;+44xw~=* zxvNz35vfsQE)@)Zsp*6_GjYD};Squ83<_?^SbALb{a`j<0Gn%6JY!zhp=Fg}Ga2|8 z52e1WU%^L1}15Ex0fF$e@eCT(()_P zvV?CA%#Sy08_U6VPt4EtmVQraWJX` zh=N|WQ>LgrvF~R&qOfB$!%D3cGv?;Xh_z$z7k&s4N)$WYf*k=|*jCEkO19{h_(%W4 zPuOqbCw`SeAX*R}UUsbVsgtuG?xs(#Ikx9`JZoQFz0n*7ZG@Fv@kZk`gzO$HoA9kN z8U5{-yY zvV{`&WKU2$mZeoBmiJrEdzUZAv1sRxpePdg1)F*X^Y)zp^Y*R;;z~vOv-z&)&G)JQ{m!C9cmziu1^nHA z`#`0c>@PnQ9CJKgC5NjJD8HM3|KC(g5nnCq$n0Gsu_DXk36@ql%npEye|?%RmG)

FJ$wK}0tWNB{uH;AM~i diff --git a/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp deleted file mode 100644 index 948a3070fe34c611c42c0d3ad3013a0dce358be0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1900 zcmV-y2b1_xNk&Fw2LJ$9MM6+kP&il$0000G0001A003VA06|PpNH75a00DqwTbm-~ zullQTcXxO9ki!OCRx^i?oR|n!<8G0=kI^!JSjFi-LL*`V;ET0H2IXfU0*i>o6o6Gy zRq6Ap5(_{XLdXcL-MzlN`ugSdZY_`jXhcENAu)N_0?GhF))9R;E`!bo9p?g?SRgw_ zEXHhFG$0{qYOqhdX<(wE4N@es3VIo$%il%6xP9gjiBri+2pI6aY4 zJbgh-Ud|V%3O!IcHKQx1FQH(_*TK;1>FQWbt^$K1zNn^cczkBs=QHCYZ8b&l!UV{K z{L0$KCf_&KR^}&2Fe|L&?1I7~pBENnCtCuH3sjcx6$c zwqkNkru);ie``q+_QI;IYLD9OV0ZxkuyBz|5<$1BH|vtey$> z5oto4=l-R-Aaq`Dk0}o9N0VrkqW_#;!u{!bJLDq%0092{Ghe=F;(kn} z+sQ@1=UlX30+2nWjkL$B^b!H2^QYO@iFc0{(-~yXj2TWz?VG{v`Jg zg}WyYnwGgn>{HFaG7E~pt=)sOO}*yd(UU-D(E&x{xKEl6OcU?pl)K%#U$dn1mDF19 zSw@l8G!GNFB3c3VVK0?uyqN&utT-D5%NM4g-3@Sii9tSXKtwce~uF zS&Jn746EW^wV~8zdQ1XC28~kXu8+Yo9p!<8h&(Q({J*4DBglPdpe4M_mD8AguZFn~ ztiuO~{6Bx?SfO~_ZV(GIboeR9~hAym{{fV|VM=77MxDrbW6`ujX z<3HF(>Zr;#*uCvC*bpoSr~C$h?_%nXps@A)=l_;({Fo#6Y1+Zv`!T5HB+)#^-Ud_; zBwftPN=d8Vx)*O1Mj+0oO=mZ+NVH*ptNDC-&zZ7Hwho6UQ#l-yNvc0Cm+2$$6YUk2D2t#vdZX-u3>-Be1u9gtTBiMB^xwWQ_rgvGpZ6(C@e23c!^K=>ai-Rqu zhqT`ZQof;9Bu!AD(i^PCbYV%yha9zuoKMp`U^z;3!+&d@Hud&_iy!O-$b9ZLcSRh? z)R|826w}TU!J#X6P%@Zh=La$I6zXa#h!B;{qfug}O%z@K{EZECu6zl)7CiNi%xti0 zB{OKfAj83~iJvmpTU|&q1^?^cIMn2RQ?jeSB95l}{DrEPTW{_gmU_pqTc)h@4T>~& zluq3)GM=xa(#^VU5}@FNqpc$?#SbVsX!~RH*5p0p@w z;~v{QMX0^bFT1!cXGM8K9FP+=9~-d~#TK#ZE{4umGT=;dfvWi?rYj;^l_Zxywze`W z^Cr{55U@*BalS}K%Czii_80e0#0#Zkhlij4-~I@}`-JFJ7$5{>LnoJSs??J8kWVl6|8A}RCGAu9^rAsfCE=2}tHwl93t0C?#+jMpvr7O3`2=tr{Hg$=HlnjVG^ewm|Js0J*kfPa6*GhtB>`fN!m#9J(sU!?(OSfzY*zS(FJ<-Vb zfAIg+`U)YaXv#sY(c--|X zEB+TVyZ%Ie4L$gi#Fc++`h6%vzsS$pjz9aLt+ZL(g;n$Dzy5=m=_TV(3H8^C{r0xd zp#a%}ht55dOq?yhwYPrtp-m1xXp;4X;)NhxxUpgP%XTLmO zcjaFva^}dP3$&sfFTIR_jC=2pHh9kpI@2(6V*GQo7Ws)`j)hd+tr@P~gR*2gO@+1? zG<`_tB+LJuF|SZ9tIec;h%}}6WClT`L>HSW?E{Hp1h^+mlbf_$9zA>!ug>NALJsO{ mU%z=YwVD?}XMya)Bp;vlyE5&E_6!fzx9pwrdz474!~g(M6R?N? diff --git a/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp deleted file mode 100644 index 1b9a6956b3acdc11f40ce2bb3f6efbd845cc243f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3918 zcmV-U53%r4Nk&FS4*&pHMM6+kP&il$0000G0001A003VA06|PpNSy@$00HoY|G(*G z+qV7x14$dSO^Re!iqt-AAIE9iwr$(CZQJL$blA4B`>;C3fBY6Q8_YSjb2%a=fc}4E zrSzssacq<^nmW|Rs93PJni30R<8w<(bK_$LO4L?!_OxLl$}K$MUEllnMK|rg=f3;y z*?;3j|Nh>)p0JQ3A~rf(MibH2r+)3cyV1qF&;8m{w-S*y+0mM){KTK^M5}ksc`qX3 zy>rf^b>~l>SSHds8(I@hz3&PD@LmEs4&prkT=BjsBCXTMhN$_)+kvnl0bLKW5rEsj z*d#KXGDB4P&>etx0X+`R19yC=LS)j!mgs5M0L~+o-T~Jl!p!AJxnGAhV%~rhYUL4hlWhgES3Kb5oA&X z{}?3OBSS-{!v$nCIGj->(-TAG)8LR{htr41^gxsT8yqt2@DEG6Yl`Uma3Nd4;YUoW zTbkYl3CMU5ypMF3EIkYmWL|*BknM`0+Kq6CpvO(y$#j94e+q{vI{Zp8cV_6RK!`&C zob$*5Q|$IZ09dW=L!V zw@#2wviu|<#3lgGE8GEhcx+zBt`} zOwP8j9X%^f7i_bth4PiJ$LYtFJSCN$3xwDN;8mr*B;CJwBP2G0TMq0uNt7S^DO_wE zepk!Wrn#Z#03j{`c*Rf~y3o7?J}w?tEELRUR2cgxB*Y{LzA#pxHgf}q?u5idu>077 zd^=p)`nA}6e`|@`p?u}YU66PP_MA}Zqqe!c{nK&z%Jwq1N4e_q<#4g^xaz=ao;u|6 zwpRcW2Lax=ZGbx=Q*HhlJ`Ns#Y*r0*%!T?P*TTiX;rb)$CGLz=rSUum$)3Qyv{BL2 zO*=OI2|%(Yz~`pNEOnLp>+?T@glq-DujlIp?hdJeZ7ctP4_OKx|5@EOps3rr(pWzg zK4d3&oN-X2qN(d_MkfwB4I)_)!I_6nj2iA9u^pQ{;GckGLxBGrJUM2Wdda!k)Y>lq zmjws>dVQ*vW9lvEMkiN3wE-__6OWD0txS&Qn0n22cyj4Q*8(nG4!G{6OOwNvsrPIL zCl-$W9UwkEUVuLwyD%|inbOF*xMODZ4VMEVAq_zUxZ+K#Gdqf!DW$5f)?7UNOFMz! zrB~tuu=6X2FE(p^iqgxr+?ZK;=yz`e;C$#_@D9Lj-+TDVOrva>(#*PVbaHO>A)mhl z07OJWCqYC60518$!&c`eNBcBW%GnfaQ*$eazV^2_AW?j)h;J1nUjN(I9=0+!RVx~% z3@Tf!P0TE+98jA?WceK-}A1% zW!K)lyKcGqy#M~})315-A#2NXQ`?6NR#Apo=S!oF=JfpX>iR*49ec{7AN$xxpK{D$ z2d%Fz&rdfSqourN$~Y^NFIMV1CZ?J*bMx~H3k&meGtH@q9ra2vZxmA$S(#jaaj-g4 ztJmxG+DLV<*q<|sDXPp$X>E)#S}Vm&sRaO5P&goh2><}FEdZSXDqsL$06sAkh(e+v zAsBhKSRexgwg6tIy~GFJzaTxXD(}|+0eOwFDA%rn`X;MVwDHT9=4=g%OaJ9s%3b9>9EUTnnp0t;2Zpa{*>mk~hZqItE_!dQ zOtC>8`$l|mV43Jbudf0N6&&X;{=z}Zi}d1`2qmJ}i|0*GsulD3>GgQXHN)pkR6sf1 z?5ZU%&xtL}oH;YiAA)d*^Ndw2T$+Mjuzyzz@-SM`9df7LqTxLuIwC~S0092~+=qYv z@*ja;?Wt!T!{U?c*Z0YtGe)XbI&y-?B&G2$`JDM)(dIV9G`Sc#6?sI60de6kv+)Qb zUW~2|WjvJq3TA8`0+sWA3zRhY9a~ow)O~&StBkG2{*{TGiY~S8ep{V&Vo2l<6LWsu z^#p0-v*t2?3&aA1)ozu|%efSR=XnpX$lvTeRdKlvM!@|pM5p2w3u-6 zU>}t2xiYLS+{|%C65AzX+23Mtlq?BS&YdYcYsVjoiE&rT>;Necn6l^K)T^lmE`5u{ zm1i+-a-gc;Z&v-{;8r)z6NYfBUv+=_L}ef}qa9FX01)+Aaf+;xj(mL6|JUzGJR1|fnanb%?BPPIp>SCjP|8qE5qJ{=n5ZGw?81z3(k;pzH%1CtlX50{E7h)$h{qGKfzC`e2o`*IqA#tjA z`Fz&^%$b9F*N`)U-#6>a)Z`55`$Dd0cfcs0$d13^ONrdCu9xcv_=n#WQo8stcz3jP9|2EvdI-RhJM3%Q%oM&!OlShM|0 z?gz?wHZSnm45njLtsz8PVT1S&jAlbKg5kVam$p16=EK@Sj4EP0OtH zmJDmdc^v)x>56Qg_wmYHz6h)>kl_h$>0@J!ypv%APmjZTAQVLy6Fu50RGY&JAVNhx zrF_qG6`x9MkT;1SFWo$)l{M$;3qUDn9JwE}z zRl#E_bDRJFii61kPgBybIgp8dNW!Cc1b*^YYk-#oWLJvtM_v^hQx~9?8LD4VFFxBF z3MlrsSC%f9Oupn*ctPL0U1fwfX?`tRhPD{PSLFPQOmIt$mDy0SgpNVvHS+f#Do>h1Gn?LZU9(KaN>Q_=Y*_T zvtD7%_u^^+{g`0VGzg(VZrpVQ6Ub5M=tI_p7T93R8@3Zulu3|#{iNcu!oiHxZ4Rf*( zfmiN$$ru(*_Zqn=`Gq#OuHRTSwp7uH_SokR&|)RuW5yo=Z|_4?qU-JU+tpt>!B&Is z@N(=SG;bpVc;AO@zbmMM zScqq1)b-ZQIrs={oD}|?6y{$HNB1U0^LsBh8JI&3!GBZxOXI<}&5-$lgkAaYqhOTb z?2vEnZ$-kk;*M_17(upJF3%+iH*s0-r{vttXVB2OUwI1s^+G(Ft(U8gYFXC}#P&E^ z>T@C^tS`Z7{6HT4_nF~n>JlZtk5&qDBl6r|^kzQYe`wq!C)n@$c>WOPA61NDFj<<6 zGW71NMMhwAl!U-yqrq2xrSFqRCI8acw7?}3j;ynxo*-b7Co;g5r%^j=H@9({PXXBf z@r>U>>N;E)81wx`B4f%{PB~MHka_);%kBCb(d|Jy5!MqJ%2p`t&@L)4$T2j&-WHvG zv3(uyA_gwqNu(k?jQTtv3dgPKRZoH8prxe7>pQBW5L&dpumS&5Ld2?(sCpJjvc4L5 zEnh&?91WVm)ZdTj=fjJ$pPDdgAttLXuke+?KdKxu*;kTC(r!tQk6;gxj4h%FdHAt(^M3YvYj(!tOeN)+Hvj6+< zzyJRG?^lZfWuR#t!tUKP&(?%3v&Zd$R2YN>lB(Lq`OInY48%4%yTv2 zYe1{G`3)(PDEio5Y@-I5tUf`c%%OCJMtSW56g3iEg%3`$7XSJJHyA z<|7&N)5Xrlgv~%BO24eFd;Hd;uiK%D`EdK|quUeRZDqbh9l)%j%J#0lfrZumvA<_w zu&=AVvdChf6}eqh(bUz`(`Ue*p01{fBAcTgKyDYLs_I+YyJEk+rM@avU~>fB$n)HS zM7pfJydu`i%gfS<{PF94kZDv$t>06sAkheDzu40NJ$5CMW%n^Lls?8^p^QGWURbKu3ZduZQZ((s2? zzE`}<{;Zt7<$C|9R8A~DJ~@%x>TfP zF>TX8)@v|t)q4GjRt<}5s6hLHwRel7>V@&r-O|Av(yh;Q1A{E>Ir>p+%dHD|=l+lT zpr(Dg&>#Nu=!)6bCLr-ZS%|;h)Ij$+e@r8_{qO19QvDe=&1tmpY*0lcA^Cc-#{9fQ z<~$*<&P$Q<_jy#<$40PMofM7aQ}C=jphI`4kLg}Z7CIN#26D{-4v-_CA-LiE@(%{y!BzsU%gG`Q?sjLUf%qFSl0y)2#ae*+EI>s|i`d^V$Dn)qmzqRq6VJRY|{4ujsIU%#bnqU6MR&-1I_43=|5(6Jr;Jvert) zE?S|Tmn}Tv<-??sxV5@9t}3D=>YZ0JrQe$CO~|EY=Lj9RM&4svQHPQL6%pV5fPFiH zfXDx;l@~et{*{U*#c#Dvzu)|znDO7$#CRx)Z&yp-}SrD{&|(MQtfUz~n35@RLfUy=aqrhCX0M}J_r5QsK~NmRCR|Nm&L z41UdsLjWxSUlL41r^0K&nCCK>fdR-!MYjFg(z9_mF^C|#ZQw?`)f6uVzF^`bRnVY& zo}@M06J&_+>w9@jpaO4snmU;0t-(zYW1qVBHtuD!d?%?AtN7Plp><-1Y8Rqb20ZaP zTCgn*-Sri4Q8Xn>=gNaWQ57%!D35UkA@ksOlPB*Dvw}t02ENAqw|kFhn%ZyyW%+t{ zNdM!uqEM^;2}f+tECHbwLmH*!nZVrb$-az%t50Y2pg(HqhvY-^-lb}>^6l{$jOI6} zo_kBzj%8aX|6H5M0Y<)7pzz_wLkIpRm!;PzY)9+24wk2&TT{w--phDGDCOz{cN_ca zpnm7`$oDy=HX%0i-`769*0M6(e5j-?(?24%)<)&46y0e&6@HCDZAm9W6Ib#Y#BF6- z=30crHGg+RRTe%VBC>T00OV6F+gQDAK38Ne3N9bm|62tPccBJi)5{B z4zc^Db72XiBd}v$CF|yU{Z=M|DZ%-(XarYNclODlb1Kz1_EKLy(NSLCN`eUl(rBCL zT*jx@wNvze0|TSqgE(QArOZU)_?qH(sj#TwzElLs9q)(0u!_P|R%Cy_0JFQxgGV>1 zz4?_uq<8_gM0`c*Hh|;UMz~vrg1gQXp{ufg`hM_qU;U>+zmvc5blCLSq@PrEBSGR# z&8=2Z4uXN`F3p73ueD1l{s{k$WipAvSh5W7ABe?4)t;r@V?y`bNB5FvBuE|0VRTb< zM1Hn^?DSsJY+sX@T5xW=#>T9VEV|?<(=6|ge$X6Sb05!LFdjDcoq*gM(Zq=t;_)Le&jyt(&9jzR73noru`a# zN*<`KwGa^gZU3-)MSLF0aFag#f0<>E(bYTeHmtdbns#|I)-$)mJ`q9ctQ8g0=ET?| zdO}eZ*b_p>ygRTtR^5Ggdam=Zb5wmd{}np+Jn1d_=M`~P=M67jj})fH4ztb5yQqQW z^C|C&^LHAK-u+ooIK)yM)QM?t;|<{P;;{`p=BclzAN#JzL4jCwXkQB1Dy{=^KR`=~ zTrr)y7eiYBzSNs_DvO=4A6#EgGS-zY%Vi)N*Yb`U;6o}KR}dq{r9pT5wqZ@3NOE8- z9-(}D|Nc5732CSYQbL)!gPQ#RbD8BhK3dl{sUuPvei0tkvnJBxDEAYTesU8H$)g(Plra{VH(v3u^CO1~(+ zU0O7#)jaS4{NcwA+LuSm&VBcX2#Im3xg)W}ySNw%->orn1taZ&+d)}8gJTqA!u|5P z{yv?zol_3|(1(%M(EVU=cp?L`{Pi|ixk{U)*guFML3P!OSlz;zGA#T+E@8@cgQ_mv1o7RSU=Zo_82F?&&2r;WE z@wk}JHYEZ9nYUc(Vv~iTCa3u8e4q(yq<29VoNbKk|`mq%I6u)My=gPIDuUb&lzf4`MEA9^g8u z)vp8|$$HE9m_BTV?lOosIGa4jud=jIbw)O2eCMfyw2*S8?hjWw^nqws$O*M$3I1)x zR0PWFb3$ySOcGTe1dz%N0l;RPc`x%05FtT^f^j{YCP}*Q=lvp4$ZXrTZQHhO+w%wJn3c8j%+5C3UAFD&%8dBl_qi9D5g8fry}6Ev z2_Q~)5^N$!IU`BPh1O|=BxQ#*C5*}`lluC515$lxc-vNC)IgW=K|=z7o%cWFpndn= zX}f{`!VK02_kU+Q5a3m37J;c} zTzbxteE{GNf?yLt5X=Bzc-mio^Up0nunMCgp*ZJ;%MJvPM3QK)BryP(_v@ei4UvHr z6+sbCifQaOkL6-;5fL8$W($zZ_;CZp305C;~$hhRquZr-r)jjd1z z31%ZK{-(`P#|Um_Sivn@p$-vz46uqT>QG0B1w9znfS9A8PB2LaHdzA|_)yjXVR*l{ zkcu3@vEf7bxH0nkh`q?8FmoO_Ucui*>_a~P?qQrlZ9@+D7%MTpSnztpylXrt5!-k8_QPB?YL8Kx_On8WD zgT+111d(Op$^$&KLAN5+@?>f7F4~wFi(8TL8+szgVmcMDTp5l&k6~=rA{Dt}!gb^r zSWY<)M7D|Z2P0cEodj6E42PV>&>DFmQpgt)E-|#sSUU@uKed+F680H@<;-x{p|nuH4!_mn85rx>wz;0mPi2ZkL#k6;sznu?cXh!T0S>{w6 zL^gvR05NY64l*<+_L>On$rjx9!US;l;LX6@z}yi#2XHh)F@Oo+l)h%fq$v}DNmF2> zfs^_t0)3N-W<9-N?uedVv{)-J0W5mh#29QM5R5h&KuiRM=0Zvnf#lF=K#WlCgc#9c zS;qvh(P$!_a8JwyhI^ZJV2k+B6Z^64?w|1?5gyo6y{}923CRZfYVe1#?F% z7h2SUiNO3;T#JUOyovSs@@C1GtwipycA=*x5{BpIZ_#GCMuV8XK=x;qCNy{d7?wA~ zC+=vjls;ci&zW=6$H~4^K%v{p}Ab?U%C6Z4p%eC<3ExqU$XR<}LLF67A$Sr20DR_pJ3yeBa~ z^sw{V0FI5;UpwXsScYuhbqGQ`YQ25;6p6W^+tgL&;Ml;>S3CGpSZ>VrTn0m1$y$HU z&65)I!c?oREz};c=nLCliriqQX->4uivHTgd${GqeAlf*!P^B|jkU|*IdNP(&6C>4 zqOW$)Nw9nvjy^&`?E|gotDV{JmJ9Q~vuhy<`^C4XIUDt|j4o6rK^e8_(=YqC zuaR6TRVf@tUFHB079o4MBIh{M~4>WwnGgesQH*3?w(RA%hCZ*7)b!aNV=yOQ%o_Y=Lt0Sl*(9^jfRnC210Om$=y>*o|3z} zAR&vAdrB#mWoaB0fJSw9xw|Am$fzK>rx-~R#7IFSAwdu_EI|SRfB*yl0w8oX09H^q zAjl2?0I)v*odGJ40FVGaF&2qJq9Gv`>V>2r0|c`GX8h>CX8eHcOy>S0@<;M3<_6UM z7yCEpug5NZL!H_0>Hg_HasQGxR`rY&Z{geOy?N92Z z{lER^um|$*?*G63*njwc(R?NT)Bei*3jVzR>FWUDb^gKhtL4A=kE_1p-%Fo2`!8M} z(0AjuCiS;G{?*^1tB-uY%=)SRx&D)pK4u@>f6@KPe3}2j_har$>HqzH;UCR^ssFD0 z7h+VLO4o@_Yt>>AeaZKUxqyvxWCAjKB>qjQ30UA)#w z&=RmdwlT`7a8J8Yae=7*c8XL|{@%wA8uvCqfsNX^?UZsS>wX}QD{K}ad4y~iO*p%4 z_cS{u7Ek%?WV6em2(U9#d8(&JDirb^u~7wK4+xP$iiI6IlD|a&S)6o=kG;59N|>K1 zn(0mUqbG3YIY7dQd+*4~)`!S9m7H6HP6YcKHhBc#b%1L}VIisp%;TckEkcu0>lo@u995$<*Em;XNodjTiCdC%R+TX|_ZR#|1`RR|`^@Teh zl#w@8fI1FTx2Dy+{blUT{`^kY*V-AZUd?ZZqCS4gW(kY5?retkLbF=>p=59Nl|=sf zo1Pc|{{N4>5nt#627ylGF`3n>X%`w%bw-Y~zWM_{Si$dc82|=YhISal{N7OY?O`C4 zD|qb}6nLWJ`hUyL+E>-;ricg9J@ZNYP(x(Sct&OI$Y!QWr*=^VN;G3#i>^1n4e#Je zOVhbFbLpXVu*16enDM+ic;97@R~u&kh__kgP#!R`*rQEnA+_dLkNP~L`0alC|J;c; zeiK=s8;BsLE)KbG3BD&Br@(Ha@SBT&$?xX`=$;eeel=|R_dIr6-Ro?=HEjnsJ_b`1 zK6Yg^-6;^2aW!xeTK)A~3Rm|L^FCHB_I>jIju7ZGo&N_1*QHkxH2!!%@o4iZ?vntS;&zJdPe1dH#04YD93A44o-MpfD zP{rn_aq>U%RDvC2+bp;xPlsOzauIi3*Lf42`jVKKZCRuKdYhi>FDuL2l=v{$BCN#Q6796s%r-AG$Q^t(3c@ zD?w0UhYr11@feiyl9kY_@H8~|xlmO<8PfQmj1!$@WieW@VxR@Psxfe-v9WCi1+f>F4VL?0O~K7T?m4-u|pSkBpUJZZe*16_wAp zSYZ@;k`3;W3UHKUWc8QeI}0jH5Ly=cGWQPw(Kr2fm=-5L(d`lcXofy8tJY3@Tuadz zYWXR{mW7XT!RF#RVCe%}=tM*O6!AD3^(!8un~opNI%Uko7$5t@<8+?; zTxDys(MyyGsUjtSu9$+|_-t!U3fVb1dkK?l`17<+jfl=hrBHnDSV>^R1=TnQeyqbW z>ov#l%!1|S!1>8UUxIdhQq`_klcHVx0{?#>K3#$4GlXncwldt!g17TcvKq-jo_996 z>oA=tH9CqRl6Yw?Uc`am!V?lHJbizOJaVaScf1UP5e7Dbgabq=b!B~T&_F6?ooU>w%x0A zH~&MHJ=q`fCH{U<7MDXE4SD32cDZA)WJeWkllJ`UspWaS#eDe^kg^oU_A14UE9zG-a^g{xaXf$})Wik>gT zl#dkzGr(;h0JZDuFn(+k8wNq?PZ5grQ<+sM?wBGt@JnH6v0#or-5wBQWKU~(S_> zkE!tc*ZJ1Y&*p(xX84POb3cClRMd!^qJ#CAZfIepEj-<`VURS_yCz0(?*Ixcj4 z-!zV1_QZhpm=0<;*(nm+F>T=)o?ep@CK5I%g^VAA+RB25ab?7)A~z~egru=I1S|@v zH7tXV!0wmGS^qj#e+MY;C5eUjEAp$Y?LDkS^QPZ}8WN85?r$u<-Epi;yZ1|J2J`se z$D6DpH~2F=eI0B&=UFAUnJvZAmClJlK)sutJ?M>xpZiWV&0=G4MZP+x+p>EX=HbCz zxls%Mw?*u^;LbHWIWCyq+yi)`GmFn9J112CZda_u@YIP%i;srFg_paU02Ifij*7}l z&CF-(3|>*a|+vbNR`^RP=9G?ymEJ0Z~)d&c*UE$UMepZ zcITr{0WqhxkjUnM15js_gW=e3Uh|y6ZReaXHIz-=p`x5VvB&rH9y>Amv@^WmXFEw) zQXYrk3feir=a{jMQ+wDIkkFnZ$k{sJakHn*?u za%4b!00ev8NVLM1TY=cl?KB&55BY_MU-sg?c>=Dbz_W{(Z~c?HJi*XpYL)C6Bd8WH zt+v-#0&o~@t4qESi*)+eW%@VD0|o^yF)n0hME$UtXF$*Lvh}7sso{`|pn*JDIy5^Fm3s$5*zEE=?u5<=l8FJc3r%+H} zdfoNl2J0^~!-*mOL5o-x32|e0Im*E!yY7F7E5N)W3>+v_LBydlEx?4$RL5f2oYRD# zaR0wv(-p~wO0eLDl3K=%`{5+0Gd$ktO=W)gWlGZJ0`K z$_RNA=ckrfa;H0KA~dR^p�(p-{x$&=IACIfoAR!za)F-^da-t3#0Dycnp zwO~NVXwXCl;jE<}>%@xz|=8fIJAB?>+E{7)|4l${4ngA3G|=r z2Dyv;VVWSgZx9Wj>qUjleGl3Ei9K4>h!(lPS%8VOG>Xu0%6VDz^O=bjJmuP7>DeUv zrbI}MlHB^^d?{zv6d=@_ZD2lg1&G7UjnVN{1}9WkaM3H~btX0GtSzB+tZ^qRgWo4m z!GmimlG$=wgXCnr6j@m<1gAL46#T~5Bnm=2{^@>|t&`9mkEPddj zAvG~@Tv~TAm2i%VW}R-g(Z0)z-Y|szHr@rk>4MAyG*Ma*7Yh#H7(!-5>DZ@8r;_dx z{prSe<>~099F8vsYd2xff7uAS%7{S)f(|@me3t2$iy&NEc7OUEchp@9A|X;;IA>8!oX+y(BKJ$EzV* znR$z;!L$s7uy@{OT~nG#B!NRraT8(X##Ho!0r_o@gg0CA-9H^;-uE&?$2$nHv_00o z%cbuUc-tCx$Uh&EZ4Nf4Zgqv)Y6>usG3>GeQnxx_Z6+PcbX-+ysbt1hQ`K1LDpOE? zrAhIZhSN9yVIAOa22gn577tbc&i3|3V8NWy&!tw##`}9*x}gtI^h1DzZRA>UuaJG) zaZ7j)dq!O}{?#8Y7~7i6fHh4{`pL?>-18|p!S75Y#^DM>-S3)vuZG+Q7l@ek zQP~#cBpWgg#mApc_sPYjpw8odQuRokmTkzcNl`^CcKB7e&;zViV;{Y{o^Y$%7i0m# z62%#1Lq!RC?}lK>%mp}T!3Xv;L*0v*>USLm``N%>w>@fwC+#T&Tx2bN4w(20JB}oU zuSa6v^kXi0xPs?pbaOHnyiqq6By1EZY9OZ^^QA>{q-Hsd&m`pbQ%8121aWG-F5xf zlZ%;B{;C>X19|`^_?dVyCq>n+41w7|!tUS!{9rHlbhX=SZO5CQ^;!Du_E7*`GiR^Q w)2!4MKjfSAeNo!9>IaV6aUZ*?W>} zs4%E?srLW`CJh0GCIK@hTkrW7A15Iu%N&?Q^$0+!{Tv&|t^Y@u%!L zglTg&?Q5q#ijZ;&HBQ?FNPp;k3J5!&{^+SGq?AX~SiOM9jJMRpyP?RCr@z38AQyy&WRMaC;n4una$~nJKSp?q|s8F00c9?Q! zY_ovvjTFm+DeQM^LXJ#v0}6HRt3R1%5PT*}W!k8BEM;Jrj8dIceFo2fhzTqaB3KKk zGlCLI)gU25(#u6ch6GeB1k@eHq7l{EHXv0n6xE#ws#ri}08kkCf8hUt{|Ejb`2YW* zvg}0nSSX1m=76s?sZhRY$K=3dpJ+y*eDULGnL2}4>4nvW^7_<~wIM_5fjvwt4h1|g z)g0Z6ZFq9j<~9~b8((~TN{Z?ZQfw|is&Xp~AC61sj;xItKyCHdI|tCMC_LbXF>~vR z=w6V3^H=W4CbAgR4#xw}ETTwu2guW~=Crl@SMXv85jQ=%y!s^?m4PI0My7MWICO;- z175jm%&PcPWh8QdOU(#8bp4!N7ET-+)N}N2zk2)8ch|4Q&lPFNQgT-thu053`r*h3 z_8dI@G;`zn;lH$zX3RzIk`E8~`J=BBdR}qD%n@vVG1834)!pS1Y?zVkJGtsa(sB~y zNfMYKsOJb%5J(0ivK8d+l2D2y&5X!cg3BG!AJ}910|_${nF}sC1QF^nLIhzXk-Y#x z0)&1iK!O;Og0Ky!;`b~v%b$`S4E&fB)1NB4v@8wr( z&+NX4e^&o)ecb=)dd~C!{(1e6t?&9j{l8%U*k4)?`(L3;Qjw z#w7FS+U(94MaJKS!J9O8^$)36_J8;thW#2$y9i{bB{?M{QS_inZIJ!jwqAbfXYVd$ zQ5fC$6Nc9hFi8m^;oI-%C#BS|c8vy+@{jx6hFcf^_;2VRgkoN(0h!_VSGmgNPRsxI z8$rTo0LaYq-H5i&gtj81=&xU?H-Y2==G@uQV7E`@+2E9XQW@{&j`?EOktk|Ho{HU>ZqDzvgjwBmdex z&uZNd2C1h{{}2k6Ys9$*nFP3;K%u!MhW`uZy7Sn`1M1zs@Es&;z*Z>Gsh@-3Fe6pE zQD2@cqF((NrRevgvLsvM_8;;iNyJ5nyPyy?e!kvKjGj`6diRFBEe49Oa7wwkJFV7Z z$YT&DWloYu-H?3<0BKn9L&JYDT-SK~*6c5pi18P26$JESKRYj{T7Zk6KiRJcbvOO*{P56Q6s8msbeI3>|j>K9}Q9UBeq*inXKemCm`-<5|-$ZyN4u$(3 z&HcvqehFD%5Yrmykg-^d`=BSa8(i=>ZoC77^mWY{evp(km@aHqhUECBz76YiR+VYK zY_avFC~V3$=`6C4JhfHAQ@DZtUOwH`L;oYX6zK0-uI^?hS$ALfq}A7evR;ohJHij} zHSZdW?EKv9U1s4oD*<(0oQ*;MaQ6@cvGL zuHCPgm_NhVsgp^sfr*ia^Db}swo1?O(_Q2)y+S$CBm+g=9wCOUPbz(x)_GbaKa@A7 zuI&!ynLiZRT#V%_y_-D`0Z5lT*auoe{(U5NylTzFSJW()W-#F6*&A`LNO1bV#Y;QJ zSbLBnp|B^dtK|KIWC|No>JjWBWE@n7O)x{&^E(WMeMvp57#qA8m* zeTow*U@_86B#Fm*rxyYu5PRWaWHx8y> z*qmHEp(AMDl0v)ij(AY8fnH=~ZwwjVAbu*m5;xPfidh@ov6d8g zfJsi&!QyK53Es%sC39ts;54V68koALD4b|%tNHW0bIkZAJKa=W&FomJSEDT>W1xIX z1x%Z>AvNIsSPLcn3RTcHXb@KB?cuM)=x6fcIx>&(GxqZ8w3p#jJ(GVgc*`c0HG}dv zIop&Qim!K1NFwic%07KcjWgHBPUkq7f~lj;TPqVGTiT#cUeim>;nY`>h@a*S{qQex zQ`z62WK|Mj)Y{tfF{;T4P;c8$Q|KU?Joh zIkA^z%X7z|r>4aTh@|StTi!-r1D!g=zb#3d#{{&K3CqE$Iz-UH<%37c zRfkO`&uM%#AD3PHv`g5t0e^O%nVL0d{Xlx^EjEC3#skF@`zl-7PF^0oxW)1!C!JxR zWvuAHH?)61FKA1QeT*_sY7;_Id#!GmV4n`MO{~sv}VLSK` zXRw=Y=Clz*00B(5y^K;gCZMAzjT5+c3IC=)l(9VIDdatpxj3y89WwI|bH&$!ZEvp` zPR!T@#!(|KfI-w?!&+7$N3F6>tD{YO4Qg$d_`nNEdfVCha9vaPn0jI0`)`@*72hq! zpU5ND^P*RoEkbD5o#az(-g=Y)L>HH>Oc%}$ zT3Rs_ih0;4+Lv4Y;@Iv(;fUbQ=i-G(#>vghec~*j(I#r|5mqFiJBpzi&hzEcD{u$< zRsm0BVYn=pT;0>R(itW|*D&;O%bOc7et9ACaH#J>z3A1A~6fdP>pmbM%xzm4>|;c_?B+%sl;Qs2{t!60$^u zH1t@9^6>;?!FuusnISi$f5CL&;z?EqJN$FBuWDA#D5`cy_UvCFIVvf{c?4N0teh;d zET$7aVbj08KTQS!x?Nd1Is8q8qFzs}a=!@nJ;7FSfCY^T@D-gpw`w<6e#X3+;O}1h z$%I!M)0bg|EKUA04Qjn@+x{Rj8vt6Wn!R|3A92z}^$KfF5(#CWr4y#~re1CN4i4w0 z#GsypBR{xA3Er7sgAi(|}1-W?s~n$7?K|9WL8kpVfw-;#b9 z+mn;=ep!162U5R>_t}fOt~tE?s#m( zO-S$7>Ay6*hHdZ)7_oU915WYYCIX;hFI-U2EWYX!pllONr@Q--2o~`!isi6vTPLJ4@(|o=%NHYjo0_S&q*UQIROw@*N-By@PaQ&;YxFZ0aR zX&}LeOEz);#m~Hwm^VAY8DK}b$F4bo{jMN?d!lxKPhNklzr^Cd`0f4oJr^z=I|l`* zm8AHm*fPV`0=lF3Pnnp}&J0N1X@}-D94YvmUabFrLGSnTz7Mu^21F#O5tN#CuY9Vh zUZBH=ez%h*wkf0hBtXJh1SN3d+IF{gzT7lp)j}n?03lt;XSQRAh7qd&v;RwTYDuQ# zbI2*r<>?x-G0@hM{;%{VBD7nLKt~D`T~-HAt5;h%i0_=Ifs=yHma5dhJ+QMG?Ux(a z|E?1CMy1!~oA`FP!k~iG=t&5#>bVdz=peT8HMB6Y)#7PpETtNryT^+Rv3vpJaF^zP z{H}0-LyV9Fu21ID%wO9f1IKlFr1p4c{o-?03vyB-tr5duk^&L$;m_|f$vs`^Sl{j2 z95}oY{LlY+=ZS%J+tZoXCd0*sSU7w^gjovXn+g7uyra5{cU49@yHf#Z^Jl-$9cIfo z+AJuxH$VLb=#+uBbVmUjnx zxb1pZ@-O9=AIk4@S)m6fJ2?{HrNYwwnL3a45muuNjr;6$O`bGEM0T4A2_S$t=86*- zcO+0mywg*j#A4mU}enR_!cGmIYQ;qwfchWtFEXL)AK%*;=j znYne+hS4EMy3S)C*mZ1KI>!+)0V@9!N6H$Y}~MJ{rYuf zz^KljIWvFi-?#?V@LPR&c6Nn{!=XM z>}-h$S76;$H{E{Y%@^zlmOl^efBwa%UU+jJD9UVukQ3ti_kH-?H*RC0?M1W%FCvMB zM_+v6fk$6X2sx)-p~B3&Kl{nscK}pNLM*qjtpaf9>AU{-iPKQZR8yCg!TY}Qg*(;) z)gdvCcB%kppZc$VdvsK@)3l1{&DG!d_6OHOS`y=ITLEVu`unSKA2E%JD*DVX{LJ}K z9l>hMRDqxQh0lnpGHpVYneX}eA3Pt|2v%=q;rt)``R|#bDyB)OXY&vI_@|*}h}G?^ z@aZ4_!7cQPX`!fW_?{oT1NTwHs#l5L-0`E|y@48<3Q^HFf8=Idi zpJYD%1MkII!~|7I^WGo)IF=?{>ACnjJ_WUi39C}!Q{QnheVJqeKKqq5^o5CBde(g9 zvw$X6^jz_^E2$wSw4!q5*RG(C2_^XO$HBn_55vbl44OnTTRwRaePP0vo{K)U1#99& z<>rq7V&V(<&@I%MFoN5zrY}sz=(*-L&}1QQ*a%`u25h{cFj===17eB_uGuzG&byQ< zrm8BJZl4r_E$3k|Wo6FW0-6M7>qac5uFQsQcmkLWGfeH74S3Z_rJ!jgN++!@i=HW8 zkyjI(oPH-+-N#Qc^-mpNO`bc6r=2-<%&Wy5K1vfFJB(L_IkpS6fY^NmuL8qsgj>MD zn~BHH9WM~32_3vd=W&B)k7F9q%stJx+b_L_X-4zr^LVUMCmyCTA3sWtkvsmME?Xiy z?xOSfB=_$oY06~J-HcCq&)qcW{j;uP;?Dm}=hkq?zh&n!;m((-G-u_t|6x399Q;>A zgNpxoJNj{u|MFDH7Rhq@FCAl0dE|ddnl!oh9{Lq?@JDoR6L;C941IK`ISfdE$4S zE0AUQ8+2|Ncl_q5QkSp#AODp~(^mfP&%Au@@|TBQwoP`UU+V{6u8|)6ZA{~uKmQ*M zmrMTDU8S~8Eqi{^v0Ug&5Upcm#y7Z1(RbgZAG8jB$eRwCspQ)>5;U)oGZ&E5aeR*K z8Yt`Y0$G))Yd(Y3KH}tA4`-_QmNke5hU_|nq=xtyjwW(_o?itz>B>WM&^63bNdQ)k@-IgDHW*RW$Xo9#RzrTrCn7L2H{9Amq|qNg@#eZY=|P zCoI?2s+L)zsM%WX(NbVEY^`C>lFjIBYmJ6@DKJ0ZT4&F&WHW!dwa%QzOG!?jY_2(S zDcEzZbz*2Q!43|z))9yOP9X1Xt%DXzwY(3tl-TR=Qb_MbZYRrooh;dYYmS!U_as1(=YVB?Q_A|tNu5Ut&_q3jbfDM zoFxT^uEuH`nX3*sB%K?GuHUkweYReBwnHqh3P)~`+s3+Tj!rDA1e)8vuBv5J*IsxC zkd^~b(aGzArj08{>cnzOuy04C+C`}gb|Yz-1avxeWzev3NzcHbz_&4W@QCr$z3~w=8Ua- z`;vfG1~BP8CyLb=F7t1am~ph_#|O%$khSJ9%Vtcn)YmpgQxF?xM^_Vb+5fnpB^W0I`f%X8gb9#X{Q-yJG0{Z56aWeI&zPxnf5pdJA38bM`cYnS#x)% z`n1tFf$i)W-hGm(f9mde^=X@NcV_lFb=P`4&CI&H=IArijGwdCk&X@uQ$5xmj!~^? z#$ROCI)V-~t%L%GS#wo@U27ddR`4`3)WoB{R-4snfNrfee|kI8^bu#yDgYqOwas9# zmcb`3!kRJ`Cr=_tq)8aMt{aGtUZsqwVlj6DgCGre>AEt&x8H_in!x@uwgExIh|-mA zjdaC(29~CTVSaaF7HPbql&*9Uo8P@f)>LqCXclr}peS7_1BQ28u9PO8Eq1@`l3q9o zkfKCaO2?T?ZyA6loW<#9_c^O=m<&h}CA!ineAD@=(gbq`vyT|tiJ6#^B1$P;;qax` z55k&Q?wEh#87niLo*+n4L@65J(Nz~=Ya%7^(miLb(E>A3B@|Jjl;FU&D>o|9#7PJH z?|ago!o;WC^h=|T7PVBg(DAB}72cyUS zb(f>Bwbr!F1eTCO5fpj<{PqhY5>143p?~5ZA5H40);=@M#MYvrB6gqHbU_!GSY??i z%s=>-ciA4*zOOZHds0a(kWewZ4h(k8h(ua7HX)Au&mY~H8KY6(_cb$_&fA@QjIW-*heP3%$d!m5^AdnT}`12qA^c@!g3DOwZ5WwE2?)-yU z!)Vx#Mtxt?FzFTwK!77sy7)sMzUd->w4^bxtpM2j!b1pjgyk zGKwWGeb4)^zjy{9Es&PU1}gwg?|J#L$KJB7ett9@4M%-nGtIQr0>Fl@8-yh`-+1ed zS6r}(MeSvgSoFmH*_WPu@i?}!AB~2?;i&IxrkNg~cQ9Som98tcq)k^|eeER|Zl77t za-TVUc;DNvzVXJ%w52+#weN?+;i#{f#!Oc&z?81*N>^e~ltRS%ZI@lR{rs()HmqG! zx*}ZrI-EZ}ckJMiy>A^oofwDfC~IH)z8{VHKGT@#E5I(Ll&+MnMCl>~AV7+>Gi%mF zkU1QlKASdR0B80!YhP<$Ywi0?W2Ux45oPfxv9QolWzJPD^weBfvo4SONxP35106sAmh(e+vAs0GboFD@PvNs)jNPvarhW}0YliZEg{Gazv z+JDIpoojRVPr<*C|BTq<`6ga{5q^8^!|0cxe=rZ!zxH3%f5ZO0cQ*Z<^$Yt2{|Ek0 zyT|*F+CO@K;(owBKtGg!S^xj-Z~rga2m6nxKl9J=fBSuNKW_dLKWhJKeg^-Xe`^1? z`TyJj)8E!#>_3Y?uKrwqq3LJ#SGU>AzUO|6`nR^u&3FNN_jGOc zw)Nw`wr3yIKhgcee6IaN=ws>M{6677%)hPwx&HzC(f&u~&)6@b2kNRzBDQAP0*H73 zq%McOmRk{B3i47qRe=DA*$&odrbEJZ*pV9XXa&p@wlW~@Yfs>V{yiTtplMhgM*-Bz zsSnlq&pG;z0OUN%$~$3=g1UF+G*>+17eRbBf3=y79J}KR8owon@$1Z7MIrvvWWH)34nK2SD)GsrJ{l z1Cl#oVo3A8qY3e=aF)qzms~FG#2$LzT=gs&aVMOj>(%{y<&O0cG!nCiESl~x=^dF{ zKvj8F1K8Ng171wwM5Fh4KoQw`_c6#y$(5cAm7e}~nJ#A*fx+c9;y#&W!#VukR)ugk zKp3=+;Ut+IYn%m+r4d*<`L2h%aDnX5}^!5R|H;(34AoVWjRx(msBZvk;rCI*|~ zdOijqI@9Z{Vu!~jvHW{lBa$rnl4+!s_5sfK3bCGk-B%iDe&@-}+%fOKU|(9?V1 zHE8&@4z)Kx!RAvAs z!Wic9=o#(bg?kc-G68-m(jZ`^=XGUXb)}t(%&~sjFnV^sEX%hSy6UKC4iOhgV=BHV z2w`4g7Y=s#Vu2B_?#VQ|hP39@eArgfX>-0S+dd&^mx0*wp}>)x;c4RUgxz%;oNe?& z-7-lJ@Y^2^C;=qJsxx5|xF)*pTGhch2B&kxtn;f!7=gznk}I3}Dh}(CoMXgA5-p&kS202!l?!fT3t|HG*rIP~mS* z$Wjo}jq3}z$Qq!9yrtd3fM0N629ZM?LU$nv@Tv9b7I;D|;0H2dsA~g7Z7zp1| zB)XmrkMgF6OQr|R)HHD^TE{Y#j!~SR?b`Xt3Qs`B+x<hxexYeAjMUWdZ-*n9%(1)Wb(n2U<><7&9dwGJmrob)4%H? zlQ%z+L-^$dFhhH|@u$%97Qz?*Ynh2VG@q|?8vY&L74&fs&_b&3$x&Oyjl~LQDRRap zJU4U*R+(2Dd!G+lh8!V{pT_UJn+^1Qg6$` zqkNm(a#hWyc6SP+p5=C4HL8-m`pO`5o~`-LI?_h5CsH?F_%?nDodmz&pWR20WTpJE z?N|wSzLjMUK8E)a2tI}Lf;+;*M|h3Y(U#>)g1>zk9|Hd}oZAa2 zLYBWBoSW!Ts!RwXr^8h+U*@{9{zqS^iH)Op<;r`Uw~nc}<^$V~_i%$GFjaG?X1@E|M`h)nekvFKt`Dh-f>@|0-`Xoq)o` zx;JmzDfOV9qCx|EVpogEe0LK~tGS?5$$L_i6P$P6wIsCQaP_;d{{N=iV@+8LI}o#( zvo*Ejy=IIn{rdIQh1&q-{EuohpVOjJ^Q3lD*YTp37$^RRgn8ihpdu5{Ct%5-KO!VL zcNB6dUajXI9jkm-P|i3~GB-A(X`P1Oqqb$tcku)UJw0w3GeUijb__#QT4j%64z%EeB7S?jlWwx_7&+EEvB|6N=kV}DwnyAlX=?j`) zmU#!$*^@NIu#n_d7;WoJV@*Fbv9|yJO4;n|BNF2xy(54RyB>t~8lUOUW$&2%Nwi1y zx6JxW88>U2$#qhl^6KUbtmg9}D0o5vYDT7kWJthLGkpGnN4T>{St^_EU>4;DmLF9o zr|LqsA8_MoNLQ=}w?8u!ziSZ@PC#Y<#9uJFo-ozVo6D;<8j^1$c|qAE3ZTE5i~zmE z$BU5lw6l=EWsg^y^;8>r9qH{xfL|~PZYK#md$zZ0?o11gV<*WSW~cgy2GYGQir%wf zt4iW8D+;s*;RGrmd(-T<@2&j(Cb9xhV*l-x`TpK`xq|7p?5R%5*s!69?2c!cC*VY* z2DE^9pvOPLU!1e}wA8S8opcTJ3`NB>hY=JQnL~QFXR4K8A$BqJnoEB$wn-%u@E6Mh zCfMF4kusv3N!(aHC}4)Xs^xoOwXd%e^6pi5|DZo=Q25j+6HlJ^7FodH6y1bMROR^q zGu6)fopS`h%Sw<;ZH%TEPf+#81-#_v+@8nlR0jLcIDKQtLleOC)6yLZgC!D9X3GgS zohwU{v$jl=quD#Go^hB{`@Qw*a%`(^jyT~=q^bWgGzRj;|12J55HWdCWV}EB|K=%N z3Nq-qxJJ`>^|1MNN+q}zTB&ooE3j==AgK@^UW<^oSbeALa2peF)Th6{@sj0KyMNHZ zksk1+MXN2tv+22A%cQOGpS9)77(uP9mh+!5T5ERLvF@b}$+WvXM45Z?-kCa)fb~f1 znVbTD$Gx-0Zxc`0D@YgHakge6SL0H`-vN_x?AP0>iGH0_EE&=v83hMJgaKAI0jJXm zVxVz;X<$v6WW7}fxROO7vr#YLP;;lij5VrX{;>7kK6TtOH&6|Ar^xo>00%+u$C4@# z>!jOt6*3><171+WxoZnKDTzJtDRw+T030;yI}~uV@9fCnei^I*j>Bp&mzP2d=FPb_ zCM*l_+$LDR3B*a!A$g#>xsrZvw0lckxmMg>0aQd7tPyN=t{dgXb;Ie+T8{fZH=gdu zM7Rg9c(kg(Jg0?ARRRl=AONFKrvFj)lTY$KfT%6^6s`mk*ABGhsce*LsoD>K{z_M2 ziPpnu+lw22PfF!CoId^6n*G4H(Ix+#+N{C(da7t1BYMGEaE#PdpOLxsVD5riQXHp@OX;`S`8VnpM~)I920w~<3|mo0 zf8~Az`*?2?H&gZ&*K&bRkV@qzvMlRHXys8*Ze2+1c?5o!^+$&MHxB@4Ee5cke52R! zmn7AZtY6ST%ixgU5)%$%QcwHj7Es-Qu^kLAPwy%7pGBw_4Q9#da^W2$}axNHr03)_nw z5?yuNmXrI5HgS46)c5&}B)Tts49oU92>3xBLLy}FMUW=84DQbVq^;7_e7|(Sdz|&J z73N+M`rc2rt*oSWu#7S{*s~nH6HRHJS1SmzeXk|;CA)FI4bat3<%}nkB%;;?=F>B7ms9QSxv#@+69;@>QaR?REYX4&)=itG>rM{<{A79Rmk)`5ON#GL`*KX%}Ihk3w(RtM-WLt z?f&FLF}4N^yE!(pZ&Yj&Bc`~K0@4_}*0Om?wN|}4WJ>WL;G^H2*QpgEkGA~OET-Km zkwz|5{6dnz1U<2Pe9DNL>3g5FEIvp1jzP&2K#z~j%g6!7B;^zF+o95?fV{3mnB8*RMhCDNp>Am-3e@jNfMj?jHV$MWjk!DDKP zkAz$Y?Sr)!GUOX}qTQ5aMh|wq1uq}~joWyKl=b_LboM#wi{CMuz5x6BKlA-qy++cM01D3b7`uD z#l6M4pI;JCypO8JZ6?U&wNxR!{4oB_ zlV!x9+-&Qy6{%MQ{~yoZGkKiTSC`YS_j22~G;xUV855g2&C(zm^V!(wpcm@zn{%!g z4}JGo(sGZ1O~to-}le

UmY2RIYtNPVDpE$%vda+HD#3m z&VuXJ{BK&Qe+rBa7eq}Q(bq|tn(RrJAk|ztj2(i{d>nmQnM?;HF2k&9sA6up5tmjl z7lySlzMbifH17-m-Lwa_F&e7nOH?ESi3#ckR3tsM+jsck3`oG!uMS}|eAwVXv>}qxwq?QY%QJ0}r@^;fhuUA9W z*BVl>TGo&N004@xSiwDUXUvp51sVmqO3m)=B55aPwf@0=e}cN+$-BdKxY`YrT_4)0 z_d10#i44Q*rFr8MC>*)v$EJvz``(pb{e&*6k+b zsMz%($|1+8hn8c2?P(l@;Rb&CsZeYoCI3?2!LqjbwPXW3z4G$Qfj=cT5Yb%vY0(AX oeb?AaKtwrnc|$|zzw9vfvn^aJJ!zd)XFXqqy0000001=f@-~a#s diff --git a/examples/llama.android/app/src/main/res/values/colors.xml b/examples/llama.android/app/src/main/res/values/colors.xml deleted file mode 100644 index ca1931bca..000000000 --- a/examples/llama.android/app/src/main/res/values/colors.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - #FFBB86FC - #FF6200EE - #FF3700B3 - #FF03DAC5 - #FF018786 - #FF000000 - #FFFFFFFF - diff --git a/examples/llama.android/app/src/main/res/values/strings.xml b/examples/llama.android/app/src/main/res/values/strings.xml deleted file mode 100644 index 7a9d314e2..000000000 --- a/examples/llama.android/app/src/main/res/values/strings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - LlamaAndroid - diff --git a/examples/llama.android/app/src/main/res/values/themes.xml b/examples/llama.android/app/src/main/res/values/themes.xml deleted file mode 100644 index 8a24fda56..000000000 --- a/examples/llama.android/app/src/main/res/values/themes.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - -