diff --git a/.github/workflows/kcpp-build-release-win-cuda.yaml b/.github/workflows/kcpp-build-release-win-cuda.yaml deleted file mode 100644 index e63ace77d..000000000 --- a/.github/workflows/kcpp-build-release-win-cuda.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Koboldcpp Windows CUDA - -on: workflow_dispatch -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - -jobs: - windows: - runs-on: windows-2019 - steps: - - name: Clone - id: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref || github.ref_name }} - - - uses: Jimver/cuda-toolkit@v0.2.15 - id: cuda-toolkit - with: - cuda: '11.4.4' - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. -DLLAMA_CUBLAS=ON -DCMAKE_SYSTEM_VERSION="10.0.19041.0" - cmake --build . --config Release -j 2 - - - name: Save artifact - uses: actions/upload-artifact@v3 - with: - name: kcpp_windows_cuda_binary - path: build/bin/Release/ diff --git a/.github/workflows/kcpp-build-release-win-cuda12.yaml b/.github/workflows/kcpp-build-release-win-cuda12.yaml deleted file mode 100644 index c7980bbc4..000000000 --- a/.github/workflows/kcpp-build-release-win-cuda12.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Koboldcpp Windows CUDA12 - -on: workflow_dispatch -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - -jobs: - windows: - runs-on: windows-2019 - steps: - - name: Clone - id: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref || github.ref_name }} - - - uses: Jimver/cuda-toolkit@v0.2.15 - id: cuda-toolkit - with: - cuda: '12.1.0' - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. -DLLAMA_CUBLAS=ON -DCMAKE_SYSTEM_VERSION="10.0.19041.0" - cmake --build . --config Release -j 2 - - - name: Save artifact - uses: actions/upload-artifact@v3 - with: - name: kcpp_windows_cuda_binary - path: build/bin/Release/ diff --git a/.github/workflows/kcpp-build-release-win-full-cu12.yaml b/.github/workflows/kcpp-build-release-win-full-cu12.yaml index 4bd545b6c..cb673277d 100644 --- a/.github/workflows/kcpp-build-release-win-full-cu12.yaml +++ b/.github/workflows/kcpp-build-release-win-full-cu12.yaml @@ -45,7 +45,7 @@ jobs: - name: Build Non-CUDA id: make_build run: | - make -j ${env:NUMBER_OF_PROCESSORS} + make LLAMA_CLBLAST=1 LLAMA_VULKAN=1 LLAMA_PORTABLE=1 -j ${env:NUMBER_OF_PROCESSORS} - uses: Jimver/cuda-toolkit@v0.2.15 id: cuda-toolkit diff --git a/.github/workflows/kcpp-build-release-win-full.yaml b/.github/workflows/kcpp-build-release-win-full.yaml index 576485934..018fc46f6 100644 --- a/.github/workflows/kcpp-build-release-win-full.yaml +++ b/.github/workflows/kcpp-build-release-win-full.yaml @@ -45,7 +45,7 @@ jobs: - name: Build Non-CUDA id: make_build run: | - make -j ${env:NUMBER_OF_PROCESSORS} + make LLAMA_CLBLAST=1 LLAMA_VULKAN=1 LLAMA_PORTABLE=1 -j ${env:NUMBER_OF_PROCESSORS} - uses: Jimver/cuda-toolkit@v0.2.15 id: cuda-toolkit diff --git a/.github/workflows/kcpp-build-release-win-oldcpu-full.yaml b/.github/workflows/kcpp-build-release-win-oldcpu-full.yaml index 3b8ae778d..1ef933601 100644 --- a/.github/workflows/kcpp-build-release-win-oldcpu-full.yaml +++ b/.github/workflows/kcpp-build-release-win-oldcpu-full.yaml @@ -45,7 +45,7 @@ jobs: - name: Build Non-CUDA id: make_build run: | - make -j ${env:NUMBER_OF_PROCESSORS} LLAMA_NOAVX2=1 + make LLAMA_CLBLAST=1 LLAMA_VULKAN=1 LLAMA_PORTABLE=1 -j ${env:NUMBER_OF_PROCESSORS} LLAMA_NOAVX2=1 - uses: Jimver/cuda-toolkit@v0.2.15 id: cuda-toolkit diff --git a/Makefile b/Makefile index 591b7947c..e6de6633f 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,6 @@ default: koboldcpp_default koboldcpp_failsafe koboldcpp_noavx2 koboldcpp_clblast koboldcpp_clblast_noavx2 koboldcpp_cublas koboldcpp_hipblas koboldcpp_vulkan koboldcpp_vulkan_noavx2 finishedmsg tools: quantize_gpt2 quantize_gptj quantize_gguf quantize_neox quantize_mpt quantize_clip whispermain sdmain gguf-split -dev: koboldcpp_default -dev2: koboldcpp_clblast -dev3: koboldcpp_vulkan finishedmsg ifndef UNAME_S UNAME_S := $(shell uname -s) @@ -147,6 +144,7 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) # Use all CPU extensions that are available: # old library NEEDS mf16c to work. so we must build with it. new one doesnt ifeq ($(OS),Windows_NT) + ifdef LLAMA_PORTABLE CFLAGS += NONECFLAGS += SIMPLECFLAGS += -mavx -msse3 @@ -155,8 +153,10 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) else FULLCFLAGS += -mavx2 -msse3 -mfma -mf16c -mavx endif + else + CFLAGS += -march=native -mtune=native + endif else -# if not on windows, they are clearly building it themselves, so lets just use whatever is supported ifdef LLAMA_PORTABLE CFLAGS += NONECFLAGS += @@ -373,10 +373,17 @@ NOTIFY_MSG = ifeq ($(OS),Windows_NT) DEFAULT_BUILD = $(CXX) $(CXXFLAGS) $^ -shared -o $@.dll $(LDFLAGS) + ifdef LLAMA_PORTABLE FAILSAFE_BUILD = $(CXX) $(CXXFLAGS) $^ -shared -o $@.dll $(LDFLAGS) NOAVX2_BUILD = $(CXX) $(CXXFLAGS) $^ -shared -o $@.dll $(LDFLAGS) + endif + + ifdef LLAMA_CLBLAST CLBLAST_BUILD = $(CXX) $(CXXFLAGS) $^ lib/OpenCL.lib lib/clblast.lib -shared -o $@.dll $(LDFLAGS) + endif + ifdef LLAMA_VULKAN VULKAN_BUILD = $(CXX) $(CXXFLAGS) $^ lib/vulkan-1.lib -shared -o $@.dll $(LDFLAGS) + endif ifdef LLAMA_CUBLAS CUBLAS_BUILD = $(CXX) $(CXXFLAGS) $(CUBLAS_FLAGS) $^ -shared -o $@.dll $(CUBLASLD_FLAGS) $(LDFLAGS) @@ -409,18 +416,18 @@ else ifdef LLAMA_VULKAN VULKAN_BUILD = $(CXX) $(CXXFLAGS) $^ -lvulkan -shared -o $@.so $(LDFLAGS) endif +endif - ifndef LLAMA_CLBLAST - ifndef LLAMA_CUBLAS - ifndef LLAMA_HIPBLAS - ifndef LLAMA_VULKAN - ifndef LLAMA_METAL - NOTIFY_MSG = @echo -e '\n***\nYou did a basic CPU build. For faster speeds, consider installing and linking a GPU BLAS library. For example, set LLAMA_VULKAN=1 to compile with Vulkan support. Read the KoboldCpp Wiki for more information. This is just a reminder, not an error.\n***\n' - endif - endif - endif - endif - endif +ifndef LLAMA_CLBLAST +ifndef LLAMA_CUBLAS +ifndef LLAMA_HIPBLAS +ifndef LLAMA_VULKAN +ifndef LLAMA_METAL +NOTIFY_MSG = @echo -e '\n***\nYou did a basic CPU build. For faster speeds, consider installing and linking a GPU BLAS library. For example, set LLAMA_CLBLAST=1 LLAMA_VULKAN=1 to compile with Vulkan and CLBlast support. Add LLAMA_PORTABLE=1 to make a sharable build that other devices can use. Read the KoboldCpp Wiki for more information. This is just a reminder, not an error.\n***\n' +endif +endif +endif +endif endif diff --git a/README.md b/README.md index db47c05f8..f3c6d5f5c 100644 --- a/README.md +++ b/README.md @@ -83,13 +83,16 @@ when you can't use the precompiled binary directly, we provide an automated buil - For Debian: Install `libclblast-dev`. - You can attempt a CuBLAS build with `LLAMA_CUBLAS=1`, (or `LLAMA_HIPBLAS=1` for AMD). You will need CUDA Toolkit installed. Some have also reported success with the CMake file, though that is more for windows. - For a full featured build (all backends), do `make LLAMA_CLBLAST=1 LLAMA_CUBLAS=1 LLAMA_VULKAN=1`. (Note that `LLAMA_CUBLAS=1` will not work on windows, you need visual studio) +- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1` - After all binaries are built, you can run the python script with the command `koboldcpp.py [ggml_model.gguf] [port]` ### Compiling on Windows - You're encouraged to use the .exe released, but if you want to compile your binaries from source at Windows, the easiest way is: - Get the latest release of w64devkit (https://github.com/skeeto/w64devkit). Be sure to use the "vanilla one", not i686 or other different stuff. If you try they will conflit with the precompiled libs! - Clone the repo with `git clone https://github.com/LostRuins/koboldcpp.git` - - Make sure you are using the w64devkit integrated terminal, then run `make` at the KoboldCpp source folder. This will create the .dll files. + - Make sure you are using the w64devkit integrated terminal, then run `make` at the KoboldCpp source folder. This will create the .dll files for a pure CPU native build. + - For a full featured build (all backends), do `make LLAMA_CLBLAST=1 LLAMA_VULKAN=1`. (Note that `LLAMA_CUBLAS=1` will not work on windows, you need visual studio) + - To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1` - If you want to generate the .exe file, make sure you have the python module PyInstaller installed with pip (`pip install PyInstaller`). Then run the script `make_pyinstaller.bat` - The koboldcpp.exe file will be at your dist folder. - **Building with CUDA**: Visual Studio, CMake and CUDA Toolkit is required. Clone the repo, then open the CMake file and compile it in Visual Studio. Copy the `koboldcpp_cublas.dll` generated into the same directory as the `koboldcpp.py` file. If you are bundling executables, you may need to include CUDA dynamic libraries (such as `cublasLt64_11.dll` and `cublas64_11.dll`) in order for the executable to work correctly on a different PC. @@ -104,6 +107,7 @@ when you can't use the precompiled binary directly, we provide an automated buil - You can compile your binaries from source. You can clone the repo with `git clone https://github.com/LostRuins/koboldcpp.git` - A makefile is provided, simply run `make`. - If you want Metal GPU support, instead run `make LLAMA_METAL=1`, note that MacOS metal libraries need to be installed. +- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1` - After all binaries are built, you can run the python script with the command `koboldcpp.py --model [ggml_model.gguf]` (and add `--gpulayers (number of layer)` if you wish to offload layers to GPU). ### Compiling on Android (Termux Installation) @@ -114,6 +118,7 @@ when you can't use the precompiled binary directly, we provide an automated buil - Clone the repo `git clone https://github.com/LostRuins/koboldcpp.git` - Navigate to the koboldcpp folder `cd koboldcpp` - Build the project `make` +- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1`, this disables usage of ARM instrinsics. - Grab a small GGUF model, such as `wget https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf` - Start the python server `python koboldcpp.py --model KobbleTiny-Q4_K.gguf` - Connect to `http://localhost:5001` on your mobile browser