diff --git a/.github/workflows/kcpp-build-release-linux-cuda12.yaml b/.github/workflows/kcpp-build-release-linux-cuda12.yaml index 9c8fc3de2..2d7eb3b41 100644 --- a/.github/workflows/kcpp-build-release-linux-cuda12.yaml +++ b/.github/workflows/kcpp-build-release-linux-cuda12.yaml @@ -10,7 +10,7 @@ on: env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - KCPP_CUDA: 12.1.0 + KCPP_CUDA: 12.4.0 ARCHES_CU12: 1 jobs: @@ -51,6 +51,9 @@ jobs: run: | ./koboldcpp.sh dist + - name: Rename file before upload + run: mv dist/koboldcpp-linux-x64-cuda1240 dist/koboldcpp-linux-x64-cuda12 + - name: Save artifact uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/kcpp-build-release-linux.yaml b/.github/workflows/kcpp-build-release-linux.yaml index 7614d5050..13407f641 100644 --- a/.github/workflows/kcpp-build-release-linux.yaml +++ b/.github/workflows/kcpp-build-release-linux.yaml @@ -52,6 +52,9 @@ jobs: run: | ./koboldcpp.sh dist + - name: Rename file before upload + run: mv dist/koboldcpp-linux-x64-cuda1150 dist/koboldcpp-linux-x64-cuda11 + - name: Save artifact uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/kcpp-build-release-win-full-cu12.yaml b/.github/workflows/kcpp-build-release-win-full-cu12.yaml index 77e1a86ab..ea9c7ffde 100644 --- a/.github/workflows/kcpp-build-release-win-full-cu12.yaml +++ b/.github/workflows/kcpp-build-release-win-full-cu12.yaml @@ -40,34 +40,6 @@ jobs: & "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -all -products * -format json shell: pwsh - - name: Visual Studio 2019 Reinstall - shell: cmd - run: | - @echo off - echo Preparing setup - curl -fLO https://download.visualstudio.microsoft.com/download/pr/1fbe074b-8ae1-4e9b-8e83-d1ce4200c9d1/61098e228df7ba3a6a8b4e920a415ad8878d386de6dd0f23f194fe1a55db189a/vs_Enterprise.exe - vs_Enterprise.exe --quiet --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.VC.CLI.Support --add Microsoft.VisualStudio.Component.Windows10SDK.19041 --add Microsoft.VisualStudio.Workload.UniversalBuildTools --add Microsoft.VisualStudio.Component.VC.CMake.Project - echo Waiting for VS2019 setup - set "ProcessName=setup.exe" - :CheckProcess - tasklist /FI "IMAGENAME eq %ProcessName%" | find /I "%ProcessName%" >nul - if %errorlevel%==0 ( - ping 127.0.0.1 /n 5 >nul - goto CheckProcess - ) - echo VS2019 Setup completed - exit /b 0 - - - name: Disable Visual Studio 2022 by Renaming - run: | - Rename-Item "C:\Program Files\Microsoft Visual Studio\2022\Enterprise" "Enterprise_DISABLED" - shell: pwsh - - - name: Display full Visual Studio info After - run: | - & "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -all -products * -format json - shell: pwsh - - name: Download and install win64devkit run: | curl -L https://github.com/skeeto/w64devkit/releases/download/v1.22.0/w64devkit-1.22.0.zip --output w64devkit.zip @@ -94,7 +66,7 @@ jobs: - uses: Jimver/cuda-toolkit@v0.2.15 id: cuda-toolkit with: - cuda: '12.1.0' + cuda: '12.4.0' use-github-cache: false - name: Build CUDA @@ -114,8 +86,8 @@ jobs: # ls - name: Copy CuBLAS Libraries run: | - copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\cublasLt64_12.dll" . - copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\cublas64_12.dll" . + copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\bin\cublasLt64_12.dll" . + copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\bin\cublas64_12.dll" . ls - name: Package PyInstallers diff --git a/CMakeLists.txt b/CMakeLists.txt index 33f9a1876..3c129c5e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,7 @@ if (LLAMA_CUBLAS) add_compile_definitions(GGML_CUDA_USE_GRAPHS) #try enable cuda graphs on cu12 build set(CMAKE_CUDA_ARCHITECTURES "50-virtual;61-virtual;70-virtual;75-virtual;80-virtual") # lowest CUDA 12 standard + lowest for integer intrinsics else() + add_compile_definitions(KCPP_LIMIT_CUDA_MAX_ARCH=750) #will cause issues with ggml_cuda_highest_compiled_arch if removed set(CMAKE_CUDA_ARCHITECTURES "35-virtual;50-virtual;61-virtual;70-virtual;75-virtual") # lowest CUDA 12 standard + lowest for integer intrinsics endif() endif() diff --git a/Makefile b/Makefile index 408e3ea90..cd5a13431 100644 --- a/Makefile +++ b/Makefile @@ -198,7 +198,8 @@ ifdef LLAMA_ARCHES_CU11 -gencode arch=compute_50,code=compute_50 \ -gencode arch=compute_61,code=compute_61 \ -gencode arch=compute_70,code=compute_70 \ - -gencode arch=compute_75,code=compute_75 + -gencode arch=compute_75,code=compute_75 \ + -DKCPP_LIMIT_CUDA_MAX_ARCH=750 else ifdef LLAMA_ARCHES_CU12 NVCCFLAGS += -Wno-deprecated-gpu-targets \ diff --git a/README.md b/README.md index a2d36d289..acda13677 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,11 @@ KoboldCpp is an easy-to-use AI text-generation software for GGML and GGUF models - You can also run it using the command line. For info, please check `koboldcpp.exe --help` ## Linux Usage (Precompiled Binary, Recommended) -On modern Linux systems, you should download the `koboldcpp-linux-x64-cuda1150` prebuilt PyInstaller binary for greatest compatibility on the **[releases page](https://github.com/LostRuins/koboldcpp/releases/latest)**. Simply download and run the binary (You may have to `chmod +x` it first). If you have a newer device, you can also try the `koboldcpp-linux-x64-cuda1210` instead for better speeds. +On modern Linux systems, you should download the `koboldcpp-linux-x64-cuda11` prebuilt PyInstaller binary for greatest compatibility on the **[releases page](https://github.com/LostRuins/koboldcpp/releases/latest)**. Simply download and run the binary (You may have to `chmod +x` it first). If you have a newer device, you can also try the `koboldcpp-linux-x64-cuda12` instead for better speeds. Alternatively, you can also install koboldcpp to the current directory by running the following terminal command: ``` -curl -fLo koboldcpp https://github.com/LostRuins/koboldcpp/releases/latest/download/koboldcpp-linux-x64-cuda1150 && chmod +x koboldcpp +curl -fLo koboldcpp https://github.com/LostRuins/koboldcpp/releases/latest/download/koboldcpp-linux-x64-cuda11 && chmod +x koboldcpp ``` After running this command you can launch Koboldcpp from the current directory using `./koboldcpp` in the terminal (for CLI usage, run with `--help`). Finally, obtain and load a GGUF model. See [here](#Obtaining-a-GGUF-model) diff --git a/cudart64_12.dll b/cudart64_12.dll index cbc961fce..52cb61b35 100644 Binary files a/cudart64_12.dll and b/cudart64_12.dll differ diff --git a/environment.yaml b/environment.yaml index d7aa7adcd..e65152853 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,6 +1,6 @@ name: koboldcpp channels: - - nvidia/label/cuda-12.1.0 + - nvidia/label/cuda-12.4.0 - conda-forge - defaults dependencies: diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index aee587b60..369471579 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -122,7 +122,11 @@ constexpr int ggml_cuda_highest_compiled_arch(const int arch) { } #else static int ggml_cuda_highest_compiled_arch(const int arch) { - return (arch>__CUDA_ARCH__?__CUDA_ARCH__:arch); +#ifndef KCPP_LIMIT_CUDA_MAX_ARCH + return arch; +#else + return (arch > KCPP_LIMIT_CUDA_MAX_ARCH ? KCPP_LIMIT_CUDA_MAX_ARCH : arch); +#endif } #endif // __CUDA_ARCH_LIST__ diff --git a/koboldcpp.sh b/koboldcpp.sh index 8b6144c20..de5616508 100755 --- a/koboldcpp.sh +++ b/koboldcpp.sh @@ -7,9 +7,9 @@ fi if [[ ! -f "conda/envs/linux/bin/python" && $KCPP_CUDA != "rocm" || $1 == "rebuild" && $KCPP_CUDA != "rocm" ]]; then cp environment.yaml environment.tmp.yaml if [ -n "$KCPP_CUDA" ]; then - sed -i -e "s/nvidia\/label\/cuda-12.1.0/nvidia\/label\/cuda-$KCPP_CUDA/g" environment.tmp.yaml + sed -i -e "s/nvidia\/label\/cuda-12.4.0/nvidia\/label\/cuda-$KCPP_CUDA/g" environment.tmp.yaml else - KCPP_CUDA=12.1.0 + KCPP_CUDA=12.4.0 fi bin/micromamba create --no-rc --no-shortcuts -r conda -p conda/envs/linux -f environment.tmp.yaml -y bin/micromamba create --no-rc --no-shortcuts -r conda -p conda/envs/linux -f environment.tmp.yaml -y