Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 17:25:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix for yr-rocm, large gpu scratch

Browse Source
This commit is contained in:
Concedo
2023-06-30 12:40:08 +08:00
parent 1347d3acc0
commit 86469d15c4
6 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llama.cpp
+3 -2
View File
@@ -12,7 +12,8 @@
#include "ggml.h"
#ifdef GGML_USE_CUBLAS
#include "ggml-cuda.h"
#elif defined(GGML_USE_CLBLAST)
#endif
#if defined(GGML_USE_CLBLAST)
#include "ggml-opencl.h"
#endif
@@ -1113,7 +1114,7 @@ static void llama_model_load_internal(
fprintf(stderr, "%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__);
ggml_cuda_set_scratch_size(0); // disable scratch
} else {
vram_scratch = n_batch * MB;
vram_scratch = n_batch * MB * bigctxmul;
ggml_cuda_set_scratch_size(vram_scratch);
if (n_gpu_layers > 0) {
fprintf(stderr, "%s: allocating batch_size x 1 MB = %zd MB VRAM for the scratch buffer\n",