Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

only llama can use batch sizes above 256 to prevent unacceptably high memory usage

Browse Source
This commit is contained in:
Concedo
2023-04-23 15:57:06 +08:00
parent 432cc91649
commit 9129e937f9
1 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+7 -2
View File
@@ -358,8 +358,13 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
int original_threads = params.n_threads;
if (blasmode)
{
//for gpttype, GPT2 crashes above 256.
int bbs = blasbatchsize; //(blasbatchsize>256?256:blasbatchsize);
//for non llama, limit to 256
int bbs = blasbatchsize;
if (file_format != FileFormat::GGML && file_format != FileFormat::GGHF && file_format != FileFormat::GGJT)
{
bbs = (blasbatchsize > 256 ? 256 : blasbatchsize);
}
params.n_batch = bbs; //received reports of 1024 and above crashing on some models
params.n_threads = 1;
}