Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 08:44:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

allow for single token prompt processing (actual batch size 1)

Browse Source
This commit is contained in:
Concedo
2025-04-25 16:54:46 +08:00
parent d32d0b382a 13be08daf9
commit 6b6597ebf1
24 changed files with 413 additions and 846 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+5 -1
View File
@@ -1832,10 +1832,14 @@ static int GetBatchSize(int desiredBlasBatchSize,FileFormat in_file_format)
file_format == FileFormat::GPTJ_2 ||
file_format == FileFormat::RWKV_1 ||
file_format==FileFormat::RWKV_2);
if(!approved_format || desiredBlasBatchSize<=0)
if(!approved_format && desiredBlasBatchSize>0)
{
desiredBlasBatchSize = 16;
}
if(desiredBlasBatchSize<=0)
{
desiredBlasBatchSize = 1;
}
if (file_format != FileFormat::GGML && file_format != FileFormat::GGHF && file_format != FileFormat::GGJT && file_format != FileFormat::GGJT_2 && file_format != FileFormat::GGJT_3 && file_format != FileFormat::GGUF_GENERIC)
{
desiredBlasBatchSize = (desiredBlasBatchSize > 256 ? 256 : desiredBlasBatchSize);