Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

optimize pp

Browse Source
This commit is contained in:
Concedo
2026-03-03 21:02:51 +08:00
parent ae67caa2f7
commit 707f7b37bf
3 changed files with 25 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+6 -1
View File
@@ -4465,7 +4465,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
{
skipdecodelater = true;
//decode until nearly done, then snapshot and decode the last 64
std::vector<std::vector<gpt_vocab::id>> parts = split_big_vector(embd,64);
std::vector<std::vector<gpt_vocab::id>> parts = split_big_vector_in_two(embd,64);
int temp_past = n_past;
evalres = true;
for(int p=0;p<parts.size();++p)
@@ -4477,6 +4477,11 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
std::vector<gpt_vocab::id> chunk = parts[p];
kcpp_embd_batch smallbatch = kcpp_embd_batch(chunk, temp_past, use_mrope, false);
decode_status = llama_decode(llama_ctx_v4, smallbatch.batch);
if(p==0 && decode_status==1)
{
skipdecodelater = false;
break; //big pp failed
}
evalres = (evalres && (decode_status==0));
temp_past += chunk.size();
}