Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

rnn fix adjust

Browse Source
This commit is contained in:
Concedo
2026-03-04 10:59:51 +08:00
parent 5d35193749
commit 54cf43ae64
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+4 -4
View File
@@ -4452,11 +4452,11 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
//if running rnn model in smartcache mode, save progress a little bit before the final PP is done
//this helps solve token boundary mutation issues
if(draft_ctx==nullptr && embd.size()>1 && !startedsampling && input_consumed==embd_inp.size() && input_consumed>128)
if(draft_ctx==nullptr && embd.size()>1 && !startedsampling && input_consumed==embd_inp.size() && input_consumed>64)
{
if(kcpp_data->smartcache && is_recurrent && file_format==FileFormat::GGUF_GENERIC && current_context_tokens.size() > 32)
{
if(embd.size()<=64)
if(embd.size()<=48)
{
//directly snapshot for a small batch
smartcache_quick_snapshot();
@@ -4464,8 +4464,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
else
{
skipdecodelater = true;
//decode until nearly done, then snapshot and decode the last 64
std::vector<std::vector<gpt_vocab::id>> parts = split_big_vector_in_two(embd,64);
//decode until nearly done, then snapshot and decode the last 32
std::vector<std::vector<gpt_vocab::id>> parts = split_big_vector_in_two(embd,32);
int temp_past = n_past;
evalres = true;
for(int p=0;p<parts.size();++p)