Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-12 08:08:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

improve snapshotting for rnn

Browse Source
This commit is contained in:
Concedo
2025-12-21 21:07:31 +08:00
parent d577187875
commit 80a0269dbe
1 changed files with 22 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+22
View File
@@ -4087,7 +4087,14 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
else if(embd_inp.size()==0)
{
embd_inp.push_back(current_context_tokens[current_context_tokens.size()-1]);
current_context_tokens.pop_back();
n_past -= 1;
//another dirty hack
int maxedpos = llama_memory_seq_pos_max(llama_get_memory(llama_ctx_v4),0);
if(maxedpos==n_past)
{
n_past += 1;
}
}
else if(embd_inp.size()>0 && current_context_tokens.size()>0 && last_n_tokens.size()>0)
{
@@ -4424,6 +4431,21 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
{
printf("\n");
}
//if running rnn model in smartcache mode, save progress before each gen
if(kcpp_data->smartcache && is_recurrent && file_format==FileFormat::GGUF_GENERIC && current_context_tokens.size() > 32)
{
int identical_slot = get_identical_existing_slot();
if(identical_slot==-1)
{
int oldest_slot = get_oldest_slot(-1);
gpttype_save_state_kv(oldest_slot);
}
else
{
touch_slot(identical_slot);
}
}
}
const std::vector<llama_token> eog_tokens = GetEogIDs(file_format,n_vocab);