Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-06 13:01:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

better approach when SWA window exceeded, simply refill the window. this is not 100% correct but good enough for fastforward users. Disable FF or increase window if not good enough

Browse Source
This commit is contained in:
Concedo
2026-04-17 11:44:13 +08:00
parent fa3f86ee70
commit 64ce5fca15
3 changed files with 26 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+5 -4
View File
@@ -4245,7 +4245,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
{
if(kcpp_data->use_fastforward)
{
ContextFastForward(current_context_tokens, embd_inp, n_past, last_n_tokens, nctx, smartcontext, false, true, 0);
ContextFastForward(current_context_tokens, embd_inp, n_past, last_n_tokens, nctx, smartcontext, false, true, 0, 0);
}
}
if(is_recurrent)
@@ -4297,6 +4297,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
bool triggerff = kcpp_data->use_fastforward;
if(!blank_prompt) //special case for blank prompts, no fast forward or shifts
{
int ff_swa_retain_amount = 0; //a hack for SWA to improve coherency for illegal rewinds
if(triggerff && !kcpp_data->swa_full && (file_format == FileFormat::GGUF_GENERIC))
{
const int swa_pos_min = llama_memory_seq_pos_min(llama_get_memory(llama_ctx_v4), 0); //this is the furthest back we can rewind to.
@@ -4304,10 +4305,10 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
goal_npast -= 4;
goal_npast = goal_npast < 0 ? 0 : goal_npast;
if (swa_pos_min < 0 || goal_npast <= swa_pos_min) {
triggerff = false;
ff_swa_retain_amount = kcpp_active_swa_size;
if (debugmode==1 && !is_quiet)
{
printf("\nNote: Context cannot be reused (Desired n_past=%d, SWA lowest n_past=%d), doing a full reprocess... to avoid this, disable SWA or increase SWA padding)\n", goal_npast, swa_pos_min);
printf("\nNote: SWA context cannot be reused (Desired n_past=%d, SWA lowest n_past=%d), to avoid this, disable SWA or increase SWA padding), output may degrade.\n", goal_npast, swa_pos_min);
}
}
}
@@ -4318,7 +4319,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
}
if(triggerff)
{
ContextFastForward(current_context_tokens, embd_inp, n_past, last_n_tokens, nctx, smartcontext, triggersc, false, 4);
ContextFastForward(current_context_tokens, embd_inp, n_past, last_n_tokens, nctx, smartcontext, triggersc, false, 4, ff_swa_retain_amount);
}
}
if(file_format == FileFormat::GGUF_GENERIC)