Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Allocate a small amount of extra context for GGUF to deal with KV fragmentation causing issues in some scenarios.

Browse Source
This commit is contained in:
Concedo
2023-11-28 20:55:14 +08:00
parent d2ef458b02
commit ba5c33319b
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+2 -2
View File
@@ -883,7 +883,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{
llama_model_params model_params = llama_model_default_params();
llama_context_params llama_ctx_params = llama_context_default_params();
llama_ctx_params.n_ctx = clamped_max_context_length;
llama_ctx_params.n_ctx = clamped_max_context_length + 64; //add some extra context to deal with KV fragmentation
//llama_ctx_paran_parts = -1;
llama_ctx_params.seed = -1;
llama_ctx_params.f16_kv = inputs.f16_kv;
@@ -1808,7 +1808,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
if (!evalres)
{
fprintf(stderr, "\nFailed to predict! Check your context buffer sizes!\n");
fprintf(stderr, "\nFailed to predict at %d! Check your context buffer sizes!\n",n_past);
snprintf(output.text, sizeof(output.text), "%s", "");
output.status = 0;
generation_finished = true;