Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-10 06:49:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

revamped smart context for llama models

Browse Source
This commit is contained in:
Concedo
2023-10-28 12:59:08 +08:00
parent c2f675133d
commit 15f525c580
5 changed files with 90 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llama.cpp
+11 -1
View File
@@ -9819,12 +9819,22 @@ bool llama_save_session_file(struct llama_context * ctx, const char * path_sessi
return true;
}
void printcache(struct llama_context * ctx)
{
struct llama_kv_cache & cache = ctx->kv_self;
std::string vals = "\n";
for (int32_t i = 0; i < cache.size; ++i) {
vals += std::to_string(i) + "= pos:" + std::to_string(cache.cells[i].pos) + " delta:" + std::to_string(cache.cells[i].delta) +"\n";
}
printf("%s",vals.c_str());
}
int llama_eval(
struct llama_context * ctx,
llama_token * tokens,
int32_t n_tokens,
int n_past) {
llama_kv_cache_tokens_rm(ctx->kv_self, n_past, -1);
llama_kv_cache_seq_rm(ctx->kv_self, 0, n_past, -1);
const int ret = llama_decode_internal(*ctx, llama_batch_get_one(tokens, n_tokens, n_past, 0));
if (ret < 0) {