Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-25 22:21:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

server : add LLAMA_SERVER_SLOTS_N_DIFF (#27600)

Browse Source
This commit is contained in:
Georgi Gerganov
2026-08-23 15:55:51 +03:00
committed by GitHub
parent ba8e0eddfb
commit e8eed4525a
1 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+15 -4
View File
@@ -858,8 +858,10 @@ private:
// slots / clients
std::vector<server_slot> slots;
int trace = 0;
int slots_debug = 0;
int trace = 0; // env: LLAMA_TRACE
int slots_debug = 0; // env: LLAMA_SERVER_SLOTS_DEBUG
int slots_n_diff = 0; // env: LLAMA_SERVER_SLOTS_N_DIFF
int n_empty_consecutive = 0;
std::unique_ptr<server_prompt_cache> prompt_cache;
@@ -1247,6 +1249,15 @@ private:
}
}
{
const char * LLAMA_SERVER_SLOTS_N_DIFF = getenv("LLAMA_SERVER_SLOTS_N_DIFF");
slots_n_diff = LLAMA_SERVER_SLOTS_N_DIFF ? atoi(LLAMA_SERVER_SLOTS_N_DIFF) : 0;
if (slots_n_diff) {
SRV_WRN("LLAMA_SERVER_SLOTS_N_DIFF = %d\n", slots_n_diff);
}
}
// the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens
// note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used)
{
@@ -3179,8 +3190,8 @@ private:
// when the prompt prefix does not match, print the tokens around the mismatch
// this is useful for debugging prompt caching
if (slots_debug) {
const int np0 = std::max<int>(n_past - 4, 0);
const int np1 = std::min<int>(n_past + 6, std::min(slot.prompt.tokens.size(), slot.task->tokens.size()));
const int np0 = std::max<int>(n_past - slots_n_diff, 0);
const int np1 = std::min<int>(n_past + slots_n_diff + 2, std::min(slot.prompt.tokens.size(), slot.task->tokens.size()));
std::stringstream ss0;
std::stringstream ss1;