Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-07 13:31:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

metrics: seperate cache/processed prompt tokens

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-12 13:54:36 +02:00
parent a55abe44aa
commit e0f113decb
4 changed files with 32 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+12 -4
View File
@@ -2434,11 +2434,13 @@ private:
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
res->t_start = metrics.t_start;
res->prompt_bucket = metrics.prompt_bucket;
res->predict_bucket = metrics.predict_bucket;
res->prompt_bucket = metrics.prompt_bucket;
res->predict_bucket = metrics.predict_bucket;
res->n_prompt_cached_bucket = metrics.n_prompt_cached_bucket;
res->prompt = metrics.prompt;
res->predict = metrics.predict;
res->prompt = metrics.prompt;
res->predict = metrics.predict;
res->n_prompt_cached = metrics.n_prompt_cached;
res->n_tokens_max = metrics.n_tokens_max;
@@ -3290,6 +3292,8 @@ private:
slot.stats.n_prompt_cached = n_past;
slot.stats.n_prompt_processed = 0;
metrics.add_prompt_cached(n_past);
slot.prompt.tokens.keep_first(n_past);
// this is to signal the client that the request has started processing
@@ -4402,6 +4406,10 @@ void server_routes::init_routes() {
{"name", "prompt_tokens_total"},
{"help", "Number of prompt tokens processed."},
{"value", res_task->prompt.count}
}, {
{"name", "prompt_tokens_cached_total"},
{"help", "Number of prompt tokens reused from the cache."},
{"value", res_task->n_prompt_cached}
}, {
{"name", "prompt_seconds_total"},
{"help", "Prompt process time"},