From e0f113decbe88e61763eb3a417f9c350034ec6e1 Mon Sep 17 00:00:00 2001
From: Xuan Son Nguyen <son@huggingface.co>
Date: Wed, 12 Aug 2026 13:54:36 +0200
Subject: [PATCH] metrics: seperate cache/processed prompt tokens

---
 tools/server/server-common.h    | 22 ++++++++++++++++------
 tools/server/server-context.cpp | 16 ++++++++++++----
 tools/server/server-task.cpp    |  2 ++
 tools/server/server-task.h      |  2 ++
 4 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index 52b4df4c61..474877a1f1 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -443,13 +443,17 @@ struct server_metrics {
         }
     };
 
+    // prompt tokens reused from the cache need no decode, so they only have a count
+
     // these are reset by reset_bucket()
-    bucket prompt_bucket;
-    bucket predict_bucket;
+    bucket   prompt_bucket;
+    bucket   predict_bucket;
+    uint64_t n_prompt_cached_bucket = 0;
 
     // metrics below are cumulative since the server started
-    bucket prompt;
-    bucket predict;
+    bucket   prompt; // only processed tokens, not cached tokens
+    bucket   predict;
+    uint64_t n_prompt_cached = 0;
 
     uint64_t n_tokens_max = 0;
 
@@ -472,8 +476,9 @@ struct server_metrics {
     }
 
     void reset_bucket() {
-        prompt_bucket  = {};
-        predict_bucket = {};
+        prompt_bucket          = {};
+        predict_bucket         = {};
+        n_prompt_cached_bucket = 0;
     }
 
     void add_prompt(uint64_t n_tokens, uint64_t t_us) {
@@ -481,6 +486,11 @@ struct server_metrics {
         prompt_bucket.add(n_tokens, n_tokens, t_us);
     }
 
+    void add_prompt_cached(uint64_t n_tokens) {
+        n_prompt_cached        += n_tokens;
+        n_prompt_cached_bucket += n_tokens;
+    }
+
     void on_decode_start() {
         t_decode_start = ggml_time_us();
     }
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index c3afe3d901..e23362d76b 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -2434,11 +2434,13 @@ private:
                     res->n_tasks_deferred    = queue_tasks.queue_tasks_deferred_size();
                     res->t_start             = metrics.t_start;
 
-                    res->prompt_bucket  = metrics.prompt_bucket;
-                    res->predict_bucket = metrics.predict_bucket;
+                    res->prompt_bucket          = metrics.prompt_bucket;
+                    res->predict_bucket         = metrics.predict_bucket;
+                    res->n_prompt_cached_bucket = metrics.n_prompt_cached_bucket;
 
-                    res->prompt  = metrics.prompt;
-                    res->predict = metrics.predict;
+                    res->prompt          = metrics.prompt;
+                    res->predict         = metrics.predict;
+                    res->n_prompt_cached = metrics.n_prompt_cached;
 
                     res->n_tokens_max = metrics.n_tokens_max;
 
@@ -3290,6 +3292,8 @@ private:
                         slot.stats.n_prompt_cached    = n_past;
                         slot.stats.n_prompt_processed = 0;
 
+                        metrics.add_prompt_cached(n_past);
+
                         slot.prompt.tokens.keep_first(n_past);
 
                         // this is to signal the client that the request has started processing
@@ -4402,6 +4406,10 @@ void server_routes::init_routes() {
                     {"name",  "prompt_tokens_total"},
                     {"help",  "Number of prompt tokens processed."},
                     {"value",  res_task->prompt.count}
+            }, {
+                    {"name",  "prompt_tokens_cached_total"},
+                    {"help",  "Number of prompt tokens reused from the cache."},
+                    {"value",  res_task->n_prompt_cached}
             }, {
                     {"name",  "prompt_seconds_total"},
                     {"help",  "Prompt process time"},
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 92e84b8ba5..d7bc301d93 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1518,6 +1518,7 @@ json server_task_result_metrics::to_json() {
         { "t_start",                         t_start },
 
         { "n_prompt_tokens_processed_total", prompt.count },
+        { "n_prompt_tokens_cached_total",     n_prompt_cached },
         { "t_tokens_generation_total",       predict.time / 1e3 },
         { "n_tokens_predicted_total",        predict.count },
         { "t_prompt_processing_total",       prompt.time / 1e3 },
@@ -1525,6 +1526,7 @@ json server_task_result_metrics::to_json() {
         { "n_tokens_max",                    n_tokens_max },
 
         { "n_prompt_tokens_processed",       prompt_bucket.count },
+        { "n_prompt_tokens_cached",          n_prompt_cached_bucket },
         { "t_prompt_processing",             prompt_bucket.time / 1e3 },
         { "n_tokens_predicted",              predict_bucket.count },
         { "t_tokens_generation",             predict_bucket.time / 1e3 },
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index bd6767f833..1587ae7c98 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -499,9 +499,11 @@ struct server_task_result_metrics : server_task_result {
     // note: the fields below mirror server_metrics, keep the names in-sync
     server_metrics::bucket prompt_bucket;
     server_metrics::bucket predict_bucket;
+    uint64_t n_prompt_cached_bucket = 0;
 
     server_metrics::bucket prompt;
     server_metrics::bucket predict;
+    uint64_t n_prompt_cached = 0;
 
     uint64_t n_tokens_max = 0;