should_reset_buckets

tools/server/server-context.cpp

@@ -822,6 +822,10 @@ public:
         return metrics;
     }
 
+    void reset_metrics_bucket() {
+        metrics.reset_bucket();
+    }
+
 private:
     // note: accessing these fields outside of this class is not thread-safe
     // use server_context methods instead
@@ -4583,16 +4587,18 @@ void server_routes::init_routes() {
             return res;
         }
 
-        res->content_type = "text/plain; version=0.0.4";
-        res->status = 200;
-
         // render response using cached_metrics
         auto use_cached_metrics = [&]() {
             std::unique_lock<std::mutex> lock(mutex_cache);
             res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start);
             server_task_result_metrics tmp;
             tmp.metrics = cached_metrics;
+            res->content_type = "text/plain; version=0.0.4";
+            res->status = 200;
             res->data = tmp.to_metrics();
+            // the gauges are averaged over the window between two scrapes
+            cached_metrics.reset_bucket();
+            should_reset_buckets = true;
         };
 
         if (queue_tasks.is_sleeping()) {
@@ -4628,6 +4634,8 @@ void server_routes::init_routes() {
             GGML_ASSERT(res_task != nullptr);
 
             res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);
+            res->content_type = "text/plain; version=0.0.4";
+            res->status = 200;
             res->data = res_task->to_metrics();
         }
 
@@ -5445,15 +5453,22 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
 }
 
 void server_routes::update_cached_responses(bool is_sleeping) {
-    if (is_sleeping) {
-        std::unique_lock<std::mutex> lock(mutex_cache);
+    // caller is task_queue, so ctx_server can be accessed without holding locks
+    std::unique_lock<std::mutex> lock(mutex_cache);
 
+    if (is_sleeping) {
         cached_models  = get_res_models(*meta);
         cached_props   = get_res_props(*meta, params, true);
-
-        // caller is task_queue, so we don't need to hold locks here
         cached_metrics = ctx_server.get_metrics();
 
+        should_reset_buckets = false;
+
         SRV_DBG("%s\n", "cached responses updated");
+
+    } else if (should_reset_buckets) {
+        // a scrape during sleep already reported these buckets
+        ctx_server.reset_metrics_bucket();
+
+        should_reset_buckets = false;
     }
 }

tools/server/server-context.h

@@ -175,7 +175,7 @@ private:
     std::unique_ptr<const server_context_meta> meta;
 
     const common_params & params;
-    const server_context_impl & ctx_server;
+    server_context_impl & ctx_server;
 
     server_queue & queue_tasks;
     server_response & queue_results;
@@ -186,6 +186,8 @@ private:
     json           cached_models  = nullptr;
     json           cached_props   = nullptr;
     server_metrics cached_metrics;
+    // set when a scrape during sleep already reported the throughput buckets
+    bool           should_reset_buckets = false;
     // call right before sleep to update the cached responses
     void update_cached_responses(bool is_sleeping);
 };

tools/server/tests/unit/test_sleep.py

@@ -97,3 +97,31 @@ def test_server_sleep_read_only_endpoints():
 
     # scraping /metrics must not wake the server up
     assert is_sleeping(server)
+
+
+def test_server_sleep_metrics_buckets():
+    global server
+    server.sleep_idle_seconds = 1
+    server.server_metrics = True
+    server.start()
+
+    res = server.make_request("POST", "/completion", data={
+        "n_predict": 8,
+        "prompt": "Hello",
+    })
+    assert res.status_code == 200
+
+    wait_for_sleep(server)
+
+    # the first scrape reports the throughput of the last generation
+    assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") > 0
+
+    # nothing runs while sleeping, so the next scrapes report an empty window
+    assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0
+    assert is_sleeping(server)
+
+    # waking up must not report the buckets again
+    res = server.make_request("POST", "/tokenize", data={"content": "Hello"})
+    assert res.status_code == 200
+    assert is_sleeping(server) == False
+    assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0