add cache serializer

tools/server/server-common.cpp

@@ -87,6 +87,64 @@ json server_slot_stats::to_json() const {
     return base;
 }
 
+//
+// server_metrics
+//
+
+json server_metrics::bucket::to_json() const {
+    return json {
+        {"count", count},
+        {"steps", steps},
+        {"time",  time },
+    };
+}
+
+void server_metrics::bucket::from_json(const json & data) {
+    count = data.at("count");
+    steps = data.at("steps");
+    time  = data.at("time");
+}
+
+json server_metrics::to_json() const {
+    return json {
+        {"t_start",             t_start},
+
+        {"prompt_bucket",       prompt_bucket .to_json()},
+        {"predict_bucket",      predict_bucket.to_json()},
+        {"prompt",              prompt        .to_json()},
+        {"predict",             predict       .to_json()},
+
+        {"n_prompt_cached",     n_prompt_cached},
+        {"n_tokens_max",        n_tokens_max},
+        {"n_decode",            n_decode},
+        {"n_busy_slots",        n_busy_slots},
+
+        {"n_draft_tokens",      n_draft_tokens},
+        {"n_draft_accepted",    n_draft_accepted},
+        {"n_draft_verif_steps", n_draft_verif_steps},
+        {"n_accepted_per_pos",  n_accepted_per_pos},
+    };
+}
+
+void server_metrics::from_json(const json & data) {
+    t_start = data.at("t_start");
+
+    prompt_bucket .from_json(data.at("prompt_bucket"));
+    predict_bucket.from_json(data.at("predict_bucket"));
+    prompt        .from_json(data.at("prompt"));
+    predict       .from_json(data.at("predict"));
+
+    n_prompt_cached = data.at("n_prompt_cached");
+    n_tokens_max    = data.at("n_tokens_max");
+    n_decode        = data.at("n_decode");
+    n_busy_slots    = data.at("n_busy_slots");
+
+    n_draft_tokens      = data.at("n_draft_tokens");
+    n_draft_accepted    = data.at("n_draft_accepted");
+    n_draft_verif_steps = data.at("n_draft_verif_steps");
+    n_accepted_per_pos  = data.at("n_accepted_per_pos").get<std::vector<uint64_t>>();
+}
+
 //
 // random string / id
 //

tools/server/server-common.h

@@ -450,6 +450,9 @@ struct server_metrics {
             steps += n_steps;
             time  += t_us;
         }
+
+        json to_json() const;
+        void from_json(const json & data);
     };
 
     // these are reset by reset_bucket(), only the rate is read from them
@@ -490,6 +493,10 @@ struct server_metrics {
     void add_prompt_cached(uint64_t n_tokens) {
         n_prompt_cached += n_tokens;
     }
+
+    // used to keep the metrics across a process restart, see --sleep-mode all
+    json to_json() const;
+    void from_json(const json & data);
 };
 
 //

tools/server/server-context.cpp

@@ -5472,3 +5472,28 @@ void server_routes::update_cached_responses(bool is_sleeping) {
         should_reset_buckets = false;
     }
 }
+
+json server_routes::cache_to_json() {
+    std::unique_lock<std::mutex> lock(mutex_cache);
+
+    return json {
+        {"models",  cached_models},
+        {"props",   cached_props},
+        {"metrics", cached_metrics.to_json()},
+    };
+}
+
+bool server_routes::cache_from_json(const json & data) {
+    std::unique_lock<std::mutex> lock(mutex_cache);
+
+    try {
+        cached_models = data.at("models");
+        cached_props  = data.at("props");
+        cached_metrics.from_json(data.at("metrics"));
+    } catch (const std::exception & e) {
+        SRV_ERR("failed to restore cached responses: %s\n", e.what());
+        return false;
+    }
+
+    return true;
+}

tools/server/server-context.h

@@ -158,6 +158,11 @@ struct server_routes {
     // to be used in router mode
     json get_model_info() const;
 
+    // save / restore the cached responses across a process restart, see --sleep-mode all
+    // only valid while sleeping, as the cache is only updated upon entering that state
+    json cache_to_json();
+    bool cache_from_json(const json & data);
+
 private:
     std::unique_ptr<server_res_generator> handle_completions_impl(
             const server_http_req & req,