From 9c57fd783c64c09acc7dc53ebceca60b7e2ef23d Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 12 Aug 2026 14:16:11 +0200 Subject: [PATCH] refactor server_task_result_metrics --- tools/server/server-context.cpp | 118 +--------------------------- tools/server/server-task.cpp | 135 +++++++++++++++++++++++++++----- tools/server/server-task.h | 30 +++---- 3 files changed, 130 insertions(+), 153 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e23362d76b..81e77e13c8 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2432,25 +2432,7 @@ private: res->n_idle_slots = n_idle_slots; res->n_processing_slots = n_processing_slots; res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); - res->t_start = metrics.t_start; - - res->prompt_bucket = metrics.prompt_bucket; - res->predict_bucket = metrics.predict_bucket; - res->n_prompt_cached_bucket = metrics.n_prompt_cached_bucket; - - res->prompt = metrics.prompt; - res->predict = metrics.predict; - res->n_prompt_cached = metrics.n_prompt_cached; - - res->n_tokens_max = metrics.n_tokens_max; - - res->n_decode = metrics.n_decode; - res->n_busy_slots = metrics.n_busy_slots; - - res->n_draft_tokens = metrics.n_draft_tokens; - res->n_draft_accepted = metrics.n_draft_accepted; - res->n_draft_verif_steps = metrics.n_draft_verif_steps; - res->n_accepted_per_pos = metrics.n_accepted_per_pos; + res->metrics = metrics; if (task.metrics_reset_bucket) { metrics.reset_bucket(); @@ -4400,104 +4382,10 @@ void server_routes::init_routes() { auto res_task = dynamic_cast(result.get()); GGML_ASSERT(res_task != nullptr); - // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names - json all_metrics_def = json { - {"counter", {{ - {"name", "prompt_tokens_total"}, - {"help", "Number of prompt tokens processed."}, - {"value", res_task->prompt.count} - }, { - {"name", "prompt_tokens_cached_total"}, - {"help", "Number of prompt tokens reused from the cache."}, - {"value", res_task->n_prompt_cached} - }, { - {"name", "prompt_seconds_total"}, - {"help", "Prompt process time"}, - {"value", res_task->prompt.time / 1.e6} - }, { - {"name", "tokens_predicted_total"}, - {"help", "Number of generation tokens processed."}, - {"value", res_task->predict.count} - }, { - {"name", "tokens_predicted_seconds_total"}, - {"help", "Predict process time"}, - {"value", res_task->predict.time / 1.e6} - }, { - {"name", "n_decode_total"}, - {"help", "Total number of llama_decode() calls"}, - {"value", res_task->n_decode} - }, { - {"name", "n_tokens_max"}, - {"help", "Largest observed n_tokens."}, - {"value", res_task->n_tokens_max} - }, { - {"name", "spec_decode_num_draft_tokens_total"}, - {"help", "Total draft tokens generated"}, - {"value", res_task->n_draft_tokens} - }, { - {"name", "spec_decode_num_accepted_tokens_total"}, - {"help", "Total draft tokens accepted by the target model"}, - {"value", res_task->n_draft_accepted} - }, { - {"name", "spec_decode_num_drafts_total"}, - {"help", "Total speculative decoding verification steps"}, - {"value", res_task->n_draft_verif_steps} - }}}, - {"gauge", {{ - {"name", "prompt_tokens_seconds"}, - {"help", "Average prompt throughput in tokens/s."}, - {"value", res_task->prompt_bucket.n_per_second()} - },{ - {"name", "predicted_tokens_seconds"}, - {"help", "Average generation throughput in tokens/s."}, - {"value", res_task->predict_bucket.n_per_second()} - },{ - {"name", "requests_processing"}, - {"help", "Number of requests processing."}, - {"value", (uint64_t) res_task->n_processing_slots} - },{ - {"name", "requests_deferred"}, - {"help", "Number of requests deferred."}, - {"value", (uint64_t) res_task->n_tasks_deferred} - },{ - {"name", "n_busy_slots_per_decode"}, - {"help", "Average number of busy slots per llama_decode() call"}, - {"value", (float) res_task->n_busy_slots / std::max((float) res_task->n_decode, 1.f)} - }}} - }; - - std::stringstream prometheus; - - for (const auto & el : all_metrics_def.items()) { - const auto & type = el.key(); - const auto & metrics_def = el.value(); - - for (const auto & metric_def : metrics_def) { - const std::string name = metric_def.at("name"); - const std::string help = metric_def.at("help"); - - auto value = json_value(metric_def, "value", 0.); - prometheus << "# HELP llamacpp:" << name << " " << help << "\n" - << "# TYPE llamacpp:" << name << " " << type << "\n" - << "llamacpp:" << name << " " << value << "\n"; - } - } - - // labeled counter: one time series per draft position - if (!res_task->n_accepted_per_pos.empty()) { - prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total" - " Accepted tokens per draft position\n" - << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n"; - for (size_t i = 0; i < res_task->n_accepted_per_pos.size(); i++) { - prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"" - << i << "\"} " << res_task->n_accepted_per_pos[i] << "\n"; - } - } - - res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start); + res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start); res->content_type = "text/plain; version=0.0.4"; res->status = 200; - res->data = prometheus.str(); + res->data = res_task->to_metrics(); return res; }; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index d7bc301d93..460753b995 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -10,6 +10,8 @@ #include "speculative.h" #include "server-common.h" +#include + using json = nlohmann::ordered_json; // @@ -1515,34 +1517,131 @@ json server_task_result_metrics::to_json() { { "idle", n_idle_slots }, { "processing", n_processing_slots }, { "deferred", n_tasks_deferred }, - { "t_start", t_start }, + { "t_start", metrics.t_start }, - { "n_prompt_tokens_processed_total", prompt.count }, - { "n_prompt_tokens_cached_total", n_prompt_cached }, - { "t_tokens_generation_total", predict.time / 1e3 }, - { "n_tokens_predicted_total", predict.count }, - { "t_prompt_processing_total", prompt.time / 1e3 }, + { "n_prompt_tokens_processed_total", metrics.prompt.count }, + { "n_prompt_tokens_cached_total", metrics.n_prompt_cached }, + { "t_tokens_generation_total", metrics.predict.time / 1e3 }, + { "n_tokens_predicted_total", metrics.predict.count }, + { "t_prompt_processing_total", metrics.prompt.time / 1e3 }, - { "n_tokens_max", n_tokens_max }, + { "n_tokens_max", metrics.n_tokens_max }, - { "n_prompt_tokens_processed", prompt_bucket.count }, - { "n_prompt_tokens_cached", n_prompt_cached_bucket }, - { "t_prompt_processing", prompt_bucket.time / 1e3 }, - { "n_tokens_predicted", predict_bucket.count }, - { "t_tokens_generation", predict_bucket.time / 1e3 }, + { "n_prompt_tokens_processed", metrics.prompt_bucket.count }, + { "n_prompt_tokens_cached", metrics.n_prompt_cached_bucket }, + { "t_prompt_processing", metrics.prompt_bucket.time / 1e3 }, + { "n_tokens_predicted", metrics.predict_bucket.count }, + { "t_tokens_generation", metrics.predict_bucket.time / 1e3 }, - { "n_decode_total", n_decode }, - { "n_busy_slots_total", n_busy_slots }, + { "n_decode_total", metrics.n_decode }, + { "n_busy_slots_total", metrics.n_busy_slots }, - { "n_draft_tokens_total", n_draft_tokens }, - { "n_draft_accepted_total", n_draft_accepted }, - { "n_draft_verif_steps_total", n_draft_verif_steps }, - { "n_accepted_per_pos_total", n_accepted_per_pos }, + { "n_draft_tokens_total", metrics.n_draft_tokens }, + { "n_draft_accepted_total", metrics.n_draft_accepted }, + { "n_draft_verif_steps_total", metrics.n_draft_verif_steps }, + { "n_accepted_per_pos_total", metrics.n_accepted_per_pos }, { "slots", slots_data }, }; } +// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names +std::string server_task_result_metrics::to_metrics() { + const std::vector counters = { + { + "prompt_tokens_total", + "Number of prompt tokens processed.", + metrics.prompt.count + }, { + "prompt_tokens_cached_total", + "Number of prompt tokens reused from the cache.", + metrics.n_prompt_cached + }, { + "prompt_seconds_total", + "Prompt process time", + metrics.prompt.time / 1.e6 + }, { + "tokens_predicted_total", + "Number of generation tokens processed.", + metrics.predict.count + }, { + "tokens_predicted_seconds_total", + "Predict process time", + metrics.predict.time / 1.e6 + }, { + "n_decode_total", + "Total number of llama_decode() calls", + metrics.n_decode + }, { + "n_tokens_max", + "Largest observed n_tokens.", + metrics.n_tokens_max + }, { + "spec_decode_num_draft_tokens_total", + "Total draft tokens generated", + metrics.n_draft_tokens + }, { + "spec_decode_num_accepted_tokens_total", + "Total draft tokens accepted by the target model", + metrics.n_draft_accepted + }, { + "spec_decode_num_drafts_total", + "Total speculative decoding verification steps", + metrics.n_draft_verif_steps + }, + }; + + const std::vector gauges = { + { + "prompt_tokens_seconds", + "Average prompt throughput in tokens/s.", + metrics.prompt_bucket.n_per_second() + }, { + "predicted_tokens_seconds", + "Average generation throughput in tokens/s.", + metrics.predict_bucket.n_per_second() + }, { + "requests_processing", + "Number of requests processing.", + (uint64_t) n_processing_slots + }, { + "requests_deferred", + "Number of requests deferred.", + (uint64_t) n_tasks_deferred + }, { + "n_busy_slots_per_decode", + "Average number of busy slots per llama_decode() call", + (float) metrics.n_busy_slots / std::max((float) metrics.n_decode, 1.f) + }, + }; + + std::stringstream prometheus; + + auto add_items = [&prometheus](const char * type, const std::vector & items) { + for (const auto & item : items) { + prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n" + << "# TYPE llamacpp:" << item.name << " " << type << "\n" + << "llamacpp:" << item.name << " " << item.value.get() << "\n"; + } + }; + + add_items("counter", counters); + add_items("gauge", gauges); + + // labeled counter: one time series per draft position + if (!metrics.n_accepted_per_pos.empty()) { + prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total" + " Accepted tokens per draft position\n" + << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n"; + for (size_t i = 0; i < metrics.n_accepted_per_pos.size(); i++) { + prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"" + << i << "\"} " << metrics.n_accepted_per_pos[i] << "\n"; + } + } + + return prometheus.str(); +} + // // server_task_result_slot_save_load // diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 1587ae7c98..674f99746f 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -493,33 +493,23 @@ struct server_task_result_metrics : server_task_result { int n_idle_slots; int n_processing_slots; int n_tasks_deferred; - int64_t t_start; - // TODO: somehow reuse server_metrics in the future, instead of duplicating the fields - // note: the fields below mirror server_metrics, keep the names in-sync - server_metrics::bucket prompt_bucket; - server_metrics::bucket predict_bucket; - uint64_t n_prompt_cached_bucket = 0; - - server_metrics::bucket prompt; - server_metrics::bucket predict; - uint64_t n_prompt_cached = 0; - - uint64_t n_tokens_max = 0; - - uint64_t n_decode = 0; - uint64_t n_busy_slots = 0; - - uint64_t n_draft_tokens = 0; - uint64_t n_draft_accepted = 0; - uint64_t n_draft_verif_steps = 0; - std::vector n_accepted_per_pos; + server_metrics metrics; // while we can also use std::vector this requires copying the slot object which can be quite messy // therefore, we use json to temporarily store the slot.to_json() result json slots_data = json::array(); + // used by /slots API virtual json to_json() override; + + // used by /metrics API + struct metric_item { + std::string name; + std::string description; + json value; // can be int or double + }; + std::string to_metrics(); }; struct server_task_result_slot_save_load : server_task_result {