mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-07 13:31:13 +02:00
refactor server_task_result_metrics
This commit is contained in:
Xuan Son Nguyen
@@ -2432,25 +2432,7 @@ private:
|
||||
res->n_idle_slots = n_idle_slots;
|
||||
res->n_processing_slots = n_processing_slots;
|
||||
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
|
||||
res->t_start = metrics.t_start;
|
||||
|
||||
res->prompt_bucket = metrics.prompt_bucket;
|
||||
res->predict_bucket = metrics.predict_bucket;
|
||||
res->n_prompt_cached_bucket = metrics.n_prompt_cached_bucket;
|
||||
|
||||
res->prompt = metrics.prompt;
|
||||
res->predict = metrics.predict;
|
||||
res->n_prompt_cached = metrics.n_prompt_cached;
|
||||
|
||||
res->n_tokens_max = metrics.n_tokens_max;
|
||||
|
||||
res->n_decode = metrics.n_decode;
|
||||
res->n_busy_slots = metrics.n_busy_slots;
|
||||
|
||||
res->n_draft_tokens = metrics.n_draft_tokens;
|
||||
res->n_draft_accepted = metrics.n_draft_accepted;
|
||||
res->n_draft_verif_steps = metrics.n_draft_verif_steps;
|
||||
res->n_accepted_per_pos = metrics.n_accepted_per_pos;
|
||||
res->metrics = metrics;
|
||||
|
||||
if (task.metrics_reset_bucket) {
|
||||
metrics.reset_bucket();
|
||||
@@ -4400,104 +4382,10 @@ void server_routes::init_routes() {
|
||||
auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());
|
||||
GGML_ASSERT(res_task != nullptr);
|
||||
|
||||
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
|
||||
json all_metrics_def = json {
|
||||
{"counter", {{
|
||||
{"name", "prompt_tokens_total"},
|
||||
{"help", "Number of prompt tokens processed."},
|
||||
{"value", res_task->prompt.count}
|
||||
}, {
|
||||
{"name", "prompt_tokens_cached_total"},
|
||||
{"help", "Number of prompt tokens reused from the cache."},
|
||||
{"value", res_task->n_prompt_cached}
|
||||
}, {
|
||||
{"name", "prompt_seconds_total"},
|
||||
{"help", "Prompt process time"},
|
||||
{"value", res_task->prompt.time / 1.e6}
|
||||
}, {
|
||||
{"name", "tokens_predicted_total"},
|
||||
{"help", "Number of generation tokens processed."},
|
||||
{"value", res_task->predict.count}
|
||||
}, {
|
||||
{"name", "tokens_predicted_seconds_total"},
|
||||
{"help", "Predict process time"},
|
||||
{"value", res_task->predict.time / 1.e6}
|
||||
}, {
|
||||
{"name", "n_decode_total"},
|
||||
{"help", "Total number of llama_decode() calls"},
|
||||
{"value", res_task->n_decode}
|
||||
}, {
|
||||
{"name", "n_tokens_max"},
|
||||
{"help", "Largest observed n_tokens."},
|
||||
{"value", res_task->n_tokens_max}
|
||||
}, {
|
||||
{"name", "spec_decode_num_draft_tokens_total"},
|
||||
{"help", "Total draft tokens generated"},
|
||||
{"value", res_task->n_draft_tokens}
|
||||
}, {
|
||||
{"name", "spec_decode_num_accepted_tokens_total"},
|
||||
{"help", "Total draft tokens accepted by the target model"},
|
||||
{"value", res_task->n_draft_accepted}
|
||||
}, {
|
||||
{"name", "spec_decode_num_drafts_total"},
|
||||
{"help", "Total speculative decoding verification steps"},
|
||||
{"value", res_task->n_draft_verif_steps}
|
||||
}}},
|
||||
{"gauge", {{
|
||||
{"name", "prompt_tokens_seconds"},
|
||||
{"help", "Average prompt throughput in tokens/s."},
|
||||
{"value", res_task->prompt_bucket.n_per_second()}
|
||||
},{
|
||||
{"name", "predicted_tokens_seconds"},
|
||||
{"help", "Average generation throughput in tokens/s."},
|
||||
{"value", res_task->predict_bucket.n_per_second()}
|
||||
},{
|
||||
{"name", "requests_processing"},
|
||||
{"help", "Number of requests processing."},
|
||||
{"value", (uint64_t) res_task->n_processing_slots}
|
||||
},{
|
||||
{"name", "requests_deferred"},
|
||||
{"help", "Number of requests deferred."},
|
||||
{"value", (uint64_t) res_task->n_tasks_deferred}
|
||||
},{
|
||||
{"name", "n_busy_slots_per_decode"},
|
||||
{"help", "Average number of busy slots per llama_decode() call"},
|
||||
{"value", (float) res_task->n_busy_slots / std::max((float) res_task->n_decode, 1.f)}
|
||||
}}}
|
||||
};
|
||||
|
||||
std::stringstream prometheus;
|
||||
|
||||
for (const auto & el : all_metrics_def.items()) {
|
||||
const auto & type = el.key();
|
||||
const auto & metrics_def = el.value();
|
||||
|
||||
for (const auto & metric_def : metrics_def) {
|
||||
const std::string name = metric_def.at("name");
|
||||
const std::string help = metric_def.at("help");
|
||||
|
||||
auto value = json_value(metric_def, "value", 0.);
|
||||
prometheus << "# HELP llamacpp:" << name << " " << help << "\n"
|
||||
<< "# TYPE llamacpp:" << name << " " << type << "\n"
|
||||
<< "llamacpp:" << name << " " << value << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
// labeled counter: one time series per draft position
|
||||
if (!res_task->n_accepted_per_pos.empty()) {
|
||||
prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
|
||||
" Accepted tokens per draft position\n"
|
||||
<< "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
|
||||
for (size_t i = 0; i < res_task->n_accepted_per_pos.size(); i++) {
|
||||
prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
|
||||
<< i << "\"} " << res_task->n_accepted_per_pos[i] << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start);
|
||||
res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);
|
||||
res->content_type = "text/plain; version=0.0.4";
|
||||
res->status = 200;
|
||||
res->data = prometheus.str();
|
||||
res->data = res_task->to_metrics();
|
||||
return res;
|
||||
};
|
||||
|
||||
|
||||
+117
-18
@@ -10,6 +10,8 @@
|
||||
#include "speculative.h"
|
||||
#include "server-common.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
using json = nlohmann::ordered_json;
|
||||
|
||||
//
|
||||
@@ -1515,34 +1517,131 @@ json server_task_result_metrics::to_json() {
|
||||
{ "idle", n_idle_slots },
|
||||
{ "processing", n_processing_slots },
|
||||
{ "deferred", n_tasks_deferred },
|
||||
{ "t_start", t_start },
|
||||
{ "t_start", metrics.t_start },
|
||||
|
||||
{ "n_prompt_tokens_processed_total", prompt.count },
|
||||
{ "n_prompt_tokens_cached_total", n_prompt_cached },
|
||||
{ "t_tokens_generation_total", predict.time / 1e3 },
|
||||
{ "n_tokens_predicted_total", predict.count },
|
||||
{ "t_prompt_processing_total", prompt.time / 1e3 },
|
||||
{ "n_prompt_tokens_processed_total", metrics.prompt.count },
|
||||
{ "n_prompt_tokens_cached_total", metrics.n_prompt_cached },
|
||||
{ "t_tokens_generation_total", metrics.predict.time / 1e3 },
|
||||
{ "n_tokens_predicted_total", metrics.predict.count },
|
||||
{ "t_prompt_processing_total", metrics.prompt.time / 1e3 },
|
||||
|
||||
{ "n_tokens_max", n_tokens_max },
|
||||
{ "n_tokens_max", metrics.n_tokens_max },
|
||||
|
||||
{ "n_prompt_tokens_processed", prompt_bucket.count },
|
||||
{ "n_prompt_tokens_cached", n_prompt_cached_bucket },
|
||||
{ "t_prompt_processing", prompt_bucket.time / 1e3 },
|
||||
{ "n_tokens_predicted", predict_bucket.count },
|
||||
{ "t_tokens_generation", predict_bucket.time / 1e3 },
|
||||
{ "n_prompt_tokens_processed", metrics.prompt_bucket.count },
|
||||
{ "n_prompt_tokens_cached", metrics.n_prompt_cached_bucket },
|
||||
{ "t_prompt_processing", metrics.prompt_bucket.time / 1e3 },
|
||||
{ "n_tokens_predicted", metrics.predict_bucket.count },
|
||||
{ "t_tokens_generation", metrics.predict_bucket.time / 1e3 },
|
||||
|
||||
{ "n_decode_total", n_decode },
|
||||
{ "n_busy_slots_total", n_busy_slots },
|
||||
{ "n_decode_total", metrics.n_decode },
|
||||
{ "n_busy_slots_total", metrics.n_busy_slots },
|
||||
|
||||
{ "n_draft_tokens_total", n_draft_tokens },
|
||||
{ "n_draft_accepted_total", n_draft_accepted },
|
||||
{ "n_draft_verif_steps_total", n_draft_verif_steps },
|
||||
{ "n_accepted_per_pos_total", n_accepted_per_pos },
|
||||
{ "n_draft_tokens_total", metrics.n_draft_tokens },
|
||||
{ "n_draft_accepted_total", metrics.n_draft_accepted },
|
||||
{ "n_draft_verif_steps_total", metrics.n_draft_verif_steps },
|
||||
{ "n_accepted_per_pos_total", metrics.n_accepted_per_pos },
|
||||
|
||||
{ "slots", slots_data },
|
||||
};
|
||||
}
|
||||
|
||||
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
|
||||
std::string server_task_result_metrics::to_metrics() {
|
||||
const std::vector<metric_item> counters = {
|
||||
{
|
||||
"prompt_tokens_total",
|
||||
"Number of prompt tokens processed.",
|
||||
metrics.prompt.count
|
||||
}, {
|
||||
"prompt_tokens_cached_total",
|
||||
"Number of prompt tokens reused from the cache.",
|
||||
metrics.n_prompt_cached
|
||||
}, {
|
||||
"prompt_seconds_total",
|
||||
"Prompt process time",
|
||||
metrics.prompt.time / 1.e6
|
||||
}, {
|
||||
"tokens_predicted_total",
|
||||
"Number of generation tokens processed.",
|
||||
metrics.predict.count
|
||||
}, {
|
||||
"tokens_predicted_seconds_total",
|
||||
"Predict process time",
|
||||
metrics.predict.time / 1.e6
|
||||
}, {
|
||||
"n_decode_total",
|
||||
"Total number of llama_decode() calls",
|
||||
metrics.n_decode
|
||||
}, {
|
||||
"n_tokens_max",
|
||||
"Largest observed n_tokens.",
|
||||
metrics.n_tokens_max
|
||||
}, {
|
||||
"spec_decode_num_draft_tokens_total",
|
||||
"Total draft tokens generated",
|
||||
metrics.n_draft_tokens
|
||||
}, {
|
||||
"spec_decode_num_accepted_tokens_total",
|
||||
"Total draft tokens accepted by the target model",
|
||||
metrics.n_draft_accepted
|
||||
}, {
|
||||
"spec_decode_num_drafts_total",
|
||||
"Total speculative decoding verification steps",
|
||||
metrics.n_draft_verif_steps
|
||||
},
|
||||
};
|
||||
|
||||
const std::vector<metric_item> gauges = {
|
||||
{
|
||||
"prompt_tokens_seconds",
|
||||
"Average prompt throughput in tokens/s.",
|
||||
metrics.prompt_bucket.n_per_second()
|
||||
}, {
|
||||
"predicted_tokens_seconds",
|
||||
"Average generation throughput in tokens/s.",
|
||||
metrics.predict_bucket.n_per_second()
|
||||
}, {
|
||||
"requests_processing",
|
||||
"Number of requests processing.",
|
||||
(uint64_t) n_processing_slots
|
||||
}, {
|
||||
"requests_deferred",
|
||||
"Number of requests deferred.",
|
||||
(uint64_t) n_tasks_deferred
|
||||
}, {
|
||||
"n_busy_slots_per_decode",
|
||||
"Average number of busy slots per llama_decode() call",
|
||||
(float) metrics.n_busy_slots / std::max((float) metrics.n_decode, 1.f)
|
||||
},
|
||||
};
|
||||
|
||||
std::stringstream prometheus;
|
||||
|
||||
auto add_items = [&prometheus](const char * type, const std::vector<metric_item> & items) {
|
||||
for (const auto & item : items) {
|
||||
prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n"
|
||||
<< "# TYPE llamacpp:" << item.name << " " << type << "\n"
|
||||
<< "llamacpp:" << item.name << " " << item.value.get<double>() << "\n";
|
||||
}
|
||||
};
|
||||
|
||||
add_items("counter", counters);
|
||||
add_items("gauge", gauges);
|
||||
|
||||
// labeled counter: one time series per draft position
|
||||
if (!metrics.n_accepted_per_pos.empty()) {
|
||||
prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
|
||||
" Accepted tokens per draft position\n"
|
||||
<< "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
|
||||
for (size_t i = 0; i < metrics.n_accepted_per_pos.size(); i++) {
|
||||
prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
|
||||
<< i << "\"} " << metrics.n_accepted_per_pos[i] << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
return prometheus.str();
|
||||
}
|
||||
|
||||
//
|
||||
// server_task_result_slot_save_load
|
||||
//
|
||||
|
||||
+10
-20
@@ -493,33 +493,23 @@ struct server_task_result_metrics : server_task_result {
|
||||
int n_idle_slots;
|
||||
int n_processing_slots;
|
||||
int n_tasks_deferred;
|
||||
int64_t t_start;
|
||||
|
||||
// TODO: somehow reuse server_metrics in the future, instead of duplicating the fields
|
||||
// note: the fields below mirror server_metrics, keep the names in-sync
|
||||
server_metrics::bucket prompt_bucket;
|
||||
server_metrics::bucket predict_bucket;
|
||||
uint64_t n_prompt_cached_bucket = 0;
|
||||
|
||||
server_metrics::bucket prompt;
|
||||
server_metrics::bucket predict;
|
||||
uint64_t n_prompt_cached = 0;
|
||||
|
||||
uint64_t n_tokens_max = 0;
|
||||
|
||||
uint64_t n_decode = 0;
|
||||
uint64_t n_busy_slots = 0;
|
||||
|
||||
uint64_t n_draft_tokens = 0;
|
||||
uint64_t n_draft_accepted = 0;
|
||||
uint64_t n_draft_verif_steps = 0;
|
||||
std::vector<uint64_t> n_accepted_per_pos;
|
||||
server_metrics metrics;
|
||||
|
||||
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
|
||||
// therefore, we use json to temporarily store the slot.to_json() result
|
||||
json slots_data = json::array();
|
||||
|
||||
// used by /slots API
|
||||
virtual json to_json() override;
|
||||
|
||||
// used by /metrics API
|
||||
struct metric_item {
|
||||
std::string name;
|
||||
std::string description;
|
||||
json value; // can be int or double
|
||||
};
|
||||
std::string to_metrics();
|
||||
};
|
||||
|
||||
struct server_task_result_slot_save_load : server_task_result {
|
||||
|
||||
Reference in New Issue
Block a user