Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

split metrics and slots tasks / results

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-19 17:15:02 +02:00
parent 0591080c28
commit c1fc93f297
3 changed files with 46 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+27 -17
View File
@@ -2298,8 +2298,8 @@ private:
// returns false to decline the task, it is offered again after the decode is done
bool process_single_task(server_task && task, bool is_yielding) {
// while yielding, an encode / decode is running and only accessing metrics is safe
if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS) {
// while yielding, an encode / decode is running and only reading the server state is safe
if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS && task.type != SERVER_TASK_TYPE_SLOT_GET) {
SRV_DBG("decoding, decline task, id_task = %d\n", task.id);
return false;
}
@@ -2425,28 +2425,17 @@ private:
} break;
case SERVER_TASK_TYPE_METRICS:
{
json slots_data = json::array();
int n_idle_slots = 0;
int n_processing_slots = 0;
for (server_slot & slot : slots) {
json slot_data = slot.to_json(slots_debug == 0);
if (slot.is_processing()) {
n_processing_slots++;
} else {
n_idle_slots++;
}
slots_data.push_back(slot_data);
}
SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots);
SRV_DBG("n_processing_slots = %d\n", n_processing_slots);
auto res = std::make_unique<server_task_result_metrics>();
res->id = task.id;
res->slots_data = std::move(slots_data);
res->n_idle_slots = n_idle_slots;
res->n_processing_slots = n_processing_slots;
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
res->metrics = metrics;
@@ -2454,6 +2443,28 @@ private:
if (task.metrics_reset_bucket) {
metrics.reset_bucket();
}
queue_results.send(std::move(res));
} break;
case SERVER_TASK_TYPE_SLOT_GET:
{
json slots_data = json::array();
int n_idle_slots = 0;
for (server_slot & slot : slots) {
if (!slot.is_processing()) {
n_idle_slots++;
}
slots_data.push_back(slot.to_json(slots_debug == 0));
}
SRV_DBG("n_idle_slots = %d\n", n_idle_slots);
auto res = std::make_unique<server_task_result_slots>();
res->id = task.id;
res->slots_data = std::move(slots_data);
res->n_idle_slots = n_idle_slots;
queue_results.send(std::move(res));
} break;
case SERVER_TASK_TYPE_SLOT_SAVE:
@@ -4581,7 +4592,6 @@ void server_routes::init_routes() {
res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start);
server_task_result_metrics tmp;
tmp.metrics = cached_metrics;
tmp.n_idle_slots = params.n_parallel;
res->data = tmp.to_metrics();
};
@@ -4633,7 +4643,7 @@ void server_routes::init_routes() {
// request slots data using task queue
{
server_task task(SERVER_TASK_TYPE_METRICS);
server_task task(SERVER_TASK_TYPE_SLOT_GET);
task.id = res->rd.get_new_id();
res->rd.post_task(std::move(task), true); // high-priority task
}
@@ -4651,7 +4661,7 @@ void server_routes::init_routes() {
return res;
}
auto * res_task = dynamic_cast<server_task_result_metrics*>(result.get());
auto * res_task = dynamic_cast<server_task_result_slots*>(result.get());
GGML_ASSERT(res_task != nullptr);
// optionally return "fail_on_no_slot" error
tools/server/server-task.cpp
+6 -1
View File
@@ -1512,10 +1512,15 @@ json server_task_result_error::to_json() {
//
// server_task_result_metrics
//
json server_task_result_metrics::to_json() {
json server_task_result_slots::to_json() {
return slots_data;
}
json server_task_result_metrics::to_json() {
// not used, /metrics renders prometheus text via to_metrics()
return json{};
}
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
std::string server_task_result_metrics::to_metrics() {
const std::vector<metric_item> counters = {
tools/server/server-task.h
+13 -7
View File
@@ -22,6 +22,7 @@ enum server_task_type {
SERVER_TASK_TYPE_CONTROL,
SERVER_TASK_TYPE_NEXT_RESPONSE,
SERVER_TASK_TYPE_METRICS,
SERVER_TASK_TYPE_SLOT_GET,
SERVER_TASK_TYPE_SLOT_SAVE,
SERVER_TASK_TYPE_SLOT_RESTORE,
SERVER_TASK_TYPE_SLOT_ERASE,
@@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result {
virtual json to_json() override;
};
// used by /metrics API
struct server_task_result_metrics : server_task_result {
// these are immediate stats, not accumulated (server_metrics is cumulative)
int n_idle_slots = 0;
int n_processing_slots = 0;
int n_tasks_deferred = 0;
server_metrics metrics;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
// used by /slots API
virtual json to_json() override;
// used by /metrics API
struct metric_item {
std::string name;
std::string description;
@@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result {
std::string to_metrics();
};
// used by /slots API
struct server_task_result_slots : server_task_result {
int n_idle_slots = 0;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
virtual json to_json() override;
};
struct server_task_result_slot_save_load : server_task_result {
std::string filename;
bool is_save; // true = save, false = load