diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 5f22a62f26..1c94d0d6e6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2298,8 +2298,8 @@ private: // returns false to decline the task, it is offered again after the decode is done bool process_single_task(server_task && task, bool is_yielding) { - // while yielding, an encode / decode is running and only accessing metrics is safe - if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS) { + // while yielding, an encode / decode is running and only reading the server state is safe + if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS && task.type != SERVER_TASK_TYPE_SLOT_GET) { SRV_DBG("decoding, decline task, id_task = %d\n", task.id); return false; } @@ -2425,28 +2425,17 @@ private: } break; case SERVER_TASK_TYPE_METRICS: { - json slots_data = json::array(); - - int n_idle_slots = 0; int n_processing_slots = 0; for (server_slot & slot : slots) { - json slot_data = slot.to_json(slots_debug == 0); - if (slot.is_processing()) { n_processing_slots++; - } else { - n_idle_slots++; } - - slots_data.push_back(slot_data); } - SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); + SRV_DBG("n_processing_slots = %d\n", n_processing_slots); auto res = std::make_unique(); res->id = task.id; - res->slots_data = std::move(slots_data); - res->n_idle_slots = n_idle_slots; res->n_processing_slots = n_processing_slots; res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); res->metrics = metrics; @@ -2454,6 +2443,28 @@ private: if (task.metrics_reset_bucket) { metrics.reset_bucket(); } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_GET: + { + json slots_data = json::array(); + + int n_idle_slots = 0; + + for (server_slot & slot : slots) { + if (!slot.is_processing()) { + n_idle_slots++; + } + + slots_data.push_back(slot.to_json(slots_debug == 0)); + } + SRV_DBG("n_idle_slots = %d\n", n_idle_slots); + + auto res = std::make_unique(); + res->id = task.id; + res->slots_data = std::move(slots_data); + res->n_idle_slots = n_idle_slots; + queue_results.send(std::move(res)); } break; case SERVER_TASK_TYPE_SLOT_SAVE: @@ -4581,7 +4592,6 @@ void server_routes::init_routes() { res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start); server_task_result_metrics tmp; tmp.metrics = cached_metrics; - tmp.n_idle_slots = params.n_parallel; res->data = tmp.to_metrics(); }; @@ -4633,7 +4643,7 @@ void server_routes::init_routes() { // request slots data using task queue { - server_task task(SERVER_TASK_TYPE_METRICS); + server_task task(SERVER_TASK_TYPE_SLOT_GET); task.id = res->rd.get_new_id(); res->rd.post_task(std::move(task), true); // high-priority task } @@ -4651,7 +4661,7 @@ void server_routes::init_routes() { return res; } - auto * res_task = dynamic_cast(result.get()); + auto * res_task = dynamic_cast(result.get()); GGML_ASSERT(res_task != nullptr); // optionally return "fail_on_no_slot" error diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 64afbc5edf..258cdcf8fb 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1512,10 +1512,15 @@ json server_task_result_error::to_json() { // // server_task_result_metrics // -json server_task_result_metrics::to_json() { +json server_task_result_slots::to_json() { return slots_data; } +json server_task_result_metrics::to_json() { + // not used, /metrics renders prometheus text via to_metrics() + return json{}; +} + // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names std::string server_task_result_metrics::to_metrics() { const std::vector counters = { diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 906a28dc67..25ff015122 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -22,6 +22,7 @@ enum server_task_type { SERVER_TASK_TYPE_CONTROL, SERVER_TASK_TYPE_NEXT_RESPONSE, SERVER_TASK_TYPE_METRICS, + SERVER_TASK_TYPE_SLOT_GET, SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE, @@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result { virtual json to_json() override; }; +// used by /metrics API struct server_task_result_metrics : server_task_result { // these are immediate stats, not accumulated (server_metrics is cumulative) - int n_idle_slots = 0; int n_processing_slots = 0; int n_tasks_deferred = 0; server_metrics metrics; - // while we can also use std::vector this requires copying the slot object which can be quite messy - // therefore, we use json to temporarily store the slot.to_json() result - json slots_data = json::array(); - - // used by /slots API virtual json to_json() override; - // used by /metrics API struct metric_item { std::string name; std::string description; @@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result { std::string to_metrics(); }; +// used by /slots API +struct server_task_result_slots : server_task_result { + int n_idle_slots = 0; + + // while we can also use std::vector this requires copying the slot object which can be quite messy + // therefore, we use json to temporarily store the slot.to_json() result + json slots_data = json::array(); + + virtual json to_json() override; +}; + struct server_task_result_slot_save_load : server_task_result { std::string filename; bool is_save; // true = save, false = load