From 75d2de4bac37cb5eda0cc252840becd6d6e84ad9 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 12 Aug 2026 00:33:11 +0200 Subject: [PATCH] rm result_timings --- tools/server/server-common.cpp | 27 +++++++++++ tools/server/server-common.h | 62 ++++++++++++++++++++++++ tools/server/server-context.cpp | 83 +-------------------------------- tools/server/server-task.cpp | 62 +++++++----------------- tools/server/server-task.h | 24 +--------- 5 files changed, 110 insertions(+), 148 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c9109fc962..5c4fa88677 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -58,6 +58,33 @@ json format_error_response(const std::string & message, const enum error_type ty }; } +// +// server_slot_stats +// + +json server_slot_stats::to_json() const { + json base = { + {"cache_n", n_prompt_cached}, + + {"prompt_n", n_prompt_processed}, + {"prompt_ms", t_prompt_ms()}, + {"prompt_per_token_ms", t_prompt_ms() / n_prompt_processed}, + {"prompt_per_second", n_prompt_tps()}, + + {"predicted_n", n_predict}, + {"predicted_ms", t_gen_ms()}, + {"predicted_per_token_ms", t_gen_ms() / n_predict}, + {"predicted_per_second", n_gen_tps()}, + }; + + if (n_draft_tokens > 0) { + base["draft_n"] = n_draft_tokens; + base["draft_n_accepted"] = n_draft_accepted; + } + + return base; +} + // // random string / id // diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 5fa0475b5d..39564795a9 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -449,6 +449,68 @@ struct server_pipe { struct server_slot; +// shared between server_slot and server_task_result_* +struct server_slot_stats { + uint64_t n_prompt_cached = 0; + uint64_t n_prompt_processed = 0; + uint64_t n_predict = 0; + + // Speculative decoding stats (mirror server_metrics) + uint64_t n_draft_tokens = 0; + uint64_t n_draft_accepted = 0; + uint64_t n_draft_verif_steps = 0; + std::vector n_accepted_per_pos; + + // these are absolute timestamps (in us) + uint64_t t_start = 0; + uint64_t t_prompt_last = 0; + uint64_t t_gen_last = 0; + + // can only move one direction: start -> prompt -> gen + void update_prompt_start() { + GGML_ASSERT(t_start == 0); + t_start = ggml_time_us(); + } + void update_prompt_last() { + GGML_ASSERT(t_start > 0); + t_prompt_last = ggml_time_us(); + } + void update_gen_last() { + GGML_ASSERT(t_prompt_last > 0); + t_gen_last = ggml_time_us(); + } + + // these are time durations + int64_t t_ellapsed_us() const { + return ggml_time_us() - t_start; + } + double t_prompt_ms() const { + return (t_prompt_last - t_start) / 1000.0; + } + double t_gen_ms() const { + // clamp to 1 us to avoid division by zero on the caller side + return std::max(1, t_gen_last - t_prompt_last) / 1000.0; + } + + // other derived metrics + double n_prompt_tps() const { + const double t_ms = t_prompt_ms(); + return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0; + } + double n_gen_tps() const { + const double t_ms = t_gen_ms(); + return t_ms > 0.0 ? 1e3 / t_ms * n_predict : 0.0; + } + + // false if the slot never started, i.e. the task result carries no stats + bool is_set() const { + return t_start > 0; + } + + json to_json() const; +}; + +// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot struct server_metrics { int64_t t_start = 0; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a1e69ce1e0..7a376acd2f 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -192,59 +192,6 @@ struct server_batch { } }; -struct server_slot_stats { - uint64_t n_prompt_cached = 0; - uint64_t n_prompt_processed = 0; - uint64_t n_predict = 0; - - // Speculative decoding stats (mirror server_metrics) - uint64_t n_draft_tokens = 0; - uint64_t n_draft_accepted = 0; - uint64_t n_draft_verif_steps = 0; - std::vector n_accepted_per_pos; - - // these are absolute timestamps (in us) - uint64_t t_start = 0; - uint64_t t_prompt_last = 0; - uint64_t t_gen_last = 0; - - // can only move one direction: start -> prompt -> gen - void update_prompt_start() { - GGML_ASSERT(t_start == 0); - t_start = ggml_time_us(); - } - void update_prompt_last() { - GGML_ASSERT(t_start > 0); - t_prompt_last = ggml_time_us(); - } - void update_gen_last() { - GGML_ASSERT(t_prompt_last > 0); - t_gen_last = ggml_time_us(); - } - - // these are time durations - int64_t t_ellapsed_us() const { - return ggml_time_us() - t_start; - } - double t_prompt_ms() const { - return (t_prompt_last - t_start) / 1000.0; - } - double t_gen_ms() const { - // clamp to 1 us to avoid division by zero on the caller side - return std::max(1, t_gen_last - t_prompt_last) / 1000.0; - } - - // other derived metrics - double n_prompt_tps() const { - const double t_ms = t_prompt_ms(); - return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0; - } - double n_gen_tps() const { - const double t_ms = t_gen_ms(); - return t_ms > 0.0 ? 1e3 / t_ms * n_predict : 0.0; - } -}; - struct server_slot { int id; @@ -566,32 +513,6 @@ struct server_slot { } } - result_timings get_timings() const { - const double t_prompt_processing = stats.t_prompt_ms(); - const double t_token_generation = stats.t_gen_ms(); - - result_timings timings; - timings.cache_n = stats.n_prompt_cached; - - timings.prompt_n = stats.n_prompt_processed; - timings.prompt_ms = t_prompt_processing; - timings.prompt_per_token_ms = t_prompt_processing / stats.n_prompt_processed; - timings.prompt_per_second = stats.n_prompt_tps(); - - timings.predicted_n = stats.n_predict; - timings.predicted_ms = t_token_generation; - timings.predicted_per_token_ms = t_token_generation / stats.n_predict; - timings.predicted_per_second = stats.n_gen_tps(); - - // Add speculative metrics - if (stats.n_draft_tokens > 0) { - timings.draft_n = stats.n_draft_tokens; - timings.draft_n_accepted = stats.n_draft_accepted; - } - - return timings; - } - size_t find_stopping_strings(const std::string & text, const size_t last_token_size, bool is_full_stop) { GGML_ASSERT(task); @@ -2123,7 +2044,7 @@ private: // populate timings if this is final response or timings_per_token is enabled if (slot.stop != STOP_TYPE_NONE || slot.task->params.timings_per_token) { - res->timings = slot.get_timings(); + res->stats = slot.stats; } queue_results.send(std::move(res)); @@ -2150,7 +2071,7 @@ private: res->content = std::move(slot.generated_text); res->tokens = std::move(slot.generated_tokens); } - res->timings = slot.get_timings(); + res->stats = slot.stats; res->prompt = slot.task->tokens.detokenize(ctx_tgt, true); res->response_fields = std::move(slot.task->params.response_fields); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index ddfa942087..bd9d6d51a4 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -236,34 +236,6 @@ common_chat_msg task_result_state::update_chat_msg( return chat_msg; } -// - -// result_timings -// - -json result_timings::to_json() const { - json base = { - {"cache_n", cache_n}, - - {"prompt_n", prompt_n}, - {"prompt_ms", prompt_ms}, - {"prompt_per_token_ms", prompt_per_token_ms}, - {"prompt_per_second", prompt_per_second}, - - {"predicted_n", predicted_n}, - {"predicted_ms", predicted_ms}, - {"predicted_per_token_ms", predicted_per_token_ms}, - {"predicted_per_second", predicted_per_second}, - }; - - if (draft_n > 0) { - base["draft_n"] = draft_n; - base["draft_n_accepted"] = draft_n_accepted; - } - - return base; -} - // // result_prompt_progress // @@ -382,7 +354,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() { {"stop_type", stop_type_to_str(stop)}, {"stopping_word", stopping_word}, {"tokens_cached", n_tokens_cached}, - {"timings", timings.to_json()}, + {"timings", stats.to_json()}, }; if (!stream && !probs_output.empty()) { res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs); @@ -432,8 +404,8 @@ json server_task_result_cmpl_final::to_json_oaicompat() { if (verbose) { res["__verbose"] = to_json_non_oaicompat(); } - if (timings.prompt_n >= 0) { - res.push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + res.push_back({"timings", stats.to_json()}); } return res; @@ -480,8 +452,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() { if (verbose) { res["__verbose"] = to_json_non_oaicompat(); } - if (timings.prompt_n >= 0) { - res.push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + res.push_back({"timings", stats.to_json()}); } return res; @@ -541,8 +513,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() { }); } - if (timings.prompt_n >= 0) { - deltas.back().push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + deltas.back().push_back({"timings", stats.to_json()}); } // extra fields for debugging purposes @@ -734,8 +706,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() { }} }); - if (timings.prompt_n >= 0) { - server_sent_events.back().at("data").push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + server_sent_events.back().at("data").push_back({"timings", stats.to_json()}); } return server_sent_events; @@ -1086,8 +1058,8 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() { {"tokens_evaluated", n_prompt_tokens}, }; // populate the timings object when needed (usually for the last response or with timings_per_token enabled) - if (timings.prompt_n > 0) { - res.push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + res.push_back({"timings", stats.to_json()}); } if (is_progress) { res.push_back({"prompt_progress", progress.to_json()}); @@ -1126,8 +1098,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat() { if (verbose) { res["__verbose"] = to_json_non_oaicompat(); } - if (timings.prompt_n >= 0) { - res.push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + res.push_back({"timings", stats.to_json()}); } if (is_progress) { res.push_back({"prompt_progress", progress.to_json()}); @@ -1180,8 +1152,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() { }; } - if (timings.prompt_n >= 0) { - last_json.push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + last_json.push_back({"timings", stats.to_json()}); } if (is_progress) { last_json.push_back({"prompt_progress", progress.to_json()}); @@ -1330,8 +1302,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() { if (!events.empty()) { json & data = events.back().at("data"); - if (timings.prompt_n >= 0) { - data.push_back({"timings", timings.to_json()}); + if (stats.is_set()) { + data.push_back({"timings", stats.to_json()}); } if (is_progress) { data.push_back({"prompt_progress", progress.to_json()}); diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 41640805e2..bd6767f833 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -259,26 +259,6 @@ struct server_task { } }; -struct result_timings { - int32_t cache_n = -1; - - int32_t prompt_n = -1; - double prompt_ms = 0.0; - double prompt_per_token_ms = 0.0; - double prompt_per_second = 0.0; - - int32_t predicted_n = -1; - double predicted_ms = 0.0; - double predicted_per_token_ms = 0.0; - double predicted_per_second = 0.0; - - // Optional speculative metrics - only included when > 0 - int32_t draft_n = 0; - int32_t draft_n_accepted = 0; - - json to_json() const; -}; - struct result_prompt_progress { int32_t total = 0; int32_t cache = 0; @@ -343,7 +323,7 @@ struct server_task_result_cmpl_final : server_task_result { bool stream; bool include_usage; - result_timings timings; + server_slot_stats stats; std::string prompt; bool truncated; @@ -425,7 +405,7 @@ struct server_task_result_cmpl_partial : server_task_result { bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream) // ref: https://github.com/ggml-org/llama.cpp/pull/23884 completion_token_output prob_output; - result_timings timings; + server_slot_stats stats; result_prompt_progress progress; // response formatting