Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

common: print performance in spec decoding

Browse Source
This commit is contained in:
Sascha Rogmann
2026-01-26 00:20:05 +01:00
parent 288ab50597
commit fd4d803c60
1 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/speculative.cpp
+14 -2
View File
@@ -50,6 +50,7 @@ struct common_speculative_config {
// in a subclass of common_speculative_state
struct common_speculative_state {
const enum common_speculative_type type;
const bool gen_perf = true; // whether to generate performance stats.
size_t drafts_call_count = 0; // number of times this implementation was called.
size_t drafts_generated_count = 0; // number of times a draft or part was generated by this implementation.
@@ -57,6 +58,8 @@ struct common_speculative_state {
size_t drafts_generated_tokens = 0; // number of tokens generated by this implementation.
size_t drafts_accepted_tokens = 0; // number of tokens accepted by the target model.
int64_t gen_duration_ms = 0; // total time spent in this implementation in milliseconds.
virtual ~common_speculative_state() = default;
common_speculative_state(enum common_speculative_type type) : type(type) {}
@@ -737,6 +740,7 @@ llama_tokens common_speculative_gen_draft(
// TODO: avoid dynamic casts
for (auto & impl : spec->impls) {
impl->drafts_call_count++;
const int64_t t_start_ms = impl->gen_perf ? ggml_time_ms() : 0;
switch (impl->type) {
case COMMON_SPECULATIVE_TYPE_NONE:
@@ -801,6 +805,9 @@ llama_tokens common_speculative_gen_draft(
}
}
const int64_t t_now_ms = impl->gen_perf ? ggml_time_ms() : 0;
impl->gen_duration_ms += t_now_ms - t_start_ms; // accumulate duration for this implementation
if (!result.empty()) {
LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__,
common_speculative_type_to_str(impl.get()->type).c_str(),
@@ -848,12 +855,17 @@ void common_speculative_print_stats(const struct common_speculative * spec) {
}
for (const auto & impl : spec->impls) {
LOG_INF("statistics %s: #calls = %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu\n",
// std::string via impl->gen_duration_ms (if >0, else "")
std::string performance = impl->gen_perf
? ", dur = " + std::to_string(impl->gen_duration_ms) + " ms"
: "";
LOG_INF("statistics %s: #calls = %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu%s\n",
common_speculative_type_to_str(impl->type).c_str(),
impl->drafts_call_count,
impl->drafts_generated_count,
impl->drafts_accepted_count,
impl->drafts_generated_tokens,
impl->drafts_accepted_tokens);
impl->drafts_accepted_tokens,
performance.c_str());
}
}