From 51a4f6303bf88a7d8b6dc9ac3ae4b6e99ed0dd12 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 15 Aug 2026 15:39:14 +0200 Subject: [PATCH] run common_speculative_process in worker --- tools/server/server-context.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a7d382886a..53521fc667 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2947,8 +2947,10 @@ private: }); // generate the actual drafts (if any) - { - common_speculative_draft(spec.get()); + if (!drafting.empty()) { + queue_tasks.yield_to_queue([&]() { + common_speculative_draft(spec.get()); + }); } // make checkpoints if needed @@ -3644,11 +3646,18 @@ private: // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] // for now, always re-evaluate for simplicity // ref: https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4400925384 - if (!common_speculative_process(spec.get(), batch_view)) { - SRV_ERR("%s", "failed to process speculative batch\n"); + if (spec) { + bool ok = true; + queue_tasks.yield_to_queue([&]() { + ok = common_speculative_process(spec.get(), batch_view); + }); - // TODO: handle error - throw std::runtime_error("failed to process speculative batch"); + if (!ok) { + SRV_ERR("%s", "failed to process speculative batch\n"); + + // TODO: handle error + throw std::runtime_error("failed to process speculative batch"); + } } // handle `n_cmpl > 1` tasks - when the main prompt is processed, activate all child tasks too