Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-25 22:21:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

prevent MTP drafting with batching

Browse Source
This commit is contained in:
Concedo
2026-07-01 16:17:13 +08:00
parent 0626395511
commit 983dec9a54
2 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+5 -4
View File
@@ -599,8 +599,9 @@ static size_t estimate_draft_autofit_tax_mb(
if(draft_is_mtp_estimate)
{
draft_ctx_params.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
draft_ctx_params.n_seq_max = base_ctx_params.n_seq_max;
draft_ctx_params.n_rs_seq = speculative_chunk_amt;
draft_ctx_params.n_outputs_max = base_ctx_params.n_seq_max; //match the real MTP draft context (see speculative_decoding_setup) so the autofit tax doesn't over-reserve the draft compute buffer at n_batch*n_vocab (~2GB on large-vocab models like Gemma)
draft_ctx_params.n_outputs_max = std::max<uint32_t>(1, base_ctx_params.n_seq_max); //match the real MTP draft context so the autofit tax doesn't over-reserve the draft compute buffer at n_batch*n_vocab (~2GB on large-vocab models like Gemma)
measure_model_bytes = has_draft_model;
}
@@ -885,7 +886,7 @@ static void mtp_decoding_setup(llama_model * main_model, llama_context * main_ct
mtp_ctx_params.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
mtp_ctx_params.ctx_other = main_ctx;
mtp_ctx_params.n_rs_seq = 0;
mtp_ctx_params.n_outputs_max = 1;
mtp_ctx_params.n_outputs_max = std::max<uint32_t>(1, mtp_ctx_params.n_seq_max);
printf("\nAttempting to create built-in MTP context from the main model.\n");
draft_ctx = llama_init_from_model(main_model, mtp_ctx_params);
@@ -4266,7 +4267,7 @@ public:
std::unique_lock<std::mutex> lock(batch_mutex);
batch_legacy_waiting++;
batch_cv.notify_all();
batch_cv.wait(lock, [](){ return !batch_has_live_locked(); });
batch_cv.wait(lock, [](){ return !batch_legacy_active && !batch_has_live_locked(); });
batch_legacy_waiting--;
batch_invalidate_legacy_context_locked();
batch_legacy_active = true;
@@ -4762,7 +4763,7 @@ static void batch_start_worker_locked()
bool gpttype_batch_generate_enabled()
{
return continuous_batching_slots > 1 && file_format == FileFormat::GGUF_GENERIC && llama_ctx_v4 && kcpp_data;
return continuous_batching_slots > 1 && file_format == FileFormat::GGUF_GENERIC && llama_ctx_v4 && kcpp_data && !draft_ctx && !guidance_ctx;
}
int gpttype_batch_generate_submit(const generation_inputs inputs)
koboldcpp.py
+1 -1
View File
@@ -2351,7 +2351,7 @@ def continuous_batching_python_eligible(genparams, api_format):
if model_path and not model_path.endswith(".gguf"):
utfprint("Batching disabled due to file format",2)
return False
if not getattr(args, "noshift", False) or getattr(args, "smartcontext", False) or getattr(args, "draftmodel", "") or getattr(args, "enableguidance", False):
if not getattr(args, "noshift", False) or getattr(args, "smartcontext", False) or getattr(args, "draftmodel", "") or getattr(args, "usemtp", False) or getattr(args, "enableguidance", False):
utfprint("Batching disabled due to loaded settings",2)
return False
if genparams.get("negative_prompt") or genparams.get("images") or genparams.get("audio"):