diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index f801f716c..edcae6eb6 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -599,8 +599,9 @@ static size_t estimate_draft_autofit_tax_mb( if(draft_is_mtp_estimate) { draft_ctx_params.ctx_type = LLAMA_CONTEXT_TYPE_MTP; + draft_ctx_params.n_seq_max = base_ctx_params.n_seq_max; draft_ctx_params.n_rs_seq = speculative_chunk_amt; - draft_ctx_params.n_outputs_max = base_ctx_params.n_seq_max; //match the real MTP draft context (see speculative_decoding_setup) so the autofit tax doesn't over-reserve the draft compute buffer at n_batch*n_vocab (~2GB on large-vocab models like Gemma) + draft_ctx_params.n_outputs_max = std::max(1, base_ctx_params.n_seq_max); //match the real MTP draft context so the autofit tax doesn't over-reserve the draft compute buffer at n_batch*n_vocab (~2GB on large-vocab models like Gemma) measure_model_bytes = has_draft_model; } @@ -885,7 +886,7 @@ static void mtp_decoding_setup(llama_model * main_model, llama_context * main_ct mtp_ctx_params.ctx_type = LLAMA_CONTEXT_TYPE_MTP; mtp_ctx_params.ctx_other = main_ctx; mtp_ctx_params.n_rs_seq = 0; - mtp_ctx_params.n_outputs_max = 1; + mtp_ctx_params.n_outputs_max = std::max(1, mtp_ctx_params.n_seq_max); printf("\nAttempting to create built-in MTP context from the main model.\n"); draft_ctx = llama_init_from_model(main_model, mtp_ctx_params); @@ -4266,7 +4267,7 @@ public: std::unique_lock lock(batch_mutex); batch_legacy_waiting++; batch_cv.notify_all(); - batch_cv.wait(lock, [](){ return !batch_has_live_locked(); }); + batch_cv.wait(lock, [](){ return !batch_legacy_active && !batch_has_live_locked(); }); batch_legacy_waiting--; batch_invalidate_legacy_context_locked(); batch_legacy_active = true; @@ -4762,7 +4763,7 @@ static void batch_start_worker_locked() bool gpttype_batch_generate_enabled() { - return continuous_batching_slots > 1 && file_format == FileFormat::GGUF_GENERIC && llama_ctx_v4 && kcpp_data; + return continuous_batching_slots > 1 && file_format == FileFormat::GGUF_GENERIC && llama_ctx_v4 && kcpp_data && !draft_ctx && !guidance_ctx; } int gpttype_batch_generate_submit(const generation_inputs inputs) diff --git a/koboldcpp.py b/koboldcpp.py index 7d2fd4c78..a7dd81df7 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2351,7 +2351,7 @@ def continuous_batching_python_eligible(genparams, api_format): if model_path and not model_path.endswith(".gguf"): utfprint("Batching disabled due to file format",2) return False - if not getattr(args, "noshift", False) or getattr(args, "smartcontext", False) or getattr(args, "draftmodel", "") or getattr(args, "enableguidance", False): + if not getattr(args, "noshift", False) or getattr(args, "smartcontext", False) or getattr(args, "draftmodel", "") or getattr(args, "usemtp", False) or getattr(args, "enableguidance", False): utfprint("Batching disabled due to loaded settings",2) return False if genparams.get("negative_prompt") or genparams.get("images") or genparams.get("audio"):