Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-10 14:59:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

model: MTP support for Qwen3-Next (#25589)

Browse Source 
* mtp for qwen3nex

* fix for python type-check

* Fix to compute num_mtp from directly mtp layer

* define opt_num_mtp_layers in _QwenMtpMixin and fix some comments

* Fix for python type check

* Update gguf-py/gguf/constants.py

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* rebase and add load_mtp flags

* Update src/models/qwen3next.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* Update src/models/qwen3next.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Masashi Yoshimura
2026-08-03 17:15:01 +09:00
committed by GitHub
parent 2b63e0610b
commit 4ed2b13f75
5 changed files with 389 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+6 -6
View File
@@ -2195,11 +2195,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// checks
default:
{
// The MTP head is dense-attention only on hybrid Qwen3.5/3.6, so use a plain
// The MTP head is dense-attention only on hybrid Qwen3-Next/3.5/3.6, so use a plain
// attention KV cache for the MTP context instead of the hybrid wrapper.
const bool mtp_on_hybrid_qwen35 =
const bool mtp_on_hybrid_qwen =
params.ctx_type == LLAMA_CONTEXT_TYPE_MTP &&
(arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE);
(arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE);
if (llm_arch_is_recurrent(arch)) {
res = new llama_memory_recurrent(
@@ -2211,7 +2211,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_seq_max,
cparams.n_rs_seq,
nullptr);
} else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen35) {
} else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen) {
// The main difference between hybrid architectures is the
// layer filters, so pick the right one here
llama_memory_hybrid::layer_filter_cb filter_attn = nullptr;
@@ -2226,7 +2226,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_recr = [&](uint32_t il) {
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
};
} else if (arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) {
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) {
filter_attn = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il);
};
@@ -2292,7 +2292,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
};
}
if (mtp_on_hybrid_qwen35) {
if (mtp_on_hybrid_qwen) {
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
}