Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-11 15:29:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

model: add dots3-note (#27060)

Browse Source 
* text: conversion

* init impl

* address review comments

* fix rope

* move to a new llama_kv_cache_dsa_iswa
This commit is contained in:
Xuan-Son Nguyen
2026-08-21 19:52:34 +02:00
committed by GitHub
parent 873e5d8e39
commit 5a32f7b66e
20 changed files with 1412 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+59 -1
View File
@@ -11,6 +11,7 @@
#include "llama-kv-cache.h"
#include "llama-kv-cache-iswa.h"
#include "llama-kv-cache-dsa.h"
#include "llama-kv-cache-dsa-iswa.h"
#include "llama-kv-cache-msa.h"
#include "llama-kv-cache-dsv4.h"
#include "llama-memory-hybrid.h"
@@ -194,6 +195,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_deepseek2ocr(params);
case LLM_ARCH_DEEPSEEK32:
return new llama_model_deepseek32(params);
case LLM_ARCH_DOTS3NOTE:
return new llama_model_dots3note(params);
case LLM_ARCH_DEEPSEEK4:
return new llama_model_deepseek4(params);
case LLM_ARCH_GLM_DSA:
@@ -851,6 +854,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_230B_A10B: return "230B.A10B";
case LLM_TYPE_428B_A23B: return "428B.A23B";
case LLM_TYPE_235B_A22B: return "235B.A22B";
case LLM_TYPE_288B_A19B: return "288B.A19B";
case LLM_TYPE_300B_A47B: return "300B.A47B";
case LLM_TYPE_310B_A15B: return "310B.A15B";
case LLM_TYPE_355B_A32B: return "355B.A32B";
@@ -1924,7 +1928,9 @@ void llama_model::print_info() const {
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
}
if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) {
if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR ||
arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA ||
arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
@@ -2193,6 +2199,57 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
nullptr);
}
} break;
case LLM_ARCH_DOTS3NOTE:
{
GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) {
// MTP draft context: plain attention KV cache holding only the nextn layer
llama_kv_cache::layer_filter_cb filter =
[&](uint32_t il) { return il >= hparams.n_layer(); };
res = new llama_kv_cache(
*this,
hparams,
params.type_k,
params.type_v,
!cparams.flash_attn,
cparams.offload_kqv,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
filter,
nullptr,
nullptr);
} else {
// main context: DSA cache for the trunk full-attention layers plus a window-sized SWA cache
llama_kv_cache::layer_filter_cb filter_mla = nullptr;
if (hparams.n_layer_nextn > 0) {
filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); };
}
llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && hparams.is_indexer_full(il); };
res = new llama_kv_cache_dsa_iswa(
*this,
params.type_k,
params.type_v,
!cparams.flash_attn,
cparams.offload_kqv,
params.swa_full,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
filter_mla,
filter_lid,
nullptr);
}
} break;
case LLM_ARCH_DEEPSEEK4:
{
GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
@@ -2661,6 +2718,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_LLAMA_EMBED:
case LLM_ARCH_MAINCODER:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_NANBEIGE:
case LLM_ARCH_POCKETTTS:
return LLAMA_ROPE_TYPE_NORM;