diff --git a/common/arg.cpp b/common/arg.cpp index b82bbd188..83c097116 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1899,7 +1899,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, bool value) { params.conversation_mode = value ? COMMON_CONVERSATION_MODE_ENABLED : COMMON_CONVERSATION_MODE_DISABLED; } - ).set_examples({LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); add_opt(common_arg( {"-st", "--single-turn"}, "run conversation for a single turn only, then exit when done\n" diff --git a/common/common.cpp b/common/common.cpp index 47831f64e..c71068016 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1300,11 +1300,34 @@ common_init_result::common_init_result(common_params & params, bool model_only) if (params.fit_params) { COM_TRC("%s", "fitting params to device memory ...\n"); COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n"); + + // the draft context is created from the same base params and follows the main context, fit both together + const bool has_draft = params.speculative.has_dft(); + const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + + common_params params_dft = common_base_params_to_speculative(params); + + auto mparams_dft = common_model_params_to_llama(params_dft); + auto cparams_dft = common_context_params_to_llama(params_dft); + if (spec_mtp) { + cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP; + } + cparams_dft.n_rs_seq = 0; + + const common_fit_extra_model extra = { + /*.path_model =*/ params_dft.model.path.c_str(), + /*.mparams =*/ &mparams_dft, + /*.cparams =*/ &cparams_dft, + /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model + }; + common_fit_params(params.model.path.c_str(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, + has_draft || spec_mtp ? &extra : nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); } diff --git a/common/fit.cpp b/common/fit.cpp index dd1f3ef76..c601fe405 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -178,7 +178,7 @@ common_device_memory_data_vec common_get_device_memory_data( static void common_params_fit_impl( const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, - size_t * margins_s, uint32_t n_ctx_min, enum ggml_log_level log_level) { + size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) { throw common_params_fit_exception("llama_params_fit is not implemented for SPLIT_MODE_TENSOR, abort"); } @@ -191,10 +191,92 @@ static void common_params_fit_impl( uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert + // with non-unified kv, we need to take into account n_streams + // for example, if memory can hold more than model's trained context size, we must extend the n_ctx to hold enough n_streams + const uint32_t n_streams = cparams->kv_unified ? 1 : std::max(1, cparams->n_seq_max); + const bool n_ctx_auto = cparams->n_ctx == 0; + + dmds_t dmds_extra; // memory of the extra model, laid out on the devices of the main model + uint32_t n_ctx_extra = 0; // context that memory was measured at + + // the extra model competes for the same memory as the main model, add it to every measurement + // its memory is measured again whenever the context it follows changes + auto add_extra_memory = [&](dmds_t & dmds) { + if (extra == nullptr) { + return; + } + + if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) { + std::vector devs_extra; + uint32_t ngl_extra = 0; + uint32_t nct_extra = 0; + uint32_t nex_extra = 0; + + extra->cparams->n_ctx = cparams->n_ctx; + + LOG_TRC("%s: getting device memory data for the extra model at a context size of %" PRIu32 ":\n", + __func__, cparams->n_ctx); + + dmds_t measured; + try { + measured = common_get_device_memory_data_impl( + extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level); + } catch (const std::runtime_error & e) { + // the extra model is optional, fit the main model alone rather than giving up + LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what()); + dmds_extra = dmds_t(devs.size() + 1); + n_ctx_extra = cparams->n_ctx; + return; + } + + dmds_extra = dmds_t(devs.size() + 1); + dmds_extra.back().mb = measured.back().mb; + for (size_t je = 0; je < devs_extra.size(); je++) { + for (size_t id = 0; id < devs.size(); id++) { + if (devs_extra[je] == devs[id]) { + dmds_extra[id].mb.model += measured[je].mb.model; + dmds_extra[id].mb.context += measured[je].mb.context; + dmds_extra[id].mb.compute += measured[je].mb.compute; + break; + } + } + } + if (extra->shares_model) { + for (llama_device_memory_data & dmd : dmds_extra) { + dmd.mb.model = 0; + } + } + + n_ctx_extra = cparams->n_ctx; + } + + for (size_t id = 0; id < dmds.size(); id++) { + dmds[id].mb.model += dmds_extra[id].mb.model; + dmds[id].mb.context += dmds_extra[id].mb.context; + dmds[id].mb.compute += dmds_extra[id].mb.compute; + } + }; + // step 1: get data for default parameters and check whether any changes are necessary in the first place LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__); - const dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + + // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min: + const uint32_t n_ctx_max = (uint32_t) std::min(uint64_t(hp_nct) * n_streams, UINT32_MAX); + const uint32_t n_ctx_min_total = (uint32_t) std::min(uint64_t(n_ctx_min) * n_streams, UINT32_MAX); + + // llama_context would use only hp_nct in total for n_ctx == 0, resolve the context before measuring anything else: + if (n_ctx_auto) { + cparams->n_ctx = n_ctx_max; + if (n_streams > 1) { + LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n", + __func__, n_ctx_max, n_streams); + dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + } + } + add_extra_memory(dmds_full); + const size_t nd = devs.size(); // number of devices std::vector margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits @@ -307,8 +389,8 @@ static void common_params_fit_impl( "%s: cannot meet free memory targets on all devices, need to use %" PRId64 " MiB less in total\n", __func__, -global_surplus/MiB); } - if (cparams->n_ctx == 0) { - if (hp_nct > n_ctx_min) { + if (n_ctx_auto) { + if (n_ctx_max > n_ctx_min_total) { int64_t sum_used_target = sum_free; if (nd == 0) { sum_used_target -= margins[0]; @@ -328,8 +410,9 @@ static void common_params_fit_impl( } int64_t sum_projected_used_min_ctx = 0; - cparams->n_ctx = n_ctx_min; - const dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + cparams->n_ctx = n_ctx_min_total; + dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + add_extra_memory(dmds_min_ctx); if (nd == 0) { sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total(); } else { @@ -339,14 +422,16 @@ static void common_params_fit_impl( } if (sum_used_target > sum_projected_used_min_ctx) { // linear interpolation between minimum and maximum context size: - cparams->n_ctx += (hp_nct - n_ctx_min) * (sum_used_target - sum_projected_used_min_ctx) + cparams->n_ctx += (n_ctx_max - n_ctx_min_total) * (sum_used_target - sum_projected_used_min_ctx) / (sum_projected_used - sum_projected_used_min_ctx); - cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % 256, n_ctx_min); // round down context for CUDA backend + // round down context for CUDA backend, keep it divisible by the number of streams: + const uint32_t align = 256 * n_streams; + cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % align, n_ctx_min_total); - const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (hp_nct - n_ctx_min); - const int64_t memory_reduction = (hp_nct - cparams->n_ctx) * bytes_per_ctx; + const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (n_ctx_max - n_ctx_min_total); + const int64_t memory_reduction = (n_ctx_max - cparams->n_ctx) * bytes_per_ctx; LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n", - __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB); + __func__, n_ctx_max, cparams->n_ctx, memory_reduction/MiB); if (nd <= 1) { LOG_TRC("%s: entire model can be fit by reducing context\n", __func__); return; @@ -355,14 +440,14 @@ static void common_params_fit_impl( } else { const int64_t memory_reduction = sum_projected_used - sum_projected_used_min_ctx; LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n", - __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB); + __func__, n_ctx_max, cparams->n_ctx, memory_reduction/MiB); } } else { if (n_ctx_min == UINT32_MAX) { - LOG_TRC("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, hp_nct); + LOG_TRC("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, n_ctx_max); } else { LOG_TRC("%s: default model context size is %" PRIu32 " which is <= the min. context size of %" PRIu32 " -> no change\n", - __func__, hp_nct, n_ctx_min); + __func__, n_ctx_max, n_ctx_min_total); } } } else { @@ -507,8 +592,9 @@ static void common_params_fit_impl( llama_model_params mparams_copy = *mparams; set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy); - const dmds_t dmd_nl = common_get_device_memory_data_impl( + dmds_t dmd_nl = common_get_device_memory_data_impl( path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + add_extra_memory(dmd_nl); LOG_TRC("%s: memory for test allocation by device:\n", func_name); for (size_t id = 0; id < nd; id++) { @@ -535,8 +621,9 @@ static void common_params_fit_impl( mparams->tensor_buft_overrides = tensor_buft_overrides; LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__); - const dmds_t dmds_cpu_moe = common_get_device_memory_data_impl( + dmds_t dmds_cpu_moe = common_get_device_memory_data_impl( path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + add_extra_memory(dmds_cpu_moe); for (size_t id = 0; id < nd; id++) { global_surplus_cpu_moe += dmds_cpu_moe[id].free; @@ -796,11 +883,12 @@ enum common_params_fit_status common_fit_params( llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins, uint32_t n_ctx_min, + const common_fit_extra_model * extra, ggml_log_level log_level) { const int64_t t0_us = llama_time_us(); common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS; try { - common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, log_level); + common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); LOG_TRC("%s: successfully fit params to free device memory\n", __func__); } catch (const common_params_fit_exception & e) { LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what()); diff --git a/common/fit.h b/common/fit.h index 208fc3069..824d386b0 100644 --- a/common/fit.h +++ b/common/fit.h @@ -11,6 +11,16 @@ enum common_params_fit_status { COMMON_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path }; +// a second model that shares the devices of the main model, e.g. a draft model +// - its context follows the context of the main model, so its memory is measured again whenever that context changes +// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context +struct common_fit_extra_model { + const char * path_model; + llama_model_params * mparams; + llama_context_params * cparams; + bool shares_model; +}; + // fits mparams and cparams to free device memory (assumes system memory is unlimited) // - returns true if the parameters could be successfully modified to fit device memory // - this function is NOT thread safe because it modifies the global llama logger state @@ -24,6 +34,7 @@ common_params_fit_status common_fit_params( llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements size_t * margins, // margins of memory to leave per device in bytes uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use + const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log // print estimated memory to stdout diff --git a/common/speculative.cpp b/common/speculative.cpp index 75cfd1b54..b5032afce 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2322,6 +2322,9 @@ common_params common_base_params_to_speculative(const common_params & params) { const auto & params_spec = params.speculative.draft; common_params result = params; + result.embedding = false; + result.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; + if (has_draft) { result.devices = params_spec.devices; result.model = params_spec.mparams; @@ -2385,6 +2388,9 @@ common_speculative_init_result::common_speculative_init_result( cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP; } + // the draft context holds as many tokens per sequence as the target context + cparams.n_ctx = llama_n_ctx(ctx_tgt); + // note: for small models maybe we can set this to the maximum possible draft from all speculative types // the extra memory for small models is likely negligible? cparams.n_rs_seq = 0; diff --git a/conversion/__init__.py b/conversion/__init__.py index 4b8817ead..8de97e959 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -58,12 +58,16 @@ TEXT_MODEL_MAP: dict[str, str] = { "DSparkDraftModel": "qwen", "DSparkSpeculator": "qwen", "Lfm2DSparkDraftModel": "qwen", + "LingDSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", "Dots1ForCausalLM": "dots1", + "Dots3NoteForCausalLM": "dots3", + "Dots3NoteForConditionalGeneration": "dots3", + "Dots3NoteTextForCausalLM": "dots3", "DotsOCRForCausalLM": "qwen", "DreamModel": "dream", "Ernie4_5ForCausalLM": "ernie", @@ -279,6 +283,8 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "CogVLMForCausalLM": "cogvlm", "DeepseekOCR2ForCausalLM": "deepseek", "DeepseekOCRForCausalLM": "deepseek", + "Dots3NoteForCausalLM": "dots3", + "Dots3NoteForConditionalGeneration": "dots3", "DotsOCRForCausalLM": "dotsocr", "Exaone4_5_ForConditionalGeneration": "exaone", "Gemma3ForConditionalGeneration": "gemma", diff --git a/conversion/dots3.py b/conversion/dots3.py new file mode 100644 index 000000000..c7ac2319e --- /dev/null +++ b/conversion/dots3.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import math +import re + +import torch + +from typing import TYPE_CHECKING, Any, Callable, Iterable + +if TYPE_CHECKING: + from torch import Tensor + +from .base import MmprojModel, ModelBase, gguf + +from .deepseek import DeepseekV2Model + + +@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration", "Dots3NoteTextForCausalLM") +class Dots3NoteModel(DeepseekV2Model): + model_arch = gguf.MODEL_ARCH.DOTS3NOTE + skip_mtp = False + supports_mtp_export = True + + # trunk layer count, stashed before indexing for filter_tensors (mirrors DeepseekV32Model) + _n_main_layers: int | None = None + + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + hparams = self.hparams + + # config file doesn't specify MTP block, detect it from model weight + self.n_nextn = 1 if "model.mtp.embed_tokens.weight" in self.model_tensors else 0 + if self.n_nextn: + self.block_count += self.n_nextn + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + self.layer_types = hparams["layer_types"] + if len(self.layer_types) < hparams["num_hidden_layers"]: + raise ValueError("layer_types is shorter than num_hidden_layers") + + if hparams.get("use_dsa", True) is not True: + raise ValueError("dots3-note conversion requires use_dsa=true") + if hparams.get("normalization", "RMSNorm") != "RMSNorm" or hparams.get("final_norm", "RMSNorm") != "RMSNorm": + raise ValueError("dots3-note conversion only supports RMSNorm") + if hparams.get("k_rope_only_layernorm", True) is not True: + raise ValueError("dots3-note conversion requires k_rope_only_layernorm=true") + if hparams.get("topk_method", "noaux_tc") != "noaux_tc" or hparams.get("scoring_func") != "sigmoid": + raise ValueError("dots3-note conversion only supports noaux_tc/sigmoid expert gating") + if hparams.get("n_group", 1) != 1 or hparams.get("topk_group", 1) != 1: + raise ValueError("dots3-note conversion does not support grouped expert routing") + if hparams.get("use_dynamic_rsf", False) or hparams.get("moe_gating_fp32", False): + raise ValueError("dots3-note conversion does not support use_dynamic_rsf/moe_gating_fp32") + for key in ("attention_gate_type", "swa_attention_gate_type"): + if hparams.get(key, "headwise") != "headwise": + raise ValueError(f"dots3-note conversion only supports headwise attention gate, got {key}={hparams.get(key)!r}") + if hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"] != hparams.get("swa_head_dim", 256): + raise ValueError("swa_head_dim must equal swa_qk_nope_head_dim + swa_qk_rope_head_dim") + if hparams["swa_qk_rope_head_dim"] != hparams["qk_rope_head_dim"]: + # both layer kinds share a single rope_dimension_count + raise ValueError("swa_qk_rope_head_dim must match qk_rope_head_dim") + + self.apply_lora_rescale = hparams.get("apply_mla_qkv_lora_rescale", False) + + def _is_swa_layer(self, bid: int) -> bool: + if bid >= self.hparams["num_hidden_layers"]: + # note: the NextN/MTP block uses the sliding-attention MLA + return True + return self.layer_types[bid] == "sliding_attention" + + def set_vocab(self): + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + tokens, toktypes, tokpre = self.get_vocab_base() + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"]) # ty: ignore[unresolved-attribute] + special_vocab.add_to_gguf(self.gguf_writer) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + if name.startswith(("vision_encoder.", "audio_encoder.")): + return None + + assert cls._n_main_layers is not None + is_mtp = name.startswith("model.mtp.") or \ + ((m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers) + + # --no-mtp: drop the NextN/MTP block; --mtp: keep only that block plus the shared embeddings/norm/lm_head + if is_mtp and cls.no_mtp: + return None + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + + def set_gguf_parameters(self): + hparams = self.hparams + + # head_count is a per-layer array because the two layer kinds have different head counts + n_layer = hparams["num_hidden_layers"] + hparams["num_attention_heads"] = [ + hparams["swa_num_attention_heads"] if self._is_swa_layer(il) else hparams["num_attention_heads"] + for il in range(self.block_count) + ] + + # prevent the base class from emitting key/value_length from the unused head_dim + hparams.pop("head_dim", None) + + super().set_gguf_parameters() + + # MLA geometry of the sliding-window layers (rope.freq_base_swa is emitted by the base class) + swa_kv_lora_rank = hparams["swa_kv_lora_rank"] + self.gguf_writer.add_sliding_window(hparams["sliding_window_size"]) + self.gguf_writer.add_sliding_window_pattern([self._is_swa_layer(il) for il in range(n_layer)]) + self.gguf_writer.add_kv_lora_rank_swa(swa_kv_lora_rank) + self.gguf_writer.add_key_length_swa(swa_kv_lora_rank + hparams["swa_qk_rope_head_dim"]) + self.gguf_writer.add_value_length_swa(swa_kv_lora_rank) + self.gguf_writer.add_key_length_mla_swa(hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"]) + self.gguf_writer.add_value_length_mla_swa(hparams["swa_v_head_dim"]) + if hparams["swa_q_lora_rank"] != hparams["q_lora_rank"]: + raise ValueError("dots3-note conversion assumes a shared q_lora_rank for both layer kinds") + + if self.n_nextn: + self.gguf_writer.add_nextn_predict_layers(self.n_nextn) + + # DSA indexer (full-attention layers only) + self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"]) + self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"]) + self.gguf_writer.add_indexer_top_k(hparams["index_topk"]) + self.gguf_writer.add_indexer_types([not self._is_swa_layer(il) for il in range(n_layer)]) + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # move the MTP token embedding into the NextN block so the standard nextn mapping picks it up + if name == "model.mtp.embed_tokens.weight": + name = f"model.layers.{self.hparams['num_hidden_layers']}.embed_tokens.weight" + bid = self.hparams["num_hidden_layers"] + + # fold the activation rescale sqrt(n_embd/lora_rank) into the preceding RMSNorm weight + # this also covers the indexer wq_b, which reads the same rescaled q_lora activation + if self.apply_lora_rescale and bid is not None: + if name.endswith("q_a_layernorm.weight"): + data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / self.hparams["q_lora_rank"]) + elif name.endswith("kv_a_layernorm.weight"): + rank = self.hparams["swa_kv_lora_rank"] if self._is_swa_layer(bid) else self.hparams["kv_lora_rank"] + data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / rank) + + # MLA absorption: split kv_b_proj into k_b (transposed) and v_b, per-layer-kind geometry + if name.endswith("kv_b_proj.weight"): + assert bid is not None + if self._is_swa_layer(bid): + n_head = self.hparams["swa_num_attention_heads"] + qk_nope_head_dim = self.hparams["swa_qk_nope_head_dim"] + v_head_dim = self.hparams["swa_v_head_dim"] + else: + n_head = self.hparams["num_attention_heads"] + qk_nope_head_dim = self.hparams["qk_nope_head_dim"] + v_head_dim = self.hparams["v_head_dim"] + if isinstance(n_head, list): # set_gguf_parameters turns this into a per-layer array + n_head = n_head[bid] + + assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim) + + kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1]) + k_b, v_b = kv_b.split([qk_nope_head_dim, v_head_dim], dim=1) + k_b = k_b.transpose(1, 2) + + yield from ModelBase.modify_tensors(self, k_b, name.replace("kv_b_proj", "k_b_proj"), bid) + yield from ModelBase.modify_tensors(self, v_b, name.replace("kv_b_proj", "v_b_proj"), bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration") +class Dots3NoteMmprojModel(MmprojModel): + has_vision_encoder = True + has_audio_encoder = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None + assert self.hparams_audio is not None + + # preprocessor_config.json nests the image params under vision_config + self.preprocessor_config = {**self.preprocessor_config, **self.preprocessor_config.get("vision_config", {})} + + vis = self.hparams_vision + # in this config, hidden_size is the adapter output width; embed_dim is the tower width + vis["hidden_size"] = vis["embed_dim"] + vis["image_size"] = 0 # dynamic resolution + self.pyramid = [max(0, n) for n in vis["pyramid_num_routed"]] + + if vis.get("adapter_type") != "patch_merger" or not vis.get("pre_pixel_shuffle"): + raise ValueError("dots3-note vision conversion requires adapter_type=patch_merger and pre_pixel_shuffle") + if vis.get("router_scoring_func", "sigmoid") != "sigmoid" or vis.get("router_scale", 1.0) != 1.0: + raise ValueError("dots3-note vision conversion only supports sigmoid routing with router_scale=1.0") + if vis.get("temporal_patch_size", 1) != 1 or vis.get("use_bias") or not vis.get("use_qk_norm"): + raise ValueError("unsupported dots3-note vision config variant") + + aud = self.hparams_audio + if not aud.get("use_conv2d_stem") or not aud.get("use_rope") or not aud.get("use_rms_norm") or aud.get("use_causal"): + raise ValueError("unsupported dots3-note audio config variant") + if aud["whisper_config"].get("activation_function") != "swiglu": + raise ValueError("dots3-note audio conversion requires the swiglu activation") + if aud.get("merge_factor", 1) != 1 or aud.get("chunk_seconds") != 60: + raise ValueError("unsupported dots3-note audio chunking config") + # the graph hard-codes these rope parameters + rope = aud.get("rope_parameters", {}) + if rope.get("partial_rotary_factor") != 0.5 or rope.get("rope_theta") != 10000.0: + raise ValueError("unsupported dots3-note audio rope config") + + def get_audio_config(self) -> dict[str, Any] | None: + cfg = self.global_config.get("audio_config") + if cfg is not None: + # aliases so MmprojModel.find_aparam() / n_block_keys can resolve them + whisper = cfg["whisper_config"] + cfg["hidden_size"] = whisper["d_model"] + cfg["intermediate_size"] = whisper["encoder_ffn_dim"] + cfg["num_attention_heads"] = whisper["encoder_attention_heads"] + cfg["num_hidden_layers"] = whisper["encoder_layers"] + return cfg + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + assert self.hparams_audio is not None + + self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.DOTS3NOTE_V) + self.gguf_writer.add_vision_use_silu(True) + self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["rms_norm_eps"]) + self.gguf_writer.add_vision_spatial_merge_size(self.hparams_vision["spatial_merge_size"]) + self.gguf_writer.add_vision_min_pixels(self.preprocessor_config["min_pixels"]) + self.gguf_writer.add_vision_max_pixels(self.preprocessor_config["max_pixels"]) + # pyramid MoE: per-block routed expert count, 0 = dense block + self.gguf_writer.add_vision_expert_count_per_layer(self.pyramid) + self.gguf_writer.add_vision_expert_used_count(int(self.hparams_vision["capacity_factor"])) + + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.DOTS3NOTE_A) + self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["whisper_config"]["num_mel_bins"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-6) # Dots3NoteAudioRMSNorm default + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not name.startswith(("vision_encoder.", "audio_encoder.")): + return None + return super().filter_tensors(item) + + _vis_experts: dict[int, dict[str, Tensor]] | None = None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # router params have no .weight suffix in the checkpoint, but gguf tools expect one + if name.endswith((".gate_weight", ".router_bias")): + name += ".weight" + + # audio fc1 fuses gate and up for swiglu; split it + if ".speech_encoder.layers." in name and ".fc1." in name: + gate, up = data_torch.chunk(2, dim=0) + yield from super().modify_tensors(gate, name.replace(".fc1.", ".fc1_gate."), bid) + yield from super().modify_tensors(up, name.replace(".fc1.", ".fc1_up."), bid) + return + + # vision MoE: stack per-expert weights into a single 3D tensor per block + if ".mlp.experts." in name: + assert bid is not None + n_expert = self.pyramid[bid] + if self._vis_experts is None: + self._vis_experts = {} + buf = self._vis_experts.setdefault(bid, {}) + buf[name] = data_torch + + if len(buf) >= n_expert * 3: + for w_name in ("fc1", "fc2", "fc3"): + datas: list[Tensor] = [] + for xid in range(n_expert): + ename = f"vision_encoder.blocks.{bid}.mlp.experts.{xid}.{w_name}.weight" + datas.append(buf.pop(ename)) + merged = torch.stack(datas, dim=0) + yield from super().modify_tensors(merged, f"vision_encoder.blocks.{bid}.mlp.experts.{w_name}.weight", bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + if self._vis_experts is not None: + leftover = [k for d in self._vis_experts.values() for k in d.keys()] + if leftover: + raise ValueError(f"unprocessed vision experts: {leftover}") + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # FP32 routing is load-bearing for the vision MoE (near-tied expert scores) + if ".ffn_gate_inp." in new_name or ".exp_probs_b." in new_name: + return gguf.GGMLQuantizationType.F32 + if ".conv2d" in new_name or "a.conv_out" in new_name: + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) diff --git a/conversion/qwen.py b/conversion/qwen.py index 355365763..cdba8a63e 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -709,7 +709,13 @@ class DFlashModel(Qwen3Model): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel") +@ModelBase.register( + "Qwen3DSparkModel", + "DSparkDraftModel", + "DSparkSpeculator", + "Lfm2DSparkDraftModel", + "LingDSparkModel", +) @ModelBase.example("satgeze/Qwen3.6-27B-DSpark") class DSparkModel(DFlashModel): # DSpark = DFlash + a semi-autoregressive Markov head. diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 2b5f68444..b869f4bdd 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -1896,7 +1896,6 @@ void ggml_compute_forward_repeat_back( } // ggml_compute_forward_concat - static void ggml_compute_forward_concat_any( const ggml_compute_params * params, ggml_tensor * dst) { @@ -1904,8 +1903,6 @@ static void ggml_compute_forward_concat_any( const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - const size_t len = ggml_type_size(src0->type); - const int ith = params->ith; const int nth = params->nth; @@ -1914,31 +1911,38 @@ static void ggml_compute_forward_concat_any( const int32_t dim = ggml_get_op_params_i32(dst, 0); GGML_ASSERT(dim >= 0 && dim < 4); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src1)); int64_t o[4] = {0, 0, 0, 0}; + if (dim == 0) { + GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); + GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); + o[dim] = src0->ne[dim]/ggml_blck_size(src0->type); } else { o[dim] = src0->ne[dim]; } - const char * x; + // Region 1: copy rows from src0 + for (int i3 = 0; i3 < ne03; i3++) { + for (int i2 = ith; i2 < ne02; i2 += nth) { + for (int i1 = 0; i1 < ne01; i1++) { + const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03; + char * y = ( char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3; + memcpy(y, x, ggml_row_size(src0->type, ne00)); + } + } + } - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0/ggml_blck_size(dst->type); i0++) { - if (i0 < ne00/ggml_blck_size(src0->type) && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03; - } else { - x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13; - } - - char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3; - - memcpy(y, x, len); - } + // Region 2: copy rows from src1, offset into dst by o[] + for (int i3 = 0; i3 < ne13; i3++) { + for (int i2 = ith; i2 < ne12; i2 += nth) { + for (int i1 = 0; i1 < ne11; i1++) { + const char * x = (const char *) src1->data + i1*nb11 + i2*nb12 + i3*nb13; + char * y = ( char *) dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0; + memcpy(y, x, ggml_row_size(src1->type, ne10)); } } } @@ -2078,14 +2082,6 @@ void ggml_compute_forward_concat( ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - if (ggml_is_quantized(src0->type)) { - GGML_ASSERT(ggml_is_contiguous_rows(src0)); - GGML_ASSERT(ggml_is_contiguous_rows(src1)); - GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); - GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); - } switch (src0->type) { case GGML_TYPE_F16: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index fad8d1fd8..8f6f55519 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -205,6 +205,9 @@ class Keys: VALUE_LENGTH_MLA = "{arch}.attention.value_length_mla" KEY_LENGTH_SWA = "{arch}.attention.key_length_swa" VALUE_LENGTH_SWA = "{arch}.attention.value_length_swa" + KEY_LENGTH_MLA_SWA = "{arch}.attention.key_length_mla_swa" + VALUE_LENGTH_MLA_SWA = "{arch}.attention.value_length_mla_swa" + KV_LORA_RANK_SWA = "{arch}.attention.kv_lora_rank_swa" SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" @@ -361,6 +364,8 @@ class Keys: IMAGE_MEAN = "clip.vision.image_mean" IMAGE_STD = "clip.vision.image_std" SPATIAL_MERGE_SIZE = "clip.vision.spatial_merge_size" + EXPERT_COUNT_PER_LAYER = "clip.vision.expert_count_per_layer" # dots3note pyramid MoE, 0 = dense layer + EXPERT_USED_COUNT = "clip.vision.expert_used_count" USE_GELU = "clip.use_gelu" USE_SILU = "clip.use_silu" N_WA_PATTERN = "clip.vision.n_wa_pattern" # used by qwen2.5vl @@ -558,6 +563,7 @@ class MODEL_ARCH(IntEnum): BAILINGMOE2 = auto() BAILINGMOE3 = auto() DOTS1 = auto() + DOTS3NOTE = auto() ARCEE = auto() AFMOE = auto() LAGUNA = auto() @@ -870,6 +876,11 @@ class MODEL_TENSOR(IntEnum): V_ENC_FFN_UP = auto() V_ENC_FFN_GATE = auto() V_ENC_FFN_DOWN = auto() + V_ENC_FFN_GATE_INP = auto() # dots3note vision MoE router + V_ENC_FFN_GATE_EXPS = auto() + V_ENC_FFN_UP_EXPS = auto() + V_ENC_FFN_DOWN_EXPS = auto() + V_ENC_FFN_EXP_PROBS_B = auto() V_ENC_ATTN_POST_NORM = auto() # gemma4 V_ENC_FFN_POST_NORM = auto() V_LAYER_SCALE_1 = auto() @@ -1275,6 +1286,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.BAILINGMOE2: "bailingmoe2", MODEL_ARCH.BAILINGMOE3: "bailingmoe3", MODEL_ARCH.DOTS1: "dots1", + MODEL_ARCH.DOTS3NOTE: "dots3note", MODEL_ARCH.ARCEE: "arcee", MODEL_ARCH.AFMOE: "afmoe", MODEL_ARCH.LAGUNA: "laguna", @@ -1586,6 +1598,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up", MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate", MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down", + MODEL_TENSOR.V_ENC_FFN_GATE_INP: "v.blk.{bid}.ffn_gate_inp", + MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: "v.blk.{bid}.ffn_gate_exps", + MODEL_TENSOR.V_ENC_FFN_UP_EXPS: "v.blk.{bid}.ffn_up_exps", + MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: "v.blk.{bid}.ffn_down_exps", + MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: "v.blk.{bid}.exp_probs_b", MODEL_TENSOR.V_ENC_ATTN_POST_NORM: "v.blk.{bid}.attn_post_norm", MODEL_TENSOR.V_ENC_FFN_POST_NORM: "v.blk.{bid}.ffn_post_norm", MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1", @@ -1908,6 +1925,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.V_ENC_FFN_UP, MODEL_TENSOR.V_ENC_FFN_GATE, MODEL_TENSOR.V_ENC_FFN_DOWN, + MODEL_TENSOR.V_ENC_FFN_GATE_INP, + MODEL_TENSOR.V_ENC_FFN_GATE_EXPS, + MODEL_TENSOR.V_ENC_FFN_UP_EXPS, + MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS, + MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B, MODEL_TENSOR.V_ENC_ATTN_POST_NORM, MODEL_TENSOR.V_ENC_FFN_POST_NORM, MODEL_TENSOR.V_LAYER_SCALE_1, @@ -4334,6 +4356,44 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR.FFN_UP_SHEXP, ], + MODEL_ARCH.DOTS3NOTE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_KV_A_MQA, + MODEL_TENSOR.ATTN_K_B, + MODEL_TENSOR.ATTN_V_B, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_KV_A_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.INDEXER_K_NORM, + MODEL_TENSOR.INDEXER_PROJ, + MODEL_TENSOR.INDEXER_ATTN_K, + MODEL_TENSOR.INDEXER_ATTN_Q_B, + # NextN/MTP tensors - preserved but unused + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, + ], MODEL_ARCH.ARCEE: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -5454,6 +5514,8 @@ class VisionProjectorType: COGVLM = "cogvlm" JANUS_PRO = "janus_pro" DOTSOCR = "dots_ocr" + DOTS3NOTE_V = "dots3note_v" + DOTS3NOTE_A = "dots3note_a" # audio DEEPSEEKOCR = "deepseekocr" DEEPSEEKOCR2 = "deepseekocr2" LFM2A = "lfm2a" # audio diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 16ae9f999..d8a96a27b 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -785,6 +785,15 @@ class GGUFWriter: def add_key_length_swa(self, length: int) -> None: self.add_uint32(Keys.Attention.KEY_LENGTH_SWA.format(arch=self.arch), length) + def add_key_length_mla_swa(self, length: int) -> None: + self.add_uint32(Keys.Attention.KEY_LENGTH_MLA_SWA.format(arch=self.arch), length) + + def add_value_length_mla_swa(self, length: int) -> None: + self.add_uint32(Keys.Attention.VALUE_LENGTH_MLA_SWA.format(arch=self.arch), length) + + def add_kv_lora_rank_swa(self, length: int) -> None: + self.add_uint32(Keys.Attention.KV_LORA_RANK_SWA.format(arch=self.arch), length) + def add_value_length_swa(self, length: int) -> None: self.add_uint32(Keys.Attention.VALUE_LENGTH_SWA.format(arch=self.arch), length) @@ -1318,6 +1327,12 @@ class GGUFWriter: def add_vision_spatial_merge_size(self, value: int) -> None: self.add_uint32(Keys.ClipVision.SPATIAL_MERGE_SIZE, value) + def add_vision_expert_count_per_layer(self, value: Sequence[int]) -> None: + self.add_array(Keys.ClipVision.EXPERT_COUNT_PER_LAYER, value) + + def add_vision_expert_used_count(self, value: int) -> None: + self.add_uint32(Keys.ClipVision.EXPERT_USED_COUNT, value) + def add_vision_use_gelu(self, value: bool) -> None: self.add_bool(Keys.ClipVision.USE_GELU, value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index a0571ccd3..ef580518e 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -723,6 +723,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mixer.k", # plamo2 "model.layers.layers.{bid}.mixer.k_norm", # plamo3 "layers.{bid}.self_attn.k_norm", # qwen3-embedding + "model.layers.{bid}.self_attn.k_rope_only_layernorm", # dots3note "model.layers.{bid}.attention.key_layernorm", # apertus ), @@ -1453,6 +1454,7 @@ class TensorNameMap: "mlp_AR.linear_{bid}", # PaddleOCR-VL "merger.mlp.{bid}", "vision_tower.merger.mlp.{bid}", # dots.ocr + "vision_encoder.adapter.mlp.{bid}", # dots3note "vit.perceive.proj.{bid}", # HunyuanVL (proj.0 = conv1, proj.2 = conv2) ), @@ -1503,6 +1505,7 @@ class TensorNameMap: "vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL "model.vision_tower.patch_embedder.input_proj", # gemma4 "vision_tower.patch_embed.patchifier.proj", # dots.ocr + "vision_encoder.patch_embed.proj", # dots3note "vision_model.conv1", # Step3-VL "model.vision_embedder.patch_dense", # gemma4 unified "model.vision_tower.patch_embedder.patch_embedding", # muse-glimmer @@ -1511,6 +1514,7 @@ class TensorNameMap: MODEL_TENSOR.V_ENC_EMBD_NORM: ( "visual.post_conv_layernorm", # glm4v "vision_tower.patch_embed.patchifier.norm", # dots.ocr + "vision_encoder.patch_embed.norm", # dots3note ), MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: ( @@ -1550,6 +1554,7 @@ class TensorNameMap: MODEL_TENSOR.V_ENC_ATTN_QKV: ( "visual.blocks.{bid}.attn.qkv", # qwen3vl "vision_tower.blocks.{bid}.attn.qkv", # dots.ocr + "vision_encoder.blocks.{bid}.attn.qkv", # dots3note "model.vision.transformer.layers.{bid}.attention.query_key_value", # cogvlm "model.vision_model.transformer.layers.{bid}.self_attn.qkv_proj", # Deepseek-OCR CLIP "vision_tower.encoder.blocks.{bid}.wqkv", # Kimi-K2.5 @@ -1578,6 +1583,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_Q_NORM: ( + "vision_encoder.blocks.{bid}.attn.q_norm", # dots3note "vision_tower.vision_model.encoder.layers.{bid}.attn.q_norm", # InternVL "model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1 "visual.blocks.{bid}.attn.q_norm", # GLM-OCR @@ -1605,6 +1611,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_K_NORM: ( + "vision_encoder.blocks.{bid}.attn.k_norm", # dots3note "vision_tower.vision_model.encoder.layers.{bid}.attn.k_norm", # InternVL "model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1 "visual.blocks.{bid}.attn.k_norm", # GLM-OCR @@ -1650,6 +1657,7 @@ class TensorNameMap: "siglip2.vision_model.encoder.layers.{bid}.layer_norm1", "vision_model.radio_model.model.blocks.{bid}.norm1", # Nemotron Nano v2 VL "vision_tower.blocks.{bid}.norm1", # dots.ocr + "vision_encoder.blocks.{bid}.norm_1", # dots3note "vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL "model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2 "model.vision_tower.layers.{bid}.norm1", # muse-glimmer @@ -1677,6 +1685,7 @@ class TensorNameMap: "model.qwen2_model.model.model.layers.{bid}.self_attn.o_proj", # Deepseek-OCR-2 qwen2 "vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4 "vision_tower.blocks.{bid}.attn.proj", # dots.ocr + "vision_encoder.blocks.{bid}.attn.proj", # dots3note "vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL "model.vision_tower.layers.{bid}.attn.proj", # muse-glimmer ), @@ -1705,12 +1714,14 @@ class TensorNameMap: "vision_model.radio_model.model.blocks.{bid}.norm2", # Nemotron Nano v2 VL "vision_model.model.layers.{bid}.pre_feedforward_layernorm", # gemma4 "vision_tower.blocks.{bid}.norm2", # dots.ocr + "vision_encoder.blocks.{bid}.norm_2", # dots3note "vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL "model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2 "model.vision_tower.layers.{bid}.norm2", # muse-glimmer ), MODEL_TENSOR.V_ENC_FFN_UP: ( + "vision_encoder.blocks.{bid}.mlp.fc3", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", "model.vision_tower.encoder.layers.{bid}.mlp.fc1", # minicpmv4_6 @@ -1736,6 +1747,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_GATE: ( + "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral "visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl @@ -1744,6 +1756,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_DOWN: ( + "vision_encoder.blocks.{bid}.mlp.fc2", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", "model.vision_tower.encoder.layers.{bid}.mlp.fc2", # minicpmv4_6 @@ -1768,6 +1781,29 @@ class TensorNameMap: "model.vision_tower.layers.{bid}.mlp.fc2", # muse-glimmer ), + + MODEL_TENSOR.V_ENC_FFN_GATE_INP: ( + "vision_encoder.blocks.{bid}.mlp.gate_weight", # dots3note + ), + + MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: ( + "vision_encoder.blocks.{bid}.mlp.router_bias", # dots3note + ), + + # note: expert weights are stacked into a single 3D tensor in conversion code, + # which emits the pseudo-names below + MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: ( + "vision_encoder.blocks.{bid}.mlp.experts.fc1", # dots3note + ), + + MODEL_TENSOR.V_ENC_FFN_UP_EXPS: ( + "vision_encoder.blocks.{bid}.mlp.experts.fc3", # dots3note + ), + + MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: ( + "vision_encoder.blocks.{bid}.mlp.experts.fc2", # dots3note + ), + MODEL_TENSOR.V_ENC_ATTN_POST_NORM: ( "vision_model.model.layers.{bid}.post_attention_layernorm", # gemma4 ), @@ -1799,6 +1835,7 @@ class TensorNameMap: "vision_model.layernorm_pre", # llama4 "model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP "vision_tower.patch_embed.patchifier.norm", # dots.ocr + "vision_encoder.patch_embed.norm", # dots3note "vision_model.ln_pre", # Step3-VL "model.vision_tower.ln_pre", # muse-glimmer ), @@ -1820,6 +1857,7 @@ class TensorNameMap: MODEL_TENSOR.V_MM_POST_NORM: ( "visual.merger.post_projection_norm", # glm4v "vision_tower.post_trunk_norm", # dots.ocr + "vision_encoder.post_trunk_norm", # dots3note "vit.perceive.after_rms", # HunyuanVL ), @@ -1837,6 +1875,7 @@ class TensorNameMap: "mlp_AR.pre_norm", # PaddleOCR-VL "merger.ln_q", "vision_tower.merger.ln_q", # dots.ocr + "vision_encoder.adapter.ln_q", # dots3note "model.merger.mlp.0.pre_norm", # minicpmv4_6 ), @@ -2172,10 +2211,12 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_CONV2D: ( "audio_tower.conv2d{bid}", # qwen3omni + "audio_encoder.dots_encoder.speech_encoder.conv2d{bid}", # dots3note ), MODEL_TENSOR.A_ENC_CONV_OUT: ( "audio_tower.conv_out", # qwen3omni + "audio_encoder.dots_encoder.speech_encoder.conv_out", # dots3note "speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation ), @@ -2183,12 +2224,14 @@ class TensorNameMap: MODEL_TENSOR.A_POST_NORM: ( "audio_tower.layer_norm", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note "audio_tower.ln_post", # qwen2omni "encoder.layer_norm", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_ATTN_Q: ( "audio_tower.layers.{bid}.self_attn.q_proj", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.q_proj", # dots3note "conformer.layers.{bid}.self_attn.linear_q", # lfm2 "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n "conformer.layers.{bid}.self_attn.q_proj", # gemma4 @@ -2199,6 +2242,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_ATTN_K: ( "audio_tower.layers.{bid}.self_attn.k_proj", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.k_proj", # dots3note "conformer.layers.{bid}.self_attn.linear_k", # lfm2 "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n "conformer.layers.{bid}.self_attn.k_proj", # gemma4 @@ -2209,6 +2253,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_ATTN_V: ( "audio_tower.layers.{bid}.self_attn.v_proj", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.v_proj", # dots3note "conformer.layers.{bid}.self_attn.linear_v", # lfm2 "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n "conformer.layers.{bid}.self_attn.v_proj", # gemma4 @@ -2240,6 +2285,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_INPUT_NORM: ( "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn_layer_norm", # dots3note "conformer.layers.{bid}.norm_self_att", # lfm2 "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet @@ -2249,6 +2295,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_OUTPUT: ( "audio_tower.layers.{bid}.self_attn.out_proj", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.out_proj", # dots3note "conformer.layers.{bid}.self_attn.linear_out", # lfm2 "conformer.layers.{bid}.attention.post", # gemma3n "conformer.layers.{bid}.self_attn.post", # gemma4 @@ -2259,6 +2306,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_OUTPUT_NORM: ( "audio_tower.layers.{bid}.final_layer_norm", # ultravox + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.final_layer_norm", # dots3note "conformer.layers.{bid}.norm_out", # lfm2 "conformer.layers.{bid}.attention.post_norm", # gemma3n "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet @@ -2284,6 +2332,7 @@ class TensorNameMap: ), MODEL_TENSOR.A_ENC_FFN_UP: ( + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_up", # dots3note (split from fc1 in conversion code) "audio_tower.layers.{bid}.fc1", # ultravox "conformer.layers.{bid}.feed_forward1.linear1", # lfm2 "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n @@ -2293,9 +2342,12 @@ class TensorNameMap: "encoder.layers.{bid}.fc1", # mimo-audio-tokenizer ), - MODEL_TENSOR.A_ENC_FFN_GATE: (), + MODEL_TENSOR.A_ENC_FFN_GATE: ( + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_gate", # dots3note (split from fc1 in conversion code) + ), MODEL_TENSOR.A_ENC_FFN_DOWN: ( + "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc2", # dots3note "audio_tower.layers.{bid}.fc2", # ultravox "conformer.layers.{bid}.feed_forward1.linear2", # lfm2 "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n @@ -2379,6 +2431,7 @@ class TensorNameMap: MODEL_TENSOR.A_MMPROJ: ( "audio.multi_modal_projector.linear_{bid}", # ultravox, meralion + "audio_encoder.audio_adapter.proj.{bid}", # dots3note (proj.1, proj.3) "audio_adapter.model.{bid}", # lfm2 "audio_tower.proj{bid}", # qwen3omni "sound_projection.linear{bid}", # parakeet (linear1, linear2) @@ -2393,6 +2446,7 @@ class TensorNameMap: MODEL_TENSOR.A_MM_NORM_PRE: ( "audio.multi_modal_projector.ln_pre", # ultravox + "audio_encoder.audio_adapter.proj.0", # dots3note "sound_projection.norm", # parakeet ), diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 1bc941163..4f66c850a 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -3517,7 +3517,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in } fit_params_target[0] = taxmb*1024*1024; bool success = (common_fit_params(kcpp_data->model_filename.c_str(), &model_params, &llama_ctx_params, - tensor_split_temp, tenos.data(), fit_params_target.data(), kcpp_data->n_ctx, + tensor_split_temp, tenos.data(), fit_params_target.data(), kcpp_data->n_ctx, nullptr, dospam?GGML_LOG_LEVEL_DEBUG:GGML_LOG_LEVEL_NONE)==0); if(!dospam) { diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index c9b504c33..025f9fb54 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -110,6 +110,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_BAILINGMOE2, "bailingmoe2" }, { LLM_ARCH_BAILINGMOE3, "bailingmoe3" }, { LLM_ARCH_DOTS1, "dots1" }, + { LLM_ARCH_DOTS3NOTE, "dots3note" }, { LLM_ARCH_ARCEE, "arcee" }, { LLM_ARCH_AFMOE, "afmoe" }, { LLM_ARCH_LAGUNA, "laguna" }, @@ -273,6 +274,9 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_VALUE_LENGTH_MLA, "%s.attention.value_length_mla" }, { LLM_KV_ATTENTION_KEY_LENGTH_SWA, "%s.attention.key_length_swa" }, { LLM_KV_ATTENTION_VALUE_LENGTH_SWA, "%s.attention.value_length_swa" }, + { LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, "%s.attention.key_length_mla_swa" }, + { LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, "%s.attention.value_length_mla_swa" }, + { LLM_KV_ATTENTION_KV_LORA_RANK_SWA, "%s.attention.kv_lora_rank_swa" }, { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, "%s.attention.indexer.head_count" }, { LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, "%s.attention.indexer.key_length" }, { LLM_KV_ATTENTION_INDEXER_TOP_K, "%s.attention.indexer.top_k" }, @@ -1034,6 +1038,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) { case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_LFM2: case LLM_ARCH_LFM2MOE: + case LLM_ARCH_BAILINGMOE3: return true; default: return false; @@ -1056,6 +1061,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK2: case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_DEEPSEEK4: + case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_GLM_DSA: case LLM_ARCH_BITNET: case LLM_ARCH_T5: diff --git a/src/llama-arch.h b/src/llama-arch.h index 48fe051a9..7159e23bf 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -115,6 +115,7 @@ enum llm_arch { LLM_ARCH_BAILINGMOE2, LLM_ARCH_BAILINGMOE3, LLM_ARCH_DOTS1, + LLM_ARCH_DOTS3NOTE, LLM_ARCH_ARCEE, LLM_ARCH_AFMOE, LLM_ARCH_LAGUNA, @@ -278,6 +279,9 @@ enum llm_kv { LLM_KV_ATTENTION_VALUE_LENGTH_MLA, LLM_KV_ATTENTION_KEY_LENGTH_SWA, LLM_KV_ATTENTION_VALUE_LENGTH_SWA, + LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, + LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, + LLM_KV_ATTENTION_KV_LORA_RANK_SWA, LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, LLM_KV_ATTENTION_INDEXER_TOP_K, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 48d27b60a..0b66b9da6 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -9,6 +9,7 @@ #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" +#include "llama-kv-cache-dsa-iswa.h" #include "llama-kv-cache-msa.h" #include "llama-kv-cache-dsv4.h" #include "llama-memory-hybrid.h" @@ -507,10 +508,12 @@ void llm_graph_input_attn_k::set_input(const llama_ubatch * ubatch) { } bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { - const auto * mctx = static_cast(params.mctx); + mctx = static_cast(params.mctx); - this->mctx = mctx; + return can_reuse_impl(params); +} +bool llm_graph_input_attn_k::can_reuse_impl(const llm_graph_params & params) { bool res = true; res &= self_k_idxs->ne[0] == params.ubatch.n_tokens; @@ -567,10 +570,12 @@ void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) { } bool llm_graph_input_attn_k_dsa::can_reuse(const llm_graph_params & params) { - const auto * mctx = static_cast(params.mctx); + mctx = static_cast(params.mctx); - this->mctx = mctx; + return can_reuse_impl(params); +} +bool llm_graph_input_attn_k_dsa::can_reuse_impl(const llm_graph_params & params) { bool res = true; res &= self_k_idxs_mla->ne[0] == params.ubatch.n_tokens; @@ -582,6 +587,25 @@ bool llm_graph_input_attn_k_dsa::can_reuse(const llm_graph_params & params) { return res; } +void llm_graph_input_attn_k_dsa_iswa::set_input(const llama_ubatch * ubatch) { + inp_dsa->set_input(ubatch); + inp_swa->set_input(ubatch); +} + +bool llm_graph_input_attn_k_dsa_iswa::can_reuse(const llm_graph_params & params) { + mctx = static_cast(params.mctx); + + inp_dsa->mctx = mctx->get_dsa(); + inp_swa->mctx = mctx->get_swa(); + + bool res = true; + + res &= inp_dsa->can_reuse_impl(params); + res &= inp_swa->can_reuse_impl(params); + + return res; +} + void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { // base tensors may not be allocated if there are no non-SWA attention layers if (self_k_idxs && self_k_idxs->buffer) { @@ -3211,8 +3235,12 @@ ggml_tensor * llm_graph_context::build_attn( return cur; } -llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const { - const auto * mctx_cur = static_cast(mctx); +static std::unique_ptr build_attn_inp_k_dsa_impl( + ggml_context * ctx0, + const llama_ubatch & ubatch, + const llama_hparams & hparams, + const llama_cparams & cparams, + const llama_kv_cache_dsa_context * mctx_cur) { auto inp = std::make_unique(hparams, cparams, mctx_cur); @@ -3236,9 +3264,35 @@ llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const { inp->self_k_rot_lid = mctx_cur->get_lid()->build_input_k_rot(ctx0); } + return inp; +} + +llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const { + const auto * mctx_cur = static_cast(mctx); + + auto inp = build_attn_inp_k_dsa_impl(ctx0, ubatch, hparams, cparams, mctx_cur); + return (llm_graph_input_attn_k_dsa *) res->add_input(std::move(inp)); } +llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa() const { + const auto * mctx_cur = static_cast(mctx); + + auto inp_dsa = build_attn_inp_k_dsa_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_dsa()); + + // build_attn_inp_k_impl rejects SWA caches, so construct the input directly + auto inp_swa = std::make_unique(hparams, cparams, mctx_cur->get_swa()); + + inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); + + inp_swa->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); + inp_swa->self_kq_mask_cnv = inp_swa->self_kq_mask; + + auto inp = std::make_unique(std::move(inp_dsa), std::move(inp_swa), mctx_cur); + + return (llm_graph_input_attn_k_dsa_iswa *) res->add_input(std::move(inp)); +} + llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_enabled) const { const auto * mctx_cur = static_cast(mctx); diff --git a/src/llama-graph.h b/src/llama-graph.h index 94324c745..b388e028c 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -23,6 +23,7 @@ struct llama_memory_context_i; class llama_kv_cache_context; class llama_kv_cache_dsa_context; +class llama_kv_cache_dsa_iswa_context; class llama_kv_cache_msa_context; class llama_kv_cache_dsv4_raw_context; class llama_kv_cache_dsv4_context; @@ -374,6 +375,9 @@ public: bool can_reuse(const llm_graph_params & params) override; + // like can_reuse, but does not re-bind mctx + bool can_reuse_impl(const llm_graph_params & params); + ggml_tensor * get_k_idxs() const { return self_k_idxs; } ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; } @@ -405,6 +409,9 @@ public: bool can_reuse(const llm_graph_params & params) override; + // like can_reuse, but does not re-bind mctx + bool can_reuse_impl(const llm_graph_params & params); + ggml_tensor * get_k_idxs_mla() const { return self_k_idxs_mla; } ggml_tensor * get_k_idxs_lid() const { return self_k_idxs_lid; } @@ -427,6 +434,32 @@ public: const llama_kv_cache_dsa_context * mctx; }; +// DSA input (full-attention layers + indexer) with K-only input for the SWA layers +class llm_graph_input_attn_k_dsa_iswa : public llm_graph_input_i { +public: + llm_graph_input_attn_k_dsa_iswa( + std::unique_ptr inp_dsa, + std::unique_ptr inp_swa, + const llama_kv_cache_dsa_iswa_context * mctx) : + inp_dsa(std::move(inp_dsa)), + inp_swa(std::move(inp_swa)), + mctx(mctx) { + } + ~llm_graph_input_attn_k_dsa_iswa() = default; + + void set_input(const llama_ubatch * ubatch) override; + + bool can_reuse(const llm_graph_params & params) override; + + llm_graph_input_attn_k_dsa * get_dsa() const { return inp_dsa.get(); } + llm_graph_input_attn_k * get_swa() const { return inp_swa.get(); } + + std::unique_ptr inp_dsa; + std::unique_ptr inp_swa; + + const llama_kv_cache_dsa_iswa_context * mctx; +}; + // standard K/V attention input against the base cache, plus destination indices for the indexer key cache class llm_graph_input_attn_kv_msa : public llm_graph_input_attn_kv { public: @@ -1191,6 +1224,8 @@ struct llm_graph_context { llm_graph_input_attn_k_dsa * build_attn_inp_k_dsa() const; + llm_graph_input_attn_k_dsa_iswa * build_attn_inp_k_dsa_iswa() const; + llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa(bool msa_enabled) const; ggml_tensor * build_attn( diff --git a/src/llama-hparams.h b/src/llama-hparams.h index f6af36436..c3c14292c 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -101,6 +101,11 @@ struct llama_hparams { uint32_t n_group_used = 0; uint32_t n_group_experts = 0; + // MLA + SWA (i.e. dots3note) + uint32_t n_lora_kv_swa = 0; + uint32_t n_embd_head_k_mla_swa = 0; + uint32_t n_embd_head_v_mla_swa = 0; + float expert_group_scale = 0.05f; float expert_weights_scale = 0.0f; bool expert_weights_norm = false; diff --git a/src/llama-kv-cache-dsa-iswa.cpp b/src/llama-kv-cache-dsa-iswa.cpp new file mode 100644 index 000000000..dc10342a1 --- /dev/null +++ b/src/llama-kv-cache-dsa-iswa.cpp @@ -0,0 +1,341 @@ +#include "llama-kv-cache-dsa-iswa.h" + +#include "llama-impl.h" +#include "llama-batch.h" +#include "llama-model.h" + +#include +#include + +// +// llama_kv_cache_dsa_iswa +// + +llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa( + const llama_model & model, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, + const layer_filter_cb & filter_mla, + const layer_filter_cb & filter_lid, + const layer_reuse_cb & reuse) : unified(unified) { + + const auto & hparams = model.hparams; + + // chain filters + const layer_filter_cb filter_dsa = [&](int32_t il) { + if (filter_mla && !filter_mla(il)) { + return false; + } + + return !hparams.is_swa(il); + }; + + const layer_filter_cb filter_swa = [&](int32_t il) { + if (filter_mla && !filter_mla(il)) { + return false; + } + + return hparams.is_swa(il); + }; + + const uint32_t size_dsa = kv_size; + + // note: the SWA cache is always padded to 256 for performance + // https://github.com/ggml-org/llama.cpp/issues/17037 + uint32_t size_swa = GGML_PAD(std::min(size_dsa, hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256); + + // when using full-size SWA cache, we set the SWA cache size to be equal to the base cache size + if (swa_full) { + LLAMA_LOG_WARN("%s: using full-size SWA cache (ref: %s)\n", + __func__, "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055"); + + size_swa = size_dsa; + } + + LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa); + + kv_dsa = std::make_unique( + model, type_k, type_v, + v_trans, offload, unified, size_dsa, n_seq_max, n_pad, + 0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + + kv_swa = std::make_unique( + model, hparams, type_k, type_v, + v_trans, offload, unified, size_swa, n_seq_max, n_pad, + hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr); +} + +void llama_kv_cache_dsa_iswa::clear(bool data) { + kv_dsa->clear(data); + kv_swa->clear(data); +} + +bool llama_kv_cache_dsa_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + bool res = true; + + res = res & kv_dsa->seq_rm(seq_id, p0, p1); + res = res & kv_swa->seq_rm(seq_id, p0, p1); + + return res; +} + +void llama_kv_cache_dsa_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + kv_dsa->seq_cp(seq_id_src, seq_id_dst, p0, p1); + kv_swa->seq_cp(seq_id_src, seq_id_dst, p0, p1); +} + +void llama_kv_cache_dsa_iswa::seq_keep(llama_seq_id seq_id) { + kv_dsa->seq_keep(seq_id); + kv_swa->seq_keep(seq_id); +} + +void llama_kv_cache_dsa_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { + kv_dsa->seq_add(seq_id, p0, p1, shift); + kv_swa->seq_add(seq_id, p0, p1, shift); +} + +void llama_kv_cache_dsa_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + kv_dsa->seq_div(seq_id, p0, p1, d); + kv_swa->seq_div(seq_id, p0, p1, d); +} + +llama_pos llama_kv_cache_dsa_iswa::seq_pos_min(llama_seq_id seq_id) const { + // the DSA cache is a superset of the SWA cache, so we can just check the SWA cache + return kv_swa->seq_pos_min(seq_id); +} + +llama_pos llama_kv_cache_dsa_iswa::seq_pos_max(llama_seq_id seq_id) const { + return kv_swa->seq_pos_max(seq_id); +} + +std::map llama_kv_cache_dsa_iswa::memory_breakdown() const { + std::map mb = kv_dsa->memory_breakdown(); + for (const auto & buft_size : kv_swa->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + return mb; +} + +llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { + GGML_UNUSED(embd_all); + + // first try simple split + do { + if (!unified) { + // requires equal splits, so we skip the simple split + break; + } + + balloc.split_reset(); + + std::vector ubatches; + while (true) { + auto ubatch = balloc.split_simple(n_ubatch); + + if (ubatch.n_tokens == 0) { + break; + } + + ubatches.push_back(std::move(ubatch)); // NOLINT + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + // failed to find a suitable split + break; + } + + auto sinfos_mla = kv_dsa->get_mla()->prepare(ubatches); + if (sinfos_mla.empty()) { + break; + } + + auto sinfos_lid = kv_dsa->get_lid()->prepare(ubatches); + if (sinfos_lid.empty()) { + break; + } + + auto sinfos_swa = kv_swa->prepare(ubatches); + if (sinfos_swa.empty()) { + break; + } + + assert(sinfos_mla.size() == sinfos_swa.size()); + + return std::make_unique( + this, std::move(sinfos_mla), std::move(sinfos_lid), std::move(sinfos_swa), std::move(ubatches)); + } while (false); + + // if it fails, try equal split + do { + balloc.split_reset(); + + std::vector ubatches; + while (true) { + auto ubatch = balloc.split_equal(n_ubatch, !unified, 0); + + if (ubatch.n_tokens == 0) { + break; + } + + ubatches.push_back(std::move(ubatch)); // NOLINT + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + // failed to find a suitable split + break; + } + + auto sinfos_mla = kv_dsa->get_mla()->prepare(ubatches); + if (sinfos_mla.empty()) { + break; + } + + auto sinfos_lid = kv_dsa->get_lid()->prepare(ubatches); + if (sinfos_lid.empty()) { + break; + } + + auto sinfos_swa = kv_swa->prepare(ubatches); + if (sinfos_swa.empty()) { + break; + } + + assert(sinfos_mla.size() == sinfos_swa.size()); + + return std::make_unique( + this, std::move(sinfos_mla), std::move(sinfos_lid), std::move(sinfos_swa), std::move(ubatches)); + } while (false); + + return std::make_unique(LLAMA_MEMORY_STATUS_FAILED_PREPARE); +} + +llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_full() { + return std::make_unique(this); +} + +llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); +} + +bool llama_kv_cache_dsa_iswa::get_can_shift() const { + return kv_dsa->get_can_shift() && + kv_swa->get_can_shift() && + kv_dsa->get_mla()->get_size() == kv_swa->get_size(); +} + +void llama_kv_cache_dsa_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { + if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) { + kv_dsa->state_write(io, seq_id, flags); + } + + kv_swa->state_write(io, seq_id, flags); +} + +void llama_kv_cache_dsa_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { + if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) { + kv_dsa->state_read(io, seq_id, flags); + } + + kv_swa->state_read(io, seq_id, flags); +} + +llama_kv_cache_dsa * llama_kv_cache_dsa_iswa::get_dsa() const { + return kv_dsa.get(); +} + +llama_kv_cache * llama_kv_cache_dsa_iswa::get_swa() const { + return kv_swa.get(); +} + +// +// llama_kv_cache_dsa_iswa_context +// + +llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(llama_memory_status status) : status(status) {} + +llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context( + llama_kv_cache_dsa_iswa * kv) : + ctx_dsa(kv->get_dsa()->init_full()), + ctx_swa(kv->get_swa()->init_full()), + status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) { +} + +llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context( + llama_kv_cache_dsa_iswa * kv, + llama_context * lctx, + bool optimize) : + ctx_dsa(kv->get_dsa()->init_update(lctx, optimize)), + ctx_swa(kv->get_swa()->init_update(lctx, optimize)), + status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) { +} + +llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context( + llama_kv_cache_dsa_iswa * kv, + slot_info_vec_t sinfos_mla, + slot_info_vec_t sinfos_lid, + slot_info_vec_t sinfos_swa, + std::vector ubatches) : + ubatches(std::move(ubatches)), + // note: here we copy the ubatches. not sure if this is ideal + ctx_dsa(new llama_kv_cache_dsa_context(kv->get_dsa(), std::move(sinfos_mla), std::move(sinfos_lid), this->ubatches)), + ctx_swa(new llama_kv_cache_context(kv->get_swa(), std::move(sinfos_swa), this->ubatches)), + status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) { +} + +llama_kv_cache_dsa_iswa_context:: ~llama_kv_cache_dsa_iswa_context() = default; + +bool llama_kv_cache_dsa_iswa_context::next() { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + ctx_dsa->next(); + ctx_swa->next(); + + if (++i_next >= ubatches.size()) { + return false; + } + + return true; +} + +bool llama_kv_cache_dsa_iswa_context::apply() { + assert(!llama_memory_status_is_fail(status)); + + bool res = true; + + res = res & ctx_dsa->apply(); + res = res & ctx_swa->apply(); + + return res; +} + +llama_memory_status llama_kv_cache_dsa_iswa_context::get_status() const { + return status; +} + +const llama_ubatch & llama_kv_cache_dsa_iswa_context::get_ubatch() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ubatches[i_next]; +} + +const llama_kv_cache_dsa_context * llama_kv_cache_dsa_iswa_context::get_dsa() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return static_cast(ctx_dsa.get()); +} + +const llama_kv_cache_context * llama_kv_cache_dsa_iswa_context::get_swa() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return static_cast(ctx_swa.get()); +} diff --git a/src/llama-kv-cache-dsa-iswa.h b/src/llama-kv-cache-dsa-iswa.h new file mode 100644 index 000000000..28cf95bf0 --- /dev/null +++ b/src/llama-kv-cache-dsa-iswa.h @@ -0,0 +1,134 @@ +#pragma once + +#include "llama-kv-cache-dsa.h" + +#include + +// +// llama_kv_cache_dsa_iswa +// + +// utilizes two child memories: llama_kv_cache_dsa for the full-attention (DSA) layers and llama_kv_cache for the SWA layers + +class llama_kv_cache_dsa_iswa : public llama_memory_i { +public: + llama_kv_cache_dsa_iswa( + const llama_model & model, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, + const layer_filter_cb & filter_mla, + const layer_filter_cb & filter_lid, + const layer_reuse_cb & reuse); + + ~llama_kv_cache_dsa_iswa() = default; + + // + // llama_memory_i + // + + llama_memory_context_ptr init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) override; + + llama_memory_context_ptr init_full() override; + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; + + void clear(bool data) override; + + bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; + void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override; + void seq_keep(llama_seq_id seq_id) override; + void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) override; + void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override; + + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + + std::map memory_breakdown() const override; + + // state write/load + + void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; + void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; + + // + // llama_kv_cache_dsa_iswa specific API + // + + llama_kv_cache_dsa * get_dsa() const; + llama_kv_cache * get_swa() const; + +private: + const bool unified; + + std::unique_ptr kv_dsa; + std::unique_ptr kv_swa; +}; + +class llama_kv_cache_dsa_iswa_context : public llama_memory_context_i { +public: + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + // used for errors + llama_kv_cache_dsa_iswa_context(llama_memory_status status); + + // used to create a full-cache context + llama_kv_cache_dsa_iswa_context( + llama_kv_cache_dsa_iswa * kv); + + // used to create an update context + llama_kv_cache_dsa_iswa_context( + llama_kv_cache_dsa_iswa * kv, + llama_context * lctx, + bool optimize); + + // used to create a batch processing context from a batch + llama_kv_cache_dsa_iswa_context( + llama_kv_cache_dsa_iswa * kv, + slot_info_vec_t sinfos_mla, + slot_info_vec_t sinfos_lid, + slot_info_vec_t sinfos_swa, + std::vector ubatches); + + virtual ~llama_kv_cache_dsa_iswa_context(); + + // + // llama_memory_context_i + // + + bool next() override; + bool apply() override; + + llama_memory_status get_status() const override; + const llama_ubatch & get_ubatch() const override; + + // + // llama_kv_cache_dsa_iswa_context specific API + // + + const llama_kv_cache_dsa_context * get_dsa() const; + const llama_kv_cache_context * get_swa() const; + +private: + // the index of the next ubatch to process + size_t i_next = 0; + + std::vector ubatches; + + const llama_memory_context_ptr ctx_dsa; + const llama_memory_context_ptr ctx_swa; + + const llama_memory_status status; +}; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 865f720fe..01ef0255c 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -323,7 +323,8 @@ llama_kv_cache::llama_kv_cache( hparams.n_embd_head_k() % 64 == 0; // always create Hadamard rotation tensors for DeepSeek lightning indexers - if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 || model.arch == LLM_ARCH_GLM_DSA) && + if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 || + model.arch == LLM_ARCH_GLM_DSA || model.arch == LLM_ARCH_DOTS3NOTE) && hparams.n_embd_head_k_full == hparams.indexer_head_size) { attn_rot_k = true; } diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index b9e0a6009..0d39e6de8 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -31,6 +31,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_MELLUM: case LLM_ARCH_LAGUNA: case LLM_ARCH_GRANITE_SWA: + case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config return false; default: return true; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 356730df0..5b0e71847 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -11,6 +11,7 @@ #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" +#include "llama-kv-cache-dsa-iswa.h" #include "llama-kv-cache-msa.h" #include "llama-kv-cache-dsv4.h" #include "llama-memory-hybrid.h" @@ -68,6 +69,7 @@ #include "models/delta-net-base.cpp" #include "models/dflash.cpp" #include "models/dots1.cpp" +#include "models/dots3note.cpp" #include "models/dream.cpp" #include "models/eagle3.cpp" #include "models/ernie4-5-moe.cpp" @@ -344,6 +346,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_deepseek2ocr(params); case LLM_ARCH_DEEPSEEK32: return new llama_model_deepseek32(params); + case LLM_ARCH_DOTS3NOTE: + return new llama_model_dots3note(params); case LLM_ARCH_DEEPSEEK4: return new llama_model_deepseek4(params); case LLM_ARCH_GLM_DSA: @@ -1001,6 +1005,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_230B_A10B: return "230B.A10B"; case LLM_TYPE_428B_A23B: return "428B.A23B"; case LLM_TYPE_235B_A22B: return "235B.A22B"; + case LLM_TYPE_288B_A19B: return "288B.A19B"; case LLM_TYPE_300B_A47B: return "300B.A47B"; case LLM_TYPE_310B_A15B: return "310B.A15B"; case LLM_TYPE_355B_A32B: return "355B.A32B"; @@ -2074,7 +2079,9 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale); } - if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) { + if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || + arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || + arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4) { LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead); LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q); LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv); @@ -2343,6 +2350,57 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, nullptr); } } break; + case LLM_ARCH_DOTS3NOTE: + { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) { + // MTP draft context: plain attention KV cache holding only the nextn layer + llama_kv_cache::layer_filter_cb filter = + [&](uint32_t il) { return il >= hparams.n_layer(); }; + + res = new llama_kv_cache( + *this, + hparams, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + 1, + hparams.n_swa, + hparams.swa_type, + nullptr, + filter, + nullptr, + nullptr); + } else { + // main context: DSA cache for the trunk full-attention layers plus a window-sized SWA cache + llama_kv_cache::layer_filter_cb filter_mla = nullptr; + if (hparams.n_layer_nextn > 0) { + filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); }; + } + llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && hparams.is_indexer_full(il); }; + + res = new llama_kv_cache_dsa_iswa( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + 1, + filter_mla, + filter_lid, + nullptr); + } + } break; case LLM_ARCH_DEEPSEEK4: { GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); @@ -2811,6 +2869,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_LLAMA_EMBED: case LLM_ARCH_MAINCODER: case LLM_ARCH_GLM_DSA: + case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_NANBEIGE: case LLM_ARCH_POCKETTTS: return LLAMA_ROPE_TYPE_NORM; diff --git a/src/llama-model.h b/src/llama-model.h index 4412ef08e..44bd96757 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -140,6 +140,7 @@ enum llm_type { LLM_TYPE_230B_A10B, // Minimax M2 LLM_TYPE_428B_A23B, // Minimax M3 LLM_TYPE_235B_A22B, + LLM_TYPE_288B_A19B, // dots3-note LLM_TYPE_300B_A47B, // Ernie MoE big LLM_TYPE_310B_A15B, // /MiMo-V2-Flash LLM_TYPE_355B_A32B, // GLM-4.5 diff --git a/src/llama.cpp b/src/llama.cpp index cb4b9a640..c5ec926e6 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -12,6 +12,7 @@ #include "llama-sampler.cpp" #include "llama-kv-cache.cpp" #include "llama-kv-cache-dsa.cpp" +#include "llama-kv-cache-dsa-iswa.cpp" #include "llama-kv-cache-dsv4.cpp" #include "llama-kv-cache-iswa.cpp" #include "llama-kv-cache-msa.cpp" diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index f5855696e..0637931cc 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -1,6 +1,8 @@ #include "models.h" #include "llama-memory-recurrent.h" +#include + void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl); @@ -179,7 +181,9 @@ static ggml_tensor * bailingmoe3_causal_conv1d( int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, - int64_t cache_head) { + int64_t cache_head, + uint32_t mem_size, + uint32_t n_rs_seq) { const int64_t d_inner = head_dim * n_head; const int64_t conv_state_size = (d_conv - 1) * d_inner; const int64_t total_state_size = 3 * conv_state_size; @@ -193,13 +197,18 @@ static ggml_tensor * bailingmoe3_causal_conv1d( x_proj = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); ggml_tensor * conv_x = ggml_concat(ctx0, conv_state, ggml_transpose(ctx0, x_proj), 0); - ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, - conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_x, - ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, - (d_conv - 1) * ggml_element_size(conv_states_all), - total_state_size * ggml_element_size(conv_states_all), - (cache_head * total_state_size + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + const int64_t K = (int64_t) n_rs_seq + 1; + const int64_t n_written = std::min(n_seq_tokens, K); + + for (int64_t slot = 0; slot < n_written; ++slot) { + ggml_tensor * conv_snap = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, + conv_x->nb[1], conv_x->nb[2], (conv_x->ne[0] - (d_conv - 1) - slot) * conv_x->nb[0]); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap, + ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_states_all), + total_state_size * ggml_element_size(conv_states_all), + ((slot * mem_size + cache_head) * total_state_size + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + } ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); ggml_tensor * out = ggml_ssm_conv(ctx0, conv_x, conv_weight); @@ -237,6 +246,8 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + const auto & layer = model.layers[il]; ggml_tensor * inpSA = inpL; ggml_tensor * cur = build_norm(inpL, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -245,18 +256,19 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph if (hparams.is_recr(il)) { const auto * mctx_cur = inp_rs->mctx; const auto cache_head = mctx_cur->get_head(); + const auto mem_size = mctx_cur->get_size(); ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); ggml_tensor * q = bailingmoe3_causal_conv1d( gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, - d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head); + d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq); ggml_tensor * k = bailingmoe3_causal_conv1d( gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, - d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head); + d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq); ggml_tensor * v = bailingmoe3_causal_conv1d( gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, - d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head); + d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq); ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ssm_f_a, cur); gate = ggml_add(ctx0, gate, layer.ssm_dt_b); @@ -276,11 +288,8 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph ggml_tensor * state = build_rs(inp_rs, states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs); - auto result = build_delta_net(q, k, v, gate, beta, state, il); - ggml_tensor * out = ggml_cont(ctx0, result.first); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, result.second, - ggml_view_1d(ctx0, states_all, hparams.n_embd_s() * n_seqs, - cache_head * hparams.n_embd_s() * ggml_element_size(states_all)))); + ggml_tensor * out = ggml_cont(ctx0, build_recurrent_attn( + inp_rs, states_all, q, k, v, gate, beta, state, il)); ggml_tensor * out_gate = ggml_mul_mat(ctx0, layer.ssm_g_a, cur); out_gate = ggml_reshape_3d(ctx0, out_gate, head_dim, n_head, n_tokens); diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp new file mode 100644 index 000000000..00a008c2c --- /dev/null +++ b/src/models/dots3note.cpp @@ -0,0 +1,480 @@ +#include "models.h" + +#include "llama-kv-cache.h" +#include "llama-kv-cache-dsa.h" + +// note: code adapted from deepseek32.cpp (DSA indexer + absorbed MLA) and step35.cpp (head-wise output gate) + +void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + hparams.f_norm_eps = 1e-6; // eps for the indexer k_norm layer norm + + // TODO: use MTP layer + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); + GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all"); + + // MoE parameters + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + + // MLA parameters of the full-attention layers + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); + + // MLA parameters of the sliding-window layers + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa); + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + // DSA parameters + ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); + ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); + ml.get_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl); + + switch (hparams.n_layer()) { + case 46: type = LLM_TYPE_288B_A19B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + GGML_UNUSED(ml); + + if (!hparams.is_mla()) { + throw std::runtime_error("DOTS3NOTE architecture requires MLA"); + } + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_expert_shared = hparams.n_expert_shared; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (!output) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer_all; ++i) { + auto & layer = layers[i]; + + const bool is_mtp = i >= n_layer; + // the NextN/MTP block uses the sliding-attention geometry + const bool is_swa = is_mtp || hparams.is_swa(i); + + // MTP tensors are preserved in the GGUF but there is no MTP graph yet + const int flags = is_mtp ? TENSOR_SKIP | TENSOR_NOT_REQUIRED : 0; + + const int64_t n_head_l = hparams.n_head(i); + + const int64_t kv_lora_rank = is_swa ? hparams.n_lora_kv_swa : hparams.n_lora_kv; + const int64_t n_embd_head_k_mla = is_swa ? hparams.n_embd_head_k_mla_swa : hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = is_swa ? hparams.n_embd_head_v_mla_swa : hparams.n_embd_head_v_mla(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags); + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, flags); + // norm applied on the shared rope key before rope + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_qk_rope}, flags); + + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head_l * n_embd_head_k_mla}, flags); + + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags); + + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head_l}, flags); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head_l}, flags); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head_l * n_embd_head_v_mla, n_embd}, flags); + + // head-wise sigmoid output gate + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_l}, flags); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); + + // DSA indexer + if (!is_mtp && hparams.is_indexer_full(i)) { + layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {hparams.indexer_head_size}, flags); + layer.indexer_k_norm_b = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "bias", i), {hparams.indexer_head_size}, flags); + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, hparams.indexer_n_head}, flags); + layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd, hparams.indexer_head_size}, flags); + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * hparams.indexer_head_size}, flags); + } + + if (is_mtp || i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, flags); + } else { + if (n_expert == 0 || n_expert_used == 0) { + throw std::runtime_error("n_expert and n_expert_used must be > 0"); + } + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, flags); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, flags); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + } + + if (is_mtp) { + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, flags); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags); + } + } +} + +std::unique_ptr llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + GGML_ASSERT(hparams.is_mla()); + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + + const int64_t n_indexer_head = hparams.indexer_n_head; + const int64_t n_embd_indexer_head = hparams.indexer_head_size; + const uint32_t n_indexer_top_k = hparams.indexer_top_k; + + // the indexer head layout is [rope | nope] + GGML_ASSERT(hparams.n_rot() <= n_embd_indexer_head); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * inp_pos = build_inp_pos(); + + llm_graph_input_attn_k_dsa_iswa * inp_attn = build_attn_inp_k_dsa_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + const bool is_swa = hparams.is_swa(il); + + const int64_t n_head_l = hparams.n_head(il); + + const int64_t kv_lora_rank = is_swa ? hparams.n_lora_kv_swa : hparams.n_lora_kv; + const int64_t n_embd_head_k_mla = is_swa ? hparams.n_embd_head_k_mla_swa : hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = is_swa ? hparams.n_embd_head_v_mla_swa : hparams.n_embd_head_v_mla(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + + const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla)); + const float freq_base_l = model.get_rope_freq_base(cparams, il); + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + ggml_tensor * attn_inp = cur; + + ggml_tensor * qr = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur); + cb(qr, "qr", il); + + qr = build_norm(qr, model.layers[il].attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "qr", il); + + ggml_tensor * top_k = nullptr; + + // lightning indexer (full-attention layers only) + if (!is_swa) { + ggml_tensor * indexer_q = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_q_b, qr); + cb(indexer_q, "indexer_q", il); + + // {n_embd_indexer_head, n_indexer_head, n_tokens} + indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); + indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, + LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(indexer_q, "indexer_q", il); + + ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); + cb(indexer_k, "indexer_k", il); + + indexer_k = build_norm(indexer_k, model.layers[il].indexer_k_norm, model.layers[il].indexer_k_norm_b, LLM_NORM, il); + cb(indexer_k, "indexer_k", il); + + // {n_embd_indexer_head, 1, n_tokens} + indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); + indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, + LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(indexer_k, "indexer_k", il); + + // perform Hadamard transform on indexer q and k + indexer_q = ggml_mul_mat(ctx0, inp_attn->get_dsa()->self_k_rot_lid, indexer_q); + cb(indexer_q, "indexer_q", il); + indexer_k = ggml_mul_mat(ctx0, inp_attn->get_dsa()->self_k_rot_lid, indexer_k); + cb(indexer_k, "indexer_k", il); + + // store indexer keys to KV cache + const auto * mctx_lid = inp_attn->get_dsa()->mctx->get_lid(); + const auto & k_idxs_lid = inp_attn->get_dsa()->get_k_idxs_lid(); + ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, indexer_k, k_idxs_lid, il)); + + ggml_tensor * indexer_weights = ggml_mul_mat(ctx0, model.layers[il].indexer_proj, cur); + cb(indexer_weights, "indexer_weights", il); + + indexer_k = mctx_lid->get_k(ctx0, il); + + // split the batch into streams if needed + const auto n_stream = indexer_k->ne[3]; + indexer_q = ggml_view_4d(ctx0, indexer_q, indexer_q->ne[0], indexer_q->ne[1], indexer_q->ne[2]/n_stream, n_stream, indexer_q->nb[1], indexer_q->nb[2], indexer_q->nb[3]/n_stream, 0); + indexer_weights = ggml_view_4d(ctx0, indexer_weights, indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream, indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0); + + // pre-scale weights to avoid scaling operations on huge indexer_score tensor + indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head))); + cb(indexer_weights, "indexer_weights", il); + + ggml_tensor * indexer_score = nullptr; + if (cparams.fused_lid) { + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->get_kq_mask_lid()); + cb(indexer_score, "indexer_score", il); + res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); + } else { + indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3); + cb(indexer_q, "indexer_q", il); + indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3); + cb(indexer_k, "indexer_k", il); + + ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q); + cb(indexer_kq, "indexer_kq", il); + + // ReLU requires contiguous tensors + indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3)); + cb(indexer_kq, "indexer_kq", il); + + indexer_score = ggml_relu(ctx0, indexer_kq); + cb(indexer_score, "indexer_score", il); + + indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights); + cb(indexer_score, "indexer_score", il); + + // sum by q n_indexer_head dimension + indexer_score = ggml_sum_rows(ctx0, indexer_score); + cb(indexer_score, "indexer_score", il); + + // permute result to match KQ mask + indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3)); + cb(indexer_score, "indexer_score", il); + + ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->get_kq_mask_lid(); + indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); + cb(indexer_score, "indexer_score", il); + } + + // get indices of top k indexer scores + uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k; + top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k)); + cb(top_k, "top_k", il); + } + + ggml_tensor * q = ggml_mul_mat(ctx0, model.layers[il].wq_b, qr); + cb(q, "q", il); + + // split into {n_embd_head_qk_nope, n_head_l, n_tokens} + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head_l, n_tokens, ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head_l, 0); + cb(q_nope, "q_nope", il); + + // and {n_embd_head_qk_rope, n_head_l, n_tokens} + ggml_tensor * q_pe = ggml_view_3d( + ctx0, q, n_embd_head_qk_rope, n_head_l, n_tokens, ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head_l, ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); + cb(kv_cmpr_pe, "kv_cmpr_pe", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "kv_cmpr", il); + + // and {n_embd_head_qk_rope, 1, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "k_pe", il); + + // norm on the shared rope key, applied before rope + k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); + cb(k_pe, "k_pe", il); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "kv_cmpr", il); + + // MLA attention with the absorption optimization + { + // {n_embd_head_qk_nope, n_tokens, n_head_l} + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "q_nope_perm", il); + + // {n_embd_head_qk_nope, kv_lora_rank, n_head_l} x {n_embd_head_qk_nope, n_tokens, n_head_l} + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, model.layers[il].wk_b, q_nope); + cb(q_nope_absorbed, "q_nope_absorbed", il); + + // {kv_lora_rank, n_head_l, n_tokens} + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "q_nope_absorbed_perm", il); + + // {n_embd_head_qk_rope + kv_lora_rank, n_head_l, n_tokens} + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + cb(kv_cmpr, "kv_cmpr_reshape", il); + + // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens} + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "Kcur", il); + + // {kv_lora_rank, 1, n_tokens} + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "Vcur", il); + + // apply the head-wise output gate before o_proj, so wo stays out of build_attn + if (is_swa) { + cur = build_attn(inp_attn->get_swa(), + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, kq_scale, il); + } else { + cur = build_attn(inp_attn->get_dsa(), + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il); + } + cb(cur, "attn_out", il); + + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp); + cb(gate, "attn_gate", il); + + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate_sigmoid", il); + + // broadcast the per-head gate over the head dimension + ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, cur, n_embd_head_v_mla, n_head_l, n_tokens); + ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate, 1, n_head_l, n_tokens); + attn_3d = ggml_mul(ctx0, attn_3d, gate_3d); + cb(attn_3d, "attn_gated", il); + + cur = ggml_reshape_2d(ctx0, attn_3d, n_embd_head_v_mla * n_head_l, n_tokens); + + cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); + cb(cur, "attn_output", il); + } + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + model.layers[il].ffn_gate_up_exps, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = + build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = ggml_mul_mat(ctx0, model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 1dd30dfd1..157b05dc0 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1156,6 +1156,18 @@ struct llama_model_deepseek32 : public llama_model_base { }; +struct llama_model_dots3note : public llama_model_base { + llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_deepseek4 : public llama_model_base { llama_model_deepseek4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 5d897bc46..3e78c8929 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -33,6 +33,7 @@ int llama_fit_params(int argc, char ** argv) { if (!params.fit_params_print) { const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, + nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) { LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__); diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index e12140ba0..2cf1b683a 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -120,6 +120,12 @@ struct clip_graph { ffn_op_type type_op, int il) const; + ggml_tensor * build_moe_ffn( + ggml_tensor * cur, + const clip_layer & layer, + ffn_op_type type_op, + int il) const; + ggml_tensor * build_attn( ggml_tensor * wo, ggml_tensor * wo_b, diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 3fcd12233..074f7f372 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -75,6 +75,7 @@ #define KEY_SAM_N_HEAD "clip.vision.sam.head_count" #define KEY_SAM_N_BLOCK "clip.vision.sam.block_count" #define KEY_SAM_N_EMBD "clip.vision.sam.embedding_length" +#define KEY_VISION_N_EXPERT_USED "clip.vision.expert_used_count" // audio-specific #define KEY_AUDIO_PROJ_TYPE "clip.audio.projector_type" // for models with mixed modalities #define KEY_A_NUM_MEL_BINS "clip.audio.num_mel_bins" @@ -119,7 +120,11 @@ #define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s" #define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s" #define TN_FFN_UP "%s.blk.%d.ffn_up.%s" -#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s" +#define TN_FFN_GATE_INP "%s.blk.%d.ffn_gate_inp.%s" // MoE router (dots3note) +#define TN_FFN_GATE_EXPS "%s.blk.%d.ffn_gate_exps.%s" +#define TN_FFN_UP_EXPS "%s.blk.%d.ffn_up_exps.%s" +#define TN_FFN_DOWN_EXPS "%s.blk.%d.ffn_down_exps.%s" +#define TN_FFN_EXP_PROBS_B "%s.blk.%d.exp_probs_b.%s" #define TN_LN_1 "%s.blk.%d.ln1.%s" // layer norm #define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm #define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale @@ -471,6 +476,8 @@ enum projector_type { PROJECTOR_TYPE_COGVLM, PROJECTOR_TYPE_JANUS_PRO, PROJECTOR_TYPE_DOTS_OCR, + PROJECTOR_TYPE_DOTS3NOTE_V, + PROJECTOR_TYPE_DOTS3NOTE_A, PROJECTOR_TYPE_DEEPSEEKOCR, PROJECTOR_TYPE_DEEPSEEKOCR2, PROJECTOR_TYPE_LFM2A, @@ -533,6 +540,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_COGVLM, "cogvlm"}, { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"}, { PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"}, + { PROJECTOR_TYPE_DOTS3NOTE_V, "dots3note_v"}, + { PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"}, { PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"}, { PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"}, { PROJECTOR_TYPE_LFM2A, "lfm2a"}, diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index ad25c008e..fcdabd633 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -93,6 +93,7 @@ struct clip_hparams { float eps = 1e-6; float rope_theta = 0.0; + int32_t n_expert_used = 0; std::vector feature_layers; int32_t attn_window_size = 0; int32_t n_wa_pattern = 0; @@ -259,6 +260,13 @@ struct clip_layer { ggml_tensor * ff_down_w = nullptr; ggml_tensor * ff_down_b = nullptr; + // MoE FFN (dots3note vision pyramid blocks) + ggml_tensor * ff_gate_inp_w = nullptr; + ggml_tensor * ff_gate_exps_w = nullptr; + ggml_tensor * ff_up_exps_w = nullptr; + ggml_tensor * ff_down_exps_w = nullptr; + ggml_tensor * ff_exp_probs_b = nullptr; + // layernorm 2 (or pre-FFN norm) ggml_tensor * ln_2_w = nullptr; ggml_tensor * ln_2_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 9d634e338..d4a21857a 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -44,6 +44,7 @@ #include "models/cogvlm.cpp" #include "models/conformer.cpp" #include "models/dotsocr.cpp" +#include "models/dots3note.cpp" #include "models/exaone4_5.cpp" #include "models/gemma4a.cpp" #include "models/gemma4v.cpp" @@ -571,11 +572,13 @@ ggml_tensor * clip_graph::build_vit( cb(cur, "ffn_inp_normed", il); // ffn - cur = build_ffn(cur, - layer.ff_up_w, layer.ff_up_b, - layer.ff_gate_w, layer.ff_gate_b, - layer.ff_down_w, layer.ff_down_b, - ffn_t, il); + cur = layer.ff_gate_exps_w + ? build_moe_ffn(cur, layer, ffn_t, il) + : build_ffn(cur, + layer.ff_up_w, layer.ff_up_b, + layer.ff_gate_w, layer.ff_gate_b, + layer.ff_down_w, layer.ff_down_b, + ffn_t, il); cb(cur, "ffn_out", il); @@ -756,6 +759,50 @@ ggml_tensor * clip_graph::build_ffn( return cur; } +// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision) +// the router runs in fp32; exp_probs_b only affects expert selection, not the weights +ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const { + const int64_t n_tokens = cur->ne[1]; + const int64_t n_expert = layer.ff_gate_exps_w->ne[2]; + const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert); + GGML_ASSERT(n_expert_used > 0); + GGML_ASSERT(type_op == FFN_SILU); + + ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens] + cb(probs, "ffn_moe_probs", il); + + ggml_tensor * sel = layer.ff_exp_probs_b + ? ggml_add(ctx0, probs, layer.ff_exp_probs_b) + : probs; + ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens] + + ggml_tensor * weights = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected); + weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); + weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights)); + weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); + cb(weights, "ffn_moe_weights", il); + + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens); + ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens] + ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected); + cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up); + cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens] + cur = ggml_mul(ctx0, cur, weights); + + // sum over the selected experts + ggml_tensor * out = nullptr; + for (int64_t i = 0; i < n_expert_used; i++) { + ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]); + out = out ? ggml_add(ctx0, out, v) : v; + } + if (n_expert_used == 1) { + out = ggml_cont(ctx0, out); + } + cb(out, "ffn_moe_out", il); + return out; +} + ggml_tensor * clip_graph::build_attn( ggml_tensor * wo, ggml_tensor * wo_b, @@ -990,9 +1037,14 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const builder = std::make_unique(ctx, img); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: { @@ -1590,6 +1642,25 @@ struct clip_model_loader { get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup } break; + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + hparams.rope_theta = 10000.0f; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + hparams.rope_theta = 10000.0f; + hparams.audio_chunk_len = 60; // in seconds + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; case PROJECTOR_TYPE_KIMIVL: { hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; @@ -2280,12 +2351,20 @@ struct clip_model_loader { layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias"), false); layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias"), false); + // MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present + layer.ff_gate_inp_w = get_tensor(string_format(TN_FFN_GATE_INP, prefix, il, "weight"), false); + layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false); + layer.ff_up_exps_w = get_tensor(string_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false); + layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false); + layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false); + const bool is_moe = layer.ff_gate_exps_w != nullptr; + // ffn - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight")); + layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"), !is_moe); layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias"), false); layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false); layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"), false); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight")); + layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe); layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false); // mimovl per-head attention sink bias @@ -2767,6 +2846,7 @@ struct clip_model_loader { model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); @@ -2777,6 +2857,23 @@ struct clip_model_loader { // post_trunk_norm: applied after all ViT blocks, before the merger model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight")); + model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias")); + model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight")); + model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias")); + model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight")); + model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias")); + model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias + // adapter: LayerNorm -> Linear -> GELU -> Linear + model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias")); + model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); + } break; case PROJECTOR_TYPE_ULTRAVOX: { model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); @@ -4165,12 +4262,18 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { // dynamic size int n_merge = ctx->model.hparams.n_merge; int stride = n_merge * n_merge; n_patches = CLIP_ALIGN(n_patches, stride) / stride; } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + // 3x stride-2 conv2d over mel frames + n_patches = (img->nx() + 7) / 8; + } break; case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_LIGHTONOCR: { @@ -4817,6 +4920,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_input_i32("minimax_pos_w", pos_w); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { const int pw = image_size_width / patch_size; const int ph = image_size_height / patch_size; @@ -5307,6 +5411,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_i32("pos_w", pos_data); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + } break; case PROJECTOR_TYPE_GEMMA4A: { GGML_ASSERT(imgs.entries.size() == 1); @@ -5803,6 +5917,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_LIGHTONOCR: case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + case PROJECTOR_TYPE_DOTS3NOTE_A: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_MLP_NORM: return ctx->model.mm_3_b->ne[0]; diff --git a/tools/mtmd/models/dots3note.cpp b/tools/mtmd/models/dots3note.cpp new file mode 100644 index 000000000..93c14fc79 --- /dev/null +++ b/tools/mtmd/models/dots3note.cpp @@ -0,0 +1,61 @@ +#include "models.h" + +ggml_cgraph * clip_graph_dots3note_a::build() { + // inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded + // the reference impl zero-masks conv inputs beyond the valid length at each stage; + // running on exactly the valid frames with the convs' zero padding is equivalent + ggml_tensor * inp = build_inp_raw(1); + GGML_ASSERT(inp->type == GGML_TYPE_F32); + + // 3x conv2d (k=3, s=2, p=1) + gelu + { + auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) { + x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1); + x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1)); + return ggml_gelu_erf(ctx0, x); + }; + + inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b); + inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b); + inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b); + // inp: [OW=n_frames/8, OH=n_mel/8, OC=480, 1] + cb(inp, "after_conv_stem", -1); + } + + // [OW, OH, OC, 1] -> [OH*OC, OW], feature index f + OH*c (matches the reference permute+reshape) + inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); + inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2]); + + // project to d_model (no bias) + inp = ggml_mul_mat(ctx0, model.conv_out_w, inp); + cb(inp, "after_conv_out", -1); + + const int64_t n_pos = inp->ne[1]; + + ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + + // partial rotary: first half of each head, NEOX style + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2, + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + }; + + ggml_tensor * cur = build_vit(inp, n_pos, + NORM_TYPE_RMS, hparams.ffn_op, + nullptr, add_pos); + cb(cur, "after_transformer", -1); + + // adapter: LayerNorm -> Linear -> GELU -> Linear + cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, 1e-5, -1); + cur = build_ffn(cur, + model.mm_1_w, model.mm_1_b, + nullptr, nullptr, + model.mm_2_w, model.mm_2_b, + FFN_GELU_ERF, -1); + cb(cur, "projected", -1); + + ggml_build_forward_expand(gf, cur); + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 3631d849b..10546fa5d 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -119,6 +119,11 @@ struct clip_graph_dotsocr : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_dots3note_a : clip_graph { + clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_cogvlm : clip_graph { clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 98a8c11ee..ce08f9e93 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -723,6 +723,100 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa return true; } +// +// mtmd_audio_preprocessor_dots3note +// +// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets +// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames +// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many. +// + +void mtmd_audio_preprocessor_dots3note::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + cache.fill_hann_window(hparams.audio_window_len, true); + cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); +} + +bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples, + size_t n_samples, + std::vector & output) { + if (n_samples == 0) { + return false; + } + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + const int pad = hparams.audio_n_fft / 2; // center=True padding + const int hop = hparams.audio_hop_len; + const size_t chunk_samples = (size_t) hparams.audio_chunk_len * hparams.audio_sample_rate; + + for (size_t start = 0; start < n_samples; start += chunk_samples) { + const size_t n_chunk = std::min(chunk_samples, n_samples - start); + const float * chunk = samples + start; + + const int64_t n_valid = n_chunk / hop; + if (n_valid == 0) { + continue; // sub-hop tail, contributes no frames + } + + // reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT, + // so a partial chunk sees zeros past its end while a full chunk reflects its own tail + std::vector padded(n_chunk + 2 * pad, 0.0f); + for (int i = 0; i < pad; i++) { + int src = pad - i; + padded[i] = (src < (int) n_chunk) ? chunk[src] : 0.0f; + } + std::copy(chunk, chunk + n_chunk, padded.begin() + pad); + if (n_chunk == chunk_samples) { + for (int i = 0; i < pad; i++) { + int src = (int) n_chunk - 2 - i; + padded[n_chunk + pad + i] = (src >= 0) ? chunk[src] : 0.0f; + } + } + + filter_params params; + params.n_mel = hparams.n_mel_bins; + params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hop; + params.sample_rate = hparams.audio_sample_rate; + params.no_padding = true; // padding already applied above + params.use_natural_log = false; + + mtmd_audio_mel mel_full; + if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) { + return false; + } + GGML_ASSERT(mel_full.n_len >= n_valid); + + // per-chunk whisper-style normalization, then keep only the valid frames + mtmd_audio_mel out; + out.n_mel = mel_full.n_mel; + out.n_len = n_valid; + out.n_len_org = n_valid; + out.data.resize((size_t) out.n_mel * (size_t) out.n_len); + + double mmax = -1e20; + for (int64_t m = 0; m < out.n_mel; m++) { + for (int64_t t = 0; t < n_valid; t++) { + mmax = std::max(mmax, (double) mel_full.data[(size_t) m * mel_full.n_len + t]); + } + } + mmax -= 8.0; + for (int64_t m = 0; m < out.n_mel; m++) { + for (int64_t t = 0; t < n_valid; t++) { + const double v = std::max((double) mel_full.data[(size_t) m * mel_full.n_len + t], mmax); + out.data[(size_t) m * n_valid + t] = (float) ((v + 4.0) / 4.0); + } + } + + output.push_back(std::move(out)); + } + return !output.empty(); +} + // // mtmd_audio_preprocessor_mimo_audio // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 44ad098ae..0f47d4502 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -111,6 +111,15 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; +}; + struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 788089652..412b781df 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -359,6 +359,15 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers +static bool is_webp_file(const unsigned char * buf, size_t len) { + // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container + return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0; +} + +#ifdef MTMD_VIDEO +static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder); +#endif + mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) { // calculate the hash if needed std::string id; @@ -398,6 +407,19 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, // otherwise, fallthrough to video decoding (if supported) } +#ifdef MTMD_VIDEO + // stb_image does not support webp; decode it with ffmpeg as a single frame + if (!result && is_webp_file(buf, len)) { + result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder); + if (!result) { + LOG_ERR("%s: failed to decode webp buffer\n", __func__); + return {nullptr, nullptr}; + } + mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str()); + return {result, nullptr}; + } +#endif + // last try: load as video #ifdef MTMD_VIDEO if (!result) { @@ -821,6 +843,33 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) { return result; } +#ifdef MTMD_VIDEO +static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) { + auto params = mtmd_helper_video_init_params_default(); + mtmd_helper_video vctx; + vctx.mctx = mctx; + vctx.input_buf.assign(buf, buf + len); + vctx.ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); + vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); + if (!vctx.probe(0.0f)) { + return nullptr; + } + if (placeholder) { + return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr); + } + // still image: the fps filter would output no frame, so disable it + vctx.fps_target = 0.0f; + if (!vctx.start_ffmpeg(0.0f)) { + return nullptr; + } + mtmd_bitmap * frame = vctx.read_next_frame(); + if (frame) { + mtmd_bitmap_set_mergeable(frame, false); + } + return frame; +} +#endif + mtmd_helper_video * mtmd_helper_video_init( mtmd_context * mctx, const char * path, diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 5c6b92419..58dfb1525 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -45,6 +45,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm // helper function to construct a mtmd_bitmap from a buffer containing a file // supported formats: // image: formats supported by stb_image: jpg, png, bmp, gif, etc. +// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH // audio: formats supported by miniaudio: wav, mp3, flac // note: // - for now, video input is only supported via C++ helper functions diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 95f17f7af..5b306180d 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -825,6 +825,7 @@ struct mtmd_context { image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: { // <|img|> ... (image embeddings) ... <|endofimg|> img_beg = "<|img|>"; @@ -976,6 +977,13 @@ struct mtmd_context { aud_end = ""; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + // <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|> + aud_beg = "<|audio_comp_start|>"; + aud_end = "<|audio_comp_end|>"; + audio_preproc = std::make_unique(ctx_a); + } break; case PROJECTOR_TYPE_MIMO_AUDIO: { aud_beg = "<|mimo_audio_start|>"; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 1293c8640..36d982832 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1040,62 +1040,7 @@ private: } } - // optionally reserve VRAM for the draft / MTP context before fitting the target model - if (params_base.fit_params) { - if (has_spec) { - // MTP draft context lives on the target model, only context+compute are new - bool measure_model_bytes = has_draft; - - common_params params_dft = common_base_params_to_speculative(params_base); - - auto mparams_dft = common_model_params_to_llama(params_dft); - auto cparams_dft = common_context_params_to_llama(params_dft); - if (spec_mtp) { - cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP; - } - cparams_dft.n_rs_seq = 0; - - std::vector devs; - uint32_t hp_ngl = 0; - uint32_t hp_nct = 0; - uint32_t hp_nex = 0; - try { - auto dmd = common_get_device_memory_data( - params_dft.model.path.c_str(), &mparams_dft, &cparams_dft, - devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); - - GGML_ASSERT(!params_base.fit_params_target.empty()); - size_t total = 0; - - std::vector tgt_devices = params.devices; - - if (tgt_devices.empty()) { - for(size_t i = 0; i < ggml_backend_dev_count(); ++i) { - tgt_devices.push_back(ggml_backend_dev_get(i)); - } - } - - for (size_t j = 0; j < devs.size(); ++j) { - const size_t bytes = (measure_model_bytes ? dmd[j].model : 0) + dmd[j].context + dmd[j].compute; - total += bytes; - for (size_t i = 0; i < tgt_devices.size(); i++) { - if (tgt_devices[i] == devs[j]) { - SRV_DBG("[spec] adding %.2f MiB to fit_params_target for device %s\n", - bytes / (1024.0 * 1024.0), ggml_backend_dev_name(devs[j])); - params_base.fit_params_target[i] += bytes; - break; - } - } - } - SRV_TRC("[spec] estimated memory usage of %s is %.2f MiB\n", - has_draft ? "draft model" : "MTP context", - total / (1024.0 * 1024.0)); - } catch (const std::exception & e) { - SRV_WRN("[spec] failed to measure %s memory: %s\n", - has_draft ? "draft model" : "MTP context", e.what()); - } - } - } + // note: the draft / MTP context is fitted together with the target model, see common_fit_extra_model // attach a progress callback {