From 6e5b8b7e4d88b0cdbfab6d2e6bc531cddab46065 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 25 Aug 2026 14:00:29 +0000 Subject: [PATCH] gguf: add qwen4exp (Qwen3.8-Flash-Next) arch and converter Adds the GGUF-side plumbing for HF model_type qwen4_exp: - MODEL_ARCH.QWEN4EXP plus tensors for the low-rank hyper-connection variant (hc_*_norm/down/up/inject) and the PLE n-gram hash embeddings. The DeepSeek-V4 hc_*_fn/base/scale tensors are a different parameterisation, so these are separate entries rather than reuse. - Reuses the existing indexer, per_layer_token_embd, SSM and compress_ratios keys unchanged. - conversion/qwen4exp.py inherits the Qwen3.5 linear-attention V-head reorder and interleaved mrope, concatenates the 128 PLE embedding shards, and splits index_qk_proj into separate indexer q/k tensors. The PLE hash multipliers reach ~2.4e13. prepare_tensors() casts every non-float dtype to float32 before modify_tensors() runs, and GGUF array writes infer INT32 from Python ints, so both paths are bypassed: the constants are read from the pre-cast lazy tensors and written as explicit UINT64 arrays. Additive only; no existing arch changes behaviour. --- conversion/__init__.py | 3 + conversion/qwen4exp.py | 152 +++++++++++++++++++++++++++++++++ gguf-py/gguf/constants.py | 105 +++++++++++++++++++++++ gguf-py/gguf/gguf_writer.py | 32 +++++++ gguf-py/gguf/tensor_mapping.py | 59 +++++++++++++ 5 files changed, 351 insertions(+) create mode 100644 conversion/qwen4exp.py diff --git a/conversion/__init__.py b/conversion/__init__.py index 8de97e9596..74bd96fd59 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -235,6 +235,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3_5ForConditionalGeneration": "qwen", "Qwen3_5MoeForCausalLM": "qwen", "Qwen3_5MoeForConditionalGeneration": "qwen", + "Qwen4ExpForCausalLM": "qwen4exp", + "Qwen4ExpForConditionalGeneration": "qwen4exp", "RND1": "qwen", "RWForCausalLM": "falcon", "RWKV6Qwen2ForCausalLM": "rwkv", @@ -332,6 +334,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForConditionalGeneration": "qwen3vl", "Qwen3_5MoeForConditionalGeneration": "qwen3vl", + "Qwen4ExpForConditionalGeneration": "qwen4exp", "RADIOModel": "nemotron", "Sarashina2VisionForCausalLM": "sarashina2", "SmolVLMForConditionalGeneration": "smolvlm", diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py new file mode 100644 index 0000000000..c40c7971dd --- /dev/null +++ b/conversion/qwen4exp.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from typing import Iterable + +import torch +from torch import Tensor + +import gguf + +from .base import ModelBase, MmprojModel +from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin +from .qwen3vl import Qwen3VLVisionModel + + +@ModelBase.register("Qwen4ExpForConditionalGeneration", "Qwen4ExpForCausalLM") +@ModelBase.example("unsloth/Qwen3.8-Flash-Next") +class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): + """Qwen3.8-Flash-Next. + + Shares the Qwen3.5 gated delta net and interleaved mrope, and adds three things: + hyper-connections in place of every layer norm, QSA sparse attention on the full + attention layers, and PLE n-gram hash embeddings on a single layer. + """ + + model_arch = gguf.MODEL_ARCH.QWEN4EXP + + # the MTP block is a separate draft head; vLLM drops it too + supports_mtp_export = False + no_mtp = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._ple_shards: dict[int, Tensor] = {} + self._ple_row_dim: int | None = None + + def _read_hash_constants(self, suffix: str) -> list[int]: + """Read an int64 PLE constant straight from the checkpoint. + + prepare_tensors() casts every non-float dtype to float32 before + modify_tensors() sees it (base.py), which would silently round these + 45-bit multipliers. Reading the lazy tensor here bypasses that. + """ + for name, gen in self.model_tensors.items(): + if name.endswith(suffix): + t = gen() + if t.dtype != torch.int64: + t = t.to(torch.int64) + return [int(x) for x in t.tolist()] + raise ValueError(f"PLE constant {suffix!r} missing from the checkpoint") + + # -- metadata --------------------------------------------------------- + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hp = self.hparams + + self.gguf_writer.add_hyper_connection_count(hp["hc_count"]) + self.gguf_writer.add_hyper_connection_low_rank(hp["hc_lowrank"]) + + n_layer = hp["num_hidden_layers"] + self.gguf_writer.add_indexer_head_count(hp["indexer_n_heads"]) + self.gguf_writer.add_indexer_key_length(hp["indexer_head_dim"]) + self.gguf_writer.add_indexer_top_k(hp["indexer_budget"]) + ratio = hp["indexer_compress_ratio"] + layer_types = hp["layer_types"] + self.gguf_writer.add_attention_compress_ratios( + [ratio if layer_types[i] == "full_attention" else 0 for i in range(n_layer)] + ) + + # ple_layer_ids is 1-based in the HF config + ple_layers = [i - 1 for i in hp["ple_layer_ids"]] + self.gguf_writer.add_ple_layers(ple_layers) + self.gguf_writer.add_ple_ngram_size(hp["ngram_size"]) + self.gguf_writer.add_ple_heads_per_ngram(hp["heads_per_ngram"]) + self.gguf_writer.add_ple_conv_kernel(hp["ple_conv_kernel_size"]) + self.gguf_writer.add_ple_eos_token_id(self._eos_token_id()) + if self._ple_row_dim is not None: + self.gguf_writer.add_embedding_length_per_layer_input(self._ple_row_dim) + + self.gguf_writer.add_ple_layer_multipliers( + self._read_hash_constants("ple_embedding.layer_multipliers")) + self.gguf_writer.add_ple_head_offsets( + self._read_hash_constants("ple_embedding.ngram_heads_offsets")) + self.gguf_writer.add_ple_head_vocab_sizes( + self._read_hash_constants("ple_embedding.ngram_heads_vocab_sizes")) + + def _eos_token_id(self) -> int: + eos = self.hparams.get("eos_token_id") + if isinstance(eos, list): + # the PLE hash resets n-grams on the primary EOS + return int(eos[-1]) + return int(eos) + + # -- tensors ---------------------------------------------------------- + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the n-gram hash constants travel as int64 tensors; they must stay exact, + # and 1-D tensors would be forced to F32, so carry them as KV instead + if name.endswith("ple_embedding.layer_multipliers"): + self._ple_multipliers = [int(x) for x in data_torch.tolist()] + return [] + if name.endswith("ple_embedding.ngram_heads_offsets"): + self._ple_head_offsets = [int(x) for x in data_torch.tolist()] + return [] + if name.endswith("ple_embedding.ngram_heads_vocab_sizes"): + self._ple_head_vocab_sizes = [int(x) for x in data_torch.tolist()] + return [] + + if ".ngram_embedding.shard_" in name: + idx = int(name.rpartition(".shard_")[2].partition(".")[0]) + self._ple_shards[idx] = data_torch + self._ple_row_dim = int(data_torch.shape[-1]) + n_parts = self.hparams["split_ngram_parts"] + if len(self._ple_shards) < n_parts: + return [] + # shards are contiguous row ranges in index order + table = torch.cat([self._ple_shards[i] for i in range(n_parts)], dim=0) + self._ple_shards.clear() + return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD], table)] + + # one projection feeds both indexer q and k; split it so the two get + # separate tensors, matching how minimax-m3 stores them + if ".indexer.index_qk_proj.weight" in name: + n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"] + q = data_torch[:n_q] + k = data_torch[n_q:] + return [ + (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), q), + (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k), + ] + + # Gemma-style zero-centred gammas that the inherited "norm.weight" rule misses + if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight")): + return [(self.map_tensor_name(name), data_torch + 1)] + + if name.endswith(".ple.conv1d.weight"): + return [(self.map_tensor_name(name), data_torch.squeeze())] + + return super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + if self._ple_shards: + raise ValueError( + f"unprocessed PLE embedding shards: {sorted(self._ple_shards)}" + ) + + +@ModelBase.register("Qwen4ExpForConditionalGeneration") +@ModelBase.example("unsloth/Qwen3.8-Flash-Next") +class Qwen4ExpVisionModel(Qwen3VLVisionModel): + """The vision tower is an unmodified Qwen3-VL ViT.""" diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index f236a5d2c9..7ff1ded7e4 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -225,6 +225,19 @@ class Keys: COUNT = "{arch}.hyper_connection.count" SINKHORN_ITERATIONS = "{arch}.hyper_connection.sinkhorn_iterations" EPSILON = "{arch}.hyper_connection.epsilon" + # absent means the mix projection is full rank (DeepSeek-V4 behaviour) + LOW_RANK = "{arch}.hyper_connection.low_rank" + + class PLE: + # per-layer n-gram hash embeddings (qwen4_exp) + LAYERS = "{arch}.ple.layers" + NGRAM_SIZE = "{arch}.ple.ngram_size" + HEADS_PER_NGRAM = "{arch}.ple.heads_per_ngram" + CONV_KERNEL = "{arch}.ple.conv_kernel" + LAYER_MULTIPLIERS = "{arch}.ple.layer_multipliers" + HEAD_OFFSETS = "{arch}.ple.head_offsets" + HEAD_VOCAB_SIZES = "{arch}.ple.head_vocab_sizes" + EOS_TOKEN_ID = "{arch}.ple.eos_token_id" class Rope: DIMENSION_COUNT = "{arch}.rope.dimension_count" @@ -494,6 +507,7 @@ class MODEL_ARCH(IntEnum): QWEN3VLMOE = auto() QWEN35 = auto() QWEN35MOE = auto() + QWEN4EXP = auto() PHI2 = auto() PHI3 = auto() PHIMOE = auto() @@ -636,6 +650,9 @@ class MODEL_TENSOR(IntEnum): HC_HEAD_FN = auto() HC_HEAD_BASE = auto() HC_HEAD_SCALE = auto() + HC_HEAD_NORM = auto() # qwen4exp + HC_HEAD_DOWN = auto() # qwen4exp + HC_HEAD_UP = auto() # qwen4exp ROPE_FREQS = auto() ROPE_FACTORS_LONG = auto() ROPE_FACTORS_SHORT = auto() @@ -780,6 +797,20 @@ class MODEL_TENSOR(IntEnum): HC_FFN_FN = auto() HC_FFN_BASE = auto() HC_FFN_SCALE = auto() + HC_ATTN_NORM = auto() # qwen4exp + HC_ATTN_DOWN = auto() # qwen4exp + HC_ATTN_UP = auto() # qwen4exp + HC_ATTN_INJECT = auto() # qwen4exp + HC_FFN_NORM = auto() # qwen4exp + HC_FFN_DOWN = auto() # qwen4exp + HC_FFN_UP = auto() # qwen4exp + HC_FFN_INJECT = auto() # qwen4exp + PLE_KEY = auto() # qwen4exp + PLE_VALUE = auto() # qwen4exp + PLE_NORM_KEY = auto() # qwen4exp + PLE_NORM_QUERY = auto() # qwen4exp + PLE_NORM_CONV = auto() # qwen4exp + PLE_CONV1D = auto() # qwen4exp ATTN_COMPRESSOR_WKV = auto() ATTN_COMPRESSOR_WGATE = auto() ATTN_COMPRESSOR_APE = auto() @@ -1217,6 +1248,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe", MODEL_ARCH.QWEN35: "qwen35", MODEL_ARCH.QWEN35MOE: "qwen35moe", + MODEL_ARCH.QWEN4EXP: "qwen4exp", MODEL_ARCH.PHI2: "phi2", MODEL_ARCH.PHI3: "phi3", MODEL_ARCH.PHIMOE: "phimoe", @@ -1358,6 +1390,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.HC_HEAD_FN: "output_hc_fn", MODEL_TENSOR.HC_HEAD_BASE: "output_hc_base", MODEL_TENSOR.HC_HEAD_SCALE: "output_hc_scale", + MODEL_TENSOR.HC_HEAD_NORM: "output_hc_norm", # qwen4exp + MODEL_TENSOR.HC_HEAD_DOWN: "output_hc_down", # qwen4exp + MODEL_TENSOR.HC_HEAD_UP: "output_hc_up", # qwen4exp MODEL_TENSOR.ROPE_FREQS: "rope_freqs", MODEL_TENSOR.ROPE_FACTORS_LONG: "rope_factors_long", MODEL_TENSOR.ROPE_FACTORS_SHORT: "rope_factors_short", @@ -1502,6 +1537,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.HC_FFN_FN: "blk.{bid}.hc_ffn_fn", MODEL_TENSOR.HC_FFN_BASE: "blk.{bid}.hc_ffn_base", MODEL_TENSOR.HC_FFN_SCALE: "blk.{bid}.hc_ffn_scale", + MODEL_TENSOR.HC_ATTN_NORM: "blk.{bid}.hc_attn_norm", # qwen4exp + MODEL_TENSOR.HC_ATTN_DOWN: "blk.{bid}.hc_attn_down", # qwen4exp + MODEL_TENSOR.HC_ATTN_UP: "blk.{bid}.hc_attn_up", # qwen4exp + MODEL_TENSOR.HC_ATTN_INJECT: "blk.{bid}.hc_attn_inject", # qwen4exp + MODEL_TENSOR.HC_FFN_NORM: "blk.{bid}.hc_ffn_norm", # qwen4exp + MODEL_TENSOR.HC_FFN_DOWN: "blk.{bid}.hc_ffn_down", # qwen4exp + MODEL_TENSOR.HC_FFN_UP: "blk.{bid}.hc_ffn_up", # qwen4exp + MODEL_TENSOR.HC_FFN_INJECT: "blk.{bid}.hc_ffn_inject", # qwen4exp + MODEL_TENSOR.PLE_KEY: "blk.{bid}.ple_key", # qwen4exp + MODEL_TENSOR.PLE_VALUE: "blk.{bid}.ple_value", # qwen4exp + MODEL_TENSOR.PLE_NORM_KEY: "blk.{bid}.ple_norm_key", # qwen4exp + MODEL_TENSOR.PLE_NORM_QUERY: "blk.{bid}.ple_norm_query", # qwen4exp + MODEL_TENSOR.PLE_NORM_CONV: "blk.{bid}.ple_norm_conv", # qwen4exp + MODEL_TENSOR.PLE_CONV1D: "blk.{bid}.ple_conv1d", # qwen4exp MODEL_TENSOR.ATTN_COMPRESSOR_WKV: "blk.{bid}.attn_compressor_kv", MODEL_TENSOR.ATTN_COMPRESSOR_WGATE: "blk.{bid}.attn_compressor_gate", MODEL_TENSOR.ATTN_COMPRESSOR_APE: "blk.{bid}.attn_compressor_ape", @@ -2795,6 +2844,62 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.QWEN4EXP: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT, + # no OUTPUT_NORM / ATTN_NORM / ATTN_POST_NORM: hyper-connections replace every layer norm + MODEL_TENSOR.HC_HEAD_NORM, + MODEL_TENSOR.HC_HEAD_DOWN, + MODEL_TENSOR.HC_HEAD_UP, + MODEL_TENSOR.HC_ATTN_NORM, + MODEL_TENSOR.HC_ATTN_DOWN, + MODEL_TENSOR.HC_ATTN_UP, + MODEL_TENSOR.HC_ATTN_INJECT, + MODEL_TENSOR.HC_FFN_NORM, + MODEL_TENSOR.HC_FFN_DOWN, + MODEL_TENSOR.HC_FFN_UP, + MODEL_TENSOR.HC_FFN_INJECT, + # full attention layers: ATTN_Q holds [q|gate] interleaved per head + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + # QSA indexer + MODEL_TENSOR.INDEXER_Q_PROJ, + MODEL_TENSOR.INDEXER_K_PROJ, + MODEL_TENSOR.INDEXER_Q_NORM, + MODEL_TENSOR.INDEXER_K_NORM, + # gated delta net linear attention layers + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.SSM_BETA, + MODEL_TENSOR.SSM_ALPHA, + MODEL_TENSOR.SSM_OUT, + # MoE, every layer, with a gated shared expert + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_INP_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_GATE_UP_EXP, + # PLE n-gram hash embeddings, one layer only + MODEL_TENSOR.PER_LAYER_TOKEN_EMBD, + MODEL_TENSOR.PLE_KEY, + MODEL_TENSOR.PLE_VALUE, + MODEL_TENSOR.PLE_NORM_KEY, + MODEL_TENSOR.PLE_NORM_QUERY, + MODEL_TENSOR.PLE_NORM_CONV, + MODEL_TENSOR.PLE_CONV1D, + ], MODEL_ARCH.PLAMO: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index d8a96a27bd..984cef952f 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1029,6 +1029,38 @@ class GGUFWriter: def add_hyper_connection_epsilon(self, value: float) -> None: self.add_float32(Keys.HyperConnection.EPSILON.format(arch=self.arch), value) + def add_hyper_connection_low_rank(self, value: int) -> None: + self.add_uint32(Keys.HyperConnection.LOW_RANK.format(arch=self.arch), value) + + def add_ple_layers(self, values: Sequence[int]) -> None: + self.add_array(Keys.PLE.LAYERS.format(arch=self.arch), values) + + def add_ple_ngram_size(self, value: int) -> None: + self.add_uint32(Keys.PLE.NGRAM_SIZE.format(arch=self.arch), value) + + def add_ple_heads_per_ngram(self, value: int) -> None: + self.add_uint32(Keys.PLE.HEADS_PER_NGRAM.format(arch=self.arch), value) + + def add_ple_conv_kernel(self, value: int) -> None: + self.add_uint32(Keys.PLE.CONV_KERNEL.format(arch=self.arch), value) + + # the hash constants must survive exactly: multipliers reach ~2.4e13, so the + # default int inference (INT32) would truncate them and break the n-gram hash + def _add_u64_array(self, key: str, values: Sequence[int]) -> None: + self.add_key_value(key, list(values), GGUFValueType.ARRAY, GGUFValueType.UINT64) + + def add_ple_layer_multipliers(self, values: Sequence[int]) -> None: + self._add_u64_array(Keys.PLE.LAYER_MULTIPLIERS.format(arch=self.arch), values) + + def add_ple_head_offsets(self, values: Sequence[int]) -> None: + self._add_u64_array(Keys.PLE.HEAD_OFFSETS.format(arch=self.arch), values) + + def add_ple_head_vocab_sizes(self, values: Sequence[int]) -> None: + self._add_u64_array(Keys.PLE.HEAD_VOCAB_SIZES.format(arch=self.arch), values) + + def add_ple_eos_token_id(self, value: int) -> None: + self.add_uint32(Keys.PLE.EOS_TOKEN_ID.format(arch=self.arch), value) + def add_attention_scale(self, value: float) -> None: self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index ef580518e9..7adcac4183 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2680,6 +2680,65 @@ class TensorNameMap: "model.layers.{bid}.post_attention_layernorm", ), }, + MODEL_ARCH.QWEN4EXP: { + MODEL_TENSOR.HC_ATTN_NORM: ( + "model.layers.{bid}.attn_hyper_connection.hc_norm", + ), + MODEL_TENSOR.HC_ATTN_DOWN: ( + "model.layers.{bid}.attn_hyper_connection.input_mix_weight_down", + ), + MODEL_TENSOR.HC_ATTN_UP: ( + "model.layers.{bid}.attn_hyper_connection.input_mix_weight_up", + ), + MODEL_TENSOR.HC_ATTN_INJECT: ( + "model.layers.{bid}.attn_hyper_connection.block_inject_weight", + ), + MODEL_TENSOR.HC_FFN_NORM: ( + "model.layers.{bid}.mlp_hyper_connection.hc_norm", + ), + MODEL_TENSOR.HC_FFN_DOWN: ( + "model.layers.{bid}.mlp_hyper_connection.input_mix_weight_down", + ), + MODEL_TENSOR.HC_FFN_UP: ( + "model.layers.{bid}.mlp_hyper_connection.input_mix_weight_up", + ), + MODEL_TENSOR.HC_FFN_INJECT: ( + "model.layers.{bid}.mlp_hyper_connection.block_inject_weight", + ), + MODEL_TENSOR.HC_HEAD_NORM: ( + "model.hyper_connection_mixer.hc_norm", + ), + MODEL_TENSOR.HC_HEAD_DOWN: ( + "model.hyper_connection_mixer.input_mix_weight_down", + ), + MODEL_TENSOR.HC_HEAD_UP: ( + "model.hyper_connection_mixer.input_mix_weight_up", + ), + MODEL_TENSOR.INDEXER_Q_NORM: ( + "model.layers.{bid}.self_attn.indexer.q_layernorm", + ), + MODEL_TENSOR.INDEXER_K_NORM: ( + "model.layers.{bid}.self_attn.indexer.k_layernorm", + ), + MODEL_TENSOR.PLE_KEY: ( + "model.layers.{bid}.ple.key_proj", + ), + MODEL_TENSOR.PLE_VALUE: ( + "model.layers.{bid}.ple.value_proj", + ), + MODEL_TENSOR.PLE_NORM_KEY: ( + "model.layers.{bid}.ple.norm_key", + ), + MODEL_TENSOR.PLE_NORM_QUERY: ( + "model.layers.{bid}.ple.norm_query", + ), + MODEL_TENSOR.PLE_NORM_CONV: ( + "model.layers.{bid}.ple.norm_conv", + ), + MODEL_TENSOR.PLE_CONV1D: ( + "model.layers.{bid}.ple.conv1d", + ), + }, } mapping: dict[str, tuple[MODEL_TENSOR, str]]