diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py index e6ff4f0db1..1174edcb7b 100644 --- a/conversion/qwen4exp.py +++ b/conversion/qwen4exp.py @@ -48,8 +48,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): return [int(x) for x in t.tolist()] raise ValueError(f"PLE constant {suffix!r} missing from the checkpoint") - # -- metadata --------------------------------------------------------- - def set_gguf_parameters(self): super().set_gguf_parameters() hp = self.hparams @@ -91,11 +89,8 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): return int(eos[-1]) return int(eos) - # -- tensors ---------------------------------------------------------- - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # the n-gram hash constants travel as int64 tensors; they must stay exact, - # and 1-D tensors would be forced to F32, so carry them as KV instead + # int64 hash constants must stay exact; 1-D tensors force F32, so use KV if name.endswith("ple_embedding.layer_multipliers"): self._ple_multipliers = [int(x) for x in data_torch.tolist()] return [] @@ -113,14 +108,12 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): n_parts = self.hparams["split_ngram_parts"] if len(self._ple_shards) < n_parts: return [] - # shards are contiguous row ranges in index order table = torch.cat([self._ple_shards[i] for i in range(n_parts)], dim=0) self._ple_shards.clear() name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD] return [(name + ".weight", table)] - # one projection feeds both indexer q and k; split it so the two get - # separate tensors, matching how minimax-m3 stores them + # one projection feeds indexer q and k; split it, as minimax-m3 does if ".indexer.index_qk_proj.weight" in name: n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"] q = data_torch[:n_q] @@ -130,7 +123,7 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k), ] - # Gemma-style zero-centred gammas that the inherited "norm.weight" rule misses + # Gemma zero-centred gammas the inherited norm.weight rule misses if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight")): return [(self.map_tensor_name(name), data_torch + 1)] diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 7ff1ded7e4..2caff0b0cc 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -229,7 +229,6 @@ class Keys: LOW_RANK = "{arch}.hyper_connection.low_rank" class PLE: - # per-layer n-gram hash embeddings (qwen4_exp) LAYERS = "{arch}.ple.layers" NGRAM_SIZE = "{arch}.ple.ngram_size" HEADS_PER_NGRAM = "{arch}.ple.heads_per_ngram" @@ -2866,12 +2865,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, - # QSA indexer MODEL_TENSOR.INDEXER_Q_PROJ, MODEL_TENSOR.INDEXER_K_PROJ, MODEL_TENSOR.INDEXER_Q_NORM, MODEL_TENSOR.INDEXER_K_NORM, - # gated delta net linear attention layers MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_GATE, MODEL_TENSOR.SSM_A, @@ -2881,7 +2878,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.SSM_BETA, MODEL_TENSOR.SSM_ALPHA, MODEL_TENSOR.SSM_OUT, - # MoE, every layer, with a gated shared expert MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_GATE_INP_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, @@ -2891,7 +2887,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_GATE_UP_EXP, - # PLE n-gram hash embeddings, one layer only MODEL_TENSOR.PER_LAYER_TOKEN_EMBD, MODEL_TENSOR.PLE_KEY, MODEL_TENSOR.PLE_VALUE, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 984cef952f..363e9f751b 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1044,8 +1044,7 @@ class GGUFWriter: def add_ple_conv_kernel(self, value: int) -> None: self.add_uint32(Keys.PLE.CONV_KERNEL.format(arch=self.arch), value) - # the hash constants must survive exactly: multipliers reach ~2.4e13, so the - # default int inference (INT32) would truncate them and break the n-gram hash + # multipliers reach ~2.4e13; default INT32 inference would truncate them def _add_u64_array(self, key: str, values: Sequence[int]) -> None: self.add_key_value(key, list(values), GGUFValueType.ARRAY, GGUFValueType.UINT64) diff --git a/src/llama-hparams.h b/src/llama-hparams.h index aba71ef9d7..1c10dc851b 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -272,11 +272,9 @@ struct llama_hparams { float dsv4_hc_eps = 0.0f; std::array dsv4_compress_ratios; - // qwen4exp low-rank hyper-connections - // 0 means full rank, which is the DeepSeek-V4 parameterisation + // 0 = full rank (DeepSeek-V4) uint32_t hc_low_rank = 0; - // qwen4exp PLE n-gram hash embeddings uint32_t ple_ngram_size = 0; uint32_t ple_heads_per_ngram = 0; uint32_t ple_conv_kernel = 0; diff --git a/src/llama-model.h b/src/llama-model.h index 10d7c51ac5..a2c25c6381 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -556,7 +556,6 @@ struct llama_layer { struct ggml_tensor * index_q_norm = nullptr; struct ggml_tensor * index_k_norm = nullptr; - // qwen4exp low-rank hyper-connections struct ggml_tensor * hc_attn_norm = nullptr; struct ggml_tensor * hc_attn_down = nullptr; struct ggml_tensor * hc_attn_up = nullptr; @@ -566,7 +565,6 @@ struct llama_layer { struct ggml_tensor * hc_ffn_up = nullptr; struct ggml_tensor * hc_ffn_inject = nullptr; - // qwen4exp PLE struct ggml_tensor * ple_key = nullptr; struct ggml_tensor * ple_value = nullptr; struct ggml_tensor * ple_norm_key = nullptr; @@ -655,7 +653,6 @@ struct llama_model { struct ggml_tensor * altup_unembd_proj = nullptr; struct ggml_tensor * per_layer_tok_embd = nullptr; - // qwen4exp final hyper-connection mixer struct ggml_tensor * hc_head_norm = nullptr; struct ggml_tensor * hc_head_down = nullptr; struct ggml_tensor * hc_head_up = nullptr; diff --git a/src/models/models.h b/src/models/models.h index cb6325c72c..c946900df0 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2280,8 +2280,7 @@ struct llama_model_qwen4exp : public llama_model_base { struct graph : public llm_build_delta_net_base { graph(const llama_model & model, const llm_graph_params & params); private: - // hyper-connections replace every layer norm, so the residual carried - // between layers is [n_embd, hc, n_tokens] instead of [n_embd, n_tokens] + // HC replaces every layer norm: residual is [n_embd, hc, n_tokens] ggml_tensor * build_hc_mix( ggml_tensor * x, ggml_tensor * w_norm, diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 63089dea41..d503e1f1e2 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -8,28 +8,24 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); - // gated delta net, same dimension aliasing as Qwen3.5 ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); - // hyper-connections. low_rank is qwen4exp specific; DeepSeek-V4 leaves it - // absent and uses a full rank mix projection instead + // HC; low_rank is qwen4exp-specific, DeepSeek-V4 leaves it absent (full rank) ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); ml.get_key(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank); GGML_ASSERT(hparams.dsv4_hc_mult > 0 && "qwen4exp needs a hyper-connection count"); GGML_ASSERT(hparams.hc_low_rank > 0 && "qwen4exp needs a hyper-connection low rank"); hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd; - // QSA indexer ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); ml.get_key_or_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, hparams.n_layer_all, false); - // PLE n-gram hash embeddings ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size); ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram); ml.get_key(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel); @@ -93,9 +89,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); } - // the n-gram table is one flat [ple_head_dim, n_rows] gather target. its row - // count is the padded sum of the per-head vocab sizes, so read it back from - // the file rather than trying to reproduce the padding rule + // flat [ple_head_dim, n_rows] gather target; n_rows is padded, so read it back const std::string ple_name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str(); const auto * ple_w = ml.get_weight(ple_name.c_str()); GGML_ASSERT(ple_w != nullptr && "qwen4exp is missing the PLE n-gram table"); @@ -117,8 +111,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; - // two hyper-connection modules per layer, one before the token mixer - // and one before the MoE + // two HC modules per layer: before the token mixer, before the MoE layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, 0); layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, 0); layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, 0); @@ -136,7 +129,6 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0); layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0); - // QSA pre-indexer, MQA with a single key head const int64_t idx_dim = hparams.indexer_head_size; layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", il), { n_embd, hparams.indexer_n_head * idx_dim }, 0); layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", il), { n_embd, idx_dim }, 0); @@ -174,7 +166,6 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { } } -// the graph lands in the next commit; loading and metadata come first std::unique_ptr llama_model_qwen4exp::build_arch_graph(const llm_graph_params & params) const { GGML_UNUSED(params); throw std::runtime_error("qwen4exp: graph not implemented yet");