Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

qwen4exp: shorten comments

Browse Source
This commit is contained in:
Daniel Han
2026-08-26 12:37:53 +00:00
parent 331fa7e560
commit 058b7a1b48
7 changed files with 9 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/qwen4exp.py
+3 -10
View File
@@ -48,8 +48,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
return [int(x) for x in t.tolist()]
raise ValueError(f"PLE constant {suffix!r} missing from the checkpoint")
# -- metadata ---------------------------------------------------------
def set_gguf_parameters(self):
super().set_gguf_parameters()
hp = self.hparams
@@ -91,11 +89,8 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
return int(eos[-1])
return int(eos)
# -- tensors ----------------------------------------------------------
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# the n-gram hash constants travel as int64 tensors; they must stay exact,
# and 1-D tensors would be forced to F32, so carry them as KV instead
# int64 hash constants must stay exact; 1-D tensors force F32, so use KV
if name.endswith("ple_embedding.layer_multipliers"):
self._ple_multipliers = [int(x) for x in data_torch.tolist()]
return []
@@ -113,14 +108,12 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
n_parts = self.hparams["split_ngram_parts"]
if len(self._ple_shards) < n_parts:
return []
# shards are contiguous row ranges in index order
table = torch.cat([self._ple_shards[i] for i in range(n_parts)], dim=0)
self._ple_shards.clear()
name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
return [(name + ".weight", table)]
# one projection feeds both indexer q and k; split it so the two get
# separate tensors, matching how minimax-m3 stores them
# one projection feeds indexer q and k; split it, as minimax-m3 does
if ".indexer.index_qk_proj.weight" in name:
n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"]
q = data_torch[:n_q]
@@ -130,7 +123,7 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
(self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k),
]
# Gemma-style zero-centred gammas that the inherited "norm.weight" rule misses
# Gemma zero-centred gammas the inherited norm.weight rule misses
if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight")):
return [(self.map_tensor_name(name), data_torch + 1)]
gguf-py/gguf/constants.py
-5
View File
@@ -229,7 +229,6 @@ class Keys:
LOW_RANK = "{arch}.hyper_connection.low_rank"
class PLE:
# per-layer n-gram hash embeddings (qwen4_exp)
LAYERS = "{arch}.ple.layers"
NGRAM_SIZE = "{arch}.ple.ngram_size"
HEADS_PER_NGRAM = "{arch}.ple.heads_per_ngram"
@@ -2866,12 +2865,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.ATTN_K_NORM,
MODEL_TENSOR.ATTN_V,
MODEL_TENSOR.ATTN_OUT,
# QSA indexer
MODEL_TENSOR.INDEXER_Q_PROJ,
MODEL_TENSOR.INDEXER_K_PROJ,
MODEL_TENSOR.INDEXER_Q_NORM,
MODEL_TENSOR.INDEXER_K_NORM,
# gated delta net linear attention layers
MODEL_TENSOR.ATTN_QKV,
MODEL_TENSOR.ATTN_GATE,
MODEL_TENSOR.SSM_A,
@@ -2881,7 +2878,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.SSM_BETA,
MODEL_TENSOR.SSM_ALPHA,
MODEL_TENSOR.SSM_OUT,
# MoE, every layer, with a gated shared expert
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_GATE_INP_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
@@ -2891,7 +2887,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_GATE_UP_EXP,
# PLE n-gram hash embeddings, one layer only
MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,
MODEL_TENSOR.PLE_KEY,
MODEL_TENSOR.PLE_VALUE,
gguf-py/gguf/gguf_writer.py
+1 -2
View File
@@ -1044,8 +1044,7 @@ class GGUFWriter:
def add_ple_conv_kernel(self, value: int) -> None:
self.add_uint32(Keys.PLE.CONV_KERNEL.format(arch=self.arch), value)
# the hash constants must survive exactly: multipliers reach ~2.4e13, so the
# default int inference (INT32) would truncate them and break the n-gram hash
# multipliers reach ~2.4e13; default INT32 inference would truncate them
def _add_u64_array(self, key: str, values: Sequence[int]) -> None:
self.add_key_value(key, list(values), GGUFValueType.ARRAY, GGUFValueType.UINT64)
src/llama-hparams.h
+1 -3
View File
@@ -272,11 +272,9 @@ struct llama_hparams {
float dsv4_hc_eps = 0.0f;
std::array<uint32_t, LLAMA_MAX_LAYERS> dsv4_compress_ratios;
// qwen4exp low-rank hyper-connections
// 0 means full rank, which is the DeepSeek-V4 parameterisation
// 0 = full rank (DeepSeek-V4)
uint32_t hc_low_rank = 0;
// qwen4exp PLE n-gram hash embeddings
uint32_t ple_ngram_size = 0;
uint32_t ple_heads_per_ngram = 0;
uint32_t ple_conv_kernel = 0;
src/llama-model.h
-3
View File
@@ -556,7 +556,6 @@ struct llama_layer {
struct ggml_tensor * index_q_norm = nullptr;
struct ggml_tensor * index_k_norm = nullptr;
// qwen4exp low-rank hyper-connections
struct ggml_tensor * hc_attn_norm = nullptr;
struct ggml_tensor * hc_attn_down = nullptr;
struct ggml_tensor * hc_attn_up = nullptr;
@@ -566,7 +565,6 @@ struct llama_layer {
struct ggml_tensor * hc_ffn_up = nullptr;
struct ggml_tensor * hc_ffn_inject = nullptr;
// qwen4exp PLE
struct ggml_tensor * ple_key = nullptr;
struct ggml_tensor * ple_value = nullptr;
struct ggml_tensor * ple_norm_key = nullptr;
@@ -655,7 +653,6 @@ struct llama_model {
struct ggml_tensor * altup_unembd_proj = nullptr;
struct ggml_tensor * per_layer_tok_embd = nullptr;
// qwen4exp final hyper-connection mixer
struct ggml_tensor * hc_head_norm = nullptr;
struct ggml_tensor * hc_head_down = nullptr;
struct ggml_tensor * hc_head_up = nullptr;
src/models/models.h
+1 -2
View File
@@ -2280,8 +2280,7 @@ struct llama_model_qwen4exp : public llama_model_base {
struct graph : public llm_build_delta_net_base {
graph(const llama_model & model, const llm_graph_params & params);
private:
// hyper-connections replace every layer norm, so the residual carried
// between layers is [n_embd, hc, n_tokens] instead of [n_embd, n_tokens]
// HC replaces every layer norm: residual is [n_embd, hc, n_tokens]
ggml_tensor * build_hc_mix(
ggml_tensor * x,
ggml_tensor * w_norm,
src/models/qwen4exp.cpp
+3 -12
View File
@@ -8,28 +8,24 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);
// gated delta net, same dimension aliasing as Qwen3.5
ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
// hyper-connections. low_rank is qwen4exp specific; DeepSeek-V4 leaves it
// absent and uses a full rank mix projection instead
// HC; low_rank is qwen4exp-specific, DeepSeek-V4 leaves it absent (full rank)
ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
ml.get_key(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
GGML_ASSERT(hparams.dsv4_hc_mult > 0 && "qwen4exp needs a hyper-connection count");
GGML_ASSERT(hparams.hc_low_rank > 0 && "qwen4exp needs a hyper-connection low rank");
hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd;
// QSA indexer
ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
ml.get_key_or_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, hparams.n_layer_all, false);
// PLE n-gram hash embeddings
ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size);
ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);
ml.get_key(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel);
@@ -93,9 +89,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);
}
// the n-gram table is one flat [ple_head_dim, n_rows] gather target. its row
// count is the padded sum of the per-head vocab sizes, so read it back from
// the file rather than trying to reproduce the padding rule
// flat [ple_head_dim, n_rows] gather target; n_rows is padded, so read it back
const std::string ple_name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str();
const auto * ple_w = ml.get_weight(ple_name.c_str());
GGML_ASSERT(ple_w != nullptr && "qwen4exp is missing the PLE n-gram table");
@@ -117,8 +111,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
const int64_t value_dim = head_v_dim * n_v_heads;
const int64_t conv_dim = key_dim * 2 + value_dim;
// two hyper-connection modules per layer, one before the token mixer
// and one before the MoE
// two HC modules per layer: before the token mixer, before the MoE
layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, 0);
layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, 0);
layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, 0);
@@ -136,7 +129,6 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0);
// QSA pre-indexer, MQA with a single key head
const int64_t idx_dim = hparams.indexer_head_size;
layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", il), { n_embd, hparams.indexer_n_head * idx_dim }, 0);
layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", il), { n_embd, idx_dim }, 0);
@@ -174,7 +166,6 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
}
}
// the graph lands in the next commit; loading and metadata come first
std::unique_ptr<llm_graph_context> llama_model_qwen4exp::build_arch_graph(const llm_graph_params & params) const {
GGML_UNUSED(params);
throw std::runtime_error("qwen4exp: graph not implemented yet");