mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-19 17:24:57 +02:00
llama: load qwen4exp (Qwen3.8-Flash-Next) hparams and tensors
Adds LLM_ARCH_QWEN4EXP with its hparams and tensor loading. The graph comes in the next commit; this makes the model load and report correct metadata. - hyper-connections set n_embd_out_impl = hc_count * n_embd, so the residual stream is 4x wide and there is no output_norm: the final mixer's hc_norm is the last norm in the model. - registered as hybrid and given the same recurrent/attention memory filters as Qwen3-Next and Qwen3.5. - reuses the existing indexer, per_layer_token_embd, SSM and compress_ratios keys as-is. - the PLE table row count is read back from the file rather than recomputing the vocab padding rule. llama-model-loader gains UINT64 array support. That branch previously threw, so no existing caller changes behaviour; it is needed because the PLE hash multipliers do not fit in int32.
This commit is contained in:
danielhanchen
committed by
Daniel Han
Daniel Han
parent
6e5b8b7e4d
commit
331fa7e560
@@ -116,7 +116,8 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
# shards are contiguous row ranges in index order
|
||||
table = torch.cat([self._ple_shards[i] for i in range(n_parts)], dim=0)
|
||||
self._ple_shards.clear()
|
||||
return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD], table)]
|
||||
name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
|
||||
return [(name + ".weight", table)]
|
||||
|
||||
# one projection feeds both indexer q and k; split it so the two get
|
||||
# separate tensors, matching how minimax-m3 stores them
|
||||
|
||||
@@ -40,6 +40,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_QWEN3VLMOE, "qwen3vlmoe" },
|
||||
{ LLM_ARCH_QWEN35, "qwen35" },
|
||||
{ LLM_ARCH_QWEN35MOE, "qwen35moe" },
|
||||
{ LLM_ARCH_QWEN4EXP, "qwen4exp" },
|
||||
{ LLM_ARCH_PHI2, "phi2" },
|
||||
{ LLM_ARCH_PHI3, "phi3" },
|
||||
{ LLM_ARCH_PHIMOE, "phimoe" },
|
||||
@@ -293,6 +294,16 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_HYPER_CONNECTION_COUNT, "%s.hyper_connection.count" },
|
||||
{ LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, "%s.hyper_connection.sinkhorn_iterations" },
|
||||
{ LLM_KV_HYPER_CONNECTION_EPSILON, "%s.hyper_connection.epsilon" },
|
||||
{ LLM_KV_HYPER_CONNECTION_LOW_RANK, "%s.hyper_connection.low_rank" },
|
||||
|
||||
{ LLM_KV_PLE_LAYERS, "%s.ple.layers" },
|
||||
{ LLM_KV_PLE_NGRAM_SIZE, "%s.ple.ngram_size" },
|
||||
{ LLM_KV_PLE_HEADS_PER_NGRAM, "%s.ple.heads_per_ngram" },
|
||||
{ LLM_KV_PLE_CONV_KERNEL, "%s.ple.conv_kernel" },
|
||||
{ LLM_KV_PLE_LAYER_MULTIPLIERS, "%s.ple.layer_multipliers" },
|
||||
{ LLM_KV_PLE_HEAD_OFFSETS, "%s.ple.head_offsets" },
|
||||
{ LLM_KV_PLE_HEAD_VOCAB_SIZES, "%s.ple.head_vocab_sizes" },
|
||||
{ LLM_KV_PLE_EOS_TOKEN_ID, "%s.ple.eos_token_id" },
|
||||
|
||||
{ LLM_KV_HASH_LAYER_COUNT, "%s.hash_layer_count" },
|
||||
|
||||
@@ -500,12 +511,29 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
|
||||
{ LLM_TENSOR_HC_HEAD_FN, "output_hc_fn" },
|
||||
{ LLM_TENSOR_HC_HEAD_BASE, "output_hc_base" },
|
||||
{ LLM_TENSOR_HC_HEAD_SCALE, "output_hc_scale" },
|
||||
{ LLM_TENSOR_HC_HEAD_NORM, "output_hc_norm" },
|
||||
{ LLM_TENSOR_HC_HEAD_DOWN, "output_hc_down" },
|
||||
{ LLM_TENSOR_HC_HEAD_UP, "output_hc_up" },
|
||||
{ LLM_TENSOR_HC_ATTN_FN, "blk.%d.hc_attn_fn" },
|
||||
{ LLM_TENSOR_HC_ATTN_BASE, "blk.%d.hc_attn_base" },
|
||||
{ LLM_TENSOR_HC_ATTN_SCALE, "blk.%d.hc_attn_scale" },
|
||||
{ LLM_TENSOR_HC_FFN_FN, "blk.%d.hc_ffn_fn" },
|
||||
{ LLM_TENSOR_HC_FFN_BASE, "blk.%d.hc_ffn_base" },
|
||||
{ LLM_TENSOR_HC_FFN_SCALE, "blk.%d.hc_ffn_scale" },
|
||||
{ LLM_TENSOR_HC_ATTN_NORM, "blk.%d.hc_attn_norm" },
|
||||
{ LLM_TENSOR_HC_ATTN_DOWN, "blk.%d.hc_attn_down" },
|
||||
{ LLM_TENSOR_HC_ATTN_UP, "blk.%d.hc_attn_up" },
|
||||
{ LLM_TENSOR_HC_ATTN_INJECT, "blk.%d.hc_attn_inject" },
|
||||
{ LLM_TENSOR_HC_FFN_NORM, "blk.%d.hc_ffn_norm" },
|
||||
{ LLM_TENSOR_HC_FFN_DOWN, "blk.%d.hc_ffn_down" },
|
||||
{ LLM_TENSOR_HC_FFN_UP, "blk.%d.hc_ffn_up" },
|
||||
{ LLM_TENSOR_HC_FFN_INJECT, "blk.%d.hc_ffn_inject" },
|
||||
{ LLM_TENSOR_PLE_KEY, "blk.%d.ple_key" },
|
||||
{ LLM_TENSOR_PLE_VALUE, "blk.%d.ple_value" },
|
||||
{ LLM_TENSOR_PLE_NORM_KEY, "blk.%d.ple_norm_key" },
|
||||
{ LLM_TENSOR_PLE_NORM_QUERY, "blk.%d.ple_norm_query" },
|
||||
{ LLM_TENSOR_PLE_NORM_CONV, "blk.%d.ple_norm_conv" },
|
||||
{ LLM_TENSOR_PLE_CONV1D, "blk.%d.ple_conv1d" },
|
||||
{ LLM_TENSOR_ATTN_COMPRESSOR_WKV, "blk.%d.attn_compressor_kv" },
|
||||
{ LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "blk.%d.attn_compressor_gate" },
|
||||
{ LLM_TENSOR_ATTN_COMPRESSOR_APE, "blk.%d.attn_compressor_ape" },
|
||||
@@ -704,12 +732,29 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
|
||||
{LLM_TENSOR_HC_HEAD_FN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_HEAD_BASE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}},
|
||||
{LLM_TENSOR_HC_HEAD_SCALE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_HC_HEAD_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_HC_HEAD_DOWN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_HEAD_UP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_FN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
|
||||
{LLM_TENSOR_HC_ATTN_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_HC_FFN_FN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
|
||||
{LLM_TENSOR_HC_FFN_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_HC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_HC_ATTN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_INJECT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_HC_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_INJECT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_PLE_KEY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_PLE_VALUE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_PLE_NORM_KEY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_PLE_NORM_QUERY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_PLE_NORM_CONV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_PLE_CONV1D, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},
|
||||
{LLM_TENSOR_ATTN_COMPRESSOR_WKV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_ATTN_COMPRESSOR_WGATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_ATTN_COMPRESSOR_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},
|
||||
@@ -1009,6 +1054,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
|
||||
case LLM_ARCH_KIMI_K3:
|
||||
case LLM_ARCH_QWEN35:
|
||||
case LLM_ARCH_QWEN35MOE:
|
||||
case LLM_ARCH_QWEN4EXP:
|
||||
case LLM_ARCH_DEEPSEEK4:
|
||||
case LLM_ARCH_MINIMAX_01:
|
||||
return true;
|
||||
|
||||
@@ -45,6 +45,7 @@ enum llm_arch {
|
||||
LLM_ARCH_QWEN3VLMOE,
|
||||
LLM_ARCH_QWEN35,
|
||||
LLM_ARCH_QWEN35MOE,
|
||||
LLM_ARCH_QWEN4EXP,
|
||||
LLM_ARCH_PHI2,
|
||||
LLM_ARCH_PHI3,
|
||||
LLM_ARCH_PHIMOE,
|
||||
@@ -298,6 +299,16 @@ enum llm_kv {
|
||||
LLM_KV_HYPER_CONNECTION_COUNT,
|
||||
LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS,
|
||||
LLM_KV_HYPER_CONNECTION_EPSILON,
|
||||
LLM_KV_HYPER_CONNECTION_LOW_RANK,
|
||||
|
||||
LLM_KV_PLE_LAYERS,
|
||||
LLM_KV_PLE_NGRAM_SIZE,
|
||||
LLM_KV_PLE_HEADS_PER_NGRAM,
|
||||
LLM_KV_PLE_CONV_KERNEL,
|
||||
LLM_KV_PLE_LAYER_MULTIPLIERS,
|
||||
LLM_KV_PLE_HEAD_OFFSETS,
|
||||
LLM_KV_PLE_HEAD_VOCAB_SIZES,
|
||||
LLM_KV_PLE_EOS_TOKEN_ID,
|
||||
|
||||
LLM_KV_HASH_LAYER_COUNT,
|
||||
|
||||
@@ -565,12 +576,29 @@ enum llm_tensor {
|
||||
LLM_TENSOR_HC_HEAD_FN,
|
||||
LLM_TENSOR_HC_HEAD_BASE,
|
||||
LLM_TENSOR_HC_HEAD_SCALE,
|
||||
LLM_TENSOR_HC_HEAD_NORM, // qwen4exp
|
||||
LLM_TENSOR_HC_HEAD_DOWN, // qwen4exp
|
||||
LLM_TENSOR_HC_HEAD_UP, // qwen4exp
|
||||
LLM_TENSOR_HC_ATTN_FN,
|
||||
LLM_TENSOR_HC_ATTN_BASE,
|
||||
LLM_TENSOR_HC_ATTN_SCALE,
|
||||
LLM_TENSOR_HC_FFN_FN,
|
||||
LLM_TENSOR_HC_FFN_BASE,
|
||||
LLM_TENSOR_HC_FFN_SCALE,
|
||||
LLM_TENSOR_HC_ATTN_NORM, // qwen4exp
|
||||
LLM_TENSOR_HC_ATTN_DOWN, // qwen4exp
|
||||
LLM_TENSOR_HC_ATTN_UP, // qwen4exp
|
||||
LLM_TENSOR_HC_ATTN_INJECT, // qwen4exp
|
||||
LLM_TENSOR_HC_FFN_NORM, // qwen4exp
|
||||
LLM_TENSOR_HC_FFN_DOWN, // qwen4exp
|
||||
LLM_TENSOR_HC_FFN_UP, // qwen4exp
|
||||
LLM_TENSOR_HC_FFN_INJECT, // qwen4exp
|
||||
LLM_TENSOR_PLE_KEY, // qwen4exp
|
||||
LLM_TENSOR_PLE_VALUE, // qwen4exp
|
||||
LLM_TENSOR_PLE_NORM_KEY, // qwen4exp
|
||||
LLM_TENSOR_PLE_NORM_QUERY, // qwen4exp
|
||||
LLM_TENSOR_PLE_NORM_CONV, // qwen4exp
|
||||
LLM_TENSOR_PLE_CONV1D, // qwen4exp
|
||||
LLM_TENSOR_ATTN_COMPRESSOR_WKV,
|
||||
LLM_TENSOR_ATTN_COMPRESSOR_WGATE,
|
||||
LLM_TENSOR_ATTN_COMPRESSOR_APE,
|
||||
|
||||
@@ -236,6 +236,14 @@ bool llama_hparams::is_recr(uint32_t il) const {
|
||||
GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all);
|
||||
}
|
||||
|
||||
bool llama_hparams::is_ple(uint32_t il) const {
|
||||
if (il < n_layer_all) {
|
||||
return is_ple_impl[il];
|
||||
}
|
||||
|
||||
GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all);
|
||||
}
|
||||
|
||||
uint32_t llama_hparams::n_pos_per_embd() const {
|
||||
return rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE ? 4 : 1;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
// bump if necessary
|
||||
#define LLAMA_MAX_LAYERS 512
|
||||
#define LLAMA_MAX_EXPERTS 1024 // Kimi K3
|
||||
#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp
|
||||
#define LLAMA_MAX_PLE_HEADS 64 // qwen4exp
|
||||
|
||||
enum llama_expert_gating_func_type {
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_NONE = 0,
|
||||
@@ -270,6 +272,24 @@ struct llama_hparams {
|
||||
float dsv4_hc_eps = 0.0f;
|
||||
std::array<uint32_t, LLAMA_MAX_LAYERS> dsv4_compress_ratios;
|
||||
|
||||
// qwen4exp low-rank hyper-connections
|
||||
// 0 means full rank, which is the DeepSeek-V4 parameterisation
|
||||
uint32_t hc_low_rank = 0;
|
||||
|
||||
// qwen4exp PLE n-gram hash embeddings
|
||||
uint32_t ple_ngram_size = 0;
|
||||
uint32_t ple_heads_per_ngram = 0;
|
||||
uint32_t ple_conv_kernel = 0;
|
||||
uint32_t ple_n_heads = 0; // (ngram_size - 1) * heads_per_ngram
|
||||
uint32_t ple_head_dim = 0;
|
||||
uint32_t ple_eos_token_id = 0;
|
||||
std::array<uint32_t, LLAMA_MAX_LAYERS> is_ple_impl;
|
||||
std::array<uint64_t, LLAMA_MAX_PLE_NGRAM> ple_layer_multipliers;
|
||||
std::array<uint64_t, LLAMA_MAX_PLE_HEADS> ple_head_offsets;
|
||||
std::array<uint64_t, LLAMA_MAX_PLE_HEADS> ple_head_vocab_sizes;
|
||||
|
||||
bool is_ple(uint32_t il) const;
|
||||
|
||||
// qwen3vl deepstack
|
||||
// When parsed from GGUF, this implies the first N layers consume the first
|
||||
// N deepstack embeddings. Use deepstack_mapping_arr if you need a more
|
||||
|
||||
@@ -321,10 +321,11 @@ namespace GGUFMeta {
|
||||
case GGUF_TYPE_UINT32:
|
||||
case GGUF_TYPE_INT32: type_ok = (std::is_same<T, int32_t>::value) ||
|
||||
(std::is_same<T, uint32_t>::value); break;
|
||||
case GGUF_TYPE_UINT64: type_ok = (std::is_same<T, uint64_t>::value); break;
|
||||
case GGUF_TYPE_FLOAT32: type_ok = (std::is_same<T, float>::value); break;
|
||||
case GGUF_TYPE_STRING: type_ok = (std::is_same<T, std::string>::value); break;
|
||||
default:
|
||||
throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
|
||||
throw std::runtime_error(format("%s is not a string/float32/uint32/int32/uint64 array", key.c_str()));
|
||||
}
|
||||
if (!type_ok) {
|
||||
throw std::runtime_error(format("%s has wrong array element type %s", key.c_str(), gguf_type_name(arr_info.gt)));
|
||||
@@ -367,10 +368,11 @@ namespace GGUFMeta {
|
||||
case GGUF_TYPE_UINT32:
|
||||
case GGUF_TYPE_INT32: type_ok = (std::is_same<T, int32_t>::value) ||
|
||||
(std::is_same<T, uint32_t>::value); break;
|
||||
case GGUF_TYPE_UINT64: type_ok = (std::is_same<T, uint64_t>::value); break;
|
||||
case GGUF_TYPE_FLOAT32: type_ok = (std::is_same<T, float>::value); break;
|
||||
case GGUF_TYPE_STRING: type_ok = (std::is_same<T, std::string>::value); break;
|
||||
default:
|
||||
throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
|
||||
throw std::runtime_error(format("%s is not a string/float32/uint32/int32/uint64 array", key.c_str()));
|
||||
}
|
||||
if (!type_ok) {
|
||||
throw std::runtime_error(format("%s has wrong array element type %s", key.c_str(), gguf_type_name(arr_info.gt)));
|
||||
@@ -410,6 +412,9 @@ namespace GGUFMeta {
|
||||
template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::vector<uint32_t>>(enum llm_kv kid, std::vector<uint32_t> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::array<uint64_t, LLAMA_MAX_PLE_NGRAM>>(enum llm_kv kid, std::array<uint64_t, LLAMA_MAX_PLE_NGRAM> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::array<uint64_t, LLAMA_MAX_PLE_HEADS>>(enum llm_kv kid, std::array<uint64_t, LLAMA_MAX_PLE_HEADS> & result, bool required);
|
||||
|
||||
template<typename T>
|
||||
bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {
|
||||
|
||||
+5
-1
@@ -319,6 +319,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_qwen35(params);
|
||||
case LLM_ARCH_QWEN35MOE:
|
||||
return new llama_model_qwen35moe(params);
|
||||
case LLM_ARCH_QWEN4EXP:
|
||||
return new llama_model_qwen4exp(params);
|
||||
case LLM_ARCH_MISTRAL3:
|
||||
return new llama_model_mistral3(params);
|
||||
case LLM_ARCH_EAGLE3:
|
||||
@@ -927,6 +929,7 @@ const char * llm_type_name(llm_type type) {
|
||||
case LLM_TYPE_35B_A3B: return "35B.A3B";
|
||||
case LLM_TYPE_48B_A3B: return "48B.A3B";
|
||||
case LLM_TYPE_80B_A3B: return "80B.A3B";
|
||||
case LLM_TYPE_A3B: return "A3B";
|
||||
case LLM_TYPE_100B_A6B: return "100B.A6B";
|
||||
case LLM_TYPE_102B_A12B: return "102B.A12B";
|
||||
case LLM_TYPE_106B_A12B: return "106B.A12B";
|
||||
@@ -2441,7 +2444,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
|
||||
filter_recr = [&](uint32_t il) {
|
||||
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
|
||||
};
|
||||
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_MINIMAX_01) {
|
||||
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
|
||||
filter_attn = [&](uint32_t il) {
|
||||
return il < hparams.n_layer() && !hparams.is_recr(il);
|
||||
};
|
||||
@@ -2891,6 +2894,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_QWEN3VLMOE:
|
||||
case LLM_ARCH_QWEN35:
|
||||
case LLM_ARCH_QWEN35MOE:
|
||||
case LLM_ARCH_QWEN4EXP:
|
||||
case LLM_ARCH_QWEN3TTS:
|
||||
return LLAMA_ROPE_TYPE_IMROPE;
|
||||
|
||||
|
||||
@@ -129,6 +129,7 @@ enum llm_type {
|
||||
LLM_TYPE_35B_A3B, // Qwen3.5
|
||||
LLM_TYPE_48B_A3B, // Kimi Linear
|
||||
LLM_TYPE_80B_A3B, // Qwen3 Next
|
||||
LLM_TYPE_A3B, // Qwen3.8 Flash Next
|
||||
LLM_TYPE_100B_A6B,
|
||||
LLM_TYPE_102B_A12B, // Solar-Open
|
||||
LLM_TYPE_106B_A12B, // GLM-4.5-Air
|
||||
@@ -555,6 +556,24 @@ struct llama_layer {
|
||||
struct ggml_tensor * index_q_norm = nullptr;
|
||||
struct ggml_tensor * index_k_norm = nullptr;
|
||||
|
||||
// qwen4exp low-rank hyper-connections
|
||||
struct ggml_tensor * hc_attn_norm = nullptr;
|
||||
struct ggml_tensor * hc_attn_down = nullptr;
|
||||
struct ggml_tensor * hc_attn_up = nullptr;
|
||||
struct ggml_tensor * hc_attn_inject = nullptr;
|
||||
struct ggml_tensor * hc_ffn_norm = nullptr;
|
||||
struct ggml_tensor * hc_ffn_down = nullptr;
|
||||
struct ggml_tensor * hc_ffn_up = nullptr;
|
||||
struct ggml_tensor * hc_ffn_inject = nullptr;
|
||||
|
||||
// qwen4exp PLE
|
||||
struct ggml_tensor * ple_key = nullptr;
|
||||
struct ggml_tensor * ple_value = nullptr;
|
||||
struct ggml_tensor * ple_norm_key = nullptr;
|
||||
struct ggml_tensor * ple_norm_query = nullptr;
|
||||
struct ggml_tensor * ple_norm_conv = nullptr;
|
||||
struct ggml_tensor * ple_conv1d = nullptr;
|
||||
|
||||
// gemma4 layer output scale, reused for talkie embedding skip scale
|
||||
struct ggml_tensor * out_scale = nullptr;
|
||||
|
||||
@@ -635,6 +654,11 @@ struct llama_model {
|
||||
struct ggml_tensor * altup_proj = nullptr;
|
||||
struct ggml_tensor * altup_unembd_proj = nullptr;
|
||||
struct ggml_tensor * per_layer_tok_embd = nullptr;
|
||||
|
||||
// qwen4exp final hyper-connection mixer
|
||||
struct ggml_tensor * hc_head_norm = nullptr;
|
||||
struct ggml_tensor * hc_head_down = nullptr;
|
||||
struct ggml_tensor * hc_head_up = nullptr;
|
||||
struct ggml_tensor * per_layer_model_proj = nullptr;
|
||||
struct ggml_tensor * per_layer_proj_norm = nullptr;
|
||||
|
||||
|
||||
@@ -2272,6 +2272,69 @@ struct llama_model_qwen35 : public llama_model_base {
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_qwen4exp : public llama_model_base {
|
||||
llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
|
||||
struct graph : public llm_build_delta_net_base {
|
||||
graph(const llama_model & model, const llm_graph_params & params);
|
||||
private:
|
||||
// hyper-connections replace every layer norm, so the residual carried
|
||||
// between layers is [n_embd, hc, n_tokens] instead of [n_embd, n_tokens]
|
||||
ggml_tensor * build_hc_mix(
|
||||
ggml_tensor * x,
|
||||
ggml_tensor * w_norm,
|
||||
ggml_tensor * w_down,
|
||||
ggml_tensor * w_up,
|
||||
ggml_tensor * w_inject,
|
||||
ggml_tensor ** inject,
|
||||
int il);
|
||||
|
||||
ggml_tensor * build_hc_combine(
|
||||
ggml_tensor * residual,
|
||||
ggml_tensor * block_out,
|
||||
ggml_tensor * inject,
|
||||
int il);
|
||||
|
||||
ggml_tensor * build_layer_attn(
|
||||
llm_graph_input_attn_kv * inp_attn,
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * inp_pos,
|
||||
int * sections,
|
||||
int il);
|
||||
|
||||
ggml_tensor * build_layer_attn_linear(
|
||||
llm_graph_input_rs * inp,
|
||||
ggml_tensor * cur,
|
||||
int il);
|
||||
|
||||
ggml_tensor * build_layer_ffn(
|
||||
ggml_tensor * cur,
|
||||
int il);
|
||||
|
||||
ggml_tensor * build_norm_gated(
|
||||
ggml_tensor * input,
|
||||
ggml_tensor * weights,
|
||||
ggml_tensor * gate,
|
||||
int layer);
|
||||
|
||||
ggml_tensor * build_ple(
|
||||
llm_graph_input_rs * inp,
|
||||
ggml_tensor * hidden,
|
||||
int il);
|
||||
|
||||
// returns pair of qkv, z
|
||||
std::pair<ggml_tensor *, ggml_tensor *> build_qkvz(
|
||||
ggml_tensor * input,
|
||||
int il);
|
||||
|
||||
const llama_model & model;
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
struct llama_model_qwen35moe : public llama_model_base {
|
||||
llama_model_qwen35moe(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
#include "models.h"
|
||||
#include "llama-memory-recurrent.h"
|
||||
|
||||
void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
|
||||
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
|
||||
ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);
|
||||
|
||||
// gated delta net, same dimension aliasing as Qwen3.5
|
||||
ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
|
||||
ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
|
||||
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
|
||||
// hyper-connections. low_rank is qwen4exp specific; DeepSeek-V4 leaves it
|
||||
// absent and uses a full rank mix projection instead
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
|
||||
GGML_ASSERT(hparams.dsv4_hc_mult > 0 && "qwen4exp needs a hyper-connection count");
|
||||
GGML_ASSERT(hparams.hc_low_rank > 0 && "qwen4exp needs a hyper-connection low rank");
|
||||
hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd;
|
||||
|
||||
// QSA indexer
|
||||
ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
|
||||
ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
|
||||
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, hparams.n_layer_all, false);
|
||||
|
||||
// PLE n-gram hash embeddings
|
||||
ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size);
|
||||
ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);
|
||||
ml.get_key(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel);
|
||||
ml.get_key(LLM_KV_PLE_EOS_TOKEN_ID, hparams.ple_eos_token_id);
|
||||
ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
|
||||
|
||||
hparams.ple_n_heads = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram;
|
||||
hparams.ple_head_dim = hparams.n_embd_per_layer;
|
||||
GGML_ASSERT(hparams.ple_ngram_size >= 2 && hparams.ple_ngram_size <= LLAMA_MAX_PLE_NGRAM);
|
||||
GGML_ASSERT(hparams.ple_n_heads > 0 && hparams.ple_n_heads <= LLAMA_MAX_PLE_HEADS);
|
||||
|
||||
ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, hparams.ple_layer_multipliers);
|
||||
ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, hparams.ple_head_offsets);
|
||||
ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, hparams.ple_head_vocab_sizes);
|
||||
|
||||
std::fill(hparams.is_ple_impl.begin(), hparams.is_ple_impl.end(), 0);
|
||||
{
|
||||
uint32_t n_ple = 0;
|
||||
ml.get_arr_n(LLM_KV_PLE_LAYERS, n_ple);
|
||||
GGML_ASSERT(n_ple > 0 && "qwen4exp needs at least one PLE layer");
|
||||
|
||||
std::vector<uint32_t> ple_layers;
|
||||
ml.get_arr(LLM_KV_PLE_LAYERS, ple_layers);
|
||||
for (uint32_t il : ple_layers) {
|
||||
GGML_ASSERT(il < hparams.n_layer_all);
|
||||
hparams.is_ple_impl[il] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// linear attention everywhere except every full_attention_interval-th layer
|
||||
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
||||
uint32_t full_attn_interval = 4;
|
||||
ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
|
||||
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
||||
hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
|
||||
}
|
||||
}
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 48: type = LLM_TYPE_A3B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const int64_t hc = hparams.dsv4_hc_mult;
|
||||
const int64_t hc_dim = hc * n_embd;
|
||||
const int64_t hc_lr = hparams.hc_low_rank;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
||||
|
||||
// there is no output_norm: the final hyper-connection mixer carries it
|
||||
hc_head_norm = create_tensor(tn(LLM_TENSOR_HC_HEAD_NORM, "weight"), { hc_dim }, 0);
|
||||
hc_head_down = create_tensor(tn(LLM_TENSOR_HC_HEAD_DOWN, "weight"), { hc_dim, hc_lr }, 0);
|
||||
hc_head_up = create_tensor(tn(LLM_TENSOR_HC_HEAD_UP, "weight"), { hc_lr, hc_dim }, 0);
|
||||
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
|
||||
if (output == NULL) {
|
||||
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);
|
||||
}
|
||||
|
||||
// the n-gram table is one flat [ple_head_dim, n_rows] gather target. its row
|
||||
// count is the padded sum of the per-head vocab sizes, so read it back from
|
||||
// the file rather than trying to reproduce the padding rule
|
||||
const std::string ple_name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str();
|
||||
const auto * ple_w = ml.get_weight(ple_name.c_str());
|
||||
GGML_ASSERT(ple_w != nullptr && "qwen4exp is missing the PLE n-gram table");
|
||||
const int64_t ple_rows = ple_w->tensor->ne[1];
|
||||
per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
|
||||
{ hparams.ple_head_dim, ple_rows }, 0);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
auto & layer = layers[il];
|
||||
|
||||
const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;
|
||||
const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;
|
||||
|
||||
const int64_t head_k_dim = hparams.ssm_d_state;
|
||||
const int64_t head_v_dim = hparams.ssm_d_state;
|
||||
const int64_t n_k_heads = hparams.ssm_n_group;
|
||||
const int64_t n_v_heads = hparams.ssm_dt_rank;
|
||||
const int64_t key_dim = head_k_dim * n_k_heads;
|
||||
const int64_t value_dim = head_v_dim * n_v_heads;
|
||||
const int64_t conv_dim = key_dim * 2 + value_dim;
|
||||
|
||||
// two hyper-connection modules per layer, one before the token mixer
|
||||
// and one before the MoE
|
||||
layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, 0);
|
||||
layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, 0);
|
||||
layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, 0);
|
||||
layer.hc_attn_inject = create_tensor(tn(LLM_TENSOR_HC_ATTN_INJECT, "weight", il), { hc_dim, hc }, 0);
|
||||
layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { hc_dim }, 0);
|
||||
layer.hc_ffn_down = create_tensor(tn(LLM_TENSOR_HC_FFN_DOWN, "weight", il), { hc_dim, hc_lr }, 0);
|
||||
layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, 0);
|
||||
layer.hc_ffn_inject = create_tensor(tn(LLM_TENSOR_HC_FFN_INJECT, "weight", il), { hc_dim, hc }, 0);
|
||||
|
||||
if (!hparams.is_recr(il)) {
|
||||
// full attention: wq holds [q|gate] interleaved per head
|
||||
create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0);
|
||||
|
||||
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0);
|
||||
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0);
|
||||
|
||||
// QSA pre-indexer, MQA with a single key head
|
||||
const int64_t idx_dim = hparams.indexer_head_size;
|
||||
layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", il), { n_embd, hparams.indexer_n_head * idx_dim }, 0);
|
||||
layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", il), { n_embd, idx_dim }, 0);
|
||||
layer.index_q_norm = create_tensor(tn(LLM_TENSOR_INDEXER_Q_NORM, "weight", il), { idx_dim }, 0);
|
||||
layer.index_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", il), { idx_dim }, 0);
|
||||
} else {
|
||||
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, 0);
|
||||
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, 0);
|
||||
layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, 0);
|
||||
layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, 0);
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, 0);
|
||||
layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, 0);
|
||||
layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, 0);
|
||||
layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, 0);
|
||||
layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, 0);
|
||||
}
|
||||
|
||||
if (hparams.is_ple(il)) {
|
||||
layer.ple_key = create_tensor(tn(LLM_TENSOR_PLE_KEY, "weight", il), { n_embd, hc_dim }, 0);
|
||||
layer.ple_value = create_tensor(tn(LLM_TENSOR_PLE_VALUE, "weight", il), { n_embd, n_embd }, 0);
|
||||
layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { hc_dim }, 0);
|
||||
layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { hc_dim }, 0);
|
||||
layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { hc_dim }, 0);
|
||||
layer.ple_conv1d = create_tensor(tn(LLM_TENSOR_PLE_CONV1D, "weight", il), { hparams.ple_conv_kernel, hc_dim }, 0);
|
||||
}
|
||||
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, 0);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, 0);
|
||||
create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, 0);
|
||||
|
||||
layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, 0);
|
||||
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0);
|
||||
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0);
|
||||
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// the graph lands in the next commit; loading and metadata come first
|
||||
std::unique_ptr<llm_graph_context> llama_model_qwen4exp::build_arch_graph(const llm_graph_params & params) const {
|
||||
GGML_UNUSED(params);
|
||||
throw std::runtime_error("qwen4exp: graph not implemented yet");
|
||||
}
|
||||
Reference in New Issue
Block a user