diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 0a27367c9..0f5155b2e 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -15,26 +15,11 @@ bool llama_model_saver_supports_arch(llm_arch arch) { switch (arch) { - case LLM_ARCH_PLAMO3: - case LLM_ARCH_GEMMA3: case LLM_ARCH_GEMMA3N: - case LLM_ARCH_COHERE2: - case LLM_ARCH_COHERE2MOE: - case LLM_ARCH_OLMO2: case LLM_ARCH_BITNET: case LLM_ARCH_T5: - case LLM_ARCH_EXAONE_MOE: - case LLM_ARCH_AFMOE: case LLM_ARCH_APERTUS: - case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: - case LLM_ARCH_SPARK2_5: - case LLM_ARCH_MUSE_GLIMMER: - case LLM_ARCH_MELLUM: - case LLM_ARCH_LAGUNA: - case LLM_ARCH_GRANITE_SWA: - case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config - case LLM_ARCH_MAPLE: return false; default: return true; @@ -290,7 +275,11 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + // never collapsed to a scalar: the loaders read a scalar as a period + add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector( + hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all)); + } add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); @@ -300,6 +289,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); + add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa); + add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa); + add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa); add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index de3b2e38f..e195f50d0 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3305,6 +3305,15 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, } } +void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) { + if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) { + return; + } + + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern, false); + hparams.set_swa_pattern(n_pattern, dense_first); +} + const int32_t * llama_model_target_layer_ids(const struct llama_model * model) { const auto & v = model->target_layer_ids; return v.empty() ? nullptr : v.data(); diff --git a/src/llama-model.h b/src/llama-model.h index d61afa2cb..984b2cf38 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -814,6 +814,9 @@ struct llama_model_base : public llama_model { int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_, int flags); + // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern + void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false); + void load_stats (llama_model_loader & ml) override; void load_hparams(llama_model_loader & ml) override; void load_vocab (llama_model_loader & ml) override; diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index cf0220367..08c22b6ed 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -14,9 +14,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { // Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4) if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp index e2b366256..7ad5244e4 100644 --- a/src/models/cohere2.cpp +++ b/src/models/cohere2.cpp @@ -2,9 +2,7 @@ void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 5e02cd56e..7704cbb87 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -25,12 +25,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period, true); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index 976ee050a..840da5f67 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -3,9 +3,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 128; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp index 9ba978956..c1c55957a 100644 --- a/src/models/exaone4.cpp +++ b/src/models/exaone4.cpp @@ -4,9 +4,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_layer() == 64) { // 32B hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp index 80ed3b1a4..6c97883d8 100644 --- a/src/models/gemma-embedding.cpp +++ b/src/models/gemma-embedding.cpp @@ -2,9 +2,7 @@ void llama_model_gemma_embedding::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); hparams.causal_attn = false; // embeddings do not use causal attention diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp index 2fbfb15a9..9e5243465 100644 --- a/src/models/gemma2.cpp +++ b/src/models/gemma2.cpp @@ -3,9 +3,7 @@ void llama_model_gemma2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; // default value of gemma 2 - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.attn_soft_cap = true; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp index 690194529..f99bbaacd 100644 --- a/src/models/gemma3.cpp +++ b/src/models/gemma3.cpp @@ -4,9 +4,7 @@ void llama_model_gemma3::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); } else { diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index bb628203a..4d47ddc62 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -1,10 +1,8 @@ #include "models.h" void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) { - uint32_t swa_period = 5; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 5); hparams.n_layer_kv_from_start = 20; hparams.f_attention_scale = 1.0f; diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index 556400bfc..2bc4fa8a0 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -36,9 +36,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0 + load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0 // Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims; // SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 8a812beff..4f79b4d1e 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -16,9 +16,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { hparams.f_attn_temp_scale = 0.1f; hparams.f_attn_temp_offset = 1.0f; - uint32_t swa_period = 4; // pattern: 3 chunked - 1 full - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); // pattern: 3 chunked - 1 full hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index 872a9c8f5..3ab79747a 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -8,13 +8,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - const auto res = ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - if (res) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp index f3e9407e0..b7542d59b 100644 --- a/src/models/modern-bert.cpp +++ b/src/models/modern-bert.cpp @@ -5,9 +5,7 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 3; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 3, true); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp index 0e9415308..0e5f75ebe 100644 --- a/src/models/muse-glimmer.cpp +++ b/src/models/muse-glimmer.cpp @@ -10,12 +10,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); switch (hparams.n_layer()) { case 52: type = LLM_TYPE_30B; break; diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp index 05b9394b8..fe5dc88d4 100644 --- a/src/models/olmo2.cpp +++ b/src/models/olmo2.cpp @@ -6,9 +6,7 @@ void llama_model_olmo2::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cpp diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index c9f9b677d..39660a325 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -6,9 +6,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp index 16d0b1dce..f8235f8a8 100644 --- a/src/models/plamo3.cpp +++ b/src/models/plamo3.cpp @@ -6,9 +6,7 @@ void llama_model_plamo3::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 8; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 8); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index 680ffb8fd..555f8b718 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -6,9 +6,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;