From c3259772c711c98da4144bdd1e0fd70f303639ae Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 27 Aug 2026 07:32:16 +0000 Subject: [PATCH] qwen4exp: shrink the PLE hparams storage llama_hparams is held by value inside llm_graph_params and every llm_graph_input_*, and llm_graph_params is a stack local in graph_reserve and process_ubatch, so its width is paid on every worker thread stack. is_ple_impl spent 2048 bytes carrying 512 bits. It is the one per-layer flag that is not moved through the loader's uint32 array templates, so a bitset costs nothing in call sites and also removes the uninitialized read that non-qwen4exp archs had, since nothing filled the array for them. The PLE head offsets and vocab sizes are token-space indices; the gather that consumes them already truncates to int32, so 64-bit storage was never reachable. The gguf arrays stay uint64 for file compatibility and are narrowed on load. sizeof(llama_hparams) 34440 -> 31944, sizeof(llm_graph_params) 34872 -> 32376. --- src/llama-hparams.h | 13 ++++++++++--- src/models/qwen4exp.cpp | 19 +++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/llama-hparams.h b/src/llama-hparams.h index d829d034cb..739156da6d 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -3,6 +3,7 @@ #include "llama.h" #include +#include #include #include @@ -283,10 +284,16 @@ struct llama_hparams { uint32_t ple_eos_token_id = 0; // the id the PLE hash stands in at image positions; 0 makes the loader fall back to EOS uint32_t ple_image_token_id = 0; - std::array is_ple_impl; + // unlike is_swa_impl and friends this is never read or written as a per-layer gguf array + // (the file lists PLE layer indices), so it is not tied to the loader's uint32 array type + // and can hold one bit per layer instead of one word + std::bitset is_ple_impl; + // the hash multipliers reach ~2e13 and have to stay 64-bit std::array ple_layer_multipliers; - std::array ple_head_offsets; - std::array ple_head_vocab_sizes; + // head offsets and vocab sizes are token-space indices; the gather that consumes them + // truncates to int32, so 64-bit storage could never have been used + std::array ple_head_offsets; + std::array ple_head_vocab_sizes; bool is_ple(uint32_t il) const; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index b49cf591a2..533e9a300b 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -33,7 +33,7 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, hparams.n_layer_all, false); // PLE n-gram hash embeddings; if the key group is absent every field stays zero - std::fill(hparams.is_ple_impl.begin(), hparams.is_ple_impl.end(), 0); + hparams.is_ple_impl.reset(); hparams.ple_n_heads = 0; uint32_t n_ple = 0; @@ -43,7 +43,7 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { ml.get_arr(LLM_KV_PLE_LAYERS, ple_layers); for (uint32_t il : ple_layers) { GGML_ASSERT(il < hparams.n_layer_all); - hparams.is_ple_impl[il] = 1; + hparams.is_ple_impl.set(il); } ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size); @@ -60,8 +60,19 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { GGML_ASSERT(hparams.ple_n_heads > 0 && hparams.ple_n_heads <= LLAMA_MAX_PLE_HEADS); ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, hparams.ple_layer_multipliers); - ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, hparams.ple_head_offsets); - ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, hparams.ple_head_vocab_sizes); + + // the file writes the head ranges as uint64 arrays, so read them at that width and + // narrow; hparams keeps them at the int32 width the row gather actually uses + std::array head_offsets = {}; + std::array head_vocab_sizes = {}; + ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, head_offsets); + ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, head_vocab_sizes); + for (uint32_t h = 0; h < hparams.ple_n_heads; ++h) { + GGML_ASSERT(head_offsets[h] + head_vocab_sizes[h] <= INT32_MAX && + "PLE head range does not fit the int32 row index"); + hparams.ple_head_offsets[h] = (uint32_t) head_offsets[h]; + hparams.ple_head_vocab_sizes[h] = (uint32_t) head_vocab_sizes[h]; + } } // linear attention everywhere except every full_attention_interval-th layer