diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py index 70c31222cd..2a67c77013 100644 --- a/conversion/qwen4exp.py +++ b/conversion/qwen4exp.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from typing import Iterable import torch @@ -81,6 +82,12 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): self.gguf_writer.add_ple_heads_per_ngram(hp["heads_per_ngram"]) self.gguf_writer.add_ple_conv_kernel(hp["ple_conv_kernel_size"]) self.gguf_writer.add_ple_eos_token_id(self._eos_token_id()) + # The PLE hash runs over token ids, but a multimodal batch arrives as embeddings + # with the placeholder consumed. Carry it so those positions hash what the + # reference sees in input_ids instead of being undefined. + _img = self._image_token_id() + if _img is not None: + self.gguf_writer.add_ple_image_token_id(int(_img)) if self._ple_row_dim is not None: self.gguf_writer.add_embedding_length_per_layer_input(self._ple_row_dim) @@ -91,6 +98,19 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): self.gguf_writer.add_ple_head_vocab_sizes( self._read_hash_constants("ple_embedding.ngram_heads_vocab_sizes")) + def _image_token_id(self) -> int | None: + # image_token_id is top-level in config.json, not in self.hparams once that is + # narrowed to text_config, and the text model has no global_config; read the file + img = self.hparams.get("image_token_id") + if img is not None: + return int(img) + try: + with open(self.dir_model / "config.json", "r", encoding="utf-8") as f: + img = json.load(f).get("image_token_id") + except Exception: + return None + return None if img is None else int(img) + def _eos_token_id(self) -> int: eos = self.hparams.get("eos_token_id") if isinstance(eos, list): diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2caff0b0cc..60ddeb5540 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -237,6 +237,7 @@ class Keys: HEAD_OFFSETS = "{arch}.ple.head_offsets" HEAD_VOCAB_SIZES = "{arch}.ple.head_vocab_sizes" EOS_TOKEN_ID = "{arch}.ple.eos_token_id" + IMAGE_TOKEN_ID = "{arch}.ple.image_token_id" class Rope: DIMENSION_COUNT = "{arch}.rope.dimension_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 363e9f751b..32b8154de5 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1060,6 +1060,9 @@ class GGUFWriter: def add_ple_eos_token_id(self, value: int) -> None: self.add_uint32(Keys.PLE.EOS_TOKEN_ID.format(arch=self.arch), value) + def add_ple_image_token_id(self, value: int) -> None: + self.add_uint32(Keys.PLE.IMAGE_TOKEN_ID.format(arch=self.arch), value) + def add_attention_scale(self, value: float) -> None: self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 870b66922d..3305f26837 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -304,6 +304,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_PLE_HEAD_OFFSETS, "%s.ple.head_offsets" }, { LLM_KV_PLE_HEAD_VOCAB_SIZES, "%s.ple.head_vocab_sizes" }, { LLM_KV_PLE_EOS_TOKEN_ID, "%s.ple.eos_token_id" }, + { LLM_KV_PLE_IMAGE_TOKEN_ID, "%s.ple.image_token_id" }, { LLM_KV_HASH_LAYER_COUNT, "%s.hash_layer_count" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index bcfbc3e99b..94cf08ae16 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -309,6 +309,7 @@ enum llm_kv { LLM_KV_PLE_HEAD_OFFSETS, LLM_KV_PLE_HEAD_VOCAB_SIZES, LLM_KV_PLE_EOS_TOKEN_ID, + LLM_KV_PLE_IMAGE_TOKEN_ID, LLM_KV_HASH_LAYER_COUNT, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 053ad1641d..a3e30754b2 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -281,6 +281,9 @@ struct llama_hparams { uint32_t ple_n_heads = 0; // (ngram_size - 1) * heads_per_ngram uint32_t ple_head_dim = 0; uint32_t ple_eos_token_id = 0; + // placeholder the PLE hash sees where an image chunk is spliced in; 0 means the + // file predates this key and the loader falls back to EOS + uint32_t ple_image_token_id = 0; std::array is_ple_impl; std::array ple_layer_multipliers; std::array ple_head_offsets; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 0a009329dc..686bae2d05 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -45,6 +45,9 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram); ml.get_key(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel); ml.get_key(LLM_KV_PLE_EOS_TOKEN_ID, hparams.ple_eos_token_id); + // optional: absent in files converted before multimodal batches + // were exercised, in which case the PLE hash falls back to EOS + ml.get_key(LLM_KV_PLE_IMAGE_TOKEN_ID, hparams.ple_image_token_id, false); ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer); hparams.ple_n_heads = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram; @@ -803,12 +806,26 @@ public: }; void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { - if (!ubatch->token) { - return; - } - const auto & hp = pmodel.hparams; + // A multimodal ubatch arrives as embeddings: the mtmd layer has already + // consumed the image placeholder ids, so ubatch->token is null. The hash + // still has to produce a row for every position, because this input feeds + // ggml_get_rows. Returning early here left the index buffer uninitialised, + // so whatever happened to be in it indexed a 320 M row table -- an + // out-of-range gather, which aborts inside ggml_compute_forward_get_rows. + // + // The reference hashes input_ids, where those positions still hold the + // image placeholder, so use that token here. A file converted before the + // key existed falls back to EOS, which is defined and simply treats the + // image as a segment boundary. + const llama_token img_tok = hp.ple_image_token_id != 0 + ? (llama_token) hp.ple_image_token_id + : (llama_token) hp.ple_eos_token_id; + auto tok_of = [&](int64_t k) -> llama_token { + return ubatch->token ? ubatch->token[k] : img_tok; + }; + const int64_t n_tokens = ubatch->n_tokens; const int64_t n_gram = hp.ple_ngram_size; const int64_t n_heads = hp.ple_n_heads; @@ -848,7 +865,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { auto prev = [&](int64_t s) -> int64_t { const int64_t j = i - s; if (j >= 0 && ubatch->seq_id[j][0] == seq && ubatch->pos[j] == pos - s) { - return ubatch->token[j]; + return tok_of(j); } // s - i positions before this ubatch started, most recent last const int64_t back = s - i; @@ -864,7 +881,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { // takes the last EOS strictly *before* this position, so a segment // boundary only hides tokens from the positions that follow it. std::vector ctx(n_gram); - ctx[0] = ubatch->token[i]; + ctx[0] = tok_of(i); bool cut = false; for (int64_t s = 1; s < n_gram; ++s) { ctx[s] = cut ? eos : prev(s); @@ -887,7 +904,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { } auto & h = hist_map[seq]; - h.toks.push_back(ubatch->token[i]); + h.toks.push_back(tok_of(i)); if ((int64_t) h.toks.size() > n_gram - 1) { h.toks.erase(h.toks.begin(), h.toks.end() - (n_gram - 1)); }