qwen4exp: hash the image placeholder for multimodal batches

The PLE row indices are computed host-side from ubatch->token, and set_input
returned early when that was null. A multimodal ubatch is exactly that case:
the mtmd layer consumes the image placeholder ids and hands llama_decode
embeddings instead. The early return left the I32 index tensor uninitialised,
so ggml_get_rows indexed a 320 M row table with whatever the buffer happened to
contain, and aborted:

  GGML_ASSERT(i01 >= 0 && i01 < ne01) failed
    ggml_compute_forward_get_rows
    mtmd_helper_decode_image_chunk -> llama_decode

Every image request crashed. Nothing caught it because the vision work had only
ever been verified by converting an mmproj, never by running one.

The reference computes the hash over input_ids, where those positions still
hold the image placeholder, so carry that id through as qwen4exp.ple.image_token_id
and hash it. The key is optional: a file converted before it existed falls back
to the PLE EOS token, which is defined and treats the image as a segment
boundary rather than crashing.

Verified end to end with llama-mtmd-cli, a Q4_K_M base and the F16 mmproj, on a
generated image with known content. The model names the red circle, the blue
square, the inverted green triangle and reads "UNSLOTH 42", each with the right
position.

conversion/qwen4exp.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from typing import Iterable
 
 import torch
@@ -81,6 +82,12 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self.gguf_writer.add_ple_heads_per_ngram(hp["heads_per_ngram"])
         self.gguf_writer.add_ple_conv_kernel(hp["ple_conv_kernel_size"])
         self.gguf_writer.add_ple_eos_token_id(self._eos_token_id())
+        # The PLE hash runs over token ids, but a multimodal batch arrives as embeddings
+        # with the placeholder consumed. Carry it so those positions hash what the
+        # reference sees in input_ids instead of being undefined.
+        _img = self._image_token_id()
+        if _img is not None:
+            self.gguf_writer.add_ple_image_token_id(int(_img))
         if self._ple_row_dim is not None:
             self.gguf_writer.add_embedding_length_per_layer_input(self._ple_row_dim)
 
@@ -91,6 +98,19 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self.gguf_writer.add_ple_head_vocab_sizes(
             self._read_hash_constants("ple_embedding.ngram_heads_vocab_sizes"))
 
+    def _image_token_id(self) -> int | None:
+        # image_token_id is top-level in config.json, not in self.hparams once that is
+        # narrowed to text_config, and the text model has no global_config; read the file
+        img = self.hparams.get("image_token_id")
+        if img is not None:
+            return int(img)
+        try:
+            with open(self.dir_model / "config.json", "r", encoding="utf-8") as f:
+                img = json.load(f).get("image_token_id")
+        except Exception:
+            return None
+        return None if img is None else int(img)
+
     def _eos_token_id(self) -> int:
         eos = self.hparams.get("eos_token_id")
         if isinstance(eos, list):

gguf-py/gguf/constants.py

@@ -237,6 +237,7 @@ class Keys:
         HEAD_OFFSETS       = "{arch}.ple.head_offsets"
         HEAD_VOCAB_SIZES   = "{arch}.ple.head_vocab_sizes"
         EOS_TOKEN_ID       = "{arch}.ple.eos_token_id"
+        IMAGE_TOKEN_ID     = "{arch}.ple.image_token_id"
 
     class Rope:
         DIMENSION_COUNT           = "{arch}.rope.dimension_count"

gguf-py/gguf/gguf_writer.py

@@ -1060,6 +1060,9 @@ class GGUFWriter:
     def add_ple_eos_token_id(self, value: int) -> None:
         self.add_uint32(Keys.PLE.EOS_TOKEN_ID.format(arch=self.arch), value)
 
+    def add_ple_image_token_id(self, value: int) -> None:
+        self.add_uint32(Keys.PLE.IMAGE_TOKEN_ID.format(arch=self.arch), value)
+
     def add_attention_scale(self, value: float) -> None:
         self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value)
 

src/llama-arch.cpp

@@ -304,6 +304,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_PLE_HEAD_OFFSETS,                       "%s.ple.head_offsets"                       },
     { LLM_KV_PLE_HEAD_VOCAB_SIZES,                   "%s.ple.head_vocab_sizes"                   },
     { LLM_KV_PLE_EOS_TOKEN_ID,                       "%s.ple.eos_token_id"                       },
+    { LLM_KV_PLE_IMAGE_TOKEN_ID,                     "%s.ple.image_token_id"                     },
 
     { LLM_KV_HASH_LAYER_COUNT,                       "%s.hash_layer_count"                       },
 

src/llama-arch.h

@@ -309,6 +309,7 @@ enum llm_kv {
     LLM_KV_PLE_HEAD_OFFSETS,
     LLM_KV_PLE_HEAD_VOCAB_SIZES,
     LLM_KV_PLE_EOS_TOKEN_ID,
+    LLM_KV_PLE_IMAGE_TOKEN_ID,
 
     LLM_KV_HASH_LAYER_COUNT,
 

src/llama-hparams.h

@@ -281,6 +281,9 @@ struct llama_hparams {
     uint32_t ple_n_heads         = 0;   // (ngram_size - 1) * heads_per_ngram
     uint32_t ple_head_dim        = 0;
     uint32_t ple_eos_token_id    = 0;
+    // placeholder the PLE hash sees where an image chunk is spliced in; 0 means the
+    // file predates this key and the loader falls back to EOS
+    uint32_t ple_image_token_id  = 0;
     std::array<uint32_t, LLAMA_MAX_LAYERS> is_ple_impl;
     std::array<uint64_t, LLAMA_MAX_PLE_NGRAM>  ple_layer_multipliers;
     std::array<uint64_t, LLAMA_MAX_PLE_HEADS>  ple_head_offsets;

src/models/qwen4exp.cpp

@@ -45,6 +45,9 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
         ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);
         ml.get_key(LLM_KV_PLE_CONV_KERNEL,     hparams.ple_conv_kernel);
         ml.get_key(LLM_KV_PLE_EOS_TOKEN_ID,    hparams.ple_eos_token_id);
+        // optional: absent in files converted before multimodal batches
+        // were exercised, in which case the PLE hash falls back to EOS
+        ml.get_key(LLM_KV_PLE_IMAGE_TOKEN_ID,  hparams.ple_image_token_id, false);
         ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
 
         hparams.ple_n_heads  = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram;
@@ -803,12 +806,26 @@ public:
 };
 
 void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
-    if (!ubatch->token) {
-        return;
-    }
-
     const auto & hp = pmodel.hparams;
 
+    // A multimodal ubatch arrives as embeddings: the mtmd layer has already
+    // consumed the image placeholder ids, so ubatch->token is null. The hash
+    // still has to produce a row for every position, because this input feeds
+    // ggml_get_rows. Returning early here left the index buffer uninitialised,
+    // so whatever happened to be in it indexed a 320 M row table -- an
+    // out-of-range gather, which aborts inside ggml_compute_forward_get_rows.
+    //
+    // The reference hashes input_ids, where those positions still hold the
+    // image placeholder, so use that token here. A file converted before the
+    // key existed falls back to EOS, which is defined and simply treats the
+    // image as a segment boundary.
+    const llama_token img_tok = hp.ple_image_token_id != 0
+        ? (llama_token) hp.ple_image_token_id
+        : (llama_token) hp.ple_eos_token_id;
+    auto tok_of = [&](int64_t k) -> llama_token {
+        return ubatch->token ? ubatch->token[k] : img_tok;
+    };
+
     const int64_t n_tokens = ubatch->n_tokens;
     const int64_t n_gram   = hp.ple_ngram_size;
     const int64_t n_heads  = hp.ple_n_heads;
@@ -848,7 +865,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
         auto prev = [&](int64_t s) -> int64_t {
             const int64_t j = i - s;
             if (j >= 0 && ubatch->seq_id[j][0] == seq && ubatch->pos[j] == pos - s) {
-                return ubatch->token[j];
+                return tok_of(j);
             }
             // s - i positions before this ubatch started, most recent last
             const int64_t back = s - i;
@@ -864,7 +881,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
         // takes the last EOS strictly *before* this position, so a segment
         // boundary only hides tokens from the positions that follow it.
         std::vector<int64_t> ctx(n_gram);
-        ctx[0] = ubatch->token[i];
+        ctx[0] = tok_of(i);
         bool cut = false;
         for (int64_t s = 1; s < n_gram; ++s) {
             ctx[s] = cut ? eos : prev(s);
@@ -887,7 +904,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
         }
 
         auto & h = hist_map[seq];
-        h.toks.push_back(ubatch->token[i]);
+        h.toks.push_back(tok_of(i));
         if ((int64_t) h.toks.size() > n_gram - 1) {
             h.toks.erase(h.toks.begin(), h.toks.end() - (n_gram - 1));
         }