From 1caf6c5711ee3204b223f2e6197b8336eb6ef331 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 27 Aug 2026 06:16:54 +0000 Subject: [PATCH] llama: give the qwen4exp full memory context its indexer cache graph_reserve() walks a full memory context, and qwen4exp builds its sparse attention only when the context exposes an indexer cache. the full-context constructor left ctx_idx null, so the reserved worst case was the dense fallback: a smaller graph than the one decode executes. ggml-alloc then had to grow the compute buffer on the first decode, past the size reported at load. with -np 4 -c 32768 -fa on -ctk q8_0 -ctv q8_0 on an IQ1_S qwen4exp, the reserved CUDA0 buffer was 217.00 MiB against 275.71 MiB actually used, and CUDA_Host 42.31 MiB against 191.14 MiB. reserving the sparse graph makes both match exactly, in unified and non-unified cache mode. Co-authored-by: Pascal Assisted-by: Claude --- src/llama-memory-hybrid-idx.cpp | 9 ++++++++- src/llama-memory-hybrid-idx.h | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp index 38e3e61c57..5cef7a032e 100644 --- a/src/llama-memory-hybrid-idx.cpp +++ b/src/llama-memory-hybrid-idx.cpp @@ -557,7 +557,14 @@ llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(llama_memory_st llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(llama_memory_hybrid_idx * mem) : llama_memory_hybrid_context(mem), - mem(mem) {} + mem(mem), + // graph reservation walks a full context, and qwen4exp builds the sparse attention only when + // this is set. without it the reserved worst case is the smaller dense graph, so ggml-alloc + // must grow the compute buffer on the first decode + ns_ubatch(mem->get_mem_idx() == nullptr ? + std::vector() : std::vector{ mem->get_mem_idx()->get_n_stream() }), + ctx_idx(mem->get_mem_idx() == nullptr ? nullptr : + new llama_kv_cache_context(mem->get_mem_idx())) {} llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context( llama_memory_hybrid_idx * mem, diff --git a/src/llama-memory-hybrid-idx.h b/src/llama-memory-hybrid-idx.h index 8c867b56eb..c16fe4eeb0 100644 --- a/src/llama-memory-hybrid-idx.h +++ b/src/llama-memory-hybrid-idx.h @@ -157,7 +157,7 @@ public: // llama_memory_hybrid_idx_context specific API // - // nullptr with no indexer, and for the full and update contexts, which build no sparse graph + // nullptr with no indexer, and for the update context, which builds no sparse graph const llama_kv_cache_context * get_idx() const; // streams in the current slot info, the `ns` of get_k/get_v; 1 if unified @@ -182,7 +182,7 @@ private: // declared first, so it is initialised while sinfos_idx is still intact const std::vector ns_ubatch; - // null unless the model has an indexer and this is a batch context + // null unless the model has an indexer and this is a batch or full context const llama_memory_context_ptr ctx_idx; // mirrors the base class's ubatch cursor, which is private there