mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-19 17:24:57 +02:00
llama: give the qwen4exp full memory context its indexer cache
graph_reserve() walks a full memory context, and qwen4exp builds its sparse attention only when the context exposes an indexer cache. the full-context constructor left ctx_idx null, so the reserved worst case was the dense fallback: a smaller graph than the one decode executes. ggml-alloc then had to grow the compute buffer on the first decode, past the size reported at load. with -np 4 -c 32768 -fa on -ctk q8_0 -ctv q8_0 on an IQ1_S qwen4exp, the reserved CUDA0 buffer was 217.00 MiB against 275.71 MiB actually used, and CUDA_Host 42.31 MiB against 191.14 MiB. reserving the sparse graph makes both match exactly, in unified and non-unified cache mode. Co-authored-by: Pascal <admin@serveurperso.com> Assisted-by: Claude
This commit is contained in:
@@ -557,7 +557,14 @@ llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(llama_memory_st
|
||||
|
||||
llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(llama_memory_hybrid_idx * mem) :
|
||||
llama_memory_hybrid_context(mem),
|
||||
mem(mem) {}
|
||||
mem(mem),
|
||||
// graph reservation walks a full context, and qwen4exp builds the sparse attention only when
|
||||
// this is set. without it the reserved worst case is the smaller dense graph, so ggml-alloc
|
||||
// must grow the compute buffer on the first decode
|
||||
ns_ubatch(mem->get_mem_idx() == nullptr ?
|
||||
std::vector<uint32_t>() : std::vector<uint32_t>{ mem->get_mem_idx()->get_n_stream() }),
|
||||
ctx_idx(mem->get_mem_idx() == nullptr ? nullptr :
|
||||
new llama_kv_cache_context(mem->get_mem_idx())) {}
|
||||
|
||||
llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(
|
||||
llama_memory_hybrid_idx * mem,
|
||||
|
||||
@@ -157,7 +157,7 @@ public:
|
||||
// llama_memory_hybrid_idx_context specific API
|
||||
//
|
||||
|
||||
// nullptr with no indexer, and for the full and update contexts, which build no sparse graph
|
||||
// nullptr with no indexer, and for the update context, which builds no sparse graph
|
||||
const llama_kv_cache_context * get_idx() const;
|
||||
|
||||
// streams in the current slot info, the `ns` of get_k/get_v; 1 if unified
|
||||
@@ -182,7 +182,7 @@ private:
|
||||
// declared first, so it is initialised while sinfos_idx is still intact
|
||||
const std::vector<uint32_t> ns_ubatch;
|
||||
|
||||
// null unless the model has an indexer and this is a batch context
|
||||
// null unless the model has an indexer and this is a batch or full context
|
||||
const llama_memory_context_ptr ctx_idx;
|
||||
|
||||
// mirrors the base class's ubatch cursor, which is private there
|
||||
|
||||
Reference in New Issue
Block a user