diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp index 93b468784a..3972ce9ce2 100644 --- a/src/llama-memory-hybrid-idx.cpp +++ b/src/llama-memory-hybrid-idx.cpp @@ -55,6 +55,10 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( // K-shift must not rotate them while the stream copies in the same update still apply hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE; + // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors + hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size; + hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size; + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); return new llama_kv_cache(