Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

qwen4exp: support a quantized KV cache in the QSA attention path

Browse Source 
(cherry picked from commit 4c30574f81dc1115d08078c47b6cf8c789c0a842)
This commit is contained in:
danielhanchen
2026-08-26 16:13:21 +00:00
committed by Daniel Han
parent d4a943f9a5
commit 0ac4b18025
1 changed files with 15 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/qwen4exp.cpp
+15 -1
View File
@@ -520,7 +520,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
ggml_tensor * top_k,
float kq_scale,
int il) {
GGML_ASSERT(inp->self_k_rot == nullptr && inp->self_v_rot == nullptr);
// rotate q/k/v before they reach a quantized cache, as the dense path does. the indexer
// has already scored with its own query in build_qsa_top_k, so top_k is unaffected.
if (inp->self_k_rot) {
q_cur = llama_mul_mat_hadamard(ctx0, q_cur, inp->self_k_rot);
k_cur = llama_mul_mat_hadamard(ctx0, k_cur, inp->self_k_rot);
}
if (inp->self_v_rot) {
v_cur = llama_mul_mat_hadamard(ctx0, v_cur, inp->self_v_rot);
}
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
@@ -575,6 +584,11 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, kq_scale, il);
cb(cur, "kqv_out", il);
// the rotation is its own inverse, so undo it on the value side of the output
if (inp->self_v_rot) {
cur = llama_mul_mat_hadamard(ctx0, cur, inp->self_v_rot);
}
return cur;
}