Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 09:15:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

qwen4exp: double the Q split granularity for tensor parallelism

Browse Source 
qwen4exp fuses the attention gate into attn_q.weight the same way qwen3next
and qwen 3.5 do, so a device boundary must fall on a whole q+gate pair or the
Q heads stop lining up with the K/V heads and attn_output rows.

(cherry picked from commit 6c9a592f0a425a459ab6efae3b897cf68460e244)
This commit is contained in:
danielhanchen
2026-08-27 03:47:07 +00:00
committed by Daniel Han
parent fbe17732bd
commit 6c5afc86ae
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+2 -1
View File
@@ -724,7 +724,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {
GGML_ASSERT(segments.size() == 1);
// some models have Q gate tensors, for those cases the granularity needs to be doubled:
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
return {std::lcm(2*n_embd_q, blck_size_perf)};
}
return {granularity_q};