Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Move gate_bias and scale into the same active-thread guard

Browse Source 
Latest perf numbers:
B6000

build: 5b7d9f272 (9578)
+ ./scripts/compare-llama-bench.py -b master -c osimons/nvfp4_fuse_mmvq --tool llama-bench -i llama-bench.sqlite
| CPU                         | Model                    | Test         |   t/s master |   t/s osimons/nvfp4_fuse_mmvq |   Speedup |
|:----------------------------|:-------------------------|:-------------|-------------:|------------------------------:|----------:|
| INTEL(R) XEON(R) GOLD 6542Y | gemma4 26B.A4B NVFP4     | tg128@d32768 |       151.79 |                        154.10 |      1.02 |
| INTEL(R) XEON(R) GOLD 6542Y | gemma4 26B.A4B Q4_K_M    | tg128@d32768 |       187.90 |                        187.27 |      1.00 |
| INTEL(R) XEON(R) GOLD 6542Y | gpt-oss 20B MXFP4 MoE    | tg128@d32768 |       303.77 |                        306.56 |      1.01 |
| INTEL(R) XEON(R) GOLD 6542Y | qwen35moe 35B.A3B NVFP4  | tg128@d32768 |       193.41 |                        207.99 |      1.08 |
| INTEL(R) XEON(R) GOLD 6542Y | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 |       217.60 |                        218.58 |      1.00 |

DGX Spark

build: 5b7d9f272 (9578)
+ ./scripts/compare-llama-bench.py -b master -c osimons/nvfp4_fuse_mmvq --tool llama-bench -i llama-bench.sqlite
| CPU   | Model                    | Test         |   t/s master |   t/s osimons/nvfp4_fuse_mmvq |   Speedup |
|:------|:-------------------------|:-------------|-------------:|------------------------------:|----------:|
| CPU   | gemma4 26B.A4B NVFP4     | tg128@d32768 |        34.61 |                         34.84 |      1.01 |
| CPU   | gemma4 26B.A4B Q4_K_M    | tg128@d32768 |        46.95 |                         46.90 |      1.00 |
| CPU   | gpt-oss 20B MXFP4 MoE    | tg128@d32768 |        64.84 |                         64.62 |      1.00 |
| CPU   | qwen35moe 35B.A3B NVFP4  | tg128@d32768 |        59.63 |                         60.72 |      1.02 |
| CPU   | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 |        56.53 |                         56.55 |      1.00 |

PPL values for 5 chunks:
this PR

model                                                                                                       mode             ppl         uncertainty  log
/mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf                                 fusion_enabled   5.2892      0.35389      ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_enabled.log
/mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf                                 fusion_disabled  5.2742      0.35215      ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_disabled.log
/mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf  fusion_enabled   5.4487      0.36866      ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_enabled.log
/mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf  fusion_disabled  5.4403      0.36782      ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_disabled.log
/mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf                              fusion_enabled   17342.4348  3703.13932   ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_enabled.log
/mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf                              fusion_disabled  18627.0624  3998.42475   ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_disabled.log
/mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf                                            fusion_enabled   363.8913    33.14007     ppl-value-checks/gpt-oss-20b-mxfp4.fusion_enabled.log
/mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf                                            fusion_disabled  363.8913    33.14007     ppl-value-checks/gpt-oss-20b-mxfp4.fusion_disabled.log
/mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf                          fusion_enabled   17330.3926  3716.70472   ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_enabled.log
/mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf                          fusion_disabled  17933.9524  3883.17066   ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_disabled.log

master:
summary: ppl-value-checks/summary.tsv
model                                                                                                       mode             ppl         uncertainty  log
/mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf                                 fusion_enabled   5.2892      0.35389      ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_enabled.log
/mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf                                 fusion_disabled  5.2742      0.35215      ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_disabled.log
/mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf  fusion_enabled   5.4487      0.36866      ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_enabled.log
/mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf  fusion_disabled  5.4403      0.36782      ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_disabled.log
/mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf                              fusion_enabled   17342.4348  3703.13932   ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_enabled.log
/mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf                              fusion_disabled  18627.0624  3998.42475   ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_disabled.log
/mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf                                            fusion_enabled   363.8913    33.14007     ppl-value-checks/gpt-oss-20b-mxfp4.fusion_enabled.log
/mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf                                            fusion_disabled  363.8913    33.14007     ppl-value-checks/gpt-oss-20b-mxfp4.fusion_disabled.log
/mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf                          fusion_enabled   17330.3926  3716.70472   ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_enabled.log
/mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf                          fusion_disabled  17933.9524  3883.17066   ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_disabled.log
This commit is contained in:
Oliver Simons
2026-06-11 14:00:40 +02:00
parent 6d815ca4d1
commit 6b9024e842
1 changed files with 12 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cuda/mmvq.cu
+12 -13
View File
@@ -555,7 +555,6 @@ static __global__ void mul_mat_vec_q(
const uint32_t channel_bias = ids ? channel_x : channel_dst;
if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&
(rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {
const uint32_t channel_bias = ids ? channel_x : channel_dst;
if (use_bias) {
x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
#pragma unroll
@@ -563,20 +562,20 @@ static __global__ void mul_mat_vec_q(
x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x];
}
}
}
if (use_gate_bias) {
gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
if (use_gate_bias) {
gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
#pragma unroll
for (int j = 0; j < ncols_dst; ++j) {
gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];
for (int j = 0; j < ncols_dst; ++j) {
gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];
}
}
}
if constexpr (type == GGML_TYPE_NVFP4) {
if (use_scale) {
x_scales = x_scale[ids ? channel_x : 0];
}
if (use_gate_scale) {
gate_scales = gate_scale[ids ? channel_x : 0];
if constexpr (type == GGML_TYPE_NVFP4) {
if (use_scale) {
x_scales = x_scale[ids ? channel_x : 0];
}
if (use_gate_scale) {
gate_scales = gate_scale[ids ? channel_x : 0];
}
}
}
}