Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

ggml : update ggml_prec specification (#26675)

Browse Source 
* ggml : update ggml_prec specification

[no ci]

* cont : add GGML_PREC_BF16

* cont : rework API

* cont : use new API

* cont : swap arg order

* cont : support for MUL_MAT_ID

* cont : fix accidental remove of "break;"

* cont : return bools, add doc TAG_GGML_PREC, clean-up

* cont : add search tag

* cont : ws
This commit is contained in:
Georgi Gerganov
2026-09-08 09:06:24 +03:00
committed by GitHub
parent 9dcf84e5ae
commit 5a6caa05fc
10 changed files with 134 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-graph.cpp
+6 -6
View File
@@ -1926,7 +1926,7 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = build_lora_mm(down, cur);
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
}
}
@@ -2024,7 +2024,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (probs_in == nullptr) {
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
ggml_mul_mat_set_prec(logits, GGML_PREC_F32);
ggml_prec_set_acc(logits, GGML_PREC_F32);
}
cb(logits, "ffn_moe_logits", il);
} else {
@@ -2636,7 +2636,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
ggml_flash_attn_ext_add_sinks(cur, sinks);
GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX);
ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max));
ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (v_mla) {
#if 0
@@ -2662,7 +2662,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
// note: this op tends to require high floating point range
// while for some models F16 is enough, for others it is not, so we default to F32 here
ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
ggml_prec_set_acc(kq, GGML_PREC_F32);
if (arch == LLM_ARCH_GROK) {
// need to do the following:
@@ -2895,7 +2895,7 @@ ggml_tensor * llm_graph_context::build_attn(
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
@@ -2982,7 +2982,7 @@ ggml_tensor * llm_graph_context::build_attn(
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
// GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators
cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}