diff --git a/conversion/minimax.py b/conversion/minimax.py index 53a9ff60f..aac340c61 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -25,7 +25,7 @@ class MiniMaxText01Model(TextModel): # they get in the way of the token sampling process and must be suppressed tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) - tokenizer_vocab_size = tokenizer.vocab_size + tokenizer_vocab_size = tokenizer.vocab_size # ty: ignore[unresolved-attribute] with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: weight_map = json.load(f)["weight_map"] diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py index b205f70a0..c86b33227 100644 --- a/conversion/muse_glimmer.py +++ b/conversion/muse_glimmer.py @@ -37,7 +37,7 @@ class MuseGlimmerModel(TextModel): from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(self.dir_model) - eot_id = tok.convert_tokens_to_ids("<|eot|>") + eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute] if isinstance(eot_id, int) and eot_id >= 0: self.gguf_writer.add_eot_token_id(eot_id) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 39529917b..312d6e625 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -439,10 +439,21 @@ extern "C" { GGML_TYPE_COUNT = 43, }; - // precision + // [TAG_GGML_PREC] + // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op + // the declared types can be: + // - result accumulation type + // - source tensor data representation type + // - etc. + // the precision parameters are stored as ggml_tensor.op_params to the respective ops enum ggml_prec { - GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default - GGML_PREC_F32 = 10, + GGML_PREC_UNDEFINED = 0, + GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED + GGML_PREC_F32 = 10, + GGML_PREC_BF16 = 15, + GGML_PREC_F16 = 20, + GGML_PREC_Q8 = 30, + GGML_PREC_Q4 = 40, }; // op hint @@ -1447,6 +1458,42 @@ extern "C" { struct ggml_tensor * b, float eps); + // [TAG_GGML_PREC] + // set the minimum required accumulator type for the implementation to use during the compute + // for example: + // - GGML_PREC_F32 - requires accumulation of the results in F32 + // - GGML_PREC_BF16 - can accumulate the results in BF16, F32 + // - GGML_PREC_F16 - can accumulate the results in F16, F32 + // - GGML_PREC_Q8 - not allowed + // - GGML_PREC_Q4 - not allowed + // + // return false on faliure + GGML_API bool ggml_prec_set_acc( + struct ggml_tensor * a, + enum ggml_prec prec); + + // [TAG_GGML_PREC] + // set the smallest rank that the implementation can use to internally convert the src[idx] data to + // ranks in decreasing order: + // - GGML_PREC_F32 - GGML_TYPE_F32 + // - GGML_PREC_BF16 - GGML_TYPE_BF16 + // - GGML_PREC_F16 - GGML_TYPE_F16, + // - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc. + // - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc. + // + // for example: + // - ggml_prec_set_src(a, GGML_PREC_Q8, 1): + // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0 + // - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4 + // - ggml_prec_set_src(a, GGML_PREC_Q4, 1): + // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc. + // + // return false on faliure + GGML_API bool ggml_prec_set_src( + struct ggml_tensor * a, + enum ggml_prec prec, + int idx); + // A: k columns, n rows => [ne03, ne02, n, k] // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] @@ -1457,9 +1504,10 @@ extern "C" { // change the precision of a matrix multiplication // set to GGML_PREC_F32 for higher precision (useful for phi-2) - GGML_API void ggml_mul_mat_set_prec( + GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec( struct ggml_tensor * a, - enum ggml_prec prec); + enum ggml_prec prec), + "use ggml_prec_set_acc() instead"); // change the hint of a matrix multiplication GGML_API void ggml_mul_mat_set_hint( @@ -2464,9 +2512,10 @@ extern "C" { float max_bias, float logit_softcap); - GGML_API void ggml_flash_attn_ext_set_prec( + GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec( struct ggml_tensor * a, - enum ggml_prec prec); + enum ggml_prec prec), + "use ggml_prec_set_acc() instead"); GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a); diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 62b76abbc..ae26e0c23 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -160,6 +160,18 @@ static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t return ((const float *)(tensor->op_params))[i]; } +// [TAG_GGML_PREC] +// - GGML_OP_MUL_MAT +// 0 - acc +// 1 - hint +// 2 - src0 precision +// 3 - src1 precision +// +// - GGML_OP_MUL_MAT_ID +// 0 - acc +// 1 - hint +// 2 - src0 precision +// 3 - src1 precision static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) { assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t)); ((int32_t *)(tensor->op_params))[i] = value; diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 3edddb835..93941dc5e 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -3293,6 +3293,57 @@ struct ggml_tensor * ggml_l2_norm_inplace( return ggml_l2_norm_impl(ctx, a, eps, true); } +// ggml_prec + +bool ggml_prec_set_acc( + struct ggml_tensor * a, + enum ggml_prec prec) { + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + { + const int32_t prec_i32 = (int32_t) prec; + ggml_set_op_params_i32(a, 0, prec_i32); + } + break; + case GGML_OP_FLASH_ATTN_EXT: + { + const int32_t prec_i32 = (int32_t) prec; + ggml_set_op_params_i32(a, 3, prec_i32); + } + break; + default: + return false; + }; + + return true; +} + +bool ggml_prec_set_src( + struct ggml_tensor * a, + enum ggml_prec prec, + int idx) { + GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC); + + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + { + if (idx != 1) { + return false; + } + + const int32_t prec_i32 = (int32_t) prec; + ggml_set_op_params_i32(a, 2 + idx, prec_i32); + } + break; + default: + return false; + }; + + return true; +} + // ggml_mul_mat static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 15f651919..b5efb7206 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1103,6 +1103,7 @@ bool llm_arch_is_diffusion(const llm_arch & arch) { bool llm_arch_supports_rs_rollback(const llm_arch & arch) { switch (arch) { + case LLM_ARCH_KIMI_K3: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: case LLM_ARCH_QWEN4EXP: diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 12033dcc0..3f402ff91 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1927,7 +1927,7 @@ ggml_tensor * llm_graph_context::build_ffn( cur = build_lora_mm(down, cur); if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); } } @@ -2025,7 +2025,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (probs_in == nullptr) { logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens] if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { - ggml_mul_mat_set_prec(logits, GGML_PREC_F32); + ggml_prec_set_acc(logits, GGML_PREC_F32); } cb(logits, "ffn_moe_logits", il); } else { @@ -2637,7 +2637,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( ggml_flash_attn_ext_add_sinks(cur, sinks); GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX); ggml_flash_attn_ext_set_n_kv_max(cur, static_cast(n_kv_max)); - ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (v_mla) { #if 0 @@ -2663,7 +2663,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( // note: this op tends to require high floating point range // while for some models F16 is enough, for others it is not, so we default to F32 here - ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + ggml_prec_set_acc(kq, GGML_PREC_F32); if (arch == LLM_ARCH_GROK) { // need to do the following: @@ -2896,7 +2896,7 @@ ggml_tensor * llm_graph_context::build_attn( if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators cur = build_lora_mm(wo, cur); - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (wo_s) { cur = ggml_mul(ctx0, cur, wo_s); } @@ -2983,7 +2983,7 @@ ggml_tensor * llm_graph_context::build_attn( if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) { // GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators cur = build_lora_mm(wo, cur); - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (wo_s) { cur = ggml_mul(ctx0, cur, wo_s); } diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index b7604cbf2..112b09849 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -1,4 +1,6 @@ #include "models.h" + +#include #include "llama-memory-recurrent.h" // @@ -357,7 +359,8 @@ static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, int64_t d_conv, int64_t head_dim, int64_t n_head, - int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) { + int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head, + int64_t mem_size, int64_t K_rs) { const int64_t d_inner = head_dim * n_head; const int64_t conv_state_size = (d_conv - 1) * d_inner; const int64_t n_embd_r_total = 3 * conv_state_size; @@ -371,14 +374,19 @@ static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); - ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, - conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]); - ggml_build_forward_expand(gf, - ggml_cpy(ctx0, last_conv_x, - ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, - (d_conv - 1) * ggml_element_size(conv_states_all), - n_embd_r_total * ggml_element_size(conv_states_all), - (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + // group s holds the conv window s tokens back. + // [TAG_RECURRENT_ROLLBACK_SPLITS]: the last K_rs tokens must share one ubatch. + for (int64_t s = 0; s < K_rs; ++s) { + const int64_t s_idx = std::max(0, n_seq_tokens - s); + ggml_tensor * conv_x_s = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, + conv_x->nb[1], conv_x->nb[2], s_idx * conv_x->nb[0]); + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, conv_x_s, + ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_states_all), + n_embd_r_total * ggml_element_size(conv_states_all), + ((s * mem_size + kv_head) * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + } ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); @@ -399,9 +407,12 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); - ggml_tensor * Qcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head); - ggml_tensor * Kcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head); - ggml_tensor * Vcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head); + const int64_t mem_size = mctx_cur->get_size(); + const int64_t K_rs = (int64_t) cparams.n_rs_seq + 1; + + ggml_tensor * Qcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); + ggml_tensor * Kcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); + ggml_tensor * Vcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); cb(Qcur, "kda_q_conv", il); cb(Kcur, "kda_k_conv", il); cb(Vcur, "kda_v_conv", il); @@ -445,16 +456,9 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); - auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); - - ggml_tensor * output = ggml_cont(ctx0, attn_out.first); + ggml_tensor * output = build_recurrent_attn(inp_rs, ssm_states_all, Qcur, Kcur, Vcur, g1, beta, state, il); + output = ggml_cont(ctx0, output); cb(output, "kda_scan_out", il); - ggml_tensor * new_state = attn_out.second; - - ggml_build_forward_expand(gf, - ggml_cpy(ctx0, new_state, - ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs, - kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all)))); // K3: single full-rank gate (kimi-linear factors this as g_b(g_a(x))) ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur_3d, cur_3d->ne[0], n_seq_tokens * n_seqs); diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 80260a629..f3b64b210 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -191,7 +191,7 @@ ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa( ggml_tensor * o = ggml_flash_attn_ext(ctx0, q, k, v, mask, kq_scale, hparams.f_max_alibi_bias, 0.0f); - ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32); + ggml_prec_set_acc(o, GGML_PREC_F32); cb(o, "msa_fattn", il); // [D, Gp, R, C] -> [D, Gp, C, R] -> [n_embd, T] @@ -389,7 +389,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns); ggml_tensor * sc = ggml_mul_mat(ctx0, ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4); - ggml_mul_mat_set_prec(sc, GGML_PREC_F32); + ggml_prec_set_acc(sc, GGML_PREC_F32); // unmapped positions come out -inf, so they can never rank into the top-k sc = ggml_add_inplace(ctx0, sc, ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns)); @@ -471,7 +471,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * sc = ggml_mul_mat(ctx0, ikp, ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps)); // indexer scores run in F32 - ggml_mul_mat_set_prec(sc, GGML_PREC_F32); + ggml_prec_set_acc(sc, GGML_PREC_F32); sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps); // unmapped positions (holes, padding, empty cells) come out -inf sc = ggml_add_inplace(ctx0, sc, pm_s); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 00724624e..64b78984a 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -839,7 +839,7 @@ ggml_tensor * clip_graph::build_attn( } cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (sinks != nullptr) { ggml_flash_attn_ext_add_sinks(cur, sinks); } @@ -852,7 +852,7 @@ ggml_tensor * clip_graph::build_attn( ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); // F32 may not needed for vision encoders? - // ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + // ggml_prec_set_acc(kq, GGML_PREC_F32); kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f); if (sinks != nullptr) { diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cpp index 6ff1124a0..e1fbe2671 100644 --- a/tools/mtmd/models/mimovl.cpp +++ b/tools/mtmd/models/mimovl.cpp @@ -2,7 +2,7 @@ ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const { ggml_tensor * cur = ggml_mul_mat(ctx0, w, x); - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); return cur; } diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp index d4659fd63..405fbb9cb 100644 --- a/tools/mtmd/models/qwen3tts-spkenc.cpp +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -27,7 +27,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC); ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out] - ggml_mul_mat_set_prec(y, GGML_PREC_F32); + ggml_prec_set_acc(y, GGML_PREC_F32); ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1); y = ggml_add(ctx0, y, b2d); diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index f90437969..3d6cbeb2c 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -56,7 +56,7 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { ggml_set_name(m, "m"); ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32); + ggml_prec_set_acc(out, GGML_PREC_F32); ggml_set_name(out, "out"); return out;