Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 17:25:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	docs/backend/SYCL.md
#	docs/backend/snapdragon/developer.md
#	examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp
#	examples/sycl/run-llama2.sh
#	examples/sycl/start-svr.sh
#	examples/sycl/test.sh
#	examples/sycl/win-run-llama2.bat
#	examples/sycl/win-start-svr.bat
#	examples/sycl/win-test.bat
#	ggml/CMakeLists.txt
#	ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c
#	ggml/src/ggml-et/ggml-et-ops.cpp
#	ggml/src/ggml-et/ggml-et-ops.h
#	ggml/src/ggml-sycl/ssm_scan.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl
#	scripts/bench-models.sh
#	scripts/snapdragon/adb/run-bench.sh
#	scripts/snapdragon/adb/run-cli.sh
#	scripts/snapdragon/adb/run-completion.sh
#	scripts/snapdragon/adb/run-mtmd.sh
#	scripts/snapdragon/windows/run-bench.ps1
#	scripts/snapdragon/windows/run-cli.ps1
#	scripts/snapdragon/windows/run-completion.ps1
#	scripts/snapdragon/windows/run-mtmd.ps1
#	scripts/sync-ggml.last
#	scripts/sync_vendor.py
#	tests/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/llama-bench/README.md
#	tools/server/README.md
This commit is contained in:
Concedo
2026-08-15 22:50:04 +08:00
parent 531cac9cd3 adb55e5148
commit 82e15c08bf
60 changed files with 1781 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-arch.cpp
+5
View File
@@ -128,6 +128,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_SEED_OSS, "seed_oss" },
{ LLM_ARCH_GROVEMOE, "grovemoe" },
{ LLM_ARCH_APERTUS, "apertus" },
{ LLM_ARCH_MINIMAX_01, "minimax-01" },
{ LLM_ARCH_MINIMAX_M2, "minimax-m2" },
{ LLM_ARCH_MINIMAX_M3, "minimax-m3" },
{ LLM_ARCH_COGVLM, "cogvlm" },
@@ -978,6 +979,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_MINIMAX_01:
return true;
default:
return false;
@@ -1001,6 +1003,8 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
return true;
default:
return false;
@@ -1031,6 +1035,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_GRANITE_HYBRID:
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_MINIMAX_M2:
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_MISTRAL4:
src/llama-arch.h
+1
View File
@@ -153,6 +153,7 @@ enum llm_arch {
LLM_ARCH_NANBEIGE,
LLM_ARCH_QWEN3TTS,
LLM_ARCH_POCKETTTS,
LLM_ARCH_MINIMAX_01,
LLM_ARCH_UNKNOWN,
};
src/llama-context.cpp
+2 -1
View File
@@ -106,7 +106,7 @@ llama_context::llama_context(
cparams.n_rs_seq = params.n_rs_seq;
if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) {
LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n",
LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model does not support recurrent partial rollback; clamping to 0\n",
__func__, cparams.n_rs_seq);
cparams.n_rs_seq = 0;
}
@@ -2310,6 +2310,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
model.arch == LLM_ARCH_DEEPSEEK4 ||
(model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) ||
model.arch == LLM_ARCH_NANBEIGE ||
model.arch == LLM_ARCH_MINIMAX_01 ||
model.arch == LLM_ARCH_MINIMAX_M3) {
res = std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());
} else {
src/llama-hparams.cpp
+7
View File
@@ -217,6 +217,13 @@ uint32_t llama_hparams::n_embd_s() const {
return n_embd_head_kda * n_embd_head_kda * n_head(); // 128 * 128 * 32 = 524288
}
if (n_embd_head_la != 0) {
// for MiniMax-Text-01 linear attention layers
// Full recurrent state: head_dim * head_dim * n_head
// tensor shape for linear attention: [head_dim, head_dim, n_head]
return n_embd_head_la * n_embd_head_la * n_head(); // 128 * 128 * 64 = 1048576
}
// corresponds to Mamba's ssm_states size
return ssm_d_state * ssm_d_inner;
}
src/llama-hparams.h
+3
View File
@@ -164,6 +164,9 @@ struct llama_hparams {
uint32_t ssm_dt_rank = 0;
uint32_t ssm_n_group = 0;
// for MiniMax-Text-01 linear attention
uint32_t n_embd_head_la = 0;
// for Kimi Linear KDA
uint32_t n_embd_head_kda = 0;
src/llama-model-loader.cpp
+18 -10
View File
@@ -316,15 +316,19 @@ namespace GGUFMeta {
struct GGUFMeta::ArrayInfo arr_info =
GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);
bool type_ok = false;
switch (arr_info.gt) {
case GGUF_TYPE_UINT32:
case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||
(std::is_same<T, uint32_t>::value)); break;
case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;
case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value)); break;
case GGUF_TYPE_INT32: type_ok = (std::is_same<T, int32_t>::value) ||
(std::is_same<T, uint32_t>::value); break;
case GGUF_TYPE_FLOAT32: type_ok = (std::is_same<T, float>::value); break;
case GGUF_TYPE_STRING: type_ok = (std::is_same<T, std::string>::value); break;
default:
throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
}
if (!type_ok) {
throw std::runtime_error(format("%s has wrong array element type %s", key.c_str(), gguf_type_name(arr_info.gt)));
}
if constexpr (std::is_same<T, std::string>::value) {
const size_t n_items = gguf_get_arr_n(ctx, kid);
@@ -357,16 +361,20 @@ namespace GGUFMeta {
struct GGUFMeta::ArrayInfo arr_info =
GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);
bool type_ok = false;
switch (arr_info.gt) {
case GGUF_TYPE_BOOL:
case GGUF_TYPE_UINT32:
case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||
(std::is_same<T, uint32_t>::value)); break;
case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;
case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value)); break;
case GGUF_TYPE_INT32: type_ok = (std::is_same<T, int32_t>::value) ||
(std::is_same<T, uint32_t>::value); break;
case GGUF_TYPE_FLOAT32: type_ok = (std::is_same<T, float>::value); break;
case GGUF_TYPE_STRING: type_ok = (std::is_same<T, std::string>::value); break;
default:
throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
}
if (!type_ok) {
throw std::runtime_error(format("%s has wrong array element type %s", key.c_str(), gguf_type_name(arr_info.gt)));
}
if (arr_info.length > N_MAX) {
throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));
@@ -1003,7 +1011,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w
ggml_tensor * B = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);
ggml_tensor * C = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);
ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs);
op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids);
op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids, /*K=*/1);
} break;
case GGML_OP_RWKV_WKV6:
{
@@ -1179,7 +1187,7 @@ struct ggml_tensor * llama_model_loader::create_tensor(
if (use_mmap) {
static std::once_flag once;
std::call_once(once, [] {
LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n");
LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --load-mode none for better performance\n");
});
}
} else {
src/llama-model.cpp
+6 -1
View File
@@ -122,6 +122,7 @@
#include "models/mimo2.cpp"
#include "models/minicpm.cpp"
#include "models/minicpm3.cpp"
#include "models/minimax-01.cpp"
#include "models/minimax-m2.cpp"
#include "models/minimax-m3.cpp"
#include "models/mistral3.cpp"
@@ -442,6 +443,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_grovemoe(params);
case LLM_ARCH_APERTUS:
return new llama_model_apertus(params);
case LLM_ARCH_MINIMAX_01:
return new llama_model_minimax_01(params);
case LLM_ARCH_MINIMAX_M2:
return new llama_model_minimax_m2(params);
case LLM_ARCH_MINIMAX_M3:
@@ -944,6 +947,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_290B: return "290B";
case LLM_TYPE_314B: return "314B";
case LLM_TYPE_405B: return "405B";
case LLM_TYPE_456B: return "456B";
case LLM_TYPE_671B: return "671B";
case LLM_TYPE_SMALL: return "0.1B";
case LLM_TYPE_MEDIUM: return "0.4B";
@@ -2429,7 +2433,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_recr = [&](uint32_t il) {
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
};
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) {
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_MINIMAX_01) {
filter_attn = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il);
};
@@ -2850,6 +2854,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_SEED_OSS:
case LLM_ARCH_GROVEMOE:
case LLM_ARCH_APERTUS:
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_MINIMAX_M2:
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_COGVLM:
src/llama-model.h
+2
View File
@@ -99,6 +99,7 @@ enum llm_type {
LLM_TYPE_290B,
LLM_TYPE_314B,
LLM_TYPE_405B,
LLM_TYPE_456B,
LLM_TYPE_671B,
LLM_TYPE_SMALL,
LLM_TYPE_MEDIUM,
@@ -271,6 +272,7 @@ struct llama_layer {
struct ggml_tensor * wv = nullptr;
struct ggml_tensor * wo = nullptr;
struct ggml_tensor * wqkv = nullptr;
struct ggml_tensor * wg = nullptr;
struct ggml_tensor * wq_a = nullptr;
struct ggml_tensor * wq_b = nullptr;
struct ggml_tensor * wkv_a_mqa = nullptr;
src/llama-vocab.cpp
+31 -1
View File
@@ -2215,6 +2215,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
// Kimi-K2 doesn't need merges, skip
LLAMA_LOG_INFO("%s: Kimi-K2 tokenizer detected, skipping BPE merges\n", __func__);
} else {
if (gguf_get_kv_type(ctx, merges_keyidx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, merges_keyidx) != GGUF_TYPE_STRING) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_MERGES).c_str()));
}
const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
for (int i = 0; i < n_merges; i++) {
const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);
@@ -2264,8 +2268,13 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
const int precompiled_charsmap_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP).c_str());
if (precompiled_charsmap_keyidx != -1) {
if (gguf_get_kv_type(ctx, precompiled_charsmap_keyidx) != GGUF_TYPE_ARRAY) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP).c_str()));
}
const gguf_type pc_type = gguf_get_arr_type(ctx, precompiled_charsmap_keyidx);
GGML_ASSERT(pc_type == GGUF_TYPE_INT8 || pc_type == GGUF_TYPE_UINT8);
if (pc_type != GGUF_TYPE_INT8 && pc_type != GGUF_TYPE_UINT8) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP).c_str()));
}
const size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx);
const char * pc = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx);
@@ -2317,6 +2326,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
throw std::runtime_error("cannot find tokenizer merges in model file\n");
}
{
if (gguf_get_kv_type(ctx, merges_keyidx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, merges_keyidx) != GGUF_TYPE_STRING) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_MERGES).c_str()));
}
const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
for (int i = 0; i < n_merges; i++) {
const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);
@@ -2643,11 +2656,20 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
throw std::runtime_error("cannot find tokenizer vocab in model file\n");
}
if (gguf_get_kv_type(ctx, token_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, token_idx) != GGUF_TYPE_STRING) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_LIST).c_str()));
}
const uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx);
const float * scores = nullptr;
const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());
if (score_idx != -1) {
if (gguf_get_kv_type(ctx, score_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, score_idx) != GGUF_TYPE_FLOAT32) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_SCORES).c_str()));
}
const uint32_t n_scores = gguf_get_arr_n(ctx, score_idx);
if (n_scores < n_tokens) {
throw std::runtime_error("Index out of array bounds for scores (" + std::to_string(n_scores) + " < " + std::to_string(n_tokens) + ")\n");
@@ -2658,6 +2680,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
const int * toktypes = nullptr;
const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());
if (toktype_idx != -1) {
if (gguf_get_kv_type(ctx, toktype_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, toktype_idx) != GGUF_TYPE_INT32) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str()));
}
const uint32_t n_toktypes = gguf_get_arr_n(ctx, toktype_idx);
if (n_toktypes < n_tokens) {
throw std::runtime_error("Index out of array bounds for toktypes (" + std::to_string(n_toktypes) + " < " + std::to_string(n_tokens) + ")\n");
@@ -2823,6 +2849,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
{
const int suppress_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SUPPRESS_TOKENS).c_str());
if (suppress_idx != -1) {
if (gguf_get_kv_type(ctx, suppress_idx) != GGUF_TYPE_ARRAY ||
gguf_get_arr_type(ctx, suppress_idx) != GGUF_TYPE_INT32) {
throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_SUPPRESS_TOKENS).c_str()));
}
const int n = gguf_get_arr_n(ctx, suppress_idx);
const int32_t * data = (const int32_t *) gguf_get_arr_data(ctx, suppress_idx);
// drop out-of-range ids
src/llama.cpp
+5 -1
View File
@@ -279,7 +279,11 @@ static bool llama_prepare_model_devices(const llama_model_params & params, llama
}
case GGML_BACKEND_DEVICE_TYPE_IGPU:
if (igpus.empty()) {
// igpus.empty() - workaround for integrated devices seen by multiple backends
// ref: https://github.com/ggml-org/llama.cpp/pull/23897
// ggml_backend_dev_backend_reg - allow devices of the same backend regardless if integrated
// ref: https://github.com/ggml-org/llama.cpp/pull/23897#issuecomment-5264222997
if (igpus.empty() || ggml_backend_dev_backend_reg(dev) == ggml_backend_dev_backend_reg(igpus.back().dev)) {
igpus.push_back({false, dev});
}
break;
src/models/dflash.cpp
+2
View File
@@ -43,6 +43,8 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);
ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false);
GGML_ASSERT(hparams.dsv4_o_group_count > 0); // avoid div by zero
if (hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
throw std::runtime_error("DSpark DSV4 draft expects sqrtsoftplus MoE scoring");
}
src/models/mamba-base.cpp
+32 -16
View File
@@ -2,6 +2,8 @@
#include "llama-memory-recurrent.h"
#include <algorithm>
llm_build_mamba_base::llm_build_mamba_base(const llm_graph_params & params) : llm_graph_context(params) {}
ggml_tensor * llm_build_mamba_base::build_mamba_layer(llm_graph_input_rs * inp,
@@ -118,7 +120,7 @@ ggml_tensor * llm_build_mamba_base::build_mamba_layer(llm_graph_input_rs * inp,
// Custom operator to optimize the parallel associative scan
// as described in the Annex D of the Mamba paper.
// => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}
return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);
return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids, /*K=*/1);
};
ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);
@@ -153,7 +155,8 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
int il) const {
const auto * mctx_cur = inp->mctx;
const auto kv_head = mctx_cur->get_head();
const auto kv_head = mctx_cur->get_head();
const auto mem_size = mctx_cur->get_size();
const int64_t d_conv = hparams.ssm_d_conv;
const int64_t d_inner = hparams.ssm_d_inner;
@@ -164,6 +167,7 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
const int64_t n_seqs = ubatch.n_seqs;
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
const int64_t K = cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1;
GGML_ASSERT(n_seqs != 0);
GGML_ASSERT(ubatch.equal_seqs());
@@ -173,6 +177,7 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
const int64_t state_slots = ssm_states_all->ne[1];
ggml_tensor * conv = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);
conv = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs);
@@ -198,15 +203,19 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
// => {d_conv - 1 + n_seq_tokens, d_inner + 2*n_group*d_state, n_seqs}
ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, xBC), 0);
// copy last (d_conv - 1) columns back into the state cache
ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs,
conv_x->nb[1], conv_x->nb[2], n_seq_tokens * (conv_x->nb[0]));
const int64_t row_count = (d_conv - 1) * (d_inner + 2 * n_group * d_state);
const size_t row_size = ggml_row_size(conv_states_all->type, row_count);
const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv,
ggml_view_1d(ctx0, conv_states_all,
(d_conv - 1) * (d_inner + 2 * n_group * d_state) * (n_seqs),
kv_head * (d_conv - 1) * (d_inner + 2 * n_group * d_state) *
ggml_element_size(conv_states_all))));
for (int64_t slot = 0; slot < n_written; ++slot) {
ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs,
conv_x->nb[1], conv_x->nb[2], (n_seq_tokens - slot) * conv_x->nb[0]);
ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv,
ggml_view_2d(ctx0, conv_states_all, row_count, n_seqs,
conv_states_all->nb[1],
((size_t) slot * mem_size + kv_head) * row_size)));
}
// 1D convolution
// The equivalent is to make a self-overlapping view of conv_x
@@ -244,20 +253,27 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
// (this is necessary in order to properly use the states before they are overwritten,
// while avoiding to make unnecessary copies of the states)
auto get_ssm_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {
ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_head, mctx_cur->get_size());
ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_head, state_slots);
// TODO: use semistructured matrices to implement state-space duality
// => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}
return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);
// K > 1 asks the backend to return rollback snapshots in addition to the final state.
return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids, K);
};
ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);
const int64_t D = d_state * d_inner;
const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
const size_t row_size = ggml_row_size(ssm_states_all->type, D);
const size_t y_row_size = ggml_row_size(y_ssm->type, D);
const size_t state_offset = ggml_nelements(x) * ggml_element_size(x);
// store last states
ggml_build_forward_expand(
gf, ggml_cpy(ctx0, ggml_view_1d(ctx0, y_ssm, d_state * d_inner * n_seqs, ggml_nelements(x) * x->nb[0]),
ggml_view_1d(ctx0, ssm_states_all, d_state * d_inner * n_seqs,
kv_head * d_state * d_inner * ggml_element_size(ssm_states_all))));
gf, ggml_cpy(ctx0,
ggml_view_3d(ctx0, y_ssm, D, n_seqs, n_written,
y_row_size, y_row_size * n_seqs, state_offset),
ggml_view_3d(ctx0, ssm_states_all, D, n_seqs, n_written,
ssm_states_all->nb[1], (size_t) mem_size * row_size, kv_head * row_size)));
ggml_tensor * y = ggml_view_4d(ctx0, y_ssm, head_dim, n_head, n_seq_tokens, n_seqs, x->nb[1], n_head * x->nb[1],
n_seq_tokens * n_head * x->nb[1], 0);
src/models/minimax-01.cpp
+520
View File
@@ -0,0 +1,520 @@
#include "models.h"
#include "llama-memory-recurrent.h"
void llama_model_minimax_01::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale);
// we use n_embd_head_la to set recurrent memory n_embd_s
hparams.n_embd_head_la = hparams.n_embd_head_k_full;
// Mark recurrent layers (lightning attention layers).
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
uint32_t full_attn_interval = 8;
ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
}
}
switch (hparams.n_layer()) {
case 80: type = LLM_TYPE_456B; break;
default: type = LLM_TYPE_UNKNOWN;
}
}
void llama_model_minimax_01::load_arch_tensors(llama_model_loader &) {
LLAMA_LOAD_LOCALS;
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
// if output is NULL, init from the input tok embed
if (output == NULL) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
}
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
if (!hparams.is_recr(i)) {
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
} else {
layer.attn_norm_2 = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd_head_k * n_head}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, 3 * n_embd_head_k * n_head}, 0);
layer.wg = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
}
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, TENSOR_NOT_REQUIRED);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0);
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0);
}
}
std::unique_ptr<llm_graph_context> llama_model_minimax_01::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(*this, params);
}
class llm_graph_input_la : public llm_graph_input_i {
public:
llm_graph_input_la(const llama_hparams & hparams) : hparams(hparams) {}
void set_input(const llama_ubatch * ubatch) override {
// this operates on assumption that we have an equal ubatch split
const int64_t n_head = hparams.n_head();
const int32_t n_seqs = ubatch->n_seqs;
const int32_t n_seqs_unq = ubatch->n_seqs_unq;
const int32_t n_tokens = ubatch->n_tokens;
const int32_t n_seq_tokens = ubatch->n_seq_tokens;
std::vector<llama_pos> p0(n_seqs_unq);
std::fill(p0.begin(), p0.end(), std::numeric_limits<llama_pos>::max());
// get lowest token position in a ubatch for each stream
for (int i = 0; i < n_tokens; ++i) {
llama_seq_id seq_id = ubatch->seq_id[i][0];
int32_t seq_idx = ubatch->seq_idx[seq_id];
llama_pos pos = ubatch->pos[i];
if (p0[seq_idx] > pos) {
p0[seq_idx] = pos;
}
}
if (inp_slopes) {
GGML_ASSERT(ggml_backend_buffer_is_host(inp_slopes->buffer));
float * data = (float *) inp_slopes->data;
float start = powf(2, -powf(2, -(log2f(n_head) - 3)));
float ratio = start;
for (int h = 0; h < n_head; ++h) {
data[h] = start * powf(ratio, h);
}
}
if (inp_q_decay) {
GGML_ASSERT(ggml_backend_buffer_is_host(inp_q_decay->buffer));
float * slopes = (float *) inp_slopes->data;
float * data = (float *) inp_q_decay->data;
for (int s = 0; s < n_seqs; ++s) {
for (int i = 0; i < n_seq_tokens; ++i) {
llama_seq_id seq_id = ubatch->seq_id[s * n_seq_tokens + i][0];
int32_t seq_idx = ubatch->seq_idx[seq_id];
llama_pos pos = ubatch->pos[s * n_seq_tokens + i];
int pos_rel = pos - p0[seq_idx];
for (int h = 0; h < n_head; ++h) {
data[seq_idx * n_head * n_seq_tokens + i * n_head + h] = -slopes[h] * (pos_rel + 1);
}
}
}
}
if (inp_k_decay) {
GGML_ASSERT(ggml_backend_buffer_is_host(inp_k_decay->buffer));
float * slopes = (float *) inp_slopes->data;
float * data = (float *) inp_k_decay->data;
for (int s = 0; s < n_seqs; ++s) {
for (int i = 0; i < n_seq_tokens; ++i) {
llama_seq_id seq_id = ubatch->seq_id[s * n_seq_tokens + i][0];
int32_t seq_idx = ubatch->seq_idx[seq_id];
llama_pos pos = ubatch->pos[s * n_seq_tokens + i];
int pos_rel = pos - p0[seq_idx];
for (int h = 0; h < n_head; ++h) {
data[seq_idx * n_head * n_seq_tokens + i * n_head + h] = -slopes[h] * (n_seq_tokens - pos_rel - 1);
}
}
}
}
if (inp_diag_decay) {
GGML_ASSERT(ggml_backend_buffer_is_host(inp_diag_decay->buffer));
float * slopes = (float *) inp_slopes->data;
float * data = (float *) inp_diag_decay->data;
for (int s = 0; s < n_seqs; ++s) {
for (int h = 0; h < n_head; ++h) {
for (int j = 0; j < n_seq_tokens; ++j) {
llama_seq_id seq_id = ubatch->seq_id[s * n_seq_tokens + j][0];
int32_t seq_idx = ubatch->seq_idx[seq_id];
llama_pos pos_j = ubatch->pos[s * n_seq_tokens + j];
int pos_rel_j = pos_j - p0[seq_idx];
for (int i = 0; i < n_seq_tokens; ++i) {
llama_pos pos_i = ubatch->pos[s * n_seq_tokens + i];
int pos_rel_i = pos_i - p0[seq_idx];
int index = pos_rel_j - pos_rel_i;
float s_index = index >= 0 ? -slopes[h] * index : -INFINITY;
data[seq_idx * n_head * n_seq_tokens * n_seq_tokens + h * n_seq_tokens * n_seq_tokens + j * n_seq_tokens + i] = s_index;
}
}
}
}
}
}
bool can_reuse(const llm_graph_params & params) override {
bool res = true;
if (params.ubatch.n_seq_tokens > 1) {
res &= ( inp_q_decay && inp_q_decay->ne[2] == params.ubatch.n_seq_tokens);
res &= ( inp_k_decay && inp_k_decay->ne[2] == params.ubatch.n_seq_tokens);
res &= (inp_diag_decay && inp_diag_decay->ne[1] == params.ubatch.n_seq_tokens);
}
return res;
}
const llama_hparams & hparams;
ggml_tensor * inp_slopes = nullptr; // F32 [n_head]
ggml_tensor * inp_q_decay = nullptr; // F32 [1, n_head, n_batch]
ggml_tensor * inp_k_decay = nullptr; // F32 [1, n_head, n_batch]
ggml_tensor * inp_diag_decay = nullptr; // F32 [n_batch, n_batch, n_head]
};
llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// GGML_ASSERT(n_embd_head == n_rot); this is wrong in case of minimax, head_dim = 128, n_rot = 64
const int64_t n_seqs = ubatch.n_seqs;
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
GGML_ASSERT(n_seqs != 0);
GGML_ASSERT(ubatch.equal_seqs());
GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
ggml_tensor * cur;
ggml_tensor * inpL;
inpL = build_inp_embd(model.tok_embd);
auto * inp_hybrid = build_inp_mem_hybrid();
auto * inp_rs = inp_hybrid->get_recr();
ggml_tensor * inp_pos = build_inp_pos();
ggml_tensor * inp_out_ids = build_inp_out_ids();
llm_graph_input_la * la = nullptr;
auto inp = std::make_unique<llm_graph_input_la>(hparams);
inp->inp_slopes = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head);
ggml_set_input(inp->inp_slopes);
cb(inp->inp_slopes, "slopes", -1);
if (n_seq_tokens != 1) {
inp->inp_q_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);
ggml_set_input(inp->inp_q_decay);
cb(inp->inp_q_decay, "q_decay_exp", -1);
inp->inp_k_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);
ggml_set_input(inp->inp_k_decay);
cb(inp->inp_k_decay, "k_decay_exp", -1);
inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs);
ggml_set_input(inp->inp_diag_decay);
cb(inp->inp_diag_decay, "diag_decay_exp", -1);
}
la = (llm_graph_input_la *) res->add_input(std::move(inp));
ggml_tensor * slopes = la->inp_slopes;
for (int il = 0; il < n_layer; ++il) {
res->t_layer_inp[il] = inpL;
ggml_tensor * inpSA = inpL;
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
ggml_tensor * residual = cur;
// self_attention
if (!hparams.is_recr(il)) {
// softmax attention layer
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow
);
Kcur = ggml_rope_ext(
ctx0, Kcur, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow
);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_hybrid->get_attn(),
model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
} else {
// lightning attention layer
const auto * mctx_cur = inp_rs->mctx;
const auto kv_head = mctx_cur->get_head();
// TODO unneeded - any way to make conv states optional in recurrent memory?
ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
ggml_build_forward_expand(gf, conv_state_all);
float slope_scale = 1.0 - 1.0 * il / (n_layer - 1) + 1e-5;
ggml_tensor * slope_rate = ggml_scale(ctx0, slopes, slope_scale);
cb(slope_rate, "slope_rate", il);
cur = ggml_reshape_4d(ctx0, cur, cur->ne[0], n_seq_tokens, 1, n_seqs);
ggml_tensor * QKVcur = build_lora_mm(model.layers[il].wqkv, cur);
cb(QKVcur, "QKVcur", il);
QKVcur = ggml_silu(ctx0, QKVcur);
cb(QKVcur, "QKVcur_silu", il);
QKVcur = ggml_reshape_4d(ctx0, QKVcur, n_embd_head * 3, n_head, n_seq_tokens, n_seqs);
ggml_tensor * Qcur = ggml_view_4d(ctx0, QKVcur, n_embd_head, n_head, n_seq_tokens, n_seqs, QKVcur->nb[1], QKVcur->nb[2], QKVcur->nb[3], 0*ggml_element_size(QKVcur)*n_embd_head);
ggml_tensor * Kcur = ggml_view_4d(ctx0, QKVcur, n_embd_head, n_head, n_seq_tokens, n_seqs, QKVcur->nb[1], QKVcur->nb[2], QKVcur->nb[3], 1*ggml_element_size(QKVcur)*n_embd_head);
ggml_tensor * Vcur = ggml_view_4d(ctx0, QKVcur, n_embd_head, n_head, n_seq_tokens, n_seqs, QKVcur->nb[1], QKVcur->nb[2], QKVcur->nb[3], 2*ggml_element_size(QKVcur)*n_embd_head);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
// get previous KV
ggml_tensor * la_states_all = mctx_cur->get_s_l(il);
ggml_tensor * state = build_rs(inp_rs, la_states_all, hparams.n_embd_s(), n_seqs);
ggml_tensor * kv_old = ggml_reshape_4d(ctx0, state, n_embd_head, n_embd_head, n_head, n_seqs);
cb(kv_old, "kv_old", il);
ggml_tensor * qkv = nullptr;
ggml_tensor * kv_new = nullptr;
if (n_seq_tokens == 1) {
// lightning attention - optimized single token case for TG
ggml_tensor * slopes_neg = ggml_scale(ctx0, slope_rate, -1.0);
cb(slopes_neg, "slopes_neg", il);
ggml_tensor * ratio = ggml_exp(ctx0, slopes_neg);
cb(ratio, "ratio", il);
ggml_tensor * ratio_3d = ggml_reshape_3d(ctx0, ratio, 1, 1, n_head);
cb(ratio_3d, "ratio3d", il);
ggml_tensor * v_trans = ggml_cont(ctx0, ggml_permute(ctx0, Vcur, 1, 2, 0, 3));
cb(v_trans, "v_trans", il);
ggml_tensor * k_trans = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 1, 2, 0, 3));
cb(k_trans, "k_trans", il);
ggml_tensor * kv_cur = ggml_mul_mat(ctx0, k_trans, v_trans);
cb(kv_cur, "kv_cur", il);
ggml_tensor * kv_old_s = ggml_mul(ctx0, kv_old, ratio_3d);
cb(kv_old_s, "kv_old_s", il);
kv_new = ggml_add(ctx0, kv_old_s, kv_cur);
cb(kv_new, "kv_new", il);
ggml_tensor * q_trans = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
cb(q_trans, "q_trans", il);
qkv = ggml_mul_mat(ctx0, kv_new, q_trans);
cb(qkv, "qkv", il);
} else if(n_seq_tokens > 1) {
// lightning attention - general multi token case for PP
ggml_tensor * q_decay_exp = la->inp_q_decay;
ggml_tensor * k_decay_exp = la->inp_k_decay;
ggml_tensor * diag_decay_exp = la->inp_diag_decay;
ggml_tensor * q_decay = ggml_exp(ctx0, ggml_scale(ctx0, q_decay_exp, slope_scale));
cb(q_decay, "q_decay", il);
ggml_tensor * k_decay = ggml_exp(ctx0, ggml_scale(ctx0, k_decay_exp, slope_scale));
cb(k_decay, "k_decay", il);
ggml_tensor * diag_decay = ggml_exp(ctx0, ggml_scale(ctx0, diag_decay_exp, slope_scale));
cb(diag_decay, "diag_decay", il);
ggml_tensor * q_s = ggml_mul(ctx0, Qcur, q_decay);
cb(q_s, "q_s", il);
ggml_tensor * q_s_trans = ggml_permute(ctx0, q_s, 0, 2, 1, 3);
cb(q_s_trans, "q_s_trans", il);
ggml_tensor * qkv_none_diag = ggml_mul_mat(ctx0, kv_old, q_s_trans);
cb(qkv_none_diag, "qkv_none_diag", il);
ggml_tensor * q_trans = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
cb(q_trans, "q_trans", il);
ggml_tensor * k_trans = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);
cb(k_trans, "k_trans", il);
ggml_tensor * qk = ggml_mul_mat(ctx0, k_trans, q_trans);
cb(qk, "qk", il);
qk = ggml_mul(ctx0, qk, diag_decay);
cb(qk, "qk_s", il);
ggml_tensor * v_trans = ggml_cont(ctx0, ggml_permute(ctx0, Vcur, 1, 2, 0, 3));
cb(v_trans, "v_trans", il);
ggml_tensor * qkv_diag = ggml_mul_mat(ctx0, v_trans, qk);
cb(qkv_diag, "qkv_diag", il);
qkv = ggml_add(ctx0, qkv_none_diag, qkv_diag);
cb(qkv, "qkv", il);
ggml_build_forward_expand(gf, qkv);
ggml_tensor * slopes_neg = ggml_scale(ctx0, slope_rate, -1.0*n_seq_tokens);
cb(slopes_neg, "slopes_neg", il);
ggml_tensor * block_decay = ggml_exp(ctx0, slopes_neg);
cb(block_decay, "block_decay", il);
ggml_tensor * block_decay_3d = ggml_reshape_3d(ctx0, block_decay, 1, 1, n_head);
cb(block_decay_3d, "block_decay_3d", il);
ggml_tensor * kv_old_s = ggml_mul(ctx0, kv_old, block_decay_3d);
cb(kv_old_s, "kv_old_s", il);
ggml_tensor * k_after_decay = ggml_mul(ctx0, Kcur, k_decay);
cb(k_after_decay, "k_after_decay", il);
ggml_tensor * k_after_decay_trans = ggml_cont(ctx0, ggml_permute(ctx0, k_after_decay, 1, 2, 0, 3));
cb(k_after_decay_trans, "k_after_decay_trans", il);
ggml_tensor * kv_cur = ggml_mul_mat(ctx0, k_after_decay_trans, v_trans);
cb(kv_cur, "kv_cur", il);
kv_new = ggml_add(ctx0, kv_old_s, kv_cur);
cb(kv_new, "kv_new", il);
}
// store new KV
ggml_build_forward_expand(gf,
ggml_cpy(ctx0, kv_new,
ggml_view_1d(ctx0, la_states_all, hparams.n_embd_s() * n_seqs,
kv_head * hparams.n_embd_s() * ggml_element_size(la_states_all))));
qkv = ggml_cont(ctx0, ggml_permute(ctx0, qkv, 0, 2, 1, 3));
cb(qkv, "qkv_permuted", il);
qkv = ggml_reshape_4d(ctx0, qkv, qkv->ne[0]*qkv->ne[1], qkv->ne[2], 1, qkv->ne[3]);
// norm
ggml_tensor * qkv_norm = build_norm(qkv,
model.layers[il].attn_norm_2, NULL,
LLM_NORM_RMS, il);
cb(qkv_norm, "qkv_norm", il);
ggml_tensor * g = build_lora_mm(model.layers[il].wg, cur);
cb(g, "g", il);
g = ggml_sigmoid(ctx0, g);
cb(g, "g_sigm", il);
cur = ggml_mul(ctx0, g, qkv_norm);
cur = build_lora_mm(model.layers[il].wo, cur);
cb(cur, "attn_out", il);
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens*n_seqs);
cb(cur, "attn_out", il);
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
residual = ggml_get_rows(ctx0, residual, inp_out_ids);
}
residual = ggml_scale(ctx0, residual, hparams.f_residual_scale);
cb(residual, "residual_scaled_attn", il);
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, residual);
cb(ffn_inp, "ffn_inp", il);
// MoE branch
cur = build_norm(ffn_inp,
model.layers[il].ffn_norm, NULL,
LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
residual = cur;
cur = build_moe_ffn(cur,
model.layers[il].ffn_gate_inp,
model.layers[il].ffn_up_exps,
model.layers[il].ffn_gate_exps,
model.layers[il].ffn_down_exps,
model.layers[il].ffn_exp_probs_b,
n_expert, n_expert_used,
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il);
cb(cur, "ffn_moe_out", il);
residual = ggml_scale(ctx0, residual, hparams.f_residual_scale);
cb(residual, "residual_scaled_ffn", il);
cur = ggml_add(ctx0, cur, residual);
cb(cur, "ffn_out", il);
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
// input for next layer
inpL = cur;
}
cur = inpL;
cur = build_norm(cur,
model.output_norm, NULL,
LLM_NORM_RMS, -1);
cb(cur, "result_norm", -1);
res->t_embd = cur;
// lm_head
cur = build_lora_mm(model.output, cur, model.output_s);
cb(cur, "result_output", -1);
res->t_logits = cur;
ggml_build_forward_expand(gf, cur);
}
src/models/minimax-m3.cpp
+2
View File
@@ -25,6 +25,8 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);
msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks };
GGML_ASSERT(hparams.indexer_block_size > 0); // avoid div by zero
switch (hparams.n_layer()) {
case 60: type = LLM_TYPE_428B_A23B; break;
default: type = LLM_TYPE_UNKNOWN;
src/models/models.h
+13
View File
@@ -2043,6 +2043,19 @@ struct llama_model_apertus : public llama_model_base {
};
struct llama_model_minimax_01 : public llama_model_base {
llama_model_minimax_01(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
void load_arch_tensors(llama_model_loader & ml) override;
struct graph : public llm_graph_context {
graph(const llama_model & model, const llm_graph_params & params);
};
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
struct llama_model_minimax_m2 : public llama_model_base {
llama_model_minimax_m2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
src/models/plamo2.cpp
+1 -1
View File
@@ -382,7 +382,7 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_mamba_layer(llm_graph_inpu
// Custom operator to optimize the parallel associative scan
// as described in the Annex D of the Mamba paper.
// => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}
return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);
return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids, /*K=*/1);
};
ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);