Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

allow offloading moe to cpu with --moecpu

Browse Source
This commit is contained in:
Concedo
2025-08-05 23:42:42 +08:00
parent 7590a0ea39
commit e40d26b9e7
3 changed files with 21 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpttype_adapter.cpp
+12
View File
@@ -2293,6 +2293,18 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
// std::string forced = "per_layer_token_embd.weight=CPU"; //this tensor on gpu is problematic on unsloth q4_0
// tensoroverrides = (tensoroverrides=="" ? forced: (forced+","+tensoroverrides));
// }
if(tensoroverrides=="" && ggml_backend_dev_count()>1 && inputs.moecpu>0)
{
for (int i = 0; i < inputs.moecpu; ++i) {
std::string tmp = string_format("blk\\.%d\\.ffn_(up|down|gate)_exps=CPU", i);
if(i>0)
{
tmp = "," + tmp;
}
tensoroverrides += tmp;
}
printf("Overriding %d MoE layers to CPU...\n",inputs.moecpu);
}
if(tensoroverrides!="" && ggml_backend_dev_count()>1)
{
printf("Handling Override Tensors for backends: ");