Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

allow overriding the devices directly

Browse Source
This commit is contained in:
Concedo
2026-01-17 19:08:06 +08:00
parent 21e6ccb8cb
commit 62bea5ef4f
7 changed files with 92 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
otherarch/embeddings_adapter.cpp
+12
View File
@@ -102,6 +102,12 @@ bool embeddingstype_load_model(const embeddings_load_model_inputs inputs)
}
}
const char* existingenv = getenv("GGML_VK_VISIBLE_DEVICES");
std::vector<ggml_backend_dev_t> devices_override;
std::string dev_override_str = inputs.devices_override;
if(dev_override_str!="")
{
devices_override = kcpp_parse_device_list(dev_override_str);
}
if(!existingenv && vulkan_info_str!="")
{
ttsvulkandeviceenv = "GGML_VK_VISIBLE_DEVICES="+vulkan_info_str;
@@ -125,6 +131,12 @@ bool embeddingstype_load_model(const embeddings_load_model_inputs inputs)
model_params.main_gpu = kcpp_parseinfo_maindevice;
model_params.split_mode = llama_split_mode::LLAMA_SPLIT_MODE_LAYER;
if(devices_override.size()>0)
{
printf("\nOverriding with %d devices...\n",devices_override.size()-1);
model_params.devices = devices_override.data();
}
llama_model * embeddingsmodel = llama_model_load_from_file(modelfile.c_str(), model_params);
const int n_ctx_train = llama_model_n_ctx_train(embeddingsmodel);