Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

allow overriding the devices directly

Browse Source
This commit is contained in:
Concedo
2026-01-17 19:08:06 +08:00
parent 21e6ccb8cb
commit 62bea5ef4f
7 changed files with 92 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
otherarch/tts_adapter.cpp
+12
View File
@@ -637,6 +637,12 @@ bool ttstype_load_model(const tts_load_model_inputs inputs)
}
}
const char* existingenv = getenv("GGML_VK_VISIBLE_DEVICES");
std::vector<ggml_backend_dev_t> devices_override;
std::string dev_override_str = inputs.devices_override;
if(dev_override_str!="")
{
devices_override = kcpp_parse_device_list(dev_override_str);
}
if(!existingenv && vulkan_info_str!="")
{
ttsvulkandeviceenv = "GGML_VK_VISIBLE_DEVICES="+vulkan_info_str;
@@ -699,6 +705,12 @@ bool ttstype_load_model(const tts_load_model_inputs inputs)
tts_ctx_params.flash_attn_type = (inputs.flash_attention?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED);
tts_ctx_params.kv_unified = true;
if(devices_override.size()>0)
{
printf("\nOverriding with %d devices...\n",devices_override.size()-1);
tts_model_params.devices = devices_override.data();
}
llama_model * ttcmodel = llama_model_load_from_file(modelfile_ttc.c_str(), tts_model_params);
ttc_ctx = llama_init_from_model(ttcmodel, tts_ctx_params);