mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 01:05:09 +02:00
refactor: handle GGML_VK_VISIBLE_DEVICES at the Python level (#2179)
All C++ handling code currently: - build a comma-separated list from the info_vulkan array - if GGML_VK_VISIBLE_DEVICES isn't set - set GGML_VK_VISIBLE_DEVICES to the list Once set, GGML_VK_VISIBLE_DEVICES affects the whole process. So this can be done in the same way at the Python level, before all loading functions. Caveat: load_model had the default `inputs.vulkan_info = "0"`, so the default GPU would be "0" only when loading a text model.
This commit is contained in:
@@ -22,7 +22,6 @@
|
||||
#endif
|
||||
|
||||
static llama_context * embeddings_ctx = nullptr; //text to codes ctx
|
||||
static std::string ttsvulkandeviceenv;
|
||||
bool embeddings_debug = false;
|
||||
static int max_batchsize = 512;
|
||||
static std::string last_output = "";
|
||||
@@ -82,27 +81,12 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu
|
||||
|
||||
bool embeddingstype_load_model(const embeddings_load_model_inputs inputs)
|
||||
{
|
||||
//duplicated from expose.cpp
|
||||
std::string vulkan_info_raw = inputs.vulkan_info;
|
||||
std::string vulkan_info_str = "";
|
||||
for (size_t i = 0; i < vulkan_info_raw.length(); ++i) {
|
||||
vulkan_info_str += vulkan_info_raw[i];
|
||||
if (i < vulkan_info_raw.length() - 1) {
|
||||
vulkan_info_str += ",";
|
||||
}
|
||||
}
|
||||
const char* existingenv = getenv("GGML_VK_VISIBLE_DEVICES");
|
||||
std::vector<ggml_backend_dev_t> devices_override;
|
||||
std::string dev_override_str = inputs.devices_override;
|
||||
if(dev_override_str!="")
|
||||
{
|
||||
devices_override = kcpp_parse_device_list(dev_override_str);
|
||||
}
|
||||
if(!existingenv && vulkan_info_str!="")
|
||||
{
|
||||
ttsvulkandeviceenv = "GGML_VK_VISIBLE_DEVICES="+vulkan_info_str;
|
||||
putenv((char*)ttsvulkandeviceenv.c_str());
|
||||
}
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user