mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-07 13:31:22 +02:00
remove CLBlast, part 1
This commit is contained in:
Concedo
+42
-160
@@ -149,9 +149,7 @@ saved_stderr_py = None
|
||||
stdout_nullfile = None
|
||||
stdout_nullfile_py = None
|
||||
|
||||
CLDevices = ["1","2","3","4"]
|
||||
CUDevices = ["1","2","3","4","All"]
|
||||
CLDevicesNames = ["","","",""]
|
||||
CUDevicesNames = ["","","","",""]
|
||||
VKDevicesNames = ["","","",""]
|
||||
VKIsDGPU = [0,0,0,0]
|
||||
@@ -201,7 +199,6 @@ class load_model_inputs(ctypes.Structure):
|
||||
("use_smartcontext", ctypes.c_bool),
|
||||
("use_contextshift", ctypes.c_bool),
|
||||
("use_fastforward", ctypes.c_bool),
|
||||
("clblast_info", ctypes.c_int),
|
||||
("kcpp_main_gpu", ctypes.c_int),
|
||||
("vulkan_info", ctypes.c_char_p),
|
||||
("batchsize", ctypes.c_int),
|
||||
@@ -297,7 +294,6 @@ class generation_outputs(ctypes.Structure):
|
||||
class sd_load_model_inputs(ctypes.Structure):
|
||||
_fields_ = [("model_filename", ctypes.c_char_p),
|
||||
("executable_path", ctypes.c_char_p),
|
||||
("clblast_info", ctypes.c_int),
|
||||
("kcpp_main_gpu", ctypes.c_int),
|
||||
("vulkan_info", ctypes.c_char_p),
|
||||
("threads", ctypes.c_int),
|
||||
@@ -368,7 +364,6 @@ class sd_info_outputs(ctypes.Structure):
|
||||
class whisper_load_model_inputs(ctypes.Structure):
|
||||
_fields_ = [("model_filename", ctypes.c_char_p),
|
||||
("executable_path", ctypes.c_char_p),
|
||||
("clblast_info", ctypes.c_int),
|
||||
("kcpp_main_gpu", ctypes.c_int),
|
||||
("vulkan_info", ctypes.c_char_p),
|
||||
("devices_override", ctypes.c_char_p),
|
||||
@@ -390,7 +385,6 @@ class tts_load_model_inputs(ctypes.Structure):
|
||||
("ttc_model_filename", ctypes.c_char_p),
|
||||
("cts_model_filename", ctypes.c_char_p),
|
||||
("executable_path", ctypes.c_char_p),
|
||||
("clblast_info", ctypes.c_int),
|
||||
("kcpp_main_gpu", ctypes.c_int),
|
||||
("vulkan_info", ctypes.c_char_p),
|
||||
("gpulayers", ctypes.c_int),
|
||||
@@ -416,7 +410,6 @@ class embeddings_load_model_inputs(ctypes.Structure):
|
||||
_fields_ = [("threads", ctypes.c_int),
|
||||
("model_filename", ctypes.c_char_p),
|
||||
("executable_path", ctypes.c_char_p),
|
||||
("clblast_info", ctypes.c_int),
|
||||
("kcpp_main_gpu", ctypes.c_int),
|
||||
("vulkan_info", ctypes.c_char_p),
|
||||
("gpulayers", ctypes.c_int),
|
||||
@@ -655,9 +648,7 @@ def pick_existant_file(ntoption,nonntoption):
|
||||
lib_default = pick_existant_file("koboldcpp_default.dll","koboldcpp_default.so")
|
||||
lib_failsafe = pick_existant_file("koboldcpp_failsafe.dll","koboldcpp_failsafe.so")
|
||||
lib_noavx2 = pick_existant_file("koboldcpp_noavx2.dll","koboldcpp_noavx2.so")
|
||||
lib_clblast = pick_existant_file("koboldcpp_clblast.dll","koboldcpp_clblast.so")
|
||||
lib_clblast_noavx2 = pick_existant_file("koboldcpp_clblast_noavx2.dll","koboldcpp_clblast_noavx2.so")
|
||||
lib_clblast_failsafe = pick_existant_file("koboldcpp_clblast_failsafe.dll","koboldcpp_clblast_failsafe.so")
|
||||
lib_vulkan_failsafe = pick_existant_file("koboldcpp_vulkan_failsafe.dll","koboldcpp_vulkan_failsafe.so")
|
||||
lib_cublas = pick_existant_file("koboldcpp_cublas.dll","koboldcpp_cublas.so")
|
||||
lib_hipblas = pick_existant_file("koboldcpp_hipblas.dll","koboldcpp_hipblas.so")
|
||||
lib_vulkan = pick_existant_file("koboldcpp_vulkan.dll","koboldcpp_vulkan.so")
|
||||
@@ -668,27 +659,22 @@ lib_option_pairs = [
|
||||
(lib_cublas, "Use CUDA"),
|
||||
(lib_hipblas, "Use hipBLAS (ROCm)"),
|
||||
(lib_vulkan, "Use Vulkan"),
|
||||
(lib_clblast, "Use CLBlast"),
|
||||
(lib_noavx2, "Use CPU (Old CPU)"),
|
||||
(lib_vulkan_noavx2, "Use Vulkan (Old CPU)"),
|
||||
(lib_clblast_noavx2, "Use CLBlast (Old CPU)"),
|
||||
(lib_clblast_failsafe, "Use CLBlast (Older CPU)"),
|
||||
(lib_vulkan_failsafe, "Use Vulkan (Older CPU)"),
|
||||
(lib_failsafe, "Failsafe Mode (Older CPU)")]
|
||||
default_option, cublas_option, hipblas_option, vulkan_option, clblast_option, noavx2_option, vulkan_noavx2_option, clblast_noavx2_option, clblast_failsafe_option, failsafe_option = (opt if file_exists(lib) or (os.name == 'nt' and file_exists(opt + ".dll")) else None for lib, opt in lib_option_pairs)
|
||||
default_option, cublas_option, hipblas_option, vulkan_option, noavx2_option, vulkan_noavx2_option, vulkan_failsafe_option, failsafe_option = (opt if file_exists(lib) or (os.name == 'nt' and file_exists(opt + ".dll")) else None for lib, opt in lib_option_pairs)
|
||||
runopts = [opt for lib, opt in lib_option_pairs if file_exists(lib)]
|
||||
|
||||
def init_library():
|
||||
global handle, args, libname
|
||||
global lib_default,lib_failsafe,lib_noavx2,lib_clblast,lib_clblast_noavx2,lib_clblast_failsafe,lib_cublas,lib_hipblas,lib_vulkan,lib_vulkan_noavx2
|
||||
global lib_default,lib_failsafe,lib_noavx2,lib_vulkan_failsafe,lib_cublas,lib_hipblas,lib_vulkan,lib_vulkan_noavx2
|
||||
|
||||
libname = lib_default
|
||||
|
||||
if args.noavx2: #failsafe implies noavx2 always
|
||||
if args.useclblast and (os.name!='nt' or file_exists("clblast.dll")):
|
||||
if file_exists(lib_clblast_noavx2) and not (args.failsafe):
|
||||
libname = lib_clblast_noavx2
|
||||
elif file_exists(lib_clblast_failsafe):
|
||||
libname = lib_clblast_failsafe
|
||||
if args.failsafe and (args.usevulkan is not None) and file_exists(lib_vulkan_failsafe):
|
||||
libname = lib_vulkan_failsafe
|
||||
elif (args.usevulkan is not None) and file_exists(lib_vulkan_noavx2):
|
||||
libname = lib_vulkan_noavx2
|
||||
elif (args.failsafe) and file_exists(lib_failsafe):
|
||||
@@ -706,13 +692,6 @@ def init_library():
|
||||
libname = lib_vulkan
|
||||
elif file_exists(lib_vulkan_noavx2):
|
||||
libname = lib_vulkan_noavx2
|
||||
elif args.useclblast and (os.name!='nt' or file_exists("clblast.dll")):
|
||||
if file_exists(lib_clblast):
|
||||
libname = lib_clblast
|
||||
elif file_exists(lib_clblast_noavx2):
|
||||
libname = lib_clblast_noavx2
|
||||
elif file_exists(lib_clblast_failsafe):
|
||||
libname = lib_clblast_failsafe
|
||||
elif libname == lib_default and not file_exists(lib_default) and file_exists(lib_noavx2):
|
||||
libname = lib_noavx2
|
||||
|
||||
@@ -798,11 +777,6 @@ def init_library():
|
||||
handle.detokenize.restype = ctypes.c_char_p
|
||||
|
||||
def set_backend_props(inputs):
|
||||
clblastids = 0
|
||||
if args.useclblast:
|
||||
clblastids = 100 + int(args.useclblast[0])*10 + int(args.useclblast[1])
|
||||
inputs.clblast_info = clblastids
|
||||
|
||||
# we must force an explicit tensor split
|
||||
# otherwise the default will divide equally and multigpu crap will slow it down badly
|
||||
inputs.kcpp_main_gpu = 0
|
||||
@@ -1517,41 +1491,7 @@ def detect_memory_vk(gpumem_ignore_limit_min, gpumem_ignore_limit_max):
|
||||
return 0
|
||||
|
||||
|
||||
def detect_memory_cl(gpumem_ignore_limit_min, gpumem_ignore_limit_max):
|
||||
|
||||
try: # Get OpenCL GPU names on windows using a special binary. overwrite at known index if found.
|
||||
basepath = os.path.abspath(os.path.dirname(__file__))
|
||||
output = ""
|
||||
data = None
|
||||
try:
|
||||
output = subprocess.run(["clinfo","--json"], capture_output=True, text=True, check=True, encoding='utf-8', timeout=10).stdout
|
||||
data = json.loads(output)
|
||||
except Exception:
|
||||
output = subprocess.run([((os.path.join(basepath, "simpleclinfo.exe")) if os.name == 'nt' else "clinfo"),"--json"], capture_output=True, text=True, check=True, creationflags=subprocess.CREATE_NO_WINDOW | subprocess.DETACHED_PROCESS, encoding='utf-8', timeout=10).stdout
|
||||
data = json.loads(output)
|
||||
plat = 0
|
||||
dev = 0
|
||||
lowestclmem = 0
|
||||
for platform in data["devices"]:
|
||||
dev = 0
|
||||
for device in platform["online"]:
|
||||
dname = device["CL_DEVICE_NAME"]
|
||||
dmem = int(device["CL_DEVICE_GLOBAL_MEM_SIZE"])
|
||||
idx = plat+dev*2
|
||||
if idx<len(CLDevices):
|
||||
CLDevicesNames[idx] = dname
|
||||
if dmem > gpumem_ignore_limit_min and dmem < gpumem_ignore_limit_max:
|
||||
lowestclmem = dmem if lowestclmem==0 else (dmem if dmem<lowestclmem else lowestclmem)
|
||||
dev += 1
|
||||
plat += 1
|
||||
return lowestclmem
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def fetch_gpu_properties(testCL,testCU,testVK,testmemory=False):
|
||||
def fetch_gpu_properties(testCU,testVK,testmemory=False):
|
||||
gpumem_ignore_limit_min = 1024*1024*600 #600 mb min
|
||||
gpumem_ignore_limit_max = 1024*1024*1024*300 #300 gb max
|
||||
|
||||
@@ -1568,13 +1508,6 @@ def fetch_gpu_properties(testCL,testCU,testVK,testmemory=False):
|
||||
if testmemory:
|
||||
print(f'detected Vulkan memory: {vkmem/(1024*1024)} MB')
|
||||
|
||||
if testCL:
|
||||
clmem = detect_memory_cl(gpumem_ignore_limit_min, gpumem_ignore_limit_max)
|
||||
MaxMemory[0] = max(clmem,MaxMemory[0])
|
||||
if testmemory:
|
||||
print(f'detected OpenCL memory: {clmem/(1024*1024)} MB')
|
||||
|
||||
|
||||
# Check VRAM detection after all backends have been tested
|
||||
if MaxMemory[0] < (1024*1024*256):
|
||||
print("Unable to detect VRAM, please set layers manually.")
|
||||
@@ -1582,7 +1515,7 @@ def fetch_gpu_properties(testCL,testCU,testVK,testmemory=False):
|
||||
return
|
||||
|
||||
def auto_set_backend_cli():
|
||||
fetch_gpu_properties(False,True,True)
|
||||
fetch_gpu_properties(True,True)
|
||||
found_new_backend = False
|
||||
|
||||
# check for avx2 and avx support
|
||||
@@ -5501,12 +5434,11 @@ def show_gui():
|
||||
batchsize_values = ["-1","16","32","64","128","256","512","1024","2048","4096"]
|
||||
batchsize_text = ["Don't Batch","16","32","64","128","256","512","1024","2048","4096"]
|
||||
contextsize_text = ["256", "512", "1024", "2048", "3072", "4096", "6144", "8192", "10240", "12288", "14336", "16384", "20480", "24576", "28672", "32768", "40960", "49152", "57344", "65536", "81920", "98304", "114688", "131072"]
|
||||
antirunopts = [opt.replace("Use ", "") for lib, opt in lib_option_pairs if opt not in runopts]
|
||||
quantkv_text = ["F16 (Off)","8-Bit","4-Bit"]
|
||||
|
||||
if not any(runopts):
|
||||
exitcounter = 999
|
||||
exit_with_error(2,"KoboldCPP couldn't locate any backends to use (i.e Default, Vulkan, CLBlast, CUDA).\n\nTo use the program, please run the 'make' command from the directory.","No Backends Available!")
|
||||
exit_with_error(2,"KoboldCPP couldn't locate any backends to use (i.e Default, Vulkan, CUDA).\n\nTo use the program, please run the 'make' command from the directory.","No Backends Available!")
|
||||
|
||||
# Vars - should be in scope to be used by multiple widgets
|
||||
gpulayers_var = ctk.StringVar(value="-1")
|
||||
@@ -5882,9 +5814,7 @@ def show_gui():
|
||||
if manual_select:
|
||||
print("\nA .kcppt template was selected from GUI - automatically selecting your backend...")
|
||||
runmode_untouched = True
|
||||
fetch_gpu_properties(False,True,True)
|
||||
else:
|
||||
fetch_gpu_properties(True,True,True)
|
||||
fetch_gpu_properties(True,True)
|
||||
found_new_backend = False
|
||||
|
||||
# check for avx2 and avx support
|
||||
@@ -5894,7 +5824,7 @@ def show_gui():
|
||||
|
||||
#autopick cublas if suitable, requires at least 3.5GB VRAM to auto pick
|
||||
#we do not want to autoselect hip/cublas if the user has already changed their desired backend!
|
||||
if eligible_cuda and exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CUDA" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and (any(CUDevicesNames) or any(CLDevicesNames)) and runmode_untouched:
|
||||
if eligible_cuda and exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CUDA" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and (any(CUDevicesNames)) and runmode_untouched:
|
||||
if "Use CUDA" in runopts:
|
||||
runopts_var.set("Use CUDA")
|
||||
gpu_choice_var.set("1")
|
||||
@@ -5935,18 +5865,6 @@ def show_gui():
|
||||
dict = json.load(f)
|
||||
import_vars(dict)
|
||||
|
||||
def setup_backend_tooltip(parent, singlerow):
|
||||
# backend count label with the tooltip function
|
||||
nl = '\n'
|
||||
tooltxt = "Number of backends you have built and available." + (f"\n\nMissing Backends: \n\n{nl.join(antirunopts)}" if len(runopts) < 8 else "")
|
||||
num_backends_built = None
|
||||
num_backends_built = makelabel(parent, str(len(runopts)) + "/9", 1, 1,tooltxt)
|
||||
if singlerow:
|
||||
num_backends_built.grid(row=1, column=0, padx=355, pady=0)
|
||||
else:
|
||||
num_backends_built.grid(row=1, column=1, padx=205, pady=0)
|
||||
num_backends_built.configure(text_color="#00ff00")
|
||||
|
||||
def gui_changed_modelfile(*args):
|
||||
global importvars_in_progress
|
||||
if not importvars_in_progress:
|
||||
@@ -5965,7 +5883,7 @@ def show_gui():
|
||||
predicted_gpu_layers = autoset_gpu_layers(int(contextsize_text[context_var.get()]),sd_quant_option(sd_quant_var.get()),int(batchsize_values[int(blas_size_var.get())]),(quantkv_var.get() if flashattention_var.get()==1 else 0))
|
||||
max_gpu_layers = (f"/{modelfile_extracted_meta[1][0]+1}" if (modelfile_extracted_meta and modelfile_extracted_meta[1] and modelfile_extracted_meta[1][0]!=0) else "")
|
||||
index = runopts_var.get()
|
||||
gpu_be = (index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use CLBlast" or index == "Use CLBlast (Old CPU)" or index == "Use CLBlast (Older CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)")
|
||||
gpu_be = (index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use Vulkan (Older CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)")
|
||||
layercounter_label.grid(row=6, column=0, padx=230, sticky="W")
|
||||
quick_layercounter_label.grid(row=6, column=1, padx=75, sticky="W")
|
||||
if sys.platform=="darwin" and gpulayers_var.get()=="-1":
|
||||
@@ -5992,12 +5910,9 @@ def show_gui():
|
||||
try:
|
||||
s = int(gpu_choice_var.get())-1
|
||||
v = runopts_var.get()
|
||||
if v == "Use Vulkan" or v == "Use Vulkan (Old CPU)":
|
||||
if v == "Use Vulkan" or v == "Use Vulkan (Old CPU)" or v == "Use Vulkan (Older CPU)":
|
||||
quick_gpuname_label.configure(text=VKDevicesNames[s])
|
||||
gpuname_label.configure(text=VKDevicesNames[s])
|
||||
elif v == "Use CLBlast" or v == "Use CLBlast (Old CPU)" or v == "Use CLBlast (Older CPU)":
|
||||
quick_gpuname_label.configure(text=CLDevicesNames[s])
|
||||
gpuname_label.configure(text=CLDevicesNames[s])
|
||||
else:
|
||||
quick_gpuname_label.configure(text=CUDevicesNames[s])
|
||||
gpuname_label.configure(text=CUDevicesNames[s])
|
||||
@@ -6056,37 +5971,22 @@ def show_gui():
|
||||
global runmode_untouched
|
||||
runmode_untouched = False
|
||||
index = runopts_var.get()
|
||||
if index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use CLBlast" or index == "Use CLBlast (Old CPU)" or index == "Use CLBlast (Older CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)":
|
||||
if index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use Vulkan (Older CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)":
|
||||
quick_gpuname_label.grid(row=3, column=1, padx=75, sticky="W")
|
||||
gpuname_label.grid(row=3, column=0, padx=230, sticky="W")
|
||||
gpu_selector_label.grid(row=3, column=0, padx = 8, pady=1, stick="nw")
|
||||
quick_gpu_selector_label.grid(row=3, column=0, padx = 8, pady=1, stick="nw")
|
||||
if index == "Use CLBlast" or index == "Use CLBlast (Old CPU)" or index == "Use CLBlast (Older CPU)":
|
||||
gpu_selector_box.grid(row=3, column=0, padx=160, pady=1, stick="nw")
|
||||
quick_gpu_selector_box.grid(row=3, column=1, padx=8, pady=1, stick="nw")
|
||||
CUDA_gpu_selector_box.grid_remove()
|
||||
CUDA_quick_gpu_selector_box.grid_remove()
|
||||
maingpu_label.grid_remove()
|
||||
maingpu_entry.grid_remove()
|
||||
if gpu_choice_var.get()=="All":
|
||||
gpu_choice_var.set("1")
|
||||
lowvram_box.grid_remove()
|
||||
elif index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)":
|
||||
gpu_selector_box.grid_remove()
|
||||
quick_gpu_selector_box.grid_remove()
|
||||
CUDA_gpu_selector_box.grid(row=3, column=0, padx=160, pady=1, stick="nw")
|
||||
CUDA_quick_gpu_selector_box.grid(row=3, column=1, padx=8, pady=1, stick="nw")
|
||||
maingpu_label.grid(row=8, column=0, padx = 270, pady=1, stick="nw")
|
||||
maingpu_entry.grid(row=8, column=0, padx = 340, pady=1, stick="nw")
|
||||
lowvram_box.grid(row=4, column=0, padx=8, pady=1, stick="nw")
|
||||
CUDA_gpu_selector_box.grid(row=3, column=0, padx=160, pady=1, stick="nw")
|
||||
CUDA_quick_gpu_selector_box.grid(row=3, column=1, padx=8, pady=1, stick="nw")
|
||||
maingpu_label.grid(row=8, column=0, padx = 270, pady=1, stick="nw")
|
||||
maingpu_entry.grid(row=8, column=0, padx = 340, pady=1, stick="nw")
|
||||
lowvram_box.grid(row=4, column=0, padx=8, pady=1, stick="nw")
|
||||
else:
|
||||
quick_gpuname_label.grid_remove()
|
||||
gpuname_label.grid_remove()
|
||||
gpu_selector_label.grid_remove()
|
||||
gpu_selector_box.grid_remove()
|
||||
CUDA_gpu_selector_box.grid_remove()
|
||||
quick_gpu_selector_label.grid_remove()
|
||||
quick_gpu_selector_box.grid_remove()
|
||||
CUDA_quick_gpu_selector_box.grid_remove()
|
||||
maingpu_label.grid_remove()
|
||||
maingpu_entry.grid_remove()
|
||||
@@ -6109,7 +6009,7 @@ def show_gui():
|
||||
tensor_split_label.grid(row=8, column=0, padx = 8, pady=1, stick="nw")
|
||||
tensor_split_entry.grid(row=8, column=0, padx = 160, pady=1, stick="nw")
|
||||
|
||||
if index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use CLBlast" or index == "Use CLBlast (Old CPU)" or index == "Use CLBlast (Older CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)":
|
||||
if index == "Use Vulkan" or index == "Use Vulkan (Old CPU)" or index == "Use Vulkan (Older CPU)" or index == "Use CUDA" or index == "Use hipBLAS (ROCm)":
|
||||
gpu_layers_label.grid(row=6, column=0, padx=8, pady=1, stick="nw")
|
||||
gpu_layers_entry.grid(row=6, column=0, padx=160, pady=1, stick="nw")
|
||||
quick_gpu_layers_label.grid(row=6, column=0, padx = 8, pady=1, stick="nw")
|
||||
@@ -6128,18 +6028,14 @@ def show_gui():
|
||||
changed_gpu_choice_var()
|
||||
|
||||
# presets selector
|
||||
makelabel(quick_tab, "Backend:", 1,0,"Select a backend to use.\nCUDA runs on Nvidia GPUs, and is much faster.\nVulkan and CLBlast works on all GPUs but is somewhat slower.\nOtherwise, runs on CPU only.\nNoAVX2 and Failsafe modes support older PCs.")
|
||||
makelabel(quick_tab, "Backend:", 1,0,"Select a backend to use.\nCUDA runs on Nvidia GPUs, and is much faster.\nVulkan works on all GPUs but is somewhat slower.\nOtherwise, runs on CPU only.\nNoAVX2 and Failsafe modes support older PCs.")
|
||||
|
||||
runoptbox = ctk.CTkComboBox(quick_tab, values=runopts, width=190,variable=runopts_var, state="readonly")
|
||||
runoptbox.grid(row=1, column=1,padx=8, stick="nw")
|
||||
runoptbox.set(runopts[0]) # Set to first available option
|
||||
|
||||
# Tell user how many backends are available
|
||||
setup_backend_tooltip(quick_tab,False)
|
||||
|
||||
# gpu options
|
||||
quick_gpu_selector_label = makelabel(quick_tab, "GPU ID:", 3,0,"Which GPU ID to load the model with.\nNormally your main GPU is #1, but it can vary for multi GPU setups.",padx=8)
|
||||
quick_gpu_selector_box = ctk.CTkComboBox(quick_tab, values=CLDevices, width=60, variable=gpu_choice_var, state="readonly")
|
||||
CUDA_quick_gpu_selector_box = ctk.CTkComboBox(quick_tab, values=CUDevices, width=60, variable=gpu_choice_var, state="readonly")
|
||||
CUDA_quick_gpu_selector_box.grid(row=3, column=1, padx=8, pady=1, stick="nw")
|
||||
quick_gpuname_label = ctk.CTkLabel(quick_tab, text="")
|
||||
@@ -6179,18 +6075,13 @@ def show_gui():
|
||||
hardware_tab = tabcontent["Hardware"]
|
||||
|
||||
# presets selector
|
||||
makelabel(hardware_tab, "Backend:", 1,0,"Select a backend to use.\nCUDA runs on Nvidia GPUs, and is much faster.\nVulkan and CLBlast works on all GPUs but is somewhat slower.\nOtherwise, runs on CPU only.\nNoAVX2 and Failsafe modes support older PCs.")
|
||||
makelabel(hardware_tab, "Backend:", 1,0,"Select a backend to use.\nCUDA runs on Nvidia GPUs, and is much faster.\nVulkan works on all GPUs but is somewhat slower.\nOtherwise, runs on CPU only.\nNoAVX2 and Failsafe modes support older PCs.")
|
||||
runoptbox = ctk.CTkComboBox(hardware_tab, values=runopts, width=180,variable=runopts_var, state="readonly")
|
||||
runoptbox.grid(row=1, column=0,padx=160, stick="nw")
|
||||
runoptbox.set(runopts[0]) # Set to first available option
|
||||
|
||||
# Tell user how many backends are available
|
||||
setup_backend_tooltip(hardware_tab,True)
|
||||
|
||||
# gpu options
|
||||
gpu_selector_label = makelabel(hardware_tab, "GPU ID:", 3,0,"Which GPU ID to load the model with.\nNormally your main GPU is #1, but it can vary for multi GPU setups.")
|
||||
gpu_selector_box = ctk.CTkComboBox(hardware_tab, values=CLDevices, width=60, variable=gpu_choice_var, state="readonly")
|
||||
gpu_selector_box.grid(row=3, column=0, padx=160, pady=1, stick="nw")
|
||||
CUDA_gpu_selector_box = ctk.CTkComboBox(hardware_tab, values=CUDevices, width=60, variable=gpu_choice_var, state="readonly")
|
||||
CUDA_gpu_selector_box.grid(row=3, column=0, padx=160, pady=1, stick="nw")
|
||||
gpuname_label = ctk.CTkLabel(hardware_tab, text="")
|
||||
@@ -6562,17 +6453,9 @@ def show_gui():
|
||||
args.usecpu = False
|
||||
args.usevulkan = None
|
||||
args.usecuda = None
|
||||
args.useclblast = None
|
||||
args.noavx2 = False
|
||||
if gpu_choice_var.get()!="All":
|
||||
gpuchoiceidx = int(gpu_choice_var.get())-1
|
||||
if runopts_var.get() == "Use CLBlast" or runopts_var.get() == "Use CLBlast (Old CPU)" or runopts_var.get() == "Use CLBlast (Older CPU)":
|
||||
args.useclblast = [[0,0], [1,0], [0,1], [1,1]][gpuchoiceidx]
|
||||
if runopts_var.get() == "Use CLBlast (Old CPU)":
|
||||
args.noavx2 = True
|
||||
elif runopts_var.get() == "Use CLBlast (Older CPU)":
|
||||
args.noavx2 = True
|
||||
args.failsafe = True
|
||||
if runopts_var.get() == "Use CUDA" or runopts_var.get() == "Use hipBLAS (ROCm)":
|
||||
if gpu_choice_var.get()=="All":
|
||||
args.usecuda = ["normal"]
|
||||
@@ -6584,13 +6467,16 @@ def show_gui():
|
||||
args.usecuda.append("nommq")
|
||||
if rowsplit_var.get()==1:
|
||||
args.usecuda.append("rowsplit")
|
||||
if runopts_var.get() == "Use Vulkan" or runopts_var.get() == "Use Vulkan (Old CPU)":
|
||||
if runopts_var.get() == "Use Vulkan" or runopts_var.get() == "Use Vulkan (Old CPU)" or runopts_var.get() == "Use Vulkan (Older CPU)":
|
||||
if gpu_choice_var.get()=="All":
|
||||
args.usevulkan = []
|
||||
else:
|
||||
args.usevulkan = [int(gpuchoiceidx)]
|
||||
if runopts_var.get() == "Use Vulkan (Old CPU)":
|
||||
args.noavx2 = True
|
||||
elif runopts_var.get() == "Use Vulkan (Older CPU)":
|
||||
args.noavx2 = True
|
||||
args.failsafe = True
|
||||
if gpulayers_var.get():
|
||||
args.gpulayers = (0 if gpulayers_var.get()=="" else int(gpulayers_var.get()))
|
||||
if runopts_var.get()=="Use CPU":
|
||||
@@ -6787,16 +6673,7 @@ def show_gui():
|
||||
lowvram_var.set(1 if "lowvram" in dict and dict["lowvram"] else 0)
|
||||
if "quantkv" in dict:
|
||||
quantkv_var.set(dict["quantkv"])
|
||||
if "useclblast" in dict and dict["useclblast"]:
|
||||
if "noavx2" in dict and dict["noavx2"]:
|
||||
if clblast_noavx2_option is not None:
|
||||
runopts_var.set(clblast_noavx2_option)
|
||||
gpu_choice_var.set(str(["0 0", "1 0", "0 1", "1 1"].index(str(dict["useclblast"][0]) + " " + str(dict["useclblast"][1])) + 1))
|
||||
else:
|
||||
if clblast_option is not None:
|
||||
runopts_var.set(clblast_option)
|
||||
gpu_choice_var.set(str(["0 0", "1 0", "0 1", "1 1"].index(str(dict["useclblast"][0]) + " " + str(dict["useclblast"][1])) + 1))
|
||||
elif "usecuda" in dict and dict["usecuda"]:
|
||||
if "usecuda" in dict and dict["usecuda"]:
|
||||
if cublas_option is not None or hipblas_option is not None:
|
||||
if cublas_option:
|
||||
runopts_var.set(cublas_option)
|
||||
@@ -6818,6 +6695,14 @@ def show_gui():
|
||||
if opt in dict["usevulkan"]:
|
||||
gpu_choice_var.set(str(opt+1))
|
||||
break
|
||||
elif "failsafe" in dict and dict["failsafe"]:
|
||||
if vulkan_failsafe_option is not None:
|
||||
runopts_var.set(vulkan_failsafe_option)
|
||||
gpu_choice_var.set("All")
|
||||
for opt in range(0,4):
|
||||
if opt in dict["usevulkan"]:
|
||||
gpu_choice_var.set(str(opt+1))
|
||||
break
|
||||
else:
|
||||
if vulkan_option is not None:
|
||||
runopts_var.set(vulkan_option)
|
||||
@@ -7493,7 +7378,7 @@ def load_config_cli(filename):
|
||||
setattr(args, key, value)
|
||||
if args.istemplate:
|
||||
print("\nA .kcppt template was selected from CLI...")
|
||||
if (args.usecuda is None) and (args.usevulkan is None) and (args.useclblast is None):
|
||||
if (args.usecuda is None) and (args.usevulkan is None):
|
||||
print("Automatically selecting your backend...")
|
||||
auto_set_backend_cli()
|
||||
|
||||
@@ -7510,7 +7395,6 @@ def convert_args_to_template(savdict):
|
||||
savdict["usemlock"] = False
|
||||
savdict["debugmode"] = 0
|
||||
savdict["ssl"] = None
|
||||
savdict["useclblast"] = None
|
||||
savdict["usecuda"] = None
|
||||
savdict["usevulkan"] = None
|
||||
savdict["usecpu"] = None
|
||||
@@ -7558,8 +7442,7 @@ def delete_old_pyinstaller():
|
||||
kobold_itemcheck3 = os.path.join(absdirpath, 'koboldcpp.py')
|
||||
kobold_itemcheck4 = os.path.join(absdirpath, 'cublasLt64_11.dll')
|
||||
kobold_itemcheck5 = os.path.join(absdirpath, 'cublas64_11.dll')
|
||||
kobold_itemcheck6 = os.path.join(absdirpath, 'clblast.dll')
|
||||
if os.path.exists(kobold_itemcheck1) or os.path.exists(kobold_itemcheck2) or os.path.exists(kobold_itemcheck3) or (os.path.exists(kobold_itemcheck4) and os.path.exists(kobold_itemcheck5) and os.path.exists(kobold_itemcheck6)):
|
||||
if os.path.exists(kobold_itemcheck1) or os.path.exists(kobold_itemcheck2) or os.path.exists(kobold_itemcheck3) or (os.path.exists(kobold_itemcheck4) and os.path.exists(kobold_itemcheck5)):
|
||||
try:
|
||||
shutil.rmtree(absdirpath)
|
||||
print(f"Deleted orphaned pyinstaller dir: {absdirpath}")
|
||||
@@ -7833,7 +7716,7 @@ def main(launch_args, default_args):
|
||||
return
|
||||
|
||||
if args.testmemory:
|
||||
fetch_gpu_properties(True, True, True, testmemory=True)
|
||||
fetch_gpu_properties(True, True, testmemory=True)
|
||||
return
|
||||
|
||||
#prevent disallowed combos
|
||||
@@ -8298,14 +8181,14 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
||||
print("MacOS detected: Auto GPU layers set to maximum")
|
||||
args.gpulayers = 200
|
||||
elif not shouldavoidgpu and args.model_param and os.path.exists(args.model_param):
|
||||
if (args.usecuda is None) and (args.usevulkan is None) and (args.useclblast is None):
|
||||
if (args.usecuda is None) and (args.usevulkan is None):
|
||||
print("No GPU or CPU backend was selected. Trying to assign one for you automatically...")
|
||||
auto_set_backend_cli()
|
||||
if MaxMemory[0] == 0: #try to get gpu vram for cuda if not picked yet
|
||||
fetch_gpu_properties(False,True,True)
|
||||
fetch_gpu_properties(True,True)
|
||||
pass
|
||||
if args.gpulayers==-1:
|
||||
if MaxMemory[0] > 0 and (not args.usecpu) and ((args.usecuda is not None) or (args.usevulkan is not None) or (args.useclblast is not None) or sys.platform=="darwin"):
|
||||
if MaxMemory[0] > 0 and (not args.usecpu) and ((args.usecuda is not None) or (args.usevulkan is not None) or sys.platform=="darwin"):
|
||||
extract_modelfile_params(args.model_param,args.sdmodel,args.whispermodel,args.mmproj,args.draftmodel,args.ttsmodel if args.ttsgpu else "",args.embeddingsmodel if args.embeddingsgpu else "")
|
||||
layeramt = autoset_gpu_layers(args.contextsize,args.sdquant,args.batchsize,(args.quantkv if args.flashattention else 0))
|
||||
print(f"Auto Recommended GPU Layers: {layeramt}")
|
||||
@@ -8829,7 +8712,6 @@ if __name__ == '__main__':
|
||||
compatgroup = parser.add_mutually_exclusive_group()
|
||||
compatgroup.add_argument("--usecuda", "--usecublas", "--usehipblas", help="Use CUDA for GPU Acceleration. Requires CUDA. Enter a number afterwards to select and use 1 GPU. Leaving no number will use all GPUs.", nargs='*',metavar=('[main GPU ID] [mmq|nommq] [rowsplit]'), choices=['normal', 'lowvram', '0', '1', '2', '3', 'all', 'mmq', 'nommq', 'rowsplit'])
|
||||
compatgroup.add_argument("--usevulkan", help="Use Vulkan for GPU Acceleration. Can optionally specify one or more GPU Device ID (e.g. --usevulkan 0), leave blank to autodetect.", metavar=('[Device IDs]'), nargs='*', type=int, default=None)
|
||||
compatgroup.add_argument("--useclblast", help="Use CLBlast for GPU Acceleration. Must specify exactly 2 arguments, platform ID and device ID (e.g. --useclblast 1 0).", type=int, choices=range(0,9), nargs=2)
|
||||
compatgroup.add_argument("--usecpu", help="Do not use any GPU acceleration (CPU Only)", action='store_true')
|
||||
parser.add_argument("--contextsize","--ctx-size", "-c", help="Controls the memory allocated for maximum context size, only change if you need more RAM for big contexts. (default 8192).",metavar=('[256 to 262144]'), type=check_range(int,256,262144), default=8192)
|
||||
parser.add_argument("--gpulayers","--gpu-layers","--n-gpu-layers","-ngl", help="Set number of layers to offload to GPU when using GPU. Requires GPU. Set to -1 to try autodetect, set to 0 to disable GPU offload.",metavar=('[GPU layers]'), nargs='?', const=1, type=int, default=-1)
|
||||
@@ -8855,7 +8737,7 @@ if __name__ == '__main__':
|
||||
compatgroup3.add_argument("--usemmap", help="If set, uses mmap to load model.", action='store_true')
|
||||
advparser.add_argument("--usemlock","--mlock", help="Enables mlock, preventing the RAM used to load the model from being paged out. Not usually recommended.", action='store_true')
|
||||
advparser.add_argument("--noavx2", help="Do not use AVX2 instructions, a slower compatibility mode for older devices.", action='store_true')
|
||||
advparser.add_argument("--failsafe", help="Use failsafe mode, extremely slow CPU only compatibility mode that should work on all devices. Can be combined with useclblast if your device supports OpenCL.", action='store_true')
|
||||
advparser.add_argument("--failsafe", help="Use failsafe mode, extremely old CPU compatibility mode that should work on all devices.", action='store_true')
|
||||
advparser.add_argument("--debugmode", help="Shows additional debug info in the terminal.", nargs='?', const=1, type=int, default=0)
|
||||
advparser.add_argument("--onready", help="An optional shell command to execute after the model has been loaded.", metavar=('[shell command]'), type=str, default="",nargs=1)
|
||||
advparser.add_argument("--benchmark", help="Do not start server, instead run benchmarks. If filename is provided, appends results to provided file.", metavar=('[filename]'), nargs='?', const="stdout", type=str, default=None)
|
||||
|
||||
Reference in New Issue
Block a user