Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-26 15:15:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

added autofit padding. autofit is now in the quick menu

Browse Source
This commit is contained in:
Concedo
2026-02-07 18:29:30 +08:00
parent 812da8b75d
commit 5cf21443bc
2 changed files with 28 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+27 -11
View File
@@ -58,6 +58,7 @@ savestate_limit = 0 #savestate slots start at 0, only set when load model
default_vae_tile_threshold = 768
default_native_ctx = 16384
overridekv_max = 4
default_autofit_padding = 1024
# abuse prevention
stop_token_max = 256
@@ -640,9 +641,9 @@ def get_default_threads():
processor = platform.processor()
if 'Intel' in processor:
default_threads = (8 if default_threads > 8 else default_threads) #this helps avoid e-cores.
if default_threads > 48:
print(f"Auto CPU Threads capped at 48 (instead of {default_threads}). You can override this by passing an explicit number of --threads.")
default_threads = 48
if default_threads > 64:
print(f"Auto CPU Threads capped at 64 (instead of {default_threads}). You can override this by passing an explicit number of --threads.")
default_threads = 64
return default_threads
def pick_existant_file(ntoption,nonntoption):
@@ -1608,7 +1609,7 @@ def load_model(model_filename):
inputs.quant_k = inputs.quant_v = 0
inputs.batchsize = args.batchsize
inputs.autofit = args.autofit
inputs.autofit_tax_mb = int(calulated_gpu_overhead/(1024*1024))
inputs.autofit_tax_mb = int(args.autofitpadding) + int(calulated_gpu_overhead/(1024*1024))
inputs.gpulayers = args.gpulayers
if args.overridenativecontext and args.overridenativecontext>0:
inputs.overridenativecontext = args.overridenativecontext
@@ -5515,6 +5516,7 @@ def show_gui():
threads_var = ctk.StringVar(value=str(default_threads))
runopts_var = ctk.StringVar()
gpu_choice_var = ctk.StringVar(value="1")
autofit_padding_var = ctk.StringVar(value=str(default_autofit_padding))
launchbrowser = ctk.IntVar(value=1)
highpriority = ctk.IntVar()
@@ -6108,6 +6110,11 @@ def show_gui():
gpu_layers_entry.grid_remove()
quick_gpu_layers_label.grid_remove()
quick_gpu_layers_entry.grid_remove()
autofit_padding_label.grid(row=6, column=0, padx=8, pady=1, stick="nw")
autofit_padding_entry.grid(row=6, column=0, padx=160, pady=1, stick="nw")
else:
autofit_padding_label.grid_remove()
autofit_padding_entry.grid_remove()
changed_gpulayers_estimate()
changed_gpu_choice_var()
@@ -6140,21 +6147,21 @@ def show_gui():
"Use MMAP": [usemmap, "Use mmap to load models if enabled, model will not be unloadable"],
"Use ContextShift": [contextshift_var, "Uses Context Shifting to reduce reprocessing.\nRecommended. Check the wiki for more info."],
"Remote Tunnel": [remotetunnel_var, "Creates a trycloudflare tunnel.\nAllows you to access koboldcpp from other devices over an internet URL."],
"Use FlashAttention": [flashattention_var, "Enable flash attention for GGUF models."],
"AutoFit": [autofit_var, "Automatically attempt to fit the model in the best possible way. Overrides everything else.\nNot recommended for multi model setups. Experimental."],
"Quiet Mode": [quietmode, "Prevents all generation related terminal output from being displayed."]
}
for idx, (name, properties) in enumerate(quick_boxes.items()):
makecheckbox(quick_tab, name, properties[0], int(idx/2) + 20, idx % 2, tooltiptxt=properties[1])
makecheckbox(quick_tab, "Use FlashAttention", flashattention_var, 22, 1, tooltiptxt="Enable flash attention for GGUF models.")
# context size
makeslider(quick_tab, "Context Size:", contextsize_text, context_var, 0, len(contextsize_text)-1, 30, width=280, set=7,tooltip="What is the maximum context size to support. Model specific. You cannot exceed it.\nLarger contexts require more memory, and not all models support it.")
makeslider(quick_tab, "Context Size:", contextsize_text, context_var, 0, len(contextsize_text)-1, 40, width=280, set=7,tooltip="What is the maximum context size to support. Model specific. You cannot exceed it.\nLarger contexts require more memory, and not all models support it.")
# load model
makefileentry(quick_tab, "GGUF Text Model:", "Select GGUF or GGML Model File", model_var, 40, 280, onchoosefile=on_picked_model_file,tooltiptxt="Select a GGUF or GGML model file on disk to be loaded.")
makefileentry(quick_tab, "GGUF Text Model:", "Select GGUF or GGML Model File", model_var, 50, 280, onchoosefile=on_picked_model_file,tooltiptxt="Select a GGUF or GGML model file on disk to be loaded.")
model_var.trace_add("write", gui_changed_modelfile)
ctk.CTkButton(quick_tab, width=70, text = "HF Search", command = model_searcher ).grid(row=41,column=1, stick="sw", padx= 202, pady=2)
ctk.CTkButton(quick_tab, width=70, text = "HF Search", command = model_searcher ).grid(row=51,column=1, stick="sw", padx= 202, pady=2)
# Hardware Tab
hardware_tab = tabcontent["Hardware"]
@@ -6175,7 +6182,8 @@ def show_gui():
lowvram_box = makecheckbox(hardware_tab, "No KV offload", lowvram_var, 4,0, tooltiptxt='Avoid offloading KV Cache or scratch buffers to VRAM.\nAllows more layers to fit, but may result in a large speed loss.')
mmq_box = makecheckbox(hardware_tab, "Use MMQ", mmq_var, 4,0,padx=160, tooltiptxt="Enable MMQ mode to use finetuned kernels instead of default CuBLAS/HipBLAS for prompt processing.\nRead the wiki. Speed may vary.")
splitmode_box = makecheckbox(hardware_tab, "Row-Split", rowsplit_var, 4,0,padx=300, tooltiptxt="Split rows across GPUs instead of splitting layers and KV across GPUs.\nUses the main GPU for small tensors and intermediate results. Speed may vary.")
gpu_layers_entry,gpu_layers_label = makelabelentry(hardware_tab,"GPU Layers:", gpulayers_var, 6, 50, padx=160,singleline=True,tooltip="How many layers to offload onto the GPU.\nVRAM intensive, usage increases with model and context size.\nRequires some trial and error to find the best fit value.\n\nCommon values for total layers, accuracy not guaranteed.\n\nLlama/Mistral 7b/8b: 33\nSolar 10.7b/11b: 49\nLlama 13b: 41\nLlama 20b(stack): 63\nLlama/Yi 34b: 61\nMixtral 8x7b: 33\nLlama 70b: 81")
gpu_layers_entry,gpu_layers_label = makelabelentry(hardware_tab,"GPU Layers:", gpulayers_var, 6, 50, padx=160,singleline=True,tooltip="How many layers to offload onto the GPU.\nUsage varies based on model type and increases with model and context size.\nRequires some trial and error to find the best fit value.\n\nNote: The auto estimation is often inaccurate! Please set layers yourself for best results!")
autofit_padding_entry,autofit_padding_label = makelabelentry(hardware_tab,"Autofit Padding (MB):", autofit_padding_var, 6, 50, padx=160,singleline=True,tooltip="How much spare allowance in MB should autofit reserve? If it's too little, the load might fail.")
layercounter_label = ctk.CTkLabel(hardware_tab, text="")
layercounter_label.grid(row=6, column=0, padx=230, sticky="W")
layercounter_label.configure(text_color="#ffff00")
@@ -6209,7 +6217,7 @@ def show_gui():
makecheckbox(hardware_tab, "Use FlashAttention", flashattention_var, 100, command=toggleflashattn, tooltiptxt="Enable flash attention for GGUF models.")
makecheckbox(hardware_tab, "AutoFit (llama.cpp mode)", autofit_var, 100,0,command=changed_autofit,padx=160, tooltiptxt="Automatically attempt to fit the model in the best possible way. Overrides everything else. Not recommended for multi model setups. Experimental.")
makecheckbox(hardware_tab, "AutoFit", autofit_var, 100,0,command=changed_autofit,padx=160, tooltiptxt="Automatically attempt to fit the model in the best possible way. Overrides everything else.\nNot recommended for multi model setups. Experimental.")
ctk.CTkButton(hardware_tab , text = "Run Benchmark", command = guibench ).grid(row=110,column=0, stick="nw", padx= 8, pady=2)
@@ -6565,6 +6573,8 @@ def show_gui():
args.failsafe = True
if gpulayers_var.get():
args.gpulayers = (0 if gpulayers_var.get()=="" else int(gpulayers_var.get()))
if autofit_padding_var.get():
args.autofitpadding = (default_autofit_padding if autofit_padding_var.get()=="" else int(autofit_padding_var.get()))
if runopts_var.get()=="Use CPU":
args.usecpu = True
if runopts_var.get()=="Use CPU (Old CPU)":
@@ -6875,6 +6885,11 @@ def show_gui():
autofit_var.set(1 if "autofit" in dict and dict["autofit"] else 0)
model_var.set(dict["model_param"] if ("model_param" in dict and dict["model_param"]) else "")
if "autofitpadding" in dict and dict["autofitpadding"]:
autofit_padding_var.set(dict["autofitpadding"])
else:
autofit_padding_var.set(str(default_autofit_padding))
lora_var.set("")
if "lora" in dict and dict["lora"]:
if len(dict["lora"]) > 1:
@@ -8954,6 +8969,7 @@ if __name__ == '__main__':
advparser.add_argument("--mcpfile", metavar=('[mcp json file]'), help="Specify path to mcp.json which contains the Cladue Desktop compatible MCP server config.", default="")
advparser.add_argument("--device", "-dev", metavar=('<dev1,dev2,..>'), help="Set llama.cpp compatible device selection override. Comma separated. Overrides normal device choices.", default="")
advparser.add_argument("--downloaddir", metavar=('[directory]'), help="Specify a directory that models will be downloaded to or searched from, if unset uses the working directory.", default="")
advparser.add_argument("--autofitpadding", metavar=('[padding in MB]'), help="How much spare allowance in MB should autofit reserve? If it's too little, the load might fail.", type=int, default=default_autofit_padding)
hordeparsergroup = parser.add_argument_group('Horde Worker Commands')
hordeparsergroup.add_argument("--hordemodelname", metavar=('[name]'), help="Sets your AI Horde display model name.", default="")