From 5cf21443bce2db00bb157f203bcc2dd16ad85009 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:29:30 +0800 Subject: [PATCH] added autofit padding. autofit is now in the quick menu --- gpttype_adapter.cpp | 2 +- koboldcpp.py | 38 +++++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 6787b55d9..feeada07a 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -2554,7 +2554,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in if(inputs.autofit) { common_params temp_params; - size_t taxmb = 1024 + inputs.autofit_tax_mb; + size_t taxmb = inputs.autofit_tax_mb; printf("\nAttempting to use llama.cpp's automating fitting code. This will override all your layer configs, may or may not work!\n"); //zero out any customizations made tenos.clear(); diff --git a/koboldcpp.py b/koboldcpp.py index 2cca8fc4b..d8644a215 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -58,6 +58,7 @@ savestate_limit = 0 #savestate slots start at 0, only set when load model default_vae_tile_threshold = 768 default_native_ctx = 16384 overridekv_max = 4 +default_autofit_padding = 1024 # abuse prevention stop_token_max = 256 @@ -640,9 +641,9 @@ def get_default_threads(): processor = platform.processor() if 'Intel' in processor: default_threads = (8 if default_threads > 8 else default_threads) #this helps avoid e-cores. - if default_threads > 48: - print(f"Auto CPU Threads capped at 48 (instead of {default_threads}). You can override this by passing an explicit number of --threads.") - default_threads = 48 + if default_threads > 64: + print(f"Auto CPU Threads capped at 64 (instead of {default_threads}). You can override this by passing an explicit number of --threads.") + default_threads = 64 return default_threads def pick_existant_file(ntoption,nonntoption): @@ -1608,7 +1609,7 @@ def load_model(model_filename): inputs.quant_k = inputs.quant_v = 0 inputs.batchsize = args.batchsize inputs.autofit = args.autofit - inputs.autofit_tax_mb = int(calulated_gpu_overhead/(1024*1024)) + inputs.autofit_tax_mb = int(args.autofitpadding) + int(calulated_gpu_overhead/(1024*1024)) inputs.gpulayers = args.gpulayers if args.overridenativecontext and args.overridenativecontext>0: inputs.overridenativecontext = args.overridenativecontext @@ -5515,6 +5516,7 @@ def show_gui(): threads_var = ctk.StringVar(value=str(default_threads)) runopts_var = ctk.StringVar() gpu_choice_var = ctk.StringVar(value="1") + autofit_padding_var = ctk.StringVar(value=str(default_autofit_padding)) launchbrowser = ctk.IntVar(value=1) highpriority = ctk.IntVar() @@ -6108,6 +6110,11 @@ def show_gui(): gpu_layers_entry.grid_remove() quick_gpu_layers_label.grid_remove() quick_gpu_layers_entry.grid_remove() + autofit_padding_label.grid(row=6, column=0, padx=8, pady=1, stick="nw") + autofit_padding_entry.grid(row=6, column=0, padx=160, pady=1, stick="nw") + else: + autofit_padding_label.grid_remove() + autofit_padding_entry.grid_remove() changed_gpulayers_estimate() changed_gpu_choice_var() @@ -6140,21 +6147,21 @@ def show_gui(): "Use MMAP": [usemmap, "Use mmap to load models if enabled, model will not be unloadable"], "Use ContextShift": [contextshift_var, "Uses Context Shifting to reduce reprocessing.\nRecommended. Check the wiki for more info."], "Remote Tunnel": [remotetunnel_var, "Creates a trycloudflare tunnel.\nAllows you to access koboldcpp from other devices over an internet URL."], + "Use FlashAttention": [flashattention_var, "Enable flash attention for GGUF models."], + "AutoFit": [autofit_var, "Automatically attempt to fit the model in the best possible way. Overrides everything else.\nNot recommended for multi model setups. Experimental."], "Quiet Mode": [quietmode, "Prevents all generation related terminal output from being displayed."] } for idx, (name, properties) in enumerate(quick_boxes.items()): makecheckbox(quick_tab, name, properties[0], int(idx/2) + 20, idx % 2, tooltiptxt=properties[1]) - makecheckbox(quick_tab, "Use FlashAttention", flashattention_var, 22, 1, tooltiptxt="Enable flash attention for GGUF models.") - # context size - makeslider(quick_tab, "Context Size:", contextsize_text, context_var, 0, len(contextsize_text)-1, 30, width=280, set=7,tooltip="What is the maximum context size to support. Model specific. You cannot exceed it.\nLarger contexts require more memory, and not all models support it.") + makeslider(quick_tab, "Context Size:", contextsize_text, context_var, 0, len(contextsize_text)-1, 40, width=280, set=7,tooltip="What is the maximum context size to support. Model specific. You cannot exceed it.\nLarger contexts require more memory, and not all models support it.") # load model - makefileentry(quick_tab, "GGUF Text Model:", "Select GGUF or GGML Model File", model_var, 40, 280, onchoosefile=on_picked_model_file,tooltiptxt="Select a GGUF or GGML model file on disk to be loaded.") + makefileentry(quick_tab, "GGUF Text Model:", "Select GGUF or GGML Model File", model_var, 50, 280, onchoosefile=on_picked_model_file,tooltiptxt="Select a GGUF or GGML model file on disk to be loaded.") model_var.trace_add("write", gui_changed_modelfile) - ctk.CTkButton(quick_tab, width=70, text = "HF Search", command = model_searcher ).grid(row=41,column=1, stick="sw", padx= 202, pady=2) + ctk.CTkButton(quick_tab, width=70, text = "HF Search", command = model_searcher ).grid(row=51,column=1, stick="sw", padx= 202, pady=2) # Hardware Tab hardware_tab = tabcontent["Hardware"] @@ -6175,7 +6182,8 @@ def show_gui(): lowvram_box = makecheckbox(hardware_tab, "No KV offload", lowvram_var, 4,0, tooltiptxt='Avoid offloading KV Cache or scratch buffers to VRAM.\nAllows more layers to fit, but may result in a large speed loss.') mmq_box = makecheckbox(hardware_tab, "Use MMQ", mmq_var, 4,0,padx=160, tooltiptxt="Enable MMQ mode to use finetuned kernels instead of default CuBLAS/HipBLAS for prompt processing.\nRead the wiki. Speed may vary.") splitmode_box = makecheckbox(hardware_tab, "Row-Split", rowsplit_var, 4,0,padx=300, tooltiptxt="Split rows across GPUs instead of splitting layers and KV across GPUs.\nUses the main GPU for small tensors and intermediate results. Speed may vary.") - gpu_layers_entry,gpu_layers_label = makelabelentry(hardware_tab,"GPU Layers:", gpulayers_var, 6, 50, padx=160,singleline=True,tooltip="How many layers to offload onto the GPU.\nVRAM intensive, usage increases with model and context size.\nRequires some trial and error to find the best fit value.\n\nCommon values for total layers, accuracy not guaranteed.\n\nLlama/Mistral 7b/8b: 33\nSolar 10.7b/11b: 49\nLlama 13b: 41\nLlama 20b(stack): 63\nLlama/Yi 34b: 61\nMixtral 8x7b: 33\nLlama 70b: 81") + gpu_layers_entry,gpu_layers_label = makelabelentry(hardware_tab,"GPU Layers:", gpulayers_var, 6, 50, padx=160,singleline=True,tooltip="How many layers to offload onto the GPU.\nUsage varies based on model type and increases with model and context size.\nRequires some trial and error to find the best fit value.\n\nNote: The auto estimation is often inaccurate! Please set layers yourself for best results!") + autofit_padding_entry,autofit_padding_label = makelabelentry(hardware_tab,"Autofit Padding (MB):", autofit_padding_var, 6, 50, padx=160,singleline=True,tooltip="How much spare allowance in MB should autofit reserve? If it's too little, the load might fail.") layercounter_label = ctk.CTkLabel(hardware_tab, text="") layercounter_label.grid(row=6, column=0, padx=230, sticky="W") layercounter_label.configure(text_color="#ffff00") @@ -6209,7 +6217,7 @@ def show_gui(): makecheckbox(hardware_tab, "Use FlashAttention", flashattention_var, 100, command=toggleflashattn, tooltiptxt="Enable flash attention for GGUF models.") - makecheckbox(hardware_tab, "AutoFit (llama.cpp mode)", autofit_var, 100,0,command=changed_autofit,padx=160, tooltiptxt="Automatically attempt to fit the model in the best possible way. Overrides everything else. Not recommended for multi model setups. Experimental.") + makecheckbox(hardware_tab, "AutoFit", autofit_var, 100,0,command=changed_autofit,padx=160, tooltiptxt="Automatically attempt to fit the model in the best possible way. Overrides everything else.\nNot recommended for multi model setups. Experimental.") ctk.CTkButton(hardware_tab , text = "Run Benchmark", command = guibench ).grid(row=110,column=0, stick="nw", padx= 8, pady=2) @@ -6565,6 +6573,8 @@ def show_gui(): args.failsafe = True if gpulayers_var.get(): args.gpulayers = (0 if gpulayers_var.get()=="" else int(gpulayers_var.get())) + if autofit_padding_var.get(): + args.autofitpadding = (default_autofit_padding if autofit_padding_var.get()=="" else int(autofit_padding_var.get())) if runopts_var.get()=="Use CPU": args.usecpu = True if runopts_var.get()=="Use CPU (Old CPU)": @@ -6875,6 +6885,11 @@ def show_gui(): autofit_var.set(1 if "autofit" in dict and dict["autofit"] else 0) model_var.set(dict["model_param"] if ("model_param" in dict and dict["model_param"]) else "") + if "autofitpadding" in dict and dict["autofitpadding"]: + autofit_padding_var.set(dict["autofitpadding"]) + else: + autofit_padding_var.set(str(default_autofit_padding)) + lora_var.set("") if "lora" in dict and dict["lora"]: if len(dict["lora"]) > 1: @@ -8954,6 +8969,7 @@ if __name__ == '__main__': advparser.add_argument("--mcpfile", metavar=('[mcp json file]'), help="Specify path to mcp.json which contains the Cladue Desktop compatible MCP server config.", default="") advparser.add_argument("--device", "-dev", metavar=(''), help="Set llama.cpp compatible device selection override. Comma separated. Overrides normal device choices.", default="") advparser.add_argument("--downloaddir", metavar=('[directory]'), help="Specify a directory that models will be downloaded to or searched from, if unset uses the working directory.", default="") + advparser.add_argument("--autofitpadding", metavar=('[padding in MB]'), help="How much spare allowance in MB should autofit reserve? If it's too little, the load might fail.", type=int, default=default_autofit_padding) hordeparsergroup = parser.add_argument_group('Horde Worker Commands') hordeparsergroup.add_argument("--hordemodelname", metavar=('[name]'), help="Sets your AI Horde display model name.", default="")