From 173d340c433a96ba8d78203dec16be7c9b8b4fad Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:15:26 +0800 Subject: [PATCH] wip on adding ubatch --- expose.h | 2 +- ggml/src/ggml-backend.cpp | 2 +- gpttype_adapter.cpp | 17 ++++++++++------- koboldcpp.py | 14 ++++++-------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/expose.h b/expose.h index d8408ae03..eb421dece 100644 --- a/expose.h +++ b/expose.h @@ -60,6 +60,7 @@ struct load_model_inputs const bool use_fastforward = false; const int kcpp_main_gpu = -1; const int batchsize = 512; + const int ubatchsize = -1; const bool autofit = false; const int autofit_tax_mb = 0; const int gpulayers = 0; @@ -84,7 +85,6 @@ struct load_model_inputs const int swa_padding = 0; const bool smartcache = false; const int smartcacheslots = 0; - const bool pipelineparallel = false; const float lora_multiplier = 1.0f; const char * devices_override = nullptr; const bool quiet = false; diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index a57d22906..e5d93dadf 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -770,7 +770,7 @@ static bool ggml_is_view_op(enum ggml_op op) { #ifndef GGML_SCHED_MAX_COPIES //kcpp can reduce this if you want to try make buffer sizes smaller on multigpu -#define GGML_SCHED_MAX_COPIES 2 +#define GGML_SCHED_MAX_COPIES 4 #endif struct ggml_backend_sched_split { diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index d0dfcf45a..aea2b8331 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -3038,9 +3038,9 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in kcpp_data->n_threads = inputs.threads; kcpp_data->n_blasthreads = inputs.blasthreads; bool isGguf = (file_format == FileFormat::GGUF_GENERIC); - kcpp_pipeline_parallelism = inputs.pipelineparallel; + kcpp_pipeline_parallelism = false; kcpp_data->n_batch = GetBatchSize(inputs.batchsize, in_file_format); - kcpp_data->n_ubatch = kcpp_data->n_batch; + kcpp_data->n_ubatch = inputs.ubatchsize > 0 ? std::min(inputs.ubatchsize, kcpp_data->n_batch) : kcpp_data->n_batch; continuous_batching_slots = (isGguf && inputs.continuous_batching_slots > 1) ? inputs.continuous_batching_slots : 0; if(continuous_batching_slots > 0) { @@ -3049,11 +3049,6 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in kcpp_data->vision_min_tokens = inputs.visionmintokens; kcpp_data->vision_max_tokens = inputs.visionmaxtokens; vision_max_res = inputs.visionmaxres; - if(isGguf && kcpp_pipeline_parallelism) - { - //double the logical batch, while keeping the physical batch the same, pipeline parallel set GGML_SCHED_MAX_COPIES to 2 - kcpp_data->n_batch *= 2; - } kcpp_data->flash_attn = inputs.flash_attention; kcpp_data->model_filename = inputs.model_filename; kcpp_data->use_smartcontext = inputs.use_smartcontext; @@ -3276,6 +3271,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in connect_rpc_servers(servers); } + if (kcpp_data->n_ubatch < kcpp_data->n_batch) { + int gpu_count = 0; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + gpu_count += ggml_backend_dev_type(ggml_backend_dev_get(i)) == GGML_BACKEND_DEVICE_TYPE_GPU; + } + kcpp_pipeline_parallelism = gpu_count > 1; + } + llama_model_params model_params = llama_model_default_params(); llama_context_params llama_ctx_params = llama_context_default_params(); llama_ctx_params.n_ctx = clamped_max_context_length; diff --git a/koboldcpp.py b/koboldcpp.py index 35aae8da8..6744e9ab6 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -252,6 +252,7 @@ deprecated_keys = { "noblas", "nommap", "pipelineparallel", + "nopipelineparallel", "sdnotile", "forceversion", "sdgendefaults", @@ -328,6 +329,7 @@ class load_model_inputs(ctypes.Structure): ("use_fastforward", ctypes.c_bool), ("kcpp_main_gpu", ctypes.c_int), ("batchsize", ctypes.c_int), + ("ubatchsize", ctypes.c_int), ("autofit", ctypes.c_bool), ("autofit_tax_mb", ctypes.c_int), ("gpulayers", ctypes.c_int), @@ -352,7 +354,6 @@ class load_model_inputs(ctypes.Structure): ("swa_padding", ctypes.c_int), ("smartcache", ctypes.c_bool), ("smartcacheslots", ctypes.c_int), - ("pipelineparallel", ctypes.c_bool), ("lora_multiplier", ctypes.c_float), ("devices_override", ctypes.c_char_p), ("quiet", ctypes.c_bool), @@ -2124,6 +2125,7 @@ def load_model(model_filename): else: inputs.quant_k = inputs.quant_v = 0 inputs.batchsize = args.batchsize + inputs.ubatchsize = args.ubatchsize inputs.autofit = args.autofit inputs.autofit_tax_mb = int(args.autofitpadding) + int(calulated_gpu_overhead/(1024*1024)) inputs.gpulayers = args.gpulayers @@ -2172,7 +2174,6 @@ def load_model(model_filename): sclimit = (savestate_limit_default if scint<=1 else scint) savestate_limit = sclimit inputs.smartcacheslots = sclimit - inputs.pipelineparallel = (not args.nopipelineparallel) inputs.continuous_batching_slots = args.parallelrequests if (args.parallelrequests>1) else 0 inputs.rpc_mode = (2 if args.rpcmode=="host" else (1 if args.rpcmode=="connect" else 0)) inputs.rpc_targets = (args.rpctargets if args.rpcmode=="connect" else "").encode("UTF-8") @@ -8768,7 +8769,6 @@ def show_gui(): debugmode = ctk.IntVar() keepforeground = ctk.IntVar() terminalonly = ctk.IntVar() - pipelineparallel = ctk.IntVar(value=1) quietmode = ctk.IntVar(value=0) nocertifymode = ctk.IntVar(value=0) @@ -9530,7 +9530,6 @@ def show_gui(): "Direct I/O": [usedirectio, "Use direct I/O when loading GGUF models. May improve cold-load times on some storage."], "Keep Foreground": [keepforeground, "Bring KoboldCpp to the foreground every time there is a new generation."], "CLI Terminal Only": [terminalonly, "Does not launch KoboldCpp HTTP server. Instead, enables KoboldCpp from the command line, accepting interactive console input and displaying responses to the terminal."], - "Pipeline Parallel": [pipelineparallel, "Enable Pipeline Parallelism for faster multigpu speeds but using more memory, only active for multigpu."], } for idx, (name, properties) in enumerate(hardware_boxes.items()): @@ -10064,7 +10063,6 @@ def show_gui(): args.remotetunnel = remotetunnel_var.get()==1 args.foreground = keepforeground.get()==1 args.cli = terminalonly.get()==1 - args.nopipelineparallel = pipelineparallel.get()==0 args.quiet = quietmode.get()==1 args.nocertify = nocertifymode.get()==1 args.nomodel = nomodel.get()==1 @@ -10315,7 +10313,6 @@ def show_gui(): remotetunnel_var.set(1 if "remotetunnel" in mydict and mydict["remotetunnel"] else 0) keepforeground.set(1 if "foreground" in mydict and mydict["foreground"] else 0) terminalonly.set(1 if "cli" in mydict and mydict["cli"] else 0) - pipelineparallel.set(0 if "nopipelineparallel" in mydict and mydict["nopipelineparallel"] else 1) quietmode.set(1 if "quiet" in mydict and mydict["quiet"] else 0) nocertifymode.set(1 if "nocertify" in mydict and mydict["nocertify"] else 0) nomodel.set(1 if "nomodel" in mydict and mydict["nomodel"] else 0) @@ -13049,7 +13046,8 @@ if __name__ == '__main__': advparser.add_argument("--analyze", metavar=('[filename]'), help="Reads the metadata, weight types and tensor names in any GGUF or safetensors file.", default="") advparser.add_argument("--autofit","--fit","-fit", help="Forces autofit, which attempts to fit the model in the best possible way. Overrides everything else.", action='store_true') advparser.add_argument("--autofitpadding", metavar=('[padding in MB]'), help="How much spare allowance in MB should autofit reserve? If it's too little, the load might fail.", type=int, default=default_autofit_padding) - advparser.add_argument("--batchsize","--blasbatchsize","--batch-size","-b", help="Sets the batch size used in batched processing (default 512). Setting it to -1 disables batched mode, but keeps other benefits like GPU offload.", type=int,choices=[-1,16,32,64,128,256,512,1024,2048,4096], default=512) + advparser.add_argument("--batchsize","--blasbatchsize","--batch-size","-b", help="Sets the logical batch size used in batched processing (default 512). Setting it to -1 disables batched mode, but keeps other benefits like GPU offload.", type=int,choices=[-1,16,32,64,128,256,512,1024,2048,4096], default=512) + advparser.add_argument("--ubatchsize","--ubatch-size","-ub", help="Sets the physical batch size used in batched processing. Setting it to -1 uses the same value as batchsize (default).", type=int,choices=[-1,16,32,64,128,256,512,1024,2048,4096], default=-1) advparser.add_argument("--benchmark", help="Do not start server, instead run benchmarks. If filename is provided, appends results to provided file.", metavar=('[filename]'), nargs='?', const="stdout", type=str, default=None) advparser.add_argument("--blasthreads","--batchthreads","--threadsbatch","--threads-batch", help="Use a different number of threads during batching if specified. Otherwise, has the same value as --threads",metavar=('[threads]'), type=int, default=0) advparser.add_argument("--chatcompletionsadapter", metavar=('[filename]'), help="Select an optional ChatCompletions Adapter JSON file to force custom instruct tags.", default="AutoGuess") @@ -13097,7 +13095,6 @@ if __name__ == '__main__': advparser.add_argument("--nobostoken", help="Prevents BOS token from being added at the start of any prompt. Usually NOT recommended for most models.", action='store_true') advparser.add_argument("--nommq", help="Disables MMQ, only used for cuda backend. This flag may be removed in future.", action='store_true') advparser.add_argument("--nomodel", help="Allows you to launch the GUI alone, without selecting any model.", action='store_true') - advparser.add_argument("--nopipelineparallel", help="Disable Pipeline Parallelism. Pipeline Parallelism provides faster multigpu speeds but using more memory, only active for multigpu.", action='store_true') advparser.add_argument("--noshift","--no-context-shift", help="If set, do not attempt to Trim and Shift the GGUF context.", action='store_true') advparser.add_argument("--noswa","--swa-full", help="If set, uses full-size SWA KV Cache. Otherwise, SWA will be enabled automatically on models that support it. SWA saves memory but cannot be used with context shifting.", action='store_true') advparser.add_argument("--onready", help="An optional shell command to execute after the model has been loaded.", metavar=('[shell command]'), type=str, default="") @@ -13216,6 +13213,7 @@ if __name__ == '__main__': deprecatedgroup.add_argument("--sdconfig", help=argparse.SUPPRESS, nargs='+') compatgroup.add_argument("--noblas", help=argparse.SUPPRESS, action='store_true') compatgroup3.add_argument("--nommap","--no-mmap", help=argparse.SUPPRESS, action='store_true') + deprecatedgroup.add_argument("--nopipelineparallel", help=argparse.SUPPRESS, action='store_true') #now to automatically enable when ubatch < batchsize and multigpu deprecatedgroup.add_argument("--pipelineparallel", help=argparse.SUPPRESS, action='store_true') #changed to nopipelineparallel deprecatedgroup.add_argument("--sdnotile", help=argparse.SUPPRESS, action='store_true') # legacy option, see sdtiledvae deprecatedgroup.add_argument("--sdvaecpu", help=argparse.SUPPRESS, action='store_true') # legacy option, see sdvaedevice