diff --git a/expose.h b/expose.h index 3d1da4fd0..f0fef3cc1 100644 --- a/expose.h +++ b/expose.h @@ -51,6 +51,7 @@ struct load_model_inputs const int visionmaxtokens = -1; const bool use_mmap = false; const bool use_mlock = false; + const bool use_direct_io = false; const bool no_host = false; const bool use_mtp = false; const bool use_smartcontext = false; diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index baf861799..1bc941163 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -3274,11 +3274,22 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in // Match llama-server's target rollback slots for speculative verification. llama_ctx_params.n_rs_seq = inputs.draft_amount; } - model_params.load_mode = inputs.use_mlock ? (inputs.use_mmap ? LLAMA_LOAD_MODE_MMAP_MLOCK : LLAMA_LOAD_MODE_MLOCK) : (inputs.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE); + if(inputs.use_direct_io) + { + model_params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; + } + else if(inputs.use_mlock) + { + model_params.load_mode = inputs.use_mmap ? LLAMA_LOAD_MODE_MMAP_MLOCK : LLAMA_LOAD_MODE_MLOCK; + } + else + { + model_params.load_mode = inputs.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE; + } model_params.n_gpu_layers = inputs.gpulayers; model_params.no_host = inputs.no_host; model_params.load_mtp = inputs.use_mtp; - kcpp_permit_any_repack = (inputs.use_mmap?false:true); + kcpp_permit_any_repack = !(inputs.use_mmap || inputs.use_direct_io); //set device overrides if needed std::vector devices_override; diff --git a/koboldcpp.py b/koboldcpp.py index e34c47dba..80e22847d 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -318,6 +318,7 @@ class load_model_inputs(ctypes.Structure): ("visionmaxtokens", ctypes.c_int), ("use_mmap", ctypes.c_bool), ("use_mlock", ctypes.c_bool), + ("use_direct_io", ctypes.c_bool), ("no_host", ctypes.c_bool), ("use_mtp", ctypes.c_bool), ("use_smartcontext", ctypes.c_bool), @@ -2020,6 +2021,7 @@ def load_model(model_filename): inputs.blasthreads = args.blasthreads inputs.use_mmap = args.usemmap inputs.use_mlock = args.usemlock + inputs.use_direct_io = args.usedirectio inputs.no_host = False inputs.use_mtp = args.usemtp inputs.lora_filename = "".encode("UTF-8") @@ -8562,6 +8564,7 @@ def show_gui(): highpriority = ctk.IntVar() usemmap = ctk.IntVar(value=0) usemlock = ctk.IntVar() + usedirectio = ctk.IntVar() debugmode = ctk.IntVar() keepforeground = ctk.IntVar() terminalonly = ctk.IntVar() @@ -9240,6 +9243,7 @@ def show_gui(): quick_boxes = { "Launch Browser": [launchbrowser, "Launches your default browser after model loading is complete"], "Use MMAP": [usemmap, "Use mmap to load models if enabled, model will not be unloadable"], + "Direct I/O": [usedirectio, "Use direct I/O when loading GGUF models. May improve cold-load times on some storage."], "Use ContextShift": [contextshift_var, "Uses Context Shifting to reduce reprocessing.\nRecommended. Check the wiki for more info."], "Remote Tunnel": [remotetunnel_var, "Creates a trycloudflare tunnel.\nAllows you to access koboldcpp from other devices over an internet URL."], "Use FlashAttention": [flashattention_var, "Enable flash attention for GGUF models."], @@ -9298,6 +9302,7 @@ def show_gui(): "High Priority": [highpriority, "Increases the koboldcpp process priority.\nMay cause lag or slowdown instead. Not recommended."], "Use MMAP": [usemmap, "Use mmap to load models if enabled, model will not be unloadable"], "Use mlock": [usemlock, "Enables mlock, preventing the RAM used to load the model from being paged out."], + "Direct I/O": [usedirectio, "Use direct I/O when loading GGUF models. May improve cold-load times on some storage."], "Debug Mode": [debugmode, "Enables debug mode, with extra info printed to the terminal."], "Keep Foreground": [keepforeground, "Bring KoboldCpp to the foreground every time there is a new generation."], "CLI Terminal Only": [terminalonly, "Does not launch KoboldCpp HTTP server. Instead, enables KoboldCpp from the command line, accepting interactive console input and displaying responses to the terminal."], @@ -9793,7 +9798,8 @@ def show_gui(): args.debugmode = debugmode.get() args.launch = launchbrowser.get()==1 args.highpriority = highpriority.get()==1 - args.usemmap = usemmap.get()==1 + args.usedirectio = usedirectio.get()==1 + args.usemmap = (usemmap.get()==1 and not args.usedirectio) args.smartcontext = smartcontext_var.get()==1 args.noflashattention = flashattention_var.get()==0 args.noshift = contextshift_var.get()==0 @@ -9849,6 +9855,7 @@ def show_gui(): args.noavx2 = True args.usecpu = True args.usemmap = False + args.usedirectio = False args.failsafe = True args.tensor_split = None if tensor_split_str_vars.get()!="": @@ -10040,7 +10047,8 @@ def show_gui(): debugmode.set(mydict["debugmode"]) launchbrowser.set(1 if "launch" in mydict and mydict["launch"] else 0) highpriority.set(1 if "highpriority" in mydict and mydict["highpriority"] else 0) - usemmap.set(1 if "usemmap" in mydict and mydict["usemmap"] else 0) + usedirectio.set(1 if (("usedirectio" in mydict and mydict["usedirectio"]) or ("directio" in mydict and mydict["directio"])) else 0) + usemmap.set(1 if "usemmap" in mydict and mydict["usemmap"] and not usedirectio.get() else 0) smartcontext_var.set(1 if "smartcontext" in mydict and mydict["smartcontext"] else 0) flashattention_var.set(0 if "noflashattention" in mydict and mydict["noflashattention"] else 1) contextshift_var.set(0 if "noshift" in mydict and mydict["noshift"] else 1) @@ -10972,6 +10980,7 @@ def convert_args_to_template(savdict): savdict["adminpassword"] = None savdict["usemmap"] = False savdict["usemlock"] = False + savdict["usedirectio"] = False savdict["debugmode"] = 0 savdict["ssl"] = None savdict["usecuda"] = None @@ -12851,6 +12860,7 @@ if __name__ == '__main__': advparser.add_argument("--usemlock","--mlock", help="Enables mlock, preventing the RAM used to load the model from being paged out. Not usually recommended.", action='store_true') compatgroup3 = advparser.add_mutually_exclusive_group() compatgroup3.add_argument("--usemmap", help="If set, uses mmap to load model.", action='store_true') + compatgroup3.add_argument("--usedirectio","--directio","--direct-io","-dio", help="Use direct I/O to load GGUF models if available. May improve cold-load times on some storage.", action='store_true') advparser.add_argument("--visionmaxres", metavar=('[max px]'), help="Clamp MMProj vision maximum allowed resolution. Allowed values are between 512 to 2048 px (default 1024).", type=int, default=default_visionmaxres) advparser.add_argument("--visionmaxtokens","--image-max-tokens", metavar=('[tokens]'), help="Override the maximum tokens for the MMProj embedding (default -1).", type=int, default=-1) advparser.add_argument("--visionmintokens","--image-min-tokens", metavar=('[tokens]'), help="Override the minimum tokens for the MMProj embedding (default -1).", type=int, default=-1)