From 79209a14a8e35684b9f26d6ded15a4bb6a16a56b Mon Sep 17 00:00:00 2001 From: Eso <65901558+esolithe@users.noreply.github.com> Date: Tue, 31 Mar 2026 09:49:59 +0100 Subject: [PATCH] feat: Autoswap functionality (#2080) * feat: Autoswap mode (cherry-picked from remoteManagement) Co-authored-by: esolithe <65901558+esolithe@users.noreply.github.com> * fix: Remove modelOverride, add triggered_sleeping to autoswap unload timeout branch Agent-Logs-Url: https://github.com/esolithe/esobold/sessions/1ddb3f88-43b4-4234-aa41-0fe6c9976db4 Co-authored-by: esolithe <65901558+esolithe@users.noreply.github.com> * fix: Remove esobold-specific GUI elements from admin tab, renumber remaining rows Agent-Logs-Url: https://github.com/esolithe/esobold/sessions/6a2e4ec3-cb19-4f98-b00f-bdb13749ead3 Co-authored-by: esolithe <65901558+esolithe@users.noreply.github.com> * fix: Removed unneeded changes --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- koboldcpp.py | 305 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 254 insertions(+), 51 deletions(-) diff --git a/koboldcpp.py b/koboldcpp.py index ea7e7f0cc..95590d8eb 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -74,13 +74,21 @@ extra_images_max = 4 # for kontext/qwen img KcppVersion = "1.111" showdebug = True kcpp_instance = None #global running instance -global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model"} +global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "swapReqType": None, "autoswapmode": False} using_gui_launcher = False handle = None friendlymodelname = "inactive" friendlysdmodelname = "inactive" friendlyembeddingsmodelname = "inactive" +autoswapmode = False +textName = None +sttName = None +ttsName = None +embedName = None +musicName = None +imageName = None +mmprojName = None lastgeneratedcomfyimg = b'' lastuploadedcomfyimg = b'' fullsdmodelpath = "" #if empty, it's not initialized @@ -1229,20 +1237,23 @@ def convert_json_to_gbnf(json_obj): def get_capabilities(): global savedata_obj, has_multiplayer, KcppVersion, friendlymodelname, friendlysdmodelname, fullsdmodelpath, password, fullwhispermodelpath, ttsmodelpath, embeddingsmodelpath, musicdiffusionmodelpath, musicllmmodelpath, has_audio_support, has_vision_support, mcp_connections - has_llm = not (friendlymodelname=="inactive") - has_txt2img = not (friendlysdmodelname=="inactive" or fullsdmodelpath=="") + global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName + has_llm = not (friendlymodelname=="inactive") or (autoswapmode and textName is not None) + has_txt2img = not (friendlysdmodelname=="inactive" or fullsdmodelpath=="") or (autoswapmode and imageName is not None) has_password = (password!="") - has_whisper = (fullwhispermodelpath!="") + has_whisper = (fullwhispermodelpath!="") or (autoswapmode and sttName is not None) has_search = True if args.websearch else False - has_tts = (ttsmodelpath!="") - has_embeddings = (embeddingsmodelpath!="") - has_music = (musicdiffusionmodelpath!="" or musicllmmodelpath!="") + has_tts = (ttsmodelpath!="") or (autoswapmode and ttsName is not None) + has_embeddings = (embeddingsmodelpath!="") or (autoswapmode and embedName is not None) + has_music = (musicdiffusionmodelpath!="" or musicllmmodelpath!="") or (autoswapmode and musicName is not None) + visionSupport = (has_vision_support) or (autoswapmode and mmprojName is not None) + audioSupport = (has_audio_support) # or (autoswapmode and mmprojName is not None) has_guidance = True if args.enableguidance else False has_jinja = True if args.jinja else False has_mcp = True if (args.mcpfile and mcp_connections and len(mcp_connections) > 0) else False admin_type = (2 if args.admin and args.admindir and args.adminpassword else (1 if args.admin and args.admindir else 0)) has_router = True if args.routermode else False - return {"result":"KoboldCpp", "version":KcppVersion, "protected":has_password, "llm":has_llm, "txt2img":has_txt2img,"vision":has_vision_support,"audio":has_audio_support,"transcribe":has_whisper,"multiplayer":has_multiplayer,"websearch":has_search,"tts":has_tts, "embeddings":has_embeddings, "music":has_music, "savedata":(savedata_obj is not None), "admin": admin_type, "router":has_router, "guidance": has_guidance, "jinja": has_jinja, "mcp":has_mcp} + return {"result":"KoboldCpp", "version":KcppVersion, "protected":has_password, "llm":has_llm, "txt2img":has_txt2img,"vision":visionSupport,"audio":audioSupport,"transcribe":has_whisper,"multiplayer":has_multiplayer,"websearch":has_search,"tts":has_tts, "embeddings":has_embeddings, "music":has_music, "savedata":(savedata_obj is not None), "admin": admin_type, "router":has_router, "guidance": has_guidance, "jinja": has_jinja, "mcp":has_mcp} def scan_directory(dirpath, valid_exts, depth): @@ -3840,7 +3851,8 @@ class KcppProxyHandler(http.server.BaseHTTPRequestHandler): wake_requests = ["/api/extra/generate/stream","/api/extra/tokencount","/api/v1/generate","/sdapi/v1/interrogate","/v1/completions","/v1/chat/completions","/v1/responses","/api/extra/transcribe","/v1/audio/transcriptions","/api/extra/tts","/v1/audio/speech","/api/extra/embeddings","/v1/embeddings","/api/extra/music/prepare","/api/extra/music/generate","/sdapi/v1/txt2img","/sdapi/v1/img2img","/sdapi/v1/upscale"] is_wake_request = self.path in wake_requests - if is_post and (is_completions_path or is_chat_completions_path or is_wake_request): + autoswapEnabled = global_memory["autoswapmode"] is not None and global_memory["autoswapmode"] + if is_post and (is_completions_path or is_chat_completions_path or (not autoswapEnabled and is_wake_request)): model_name = "" if body: try: @@ -3875,6 +3887,52 @@ class KcppProxyHandler(http.server.BaseHTTPRequestHandler): self.send_error(504, "KoboldCpp model swap reload timed out") return time.sleep(0.1) + elif autoswapEnabled: + textReqs = ["/api/extra/generate/stream","/api/extra/tokencount","/api/v1/generate","/sdapi/v1/interrogate","/v1/completions","/v1/chat/completions"] + sttReqs = ["/api/extra/transcribe","/v1/audio/transcriptions"] + ttsReqs = ["/api/extra/tts", "/v1/audio/speech"] + embedReqs = ["/api/extra/embeddings", "/v1/embeddings"] + musicReqs = ["/api/extra/music/prepare","/api/extra/music/generate"] + imageReqs = ["/sdapi/v1/txt2img", "/sdapi/v1/img2img", "/sdapi/v1/upscale"] # "/sdapi/v1/sd-models", "/sdapi/v1/options", "/sdapi/v1/samplers" + + swapModeChanged = False + if any(self.path.endswith(e) for e in textReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "text"): + global_memory["swapReqType"] = "text" + swapModeChanged = True + elif any(self.path.endswith(e) for e in sttReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "stt"): + global_memory["swapReqType"] = "stt" + swapModeChanged = True + elif any(self.path.endswith(e) for e in ttsReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "tts"): + global_memory["swapReqType"] = "tts" + swapModeChanged = True + elif any(self.path.endswith(e) for e in embedReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "embed"): + global_memory["swapReqType"] = "embed" + swapModeChanged = True + elif any(self.path.endswith(e) for e in musicReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "music"): + global_memory["swapReqType"] = "music" + swapModeChanged = True + elif any(self.path.endswith(e) for e in imageReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "image"): + global_memory["swapReqType"] = "image" + swapModeChanged = True + + if (global_memory["swapReqType"] is not None and swapModeChanged): + with proxy_reload_lock: + reqbody = json.dumps({"filename":global_memory["current_model"]}) + reqheaders = { + 'Content-Type': 'application/json', + 'Content-Length': str(len(reqbody)), + } + if args.adminpassword: + reqheaders["Authorization"] = f"Bearer {args.adminpassword}" + conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600) + conn.request("POST", "/api/admin/reload_config", body=reqbody, headers=reqheaders) + resp = conn.getresponse() + time.sleep(3) + global_memory["last_active_timestamp"] = datetime.now() + if not self.wait_for_upstream_ready(upstream_port,120,0.5): + self.send_error(504, "KoboldCpp model swap reload timed out") + return + time.sleep(0.1) try: # connect upstream conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600) @@ -4093,6 +4151,8 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): async def generate_text(self, genparams, api_format, stream_flag): global friendlymodelname, chatcompl_adapter, currfinishreason + global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName + currfinishreason = None req_id_suffix = genparams.get('oai_uniqueid',1) chatcmpl_id = f"chatcmpl-A{req_id_suffix}" @@ -4162,14 +4222,17 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): if args.debugmode: print(f"Debug ToolCall Response: {json.dumps(tool_calls)}") + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName if api_format == 1: res = {"data": {"seqs": [recvtxt]}} elif api_format == 3: - res = {"id": cmpl_id, "object": "text_completion", "created": int(time.time()), "model": friendlymodelname, + res = {"id": cmpl_id, "object": "text_completion", "created": int(time.time()), "model": modelNameToReturn, "usage": {"prompt_tokens": prompttokens, "completion_tokens": comptokens, "total_tokens": (prompttokens+comptokens)}, "choices": [{"text": recvtxt, "index": 0, "finish_reason": currfinishreason, "logprobs":logprobsdict}]} elif api_format == 4: - res = {"id": chatcmpl_id, "object": "chat.completion", "created": int(time.time()), "model": friendlymodelname, + res = {"id": chatcmpl_id, "object": "chat.completion", "created": int(time.time()), "model": modelNameToReturn, "usage": {"prompt_tokens": prompttokens, "completion_tokens": comptokens, "total_tokens": (prompttokens+comptokens)}, "choices": [{"index": 0, "message": {"role": "assistant", "content": recvtxt, "tool_calls": tool_calls}, "finish_reason": currfinishreason, "logprobs":logprobsdict}]} elif api_format == 5: @@ -4177,9 +4240,9 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): elif api_format == 6: oldprompt = genparams.get('ollamabodyprompt', "") tokarr = tokenize_ids(oldprompt+recvtxt,False) - res = {"model": friendlymodelname,"created_at": str(datetime.now(timezone.utc).isoformat()),"response":recvtxt,"done": True,"done_reason":currfinishreason,"context": tokarr,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1} + res = {"model": modelNameToReturn,"created_at": str(datetime.now(timezone.utc).isoformat()),"response":recvtxt,"done": True,"done_reason":currfinishreason,"context": tokarr,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1} elif api_format == 7: - res = {"model": friendlymodelname,"created_at": str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":recvtxt},"done": True,"done_reason":currfinishreason,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1} + res = {"model": modelNameToReturn,"created_at": str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":recvtxt},"done": True,"done_reason":currfinishreason,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1} elif api_format == 8: resp_id = f"resp-A{genparams.get('oai_uniqueid', 1)}" output_item_id = f"msg_0{genparams.get('oai_uniqueid', 1)}" @@ -4242,6 +4305,12 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): async def handle_sse_stream(self, genparams, api_format): global friendlymodelname, currfinishreason + global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName + + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName + using_openai_tools = genparams.get('using_openai_tools', False) req_id_suffix = genparams.get('oai_uniqueid',1) chatcmpl_id = f"chatcmpl-A{req_id_suffix}" @@ -4373,10 +4442,10 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): if need_split_final_msg: #we need to send one message without the finish reason, then send a finish reason with no msg to follow standards if api_format == 4: # if oai chat, set format to expected openai streaming response - event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"delta":delta}]}) + event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"delta":delta}]}) await self.send_oai_sse_event(event_str) elif api_format == 3: # non chat completions - event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"text":tokenStr}]}) + event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"text":tokenStr}]}) await self.send_oai_sse_event(event_str) else: event_str = json.dumps({"token": tokenStr, "finish_reason":None}) @@ -4388,17 +4457,17 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs lastlogprobs = handle.last_logprobs() logprobsdict = parse_last_logprobs(lastlogprobs) - addonstr = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"delta":{'role':'assistant','content':''},"logprobs":logprobsdict}]}) + addonstr = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"delta":{'role':'assistant','content':''},"logprobs":logprobsdict}]}) await self.send_oai_sse_event(addonstr) - event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":delta}]}) + event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":delta}]}) await self.send_oai_sse_event(event_str) elif api_format == 3: # non chat completions if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs lastlogprobs = handle.last_logprobs() logprobsdict = parse_last_logprobs(lastlogprobs) - addonstr = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"text":"","logprobs":logprobsdict}]}) + addonstr = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"text":"","logprobs":logprobsdict}]}) await self.send_oai_sse_event(addonstr) - event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]}) + event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]}) await self.send_oai_sse_event(event_str) elif api_format == 8: #oai-responses resp_id = f"resp-A{genparams.get('oai_uniqueid', 1)}" @@ -4475,9 +4544,9 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): if (strop and strop.get("include_usage",False)): # Send a final chunk with usage info, only if requested usage_obj = {"prompt_tokens": prompttokens, "completion_tokens": current_token, "total_tokens": (prompttokens + current_token)} if api_format == 4: - usage_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[],"usage":usage_obj}) + usage_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[],"usage":usage_obj}) else: - usage_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[],"usage":usage_obj}) + usage_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[],"usage":usage_obj}) await self.send_oai_sse_event(usage_str) await self.send_oai_sse_event('[DONE]') await asyncio.sleep(async_sleep_short) @@ -4719,7 +4788,8 @@ Change Mode
global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz global last_req_time, start_time, cached_chat_template, has_vision_support, has_audio_support, has_whisper, friendlymodelname global savedata_obj, has_multiplayer, multiplayer_turn_major, multiplayer_turn_minor, multiplayer_story_data_compressed, multiplayer_dataformat, multiplayer_lastactive, maxctx, maxhordelen, friendlymodelname, lastuploadedcomfyimg, lastgeneratedcomfyimg, KcppVersion, totalgens, preloaded_story, exitcounter, currentusergenkey, friendlysdmodelname, fullsdmodelpath, password, friendlyembeddingsmodelname, voicelist - + global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName + clean_path = self.path.split("?")[0] #for cases where we do not want query params if clean_path=="/lcpp": #fix for svelte redirect issues, browser path needs to end with slash clean_path = "/lcpp/" @@ -4760,7 +4830,10 @@ Change Mode
elif clean_path.endswith(('/api/v1/model', '/api/latest/model')): auth_ok = self.check_header_password(password, args.adminpassword) - response_body = (json.dumps({'result': (friendlymodelname if auth_ok else "koboldcpp/protected-model") }).encode()) + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName + response_body = (json.dumps({'result': (modelNameToReturn if auth_ok else "koboldcpp/protected-model") }).encode()) elif clean_path.endswith(('/api/v1/config/max_length', '/api/latest/config/max_length')): response_body = (json.dumps({"value": maxhordelen}).encode()) @@ -4856,7 +4929,11 @@ Change Mode
response_body = (json.dumps({"logprobs":logprobsdict}).encode()) elif clean_path.endswith('/v1/models') or clean_path=='/models': - mlist = [{"id":friendlymodelname,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}] + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName + + mlist = [{"id":modelNameToReturn,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}] if args.routermode: alist = get_current_admindir_list() for itm in alist: @@ -4873,20 +4950,25 @@ Change Mode
response_body = (json.dumps([]).encode()) elif clean_path.endswith('/sdapi/v1/sd-models'): - if friendlysdmodelname=="inactive" or fullsdmodelpath=="": + if autoswapmode and imageName is not None: + response_body = (json.dumps([{"title":imageName,"model_name":imageName,"hash":"8888888888","sha256":"8888888888888888888888888888888888888888888888888888888888888888","filename":imageName,"config": None}]).encode()) + elif friendlysdmodelname=="inactive" or fullsdmodelpath=="": response_body = (json.dumps([]).encode()) else: response_body = (json.dumps([{"title":friendlysdmodelname,"model_name":friendlysdmodelname,"hash":"8888888888","sha256":"8888888888888888888888888888888888888888888888888888888888888888","filename":fullsdmodelpath,"config": None}]).encode()) elif clean_path.endswith('/sdapi/v1/options'): - response_body = (json.dumps({"samples_format":"png","sd_model_checkpoint":friendlysdmodelname}).encode()) + modelNameToReturn = friendlysdmodelname + if autoswapmode and imageName is not None: + modelNameToReturn = imageName + response_body = (json.dumps({"samples_format":"png","sd_model_checkpoint":modelNameToReturn}).encode()) elif clean_path.endswith('/sdapi/v1/samplers'): - if friendlysdmodelname=="inactive" or fullsdmodelpath=="": + if (friendlysdmodelname=="inactive" or fullsdmodelpath=="") and not(autoswapmode and imageName is not None): response_body = (json.dumps([]).encode()) else: response_body = (json.dumps([{"name":"Euler","aliases":["k_euler"],"options":{}},{"name":"Euler a","aliases":["k_euler_a","k_euler_ancestral"],"options":{}},{"name":"Heun","aliases":["k_heun"],"options":{}},{"name":"DPM2","aliases":["k_dpm_2"],"options":{}},{"name":"DPM++ 2M","aliases":["k_dpmpp_2m"],"options":{}},{"name":"DDIM","aliases":["ddim"],"options":{}},{"name":"LCM","aliases":["k_lcm"],"options":{}},{"name":"Res 2s","aliases":["k_res_2s"],"options":{}},{"name":"Res Multistep","aliases":["k_res_multistep"],"options":{}}, {"name":"Default","aliases":["default"],"options":{}}]).encode()) elif clean_path.endswith('/sdapi/v1/schedulers'): - if friendlysdmodelname=="inactive" or fullsdmodelpath=="": + if (friendlysdmodelname=="inactive" or fullsdmodelpath=="") and not(autoswapmode and imageName is not None): response_body = (json.dumps([]).encode()) else: response_body = (json.dumps([{"name":name,"label":name} for name in sd_get_available_schedulers()]).encode()) @@ -4923,7 +5005,10 @@ Change Mode
response_body = (json.dumps({"temperature":0.75,"speed":1,"length_penalty":1,"repetition_penalty":1,"top_p":1,"top_k":4,"enable_text_splitting":True,"stream_chunk_size":100}).encode()) #some random voices for them to enjoy elif clean_path.endswith('/api/tags') or clean_path.endswith('/api/ps'): #ollama compatible - response_body = (json.dumps({"models":[{"name":"koboldcpp","model":f"{friendlymodelname}:latest","modified_at":"2024-07-19T15:26:55.6122841+08:00","expires_at": "2055-06-04T19:06:25.5433636+08:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}},{"name":"koboldcpp","model":friendlymodelname,"modified_at":"2025-01-01T01:00:00.0000000+00:00","expires_at": "2069-01-01T01:00:00.0000000+00:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}}]}).encode()) + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName + response_body = (json.dumps({"models":[{"name":"koboldcpp","model":f"{modelNameToReturn}:latest","modified_at":"2024-07-19T15:26:55.6122841+08:00","expires_at": "2055-06-04T19:06:25.5433636+08:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}},{"name":"koboldcpp","model":modelNameToReturn,"modified_at":"2025-01-01T01:00:00.0000000+00:00","expires_at": "2069-01-01T01:00:00.0000000+00:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}}]}).encode()) elif clean_path.endswith('/api/version'): #ollama compatible, NOT the kcpp version response_body = (json.dumps({"version":"0.7.0"}).encode()) elif clean_path=='/ping': @@ -4933,9 +5018,14 @@ Change Mode
elif clean_path=='/system_stats': response_body = (json.dumps({"system":{"os":"posix","ram_total":12345678900,"ram_free":12345678900,"comfyui_version":"v0.3.4-3-g7126ecf","python_version":"3.10.12","pytorch_version":"2.5.1","embedded_python":False,"argv":[]},"devices":[{"name":"koboldcpp","type":"cuda","index":0,"vram_total":12345678900,"vram_free":12345678900,"torch_vram_total":12345678900,"torch_vram_free":12345678900}]}).encode()) elif clean_path=='/object_info': - response_body = (json.dumps({"KSampler":{"input":{"required":{"model":["MODEL",{"tooltip":""}],"seed":["INT",{"default":0,"min":0,"max":512,"tooltip":""}],"steps":["INT",{"default":20,"min":1,"max":512,"tooltip":""}],"cfg":["FLOAT",{"default":8.0,"min":0.0,"max":100.0,"step":0.1,"round":0.01,"tooltip":"512"}],"sampler_name":[["euler"],{"tooltip":""}],"scheduler":[["normal"],{"tooltip":""}],"positive":["CONDITIONING",{"tooltip":""}],"negative":["CONDITIONING",{"tooltip":""}],"latent_image":["LATENT",{"tooltip":""}],"denoise":["FLOAT",{"default":1.0,"min":0.0,"max":1.0,"step":0.01,"tooltip":""}]}},"input_order":{"required":["model","seed","steps","cfg","sampler_name","scheduler","positive","negative","latent_image","denoise"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"KSampler","display_name":"KSampler","description":"KSampler","python_module":"nodes","category":"sampling","output_node":False,"output_tooltips":[""]},"CheckpointLoaderSimple":{"input":{"required":{"ckpt_name":[[friendlysdmodelname],{"tooltip":""}]}},"input_order":{"required":["ckpt_name"]},"output":["MODEL","CLIP","VAE"],"output_is_list":[False,False,False],"output_name":["MODEL","CLIP","VAE"],"name":"CheckpointLoaderSimple","display_name":"Load","description":"","python_module":"nodes","category":"loaders","output_node":False,"output_tooltips":["","",""]},"CLIPTextEncode":{"input":{"required":{"text":["STRING",{"multiline":True,"dynamicPrompts":True,"tooltip":""}],"clip":["CLIP",{"tooltip":""}]}},"input_order":{"required":["text","clip"]},"output":["CONDITIONING"],"output_is_list":[False],"output_name":["CONDITIONING"],"name":"CLIPTextEncode","display_name":"CLIP","description":"","python_module":"nodes","category":"conditioning","output_node":False,"output_tooltips":[""]},"CLIPSetLastLayer":{"input":{"required":{"clip":["CLIP"],"stop_at_clip_layer":["INT",{"default":-1,"min":-24,"max":-1,"step":1}]}},"input_order":{"required":["clip","stop_at_clip_layer"]},"output":["CLIP"],"output_is_list":[False],"output_name":["CLIP"],"name":"CLIPSetLastLayer","display_name":"CLIPSLL","description":"","python_module":"nodes","category":"conditioning","output_node":False},"VAEDecode":{"input":{"required":{"samples":["LATENT",{"tooltip":""}],"vae":["VAE",{"tooltip":""}]}},"input_order":{"required":["samples","vae"]},"output":["IMAGE"],"output_is_list":[False],"output_name":["IMAGE"],"name":"VAEDecode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]},"VAEEncode":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"]}},"input_order":{"required":["pixels","vae"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False},"VAEEncodeForInpaint":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"],"mask":["MASK"],"grow_mask_by":["INT",{"default":6,"min":0,"max":64,"step":1}]}},"input_order":{"required":["pixels","vae","mask","grow_mask_by"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncodeForInpaint","display_name":"VAE","description":"","python_module":"nodes","category":"latent/inpaint","output_node":False},"VAELoader":{"input":{"required":{"vae_name":[["kcpp_vae"]]}},"input_order":{"required":["vae_name"]},"output":["VAE"],"output_is_list":[False],"output_name":["VAE"],"name":"VAELoader","display_name":"Load VAE","description":"","python_module":"nodes","category":"loaders","output_node":False},"EmptyLatentImage":{"input":{"required":{"width":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"height":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"batch_size":["INT",{"default":1,"min":1,"max":1,"tooltip":""}]}},"input_order":{"required":["width","height","batch_size"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"EmptyLatentImage","display_name":"Empty Latent Image","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]}}).encode()) + modelNameToReturn = friendlysdmodelname + if autoswapmode and imageName is not None: + modelNameToReturn = imageName + response_body = (json.dumps({"KSampler":{"input":{"required":{"model":["MODEL",{"tooltip":""}],"seed":["INT",{"default":0,"min":0,"max":512,"tooltip":""}],"steps":["INT",{"default":20,"min":1,"max":512,"tooltip":""}],"cfg":["FLOAT",{"default":8.0,"min":0.0,"max":100.0,"step":0.1,"round":0.01,"tooltip":"512"}],"sampler_name":[["euler"],{"tooltip":""}],"scheduler":[["normal"],{"tooltip":""}],"positive":["CONDITIONING",{"tooltip":""}],"negative":["CONDITIONING",{"tooltip":""}],"latent_image":["LATENT",{"tooltip":""}],"denoise":["FLOAT",{"default":1.0,"min":0.0,"max":1.0,"step":0.01,"tooltip":""}]}},"input_order":{"required":["model","seed","steps","cfg","sampler_name","scheduler","positive","negative","latent_image","denoise"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"KSampler","display_name":"KSampler","description":"KSampler","python_module":"nodes","category":"sampling","output_node":False,"output_tooltips":[""]},"CheckpointLoaderSimple":{"input":{"required":{"ckpt_name":[[modelNameToReturn],{"tooltip":""}]}},"input_order":{"required":["ckpt_name"]},"output":["MODEL","CLIP","VAE"],"output_is_list":[False,False,False],"output_name":["MODEL","CLIP","VAE"],"name":"CheckpointLoaderSimple","display_name":"Load","description":"","python_module":"nodes","category":"loaders","output_node":False,"output_tooltips":["","",""]},"CLIPTextEncode":{"input":{"required":{"text":["STRING",{"multiline":True,"dynamicPrompts":True,"tooltip":""}],"clip":["CLIP",{"tooltip":""}]}},"input_order":{"required":["text","clip"]},"output":["CONDITIONING"],"output_is_list":[False],"output_name":["CONDITIONING"],"name":"CLIPTextEncode","display_name":"CLIP","description":"","python_module":"nodes","category":"conditioning","output_node":False,"output_tooltips":[""]},"CLIPSetLastLayer":{"input":{"required":{"clip":["CLIP"],"stop_at_clip_layer":["INT",{"default":-1,"min":-24,"max":-1,"step":1}]}},"input_order":{"required":["clip","stop_at_clip_layer"]},"output":["CLIP"],"output_is_list":[False],"output_name":["CLIP"],"name":"CLIPSetLastLayer","display_name":"CLIPSLL","description":"","python_module":"nodes","category":"conditioning","output_node":False},"VAEDecode":{"input":{"required":{"samples":["LATENT",{"tooltip":""}],"vae":["VAE",{"tooltip":""}]}},"input_order":{"required":["samples","vae"]},"output":["IMAGE"],"output_is_list":[False],"output_name":["IMAGE"],"name":"VAEDecode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]},"VAEEncode":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"]}},"input_order":{"required":["pixels","vae"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False},"VAEEncodeForInpaint":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"],"mask":["MASK"],"grow_mask_by":["INT",{"default":6,"min":0,"max":64,"step":1}]}},"input_order":{"required":["pixels","vae","mask","grow_mask_by"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncodeForInpaint","display_name":"VAE","description":"","python_module":"nodes","category":"latent/inpaint","output_node":False},"VAELoader":{"input":{"required":{"vae_name":[["kcpp_vae"]]}},"input_order":{"required":["vae_name"]},"output":["VAE"],"output_is_list":[False],"output_name":["VAE"],"name":"VAELoader","display_name":"Load VAE","description":"","python_module":"nodes","category":"loaders","output_node":False},"EmptyLatentImage":{"input":{"required":{"width":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"height":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"batch_size":["INT",{"default":1,"min":1,"max":1,"tooltip":""}]}},"input_order":{"required":["width","height","batch_size"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"EmptyLatentImage","display_name":"Empty Latent Image","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]}}).encode()) elif clean_path.endswith('/api/models/checkpoints') or clean_path.endswith('/models/checkpoints'): #emulate comfyui, duplication is redundant but added for clarity - if friendlysdmodelname=="inactive" or fullsdmodelpath=="": + if autoswapmode and imageName is not None: + response_body = (json.dumps([imageName]).encode()) + elif friendlysdmodelname=="inactive" or fullsdmodelpath=="": response_body = (json.dumps([]).encode()) else: response_body = (json.dumps([friendlysdmodelname]).encode()) @@ -4945,8 +5035,11 @@ Change Mode
content_type = 'image/png' response_body = lastgeneratedcomfyimg elif clean_path=='/history' or clean_path=='/api/history' or clean_path.startswith('/api/history/') or clean_path.startswith('/history/'): #emulate comfyui + modelNameToReturn = friendlysdmodelname + if autoswapmode and imageName is not None: + modelNameToReturn = imageName imgdone = (False if lastgeneratedcomfyimg==b'' else True) - response_body = (json.dumps({"12345678-0000-0000-0000-000000000001":{"prompt":[0,"12345678-0000-0000-0000-000000000001",{"3":{"class_type":"KSampler","inputs":{"cfg":5.0,"denoise":1.0,"latent_image":["5",0],"model":["4",0],"negative":["7",0],"positive":["6",0],"sampler_name":"euler","scheduler":"normal","seed":1,"steps":20}},"4":{"class_type":"CheckpointLoaderSimple","inputs":{"ckpt_name":friendlysdmodelname}},"5":{"class_type":"EmptyLatentImage","inputs":{"batch_size":1,"height":512,"width":512}},"6":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":"prompt"}},"7":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":""}},"8":{"class_type":"VAEDecode","inputs":{"samples":["3",0],"vae":["4",2]}},"9":{"class_type":"SaveImage","inputs":{"filename_prefix":"kliteimg","images":["8",0]}}},{},["9"]],"outputs":{"9":{"images":[{"filename":"kliteimg_00001_.png","subfolder":"","type":"output"}]}},"status":{"status_str":"success","completed":imgdone,"messages":[["execution_start",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_cached",{"nodes":[],"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_success",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}]]},"meta":{"9":{"node_id":"9","display_node":"9","parent_node":None,"real_node_id":"9"}}}}).encode()) + response_body = (json.dumps({"12345678-0000-0000-0000-000000000001":{"prompt":[0,"12345678-0000-0000-0000-000000000001",{"3":{"class_type":"KSampler","inputs":{"cfg":5.0,"denoise":1.0,"latent_image":["5",0],"model":["4",0],"negative":["7",0],"positive":["6",0],"sampler_name":"euler","scheduler":"normal","seed":1,"steps":20}},"4":{"class_type":"CheckpointLoaderSimple","inputs":{"ckpt_name":modelNameToReturn}},"5":{"class_type":"EmptyLatentImage","inputs":{"batch_size":1,"height":512,"width":512}},"6":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":"prompt"}},"7":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":""}},"8":{"class_type":"VAEDecode","inputs":{"samples":["3",0],"vae":["4",2]}},"9":{"class_type":"SaveImage","inputs":{"filename_prefix":"kliteimg","images":["8",0]}}},{},["9"]],"outputs":{"9":{"images":[{"filename":"kliteimg_00001_.png","subfolder":"","type":"output"}]}},"status":{"status_str":"success","completed":imgdone,"messages":[["execution_start",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_cached",{"nodes":[],"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_success",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}]]},"meta":{"9":{"node_id":"9","display_node":"9","parent_node":None,"real_node_id":"9"}}}}).encode()) elif clean_path=='/ws' and ('Upgrade' in self.headers and self.headers['Upgrade'].lower() == 'websocket' and 'Sec-WebSocket-Key' in self.headers): ws_key = self.headers['Sec-WebSocket-Key'] @@ -4976,16 +5069,22 @@ Change Mode
response_body = (json.dumps({"version":"0.2","software":{"name":"KoboldCpp","version":KcppVersion,"repository":"https://github.com/LostRuins/koboldcpp","homepage":"https://github.com/LostRuins/koboldcpp","logo":"https://raw.githubusercontent.com/LostRuins/koboldcpp/refs/heads/concedo/niko.ico"},"api":{"koboldai":{"name":"KoboldAI API","rel_url":"/api","documentation":"https://lite.koboldai.net/koboldcpp_api","version":KcppVersion},"openai":{"name":"OpenAI API","rel_url ":"/v1","documentation":"https://openai.com/documentation/api","version":KcppVersion}}}).encode()) elif clean_path=="/props": + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName + mmprojOverride = False + if autoswapmode and mmprojName is not None: + mmprojOverride = True response_body = (json.dumps({ "chat_template": cached_chat_template, "id": 0, "id_task": -1, "total_slots": 1, "modalities": { - "vision": has_vision_support, + "vision": mmprojOverride or has_vision_support, "audio": has_audio_support }, - "model_path": friendlymodelname, + "model_path": modelNameToReturn, "n_ctx": maxctx, "default_generation_settings": { "n_ctx": maxctx, @@ -5067,6 +5166,7 @@ Change Mode
def do_POST(self): global modelbusy, requestsinqueue, currentusergenkey, totalgens, pendingabortkey, lastuploadedcomfyimg, lastgeneratedcomfyimg, multiplayer_turn_major, multiplayer_turn_minor, multiplayer_story_data_compressed, multiplayer_dataformat, multiplayer_lastactive, net_save_slots, has_vision_support, savestate_limit, mcp_lock + global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName contlenstr = self.headers['content-length'] content_length = 0 body = None @@ -5620,7 +5720,10 @@ Change Mode
elif self.path.endswith('/v1/chat/completions') or self.path=='/chat/completions': api_format = 4 elif self.path.endswith('/sdapi/v1/interrogate'): - if not has_vision_support: + mmprojOverride = False + if autoswapmode and mmprojName is not None: + mmprojOverride = True + if not mmprojOverride and not has_vision_support: self.send_response(503) self.end_headers(content_type='application/json') self.wfile.write(json.dumps({"detail": { @@ -5747,6 +5850,9 @@ Change Mode
gendat = asyncio.run(self.handle_request(genparams, api_format, sse_stream_flag)) try: + modelNameToReturn = friendlymodelname + if autoswapmode and textName is not None: + modelNameToReturn = textName # Headers are already sent when streaming if (api_format == 6 or api_format == 7) and genparams.get('stream', True): #ollama fake streaming @@ -5758,7 +5864,7 @@ Change Mode
if api_format == 6: bodytxt = gendat.get("response","") # extract and erase the AI response from the sync payload. gendat["response"] = "" - pl = {"model":friendlymodelname,"created_at":str(datetime.now(timezone.utc).isoformat()),"response":bodytxt,"done":False} + pl = {"model":modelNameToReturn,"created_at":str(datetime.now(timezone.utc).isoformat()),"response":bodytxt,"done":False} self.wfile.write(f'{json.dumps(pl)}\n'.encode()) self.wfile.flush() time.sleep(0.05) #short delay @@ -5768,7 +5874,7 @@ Change Mode
else: bodytxt = gendat.get("message",{}).get("content","") # extract and erase the AI response from the sync payload. gendat["message"] = {"role":"assistant","content":""} - pl = {"model":friendlymodelname,"created_at":str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":bodytxt},"done":False} + pl = {"model":modelNameToReturn,"created_at":str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":bodytxt},"done":False} self.wfile.write(f'{json.dumps(pl)}\n'.encode()) self.wfile.flush() time.sleep(0.05) #short delay @@ -5801,7 +5907,7 @@ Change Mode
"id": "koboldcpp", "object": "chat.completion.chunk", "created": int(time.time()), - "model": friendlymodelname, + "model": modelNameToReturn, "choices": [{"index": 0, "finish_reason": None, "delta": {"role": "assistant"}}] }) self.wfile.write(f"data: {chunk_role}\n\n".encode()) @@ -5813,7 +5919,7 @@ Change Mode
"id": "koboldcpp", "object": "chat.completion.chunk", "created": int(time.time()), - "model": friendlymodelname, + "model": modelNameToReturn, "choices": [{"index": 0, "finish_reason": None, "delta": {"content": content_text}}] }) self.wfile.write(f"data: {chunk_content}\n\n".encode()) @@ -5835,7 +5941,7 @@ Change Mode
"id": "koboldcpp", "object": "chat.completion.chunk", "created": int(time.time()), - "model": friendlymodelname, + "model": modelNameToReturn, "choices": [{"index": 0, "finish_reason": None, "delta": {"tool_calls": [tc_meta]}}] }) self.wfile.write(f"data: {chunk_meta}\n\n".encode()) @@ -5852,7 +5958,7 @@ Change Mode
"id": "koboldcpp", "object": "chat.completion.chunk", "created": int(time.time()), - "model": friendlymodelname, + "model": modelNameToReturn, "choices": [{"index": 0, "finish_reason": None, "delta": {"tool_calls": [tc_args]}}] }) self.wfile.write(f"data: {chunk_args}\n\n".encode()) @@ -5863,7 +5969,7 @@ Change Mode
"id": "koboldcpp", "object": "chat.completion.chunk", "created": int(time.time()), - "model": friendlymodelname, + "model": modelNameToReturn, "choices": [{"index": 0, "finish_reason": "tool_calls", "delta": {}}] }) self.wfile.write(f"data: {chunk_final}\n\n".encode()) @@ -5980,6 +6086,9 @@ Change Mode
return elif is_embeddings: try: + modelNameToReturn = friendlyembeddingsmodelname + if autoswapmode and embedName is not None: + modelNameToReturn = embedName gendat = embeddings_generate(genparams) outdatas = [] odidx = 0 @@ -5991,7 +6100,7 @@ Change Mode
else: outdatas.append({"object":"embedding","index":odidx,"embedding":od}) odidx += 1 - genresp = (json.dumps({"object":"list","data":outdatas,"model":friendlyembeddingsmodelname,"usage":{"prompt_tokens":gendat["count"],"total_tokens":gendat["count"]}}).encode()) + genresp = (json.dumps({"object":"list","data":outdatas,"model":modelNameToReturn,"usage":{"prompt_tokens":gendat["count"],"total_tokens":gendat["count"]}}).encode()) self.send_response(200) self.send_header('content-length', str(len(genresp))) self.end_headers(content_type='application/json') @@ -6660,6 +6769,7 @@ def show_gui(): admin_password_var = ctk.StringVar() singleinstance_var = ctk.IntVar(value=0) router_mode_var = ctk.IntVar(value=0) + autoswap_mode_var = ctk.IntVar(value=0) admin_unload_timeout_var = ctk.StringVar(value=str(0)) nozenity_var = ctk.IntVar(value=0) @@ -7486,12 +7596,19 @@ def show_gui(): router_mode_box.grid() else: router_mode_box.grid_remove() + def togglerouter(a,b,c): + if router_mode_var.get()==1: + autoswap_mode_box.grid() + else: + autoswap_mode_box.grid_remove() + makecheckbox(admin_tab, "Enable Model Administration", admin_var, 1, 0, command=toggleadmin,tooltiptxt="Enable a admin server, allowing you to remotely relaunch and swap models and configs.") makelabelentry(admin_tab, "Admin Password:" , admin_password_var, 3, 150,padx=(120),singleline=True,tooltip="Require a password to access admin functions. You are strongly advised to use one for publically accessible instances!") makefileentry(admin_tab, "Config Directory (Required):", "Select directory containing .gguf or .kcpps files to relaunch from", admin_dir_var, 5, width=280, dialog_type=2, tooltiptxt="Specify a directory to look for .kcpps configs in, which can be used to swap models.") makelabelentry(admin_tab, "Auto Unload Timeout:" , admin_unload_timeout_var, 7, 70,padx=(150),singleline=True,tooltip="Set an idle timeout in seconds after which KoboldCpp will automatically unload the current model.") - makecheckbox(admin_tab, "SingleInstance Mode", singleinstance_var, 10, 0,tooltiptxt="Allows this server to be shut down by another KoboldCpp instance with singleinstance starting on the same port.") - router_mode_box = makecheckbox(admin_tab, "Router Mode", router_mode_var, 15, 0,tooltiptxt="Router mode uses a reverse proxy router, allowing you to easily hotswap models and configs within a single request. Requires admin mode.") + makecheckbox(admin_tab, "SingleInstance Mode", singleinstance_var, 9, 0,tooltiptxt="Allows this server to be shut down by another KoboldCpp instance with singleinstance starting on the same port.") + router_mode_box = makecheckbox(admin_tab, "Router Mode", router_mode_var, 11, 0, command=togglerouter, tooltiptxt="Router mode uses a reverse proxy router, allowing you to easily hotswap models and configs within a single request. Requires admin mode.") + autoswap_mode_box = makecheckbox(admin_tab, "Autoswap Mode", autoswap_mode_var, 13, 0,tooltiptxt="Autoswap mode builds on router mode to allow switching of model types within the same config automatically. Requires admin mode and router mode. All models desired must be defined within the same config.") def kcpp_export_template(): nonlocal kcpp_exporting_template @@ -7811,6 +7928,7 @@ def show_gui(): args.adminpassword = admin_password_var.get() args.singleinstance = (singleinstance_var.get()==1) args.routermode = router_mode_var.get()==1 + args.autoswapmode = autoswap_mode_var.get()==1 args.adminunloadtimeout = (0 if admin_unload_timeout_var.get()=="" else int(admin_unload_timeout_var.get())) args.showgui = False #prevent showgui from leaking into configs, its cli only @@ -8069,6 +8187,7 @@ def show_gui(): admin_var.set(dict["admin"] if ("admin" in dict) else 0) router_mode_var.set(dict["routermode"] if ("routermode" in dict) else 0) + autoswap_mode_var.set(dict["autoswapmode"] if ("autoswapmode" in dict) else 0) admin_dir_var.set(dict["admindir"] if ("admindir" in dict and dict["admindir"]) else "") admin_password_var.set(dict["adminpassword"] if ("adminpassword" in dict and dict["adminpassword"]) else "") admin_unload_timeout_var.set(dict["adminunloadtimeout"] if ("adminunloadtimeout" in dict and dict["adminunloadtimeout"]) else 0) @@ -9102,6 +9221,10 @@ def main(launch_args, default_args): sslvalid = True args.proxy_port = None #normally unused + if args.autoswapmode: + if not args.routermode: + print("\nWARNING: Autoswap mode requires router, enabling router...") + args.routermode = True if args.routermode: if not args.admin: print("\nWARNING: Router mode requires admin, enabling admin...") @@ -9146,7 +9269,7 @@ def main(launch_args, default_args): input() else: # manager command queue for admin mode with multiprocessing.Manager() as mp_manager: - global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model"}) + global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "swapReqType": None, "autoswapmode": False}) if args.remotetunnel and not args.prompt and not args.benchmark and not args.cli: setuptunnel(global_memory, True if args.sdmodel else False) @@ -9179,6 +9302,7 @@ def main(launch_args, default_args): kcpp_instance.start() global_memory["restart_target"] = "" global_memory["restart_override_config_target"] = "" + global_memory["swapReqType"] = None time.sleep(3) else: break # kill the program @@ -9191,10 +9315,16 @@ def main(launch_args, default_args): curtime = datetime.now() elapsedtime = curtime - last_active time_since_last_active = elapsedtime.total_seconds() - if time_since_last_active > args.adminunloadtimeout and global_memory["current_model"]!="unload_model": - print(f"[Unload Timeout] Inactive for over {time_since_last_active}s, unloading models...") - restart_target = "unload_model" - global_memory["triggered_sleeping"] = True + if time_since_last_active > args.adminunloadtimeout: + if args.autoswapmode: + if global_memory["swapReqType"] is not None and global_memory["swapReqType"] != "nomodel": + print(f"[Unload Timeout] Inactive for over {time_since_last_active}s, unloading models via autoswap...") + global_memory["swapReqType"] = "nomodel" + global_memory["triggered_sleeping"] = True + elif global_memory["current_model"]!="unload_model": + print(f"[Unload Timeout] Inactive for over {time_since_last_active}s, unloading models...") + restart_target = "unload_model" + global_memory["triggered_sleeping"] = True if restart_target!="": overridetxt = ("" if not restart_override_config_target else f" with override config {restart_override_config_target}") print(f"Reloading new model/config: {restart_target}{overridetxt}") @@ -9235,6 +9365,7 @@ def main(launch_args, default_args): args.model_param = targetfilepath else: reload_new_config(targetfilepath,defaultargs) + global_memory["autoswapmode"] = args.autoswapmode kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": False}) kcpp_instance.daemon = True kcpp_instance.start() @@ -9341,6 +9472,27 @@ def mk_lora_info(imgloras, multipliers, mock_filesystem=False): preloaded_table.append(lora_entry) return preloaded_table, lora_path_map, lora_name_map +def disableSwappedFieldsInConfig(args, swapReqType): + print(f"Swapping to type: {swapReqType}") + if swapReqType != "text": + for e in ["model", "model_param", "lora", "mmproj"]: + setattr(args, e, "") + if swapReqType != "stt": + for e in ["whispermodel"]: + setattr(args, e, "") + if swapReqType != "tts": + for e in ["ttsmodel", "ttswavtokenizer"]: + setattr(args, e, "") + if swapReqType != "embed": + for e in ["embeddingsmodel"]: + setattr(args, e, "") + if swapReqType != "music": + for e in ["musicllm", "musicembeddings", "musicdiffusion", "musicvae"]: + setattr(args, e, "") + if swapReqType != "image": + for e in ["sdmodel", "sdt5xxl", "sdclip1", "sdclip2", "sdphotomaker", "sdupscaler", "sdvae", "sdlora"]: + setattr(args, e, "") + def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz, start_time, exitcounter, global_memory, using_gui_launcher @@ -9356,6 +9508,56 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): if args.model_param and (args.prompt and not args.cli) and not args.benchmark and not (args.debugmode >= 1): suppress_stdout() + + global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName + autoswapmode = False + textName = None + mmprojName = None + sttName = None + ttsName = None + embedName = None + musicName = None + imageName = None + if args.autoswapmode is not None and args.autoswapmode: + autoswapmode = True + global_memory["autoswapmode"] = True + if args.model_param and args.model_param!="": + tempName = os.path.basename(os.path.abspath(args.model_param)) + tempName = os.path.splitext(tempName)[0] + textName = "koboldcpp/" + sanitize_string(tempName) + if args.mmproj and args.mmproj!="": # multimodal vision and audio support is assumed to work with mmproj - this may be incorrect! + tempName = os.path.basename(os.path.abspath(args.mmproj)) + tempName = os.path.splitext(tempName)[0] + mmprojName = sanitize_string(tempName) + if args.whispermodel and args.whispermodel!="": + tempName = os.path.basename(os.path.abspath(args.whispermodel)) + tempName = os.path.splitext(tempName)[0] + sttName = sanitize_string(tempName) + if args.ttsmodel and args.ttsmodel!="": + tempName = os.path.basename(os.path.abspath(args.ttsmodel)) + tempName = os.path.splitext(tempName)[0] + ttsName = sanitize_string(tempName) + if args.embeddingsmodel and args.embeddingsmodel!="": + tempName = os.path.basename(os.path.abspath(args.embeddingsmodel)) + tempName = os.path.splitext(tempName)[0] + embedName = sanitize_string(tempName) + if args.musicdiffusion and args.musicdiffusion!="": + tempName = os.path.basename(os.path.abspath(args.musicdiffusion)) + tempName = os.path.splitext(tempName)[0] + musicName = sanitize_string(tempName) + if args.sdmodel and args.sdmodel!="": + tempName = os.path.basename(os.path.abspath(args.sdmodel)) + tempName = os.path.splitext(tempName)[0] + imageName = sanitize_string(tempName) + if global_memory["swapReqType"] is not None: + disableSwappedFieldsInConfig(args, global_memory["swapReqType"]) + else: + global_memory["swapReqType"] = "nomodel" + setattr(args, "nomodel", True) + disableSwappedFieldsInConfig(args, "nomodel") + else: + global_memory["autoswapmode"] = False + if args.model_param and (args.benchmark or args.prompt or args.cli): start_server = False @@ -9997,7 +10199,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): print("Could not find Embedded MusicUI.") # load all TTS audio files - if args.ttsmodel: + if args.ttsmodel or ttsName is not None: try: global voicebank, voicelist voicebank = {} @@ -10428,6 +10630,7 @@ if __name__ == '__main__': admingroup.add_argument("--admindir", metavar=('[directory]'), help="Specify a directory to look for .kcpps configs in, which can be used to swap models.", default="") admingroup.add_argument("--adminunloadtimeout", help="Set an idle timeout in seconds after which KoboldCpp will automatically unload the current model.", type=int, default=0) admingroup.add_argument("--routermode", help="Router mode uses a reverse proxy router, allowing you to easily hotswap models and configs within a single request. Requires admin mode.", action='store_true') + admingroup.add_argument("--autoswapmode", help="Autoswap mode builds on router mode to allow switching of model types within the same config automatically. Requires admin mode and router mode. All models desired must be defined within the same config.", action='store_true') deprecatedgroup = parser.add_argument_group('Deprecated Commands, DO NOT USE!') deprecatedgroup.add_argument("--hordeconfig", help=argparse.SUPPRESS, nargs='+')