mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 17:25:07 +02:00
feat: Autoswap functionality (#2080)
* feat: Autoswap mode (cherry-picked from remoteManagement) Co-authored-by: esolithe <65901558+esolithe@users.noreply.github.com> * fix: Remove modelOverride, add triggered_sleeping to autoswap unload timeout branch Agent-Logs-Url: https://github.com/esolithe/esobold/sessions/1ddb3f88-43b4-4234-aa41-0fe6c9976db4 Co-authored-by: esolithe <65901558+esolithe@users.noreply.github.com> * fix: Remove esobold-specific GUI elements from admin tab, renumber remaining rows Agent-Logs-Url: https://github.com/esolithe/esobold/sessions/6a2e4ec3-cb19-4f98-b00f-bdb13749ead3 Co-authored-by: esolithe <65901558+esolithe@users.noreply.github.com> * fix: Removed unneeded changes --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
This commit is contained in:
Eso
GitHub
+254
-51
@@ -74,13 +74,21 @@ extra_images_max = 4 # for kontext/qwen img
|
||||
KcppVersion = "1.111"
|
||||
showdebug = True
|
||||
kcpp_instance = None #global running instance
|
||||
global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model"}
|
||||
global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "swapReqType": None, "autoswapmode": False}
|
||||
using_gui_launcher = False
|
||||
|
||||
handle = None
|
||||
friendlymodelname = "inactive"
|
||||
friendlysdmodelname = "inactive"
|
||||
friendlyembeddingsmodelname = "inactive"
|
||||
autoswapmode = False
|
||||
textName = None
|
||||
sttName = None
|
||||
ttsName = None
|
||||
embedName = None
|
||||
musicName = None
|
||||
imageName = None
|
||||
mmprojName = None
|
||||
lastgeneratedcomfyimg = b''
|
||||
lastuploadedcomfyimg = b''
|
||||
fullsdmodelpath = "" #if empty, it's not initialized
|
||||
@@ -1229,20 +1237,23 @@ def convert_json_to_gbnf(json_obj):
|
||||
|
||||
def get_capabilities():
|
||||
global savedata_obj, has_multiplayer, KcppVersion, friendlymodelname, friendlysdmodelname, fullsdmodelpath, password, fullwhispermodelpath, ttsmodelpath, embeddingsmodelpath, musicdiffusionmodelpath, musicllmmodelpath, has_audio_support, has_vision_support, mcp_connections
|
||||
has_llm = not (friendlymodelname=="inactive")
|
||||
has_txt2img = not (friendlysdmodelname=="inactive" or fullsdmodelpath=="")
|
||||
global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName
|
||||
has_llm = not (friendlymodelname=="inactive") or (autoswapmode and textName is not None)
|
||||
has_txt2img = not (friendlysdmodelname=="inactive" or fullsdmodelpath=="") or (autoswapmode and imageName is not None)
|
||||
has_password = (password!="")
|
||||
has_whisper = (fullwhispermodelpath!="")
|
||||
has_whisper = (fullwhispermodelpath!="") or (autoswapmode and sttName is not None)
|
||||
has_search = True if args.websearch else False
|
||||
has_tts = (ttsmodelpath!="")
|
||||
has_embeddings = (embeddingsmodelpath!="")
|
||||
has_music = (musicdiffusionmodelpath!="" or musicllmmodelpath!="")
|
||||
has_tts = (ttsmodelpath!="") or (autoswapmode and ttsName is not None)
|
||||
has_embeddings = (embeddingsmodelpath!="") or (autoswapmode and embedName is not None)
|
||||
has_music = (musicdiffusionmodelpath!="" or musicllmmodelpath!="") or (autoswapmode and musicName is not None)
|
||||
visionSupport = (has_vision_support) or (autoswapmode and mmprojName is not None)
|
||||
audioSupport = (has_audio_support) # or (autoswapmode and mmprojName is not None)
|
||||
has_guidance = True if args.enableguidance else False
|
||||
has_jinja = True if args.jinja else False
|
||||
has_mcp = True if (args.mcpfile and mcp_connections and len(mcp_connections) > 0) else False
|
||||
admin_type = (2 if args.admin and args.admindir and args.adminpassword else (1 if args.admin and args.admindir else 0))
|
||||
has_router = True if args.routermode else False
|
||||
return {"result":"KoboldCpp", "version":KcppVersion, "protected":has_password, "llm":has_llm, "txt2img":has_txt2img,"vision":has_vision_support,"audio":has_audio_support,"transcribe":has_whisper,"multiplayer":has_multiplayer,"websearch":has_search,"tts":has_tts, "embeddings":has_embeddings, "music":has_music, "savedata":(savedata_obj is not None), "admin": admin_type, "router":has_router, "guidance": has_guidance, "jinja": has_jinja, "mcp":has_mcp}
|
||||
return {"result":"KoboldCpp", "version":KcppVersion, "protected":has_password, "llm":has_llm, "txt2img":has_txt2img,"vision":visionSupport,"audio":audioSupport,"transcribe":has_whisper,"multiplayer":has_multiplayer,"websearch":has_search,"tts":has_tts, "embeddings":has_embeddings, "music":has_music, "savedata":(savedata_obj is not None), "admin": admin_type, "router":has_router, "guidance": has_guidance, "jinja": has_jinja, "mcp":has_mcp}
|
||||
|
||||
|
||||
def scan_directory(dirpath, valid_exts, depth):
|
||||
@@ -3840,7 +3851,8 @@ class KcppProxyHandler(http.server.BaseHTTPRequestHandler):
|
||||
wake_requests = ["/api/extra/generate/stream","/api/extra/tokencount","/api/v1/generate","/sdapi/v1/interrogate","/v1/completions","/v1/chat/completions","/v1/responses","/api/extra/transcribe","/v1/audio/transcriptions","/api/extra/tts","/v1/audio/speech","/api/extra/embeddings","/v1/embeddings","/api/extra/music/prepare","/api/extra/music/generate","/sdapi/v1/txt2img","/sdapi/v1/img2img","/sdapi/v1/upscale"]
|
||||
is_wake_request = self.path in wake_requests
|
||||
|
||||
if is_post and (is_completions_path or is_chat_completions_path or is_wake_request):
|
||||
autoswapEnabled = global_memory["autoswapmode"] is not None and global_memory["autoswapmode"]
|
||||
if is_post and (is_completions_path or is_chat_completions_path or (not autoswapEnabled and is_wake_request)):
|
||||
model_name = ""
|
||||
if body:
|
||||
try:
|
||||
@@ -3875,6 +3887,52 @@ class KcppProxyHandler(http.server.BaseHTTPRequestHandler):
|
||||
self.send_error(504, "KoboldCpp model swap reload timed out")
|
||||
return
|
||||
time.sleep(0.1)
|
||||
elif autoswapEnabled:
|
||||
textReqs = ["/api/extra/generate/stream","/api/extra/tokencount","/api/v1/generate","/sdapi/v1/interrogate","/v1/completions","/v1/chat/completions"]
|
||||
sttReqs = ["/api/extra/transcribe","/v1/audio/transcriptions"]
|
||||
ttsReqs = ["/api/extra/tts", "/v1/audio/speech"]
|
||||
embedReqs = ["/api/extra/embeddings", "/v1/embeddings"]
|
||||
musicReqs = ["/api/extra/music/prepare","/api/extra/music/generate"]
|
||||
imageReqs = ["/sdapi/v1/txt2img", "/sdapi/v1/img2img", "/sdapi/v1/upscale"] # "/sdapi/v1/sd-models", "/sdapi/v1/options", "/sdapi/v1/samplers"
|
||||
|
||||
swapModeChanged = False
|
||||
if any(self.path.endswith(e) for e in textReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "text"):
|
||||
global_memory["swapReqType"] = "text"
|
||||
swapModeChanged = True
|
||||
elif any(self.path.endswith(e) for e in sttReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "stt"):
|
||||
global_memory["swapReqType"] = "stt"
|
||||
swapModeChanged = True
|
||||
elif any(self.path.endswith(e) for e in ttsReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "tts"):
|
||||
global_memory["swapReqType"] = "tts"
|
||||
swapModeChanged = True
|
||||
elif any(self.path.endswith(e) for e in embedReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "embed"):
|
||||
global_memory["swapReqType"] = "embed"
|
||||
swapModeChanged = True
|
||||
elif any(self.path.endswith(e) for e in musicReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "music"):
|
||||
global_memory["swapReqType"] = "music"
|
||||
swapModeChanged = True
|
||||
elif any(self.path.endswith(e) for e in imageReqs) and (global_memory["swapReqType"] is None or global_memory["swapReqType"] != "image"):
|
||||
global_memory["swapReqType"] = "image"
|
||||
swapModeChanged = True
|
||||
|
||||
if (global_memory["swapReqType"] is not None and swapModeChanged):
|
||||
with proxy_reload_lock:
|
||||
reqbody = json.dumps({"filename":global_memory["current_model"]})
|
||||
reqheaders = {
|
||||
'Content-Type': 'application/json',
|
||||
'Content-Length': str(len(reqbody)),
|
||||
}
|
||||
if args.adminpassword:
|
||||
reqheaders["Authorization"] = f"Bearer {args.adminpassword}"
|
||||
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)
|
||||
conn.request("POST", "/api/admin/reload_config", body=reqbody, headers=reqheaders)
|
||||
resp = conn.getresponse()
|
||||
time.sleep(3)
|
||||
global_memory["last_active_timestamp"] = datetime.now()
|
||||
if not self.wait_for_upstream_ready(upstream_port,120,0.5):
|
||||
self.send_error(504, "KoboldCpp model swap reload timed out")
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
try: # connect upstream
|
||||
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)
|
||||
@@ -4093,6 +4151,8 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
|
||||
async def generate_text(self, genparams, api_format, stream_flag):
|
||||
global friendlymodelname, chatcompl_adapter, currfinishreason
|
||||
global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName
|
||||
|
||||
currfinishreason = None
|
||||
req_id_suffix = genparams.get('oai_uniqueid',1)
|
||||
chatcmpl_id = f"chatcmpl-A{req_id_suffix}"
|
||||
@@ -4162,14 +4222,17 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
if args.debugmode:
|
||||
print(f"Debug ToolCall Response: {json.dumps(tool_calls)}")
|
||||
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
if api_format == 1:
|
||||
res = {"data": {"seqs": [recvtxt]}}
|
||||
elif api_format == 3:
|
||||
res = {"id": cmpl_id, "object": "text_completion", "created": int(time.time()), "model": friendlymodelname,
|
||||
res = {"id": cmpl_id, "object": "text_completion", "created": int(time.time()), "model": modelNameToReturn,
|
||||
"usage": {"prompt_tokens": prompttokens, "completion_tokens": comptokens, "total_tokens": (prompttokens+comptokens)},
|
||||
"choices": [{"text": recvtxt, "index": 0, "finish_reason": currfinishreason, "logprobs":logprobsdict}]}
|
||||
elif api_format == 4:
|
||||
res = {"id": chatcmpl_id, "object": "chat.completion", "created": int(time.time()), "model": friendlymodelname,
|
||||
res = {"id": chatcmpl_id, "object": "chat.completion", "created": int(time.time()), "model": modelNameToReturn,
|
||||
"usage": {"prompt_tokens": prompttokens, "completion_tokens": comptokens, "total_tokens": (prompttokens+comptokens)},
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": recvtxt, "tool_calls": tool_calls}, "finish_reason": currfinishreason, "logprobs":logprobsdict}]}
|
||||
elif api_format == 5:
|
||||
@@ -4177,9 +4240,9 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
elif api_format == 6:
|
||||
oldprompt = genparams.get('ollamabodyprompt', "")
|
||||
tokarr = tokenize_ids(oldprompt+recvtxt,False)
|
||||
res = {"model": friendlymodelname,"created_at": str(datetime.now(timezone.utc).isoformat()),"response":recvtxt,"done": True,"done_reason":currfinishreason,"context": tokarr,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1}
|
||||
res = {"model": modelNameToReturn,"created_at": str(datetime.now(timezone.utc).isoformat()),"response":recvtxt,"done": True,"done_reason":currfinishreason,"context": tokarr,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1}
|
||||
elif api_format == 7:
|
||||
res = {"model": friendlymodelname,"created_at": str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":recvtxt},"done": True,"done_reason":currfinishreason,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1}
|
||||
res = {"model": modelNameToReturn,"created_at": str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":recvtxt},"done": True,"done_reason":currfinishreason,"total_duration": 1,"load_duration": 1,"prompt_eval_count": prompttokens,"prompt_eval_duration": 1,"eval_count": comptokens,"eval_duration": 1}
|
||||
elif api_format == 8:
|
||||
resp_id = f"resp-A{genparams.get('oai_uniqueid', 1)}"
|
||||
output_item_id = f"msg_0{genparams.get('oai_uniqueid', 1)}"
|
||||
@@ -4242,6 +4305,12 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
|
||||
async def handle_sse_stream(self, genparams, api_format):
|
||||
global friendlymodelname, currfinishreason
|
||||
global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName
|
||||
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
|
||||
using_openai_tools = genparams.get('using_openai_tools', False)
|
||||
req_id_suffix = genparams.get('oai_uniqueid',1)
|
||||
chatcmpl_id = f"chatcmpl-A{req_id_suffix}"
|
||||
@@ -4373,10 +4442,10 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
|
||||
if need_split_final_msg: #we need to send one message without the finish reason, then send a finish reason with no msg to follow standards
|
||||
if api_format == 4: # if oai chat, set format to expected openai streaming response
|
||||
event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"delta":delta}]})
|
||||
event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"delta":delta}]})
|
||||
await self.send_oai_sse_event(event_str)
|
||||
elif api_format == 3: # non chat completions
|
||||
event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"text":tokenStr}]})
|
||||
event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"text":tokenStr}]})
|
||||
await self.send_oai_sse_event(event_str)
|
||||
else:
|
||||
event_str = json.dumps({"token": tokenStr, "finish_reason":None})
|
||||
@@ -4388,17 +4457,17 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs
|
||||
lastlogprobs = handle.last_logprobs()
|
||||
logprobsdict = parse_last_logprobs(lastlogprobs)
|
||||
addonstr = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"delta":{'role':'assistant','content':''},"logprobs":logprobsdict}]})
|
||||
addonstr = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"delta":{'role':'assistant','content':''},"logprobs":logprobsdict}]})
|
||||
await self.send_oai_sse_event(addonstr)
|
||||
event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":delta}]})
|
||||
event_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":delta}]})
|
||||
await self.send_oai_sse_event(event_str)
|
||||
elif api_format == 3: # non chat completions
|
||||
if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs
|
||||
lastlogprobs = handle.last_logprobs()
|
||||
logprobsdict = parse_last_logprobs(lastlogprobs)
|
||||
addonstr = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"text":"","logprobs":logprobsdict}]})
|
||||
addonstr = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":None,"text":"","logprobs":logprobsdict}]})
|
||||
await self.send_oai_sse_event(addonstr)
|
||||
event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]})
|
||||
event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]})
|
||||
await self.send_oai_sse_event(event_str)
|
||||
elif api_format == 8: #oai-responses
|
||||
resp_id = f"resp-A{genparams.get('oai_uniqueid', 1)}"
|
||||
@@ -4475,9 +4544,9 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
if (strop and strop.get("include_usage",False)): # Send a final chunk with usage info, only if requested
|
||||
usage_obj = {"prompt_tokens": prompttokens, "completion_tokens": current_token, "total_tokens": (prompttokens + current_token)}
|
||||
if api_format == 4:
|
||||
usage_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[],"usage":usage_obj})
|
||||
usage_str = json.dumps({"id":chatcmpl_id,"object":"chat.completion.chunk","created":int(time.time()),"model":modelNameToReturn,"choices":[],"usage":usage_obj})
|
||||
else:
|
||||
usage_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[],"usage":usage_obj})
|
||||
usage_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":modelNameToReturn,"choices":[],"usage":usage_obj})
|
||||
await self.send_oai_sse_event(usage_str)
|
||||
await self.send_oai_sse_event('[DONE]')
|
||||
await asyncio.sleep(async_sleep_short)
|
||||
@@ -4719,7 +4788,8 @@ Change Mode<br>
|
||||
global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz
|
||||
global last_req_time, start_time, cached_chat_template, has_vision_support, has_audio_support, has_whisper, friendlymodelname
|
||||
global savedata_obj, has_multiplayer, multiplayer_turn_major, multiplayer_turn_minor, multiplayer_story_data_compressed, multiplayer_dataformat, multiplayer_lastactive, maxctx, maxhordelen, friendlymodelname, lastuploadedcomfyimg, lastgeneratedcomfyimg, KcppVersion, totalgens, preloaded_story, exitcounter, currentusergenkey, friendlysdmodelname, fullsdmodelpath, password, friendlyembeddingsmodelname, voicelist
|
||||
|
||||
global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName
|
||||
|
||||
clean_path = self.path.split("?")[0] #for cases where we do not want query params
|
||||
if clean_path=="/lcpp": #fix for svelte redirect issues, browser path needs to end with slash
|
||||
clean_path = "/lcpp/"
|
||||
@@ -4760,7 +4830,10 @@ Change Mode<br>
|
||||
|
||||
elif clean_path.endswith(('/api/v1/model', '/api/latest/model')):
|
||||
auth_ok = self.check_header_password(password, args.adminpassword)
|
||||
response_body = (json.dumps({'result': (friendlymodelname if auth_ok else "koboldcpp/protected-model") }).encode())
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
response_body = (json.dumps({'result': (modelNameToReturn if auth_ok else "koboldcpp/protected-model") }).encode())
|
||||
|
||||
elif clean_path.endswith(('/api/v1/config/max_length', '/api/latest/config/max_length')):
|
||||
response_body = (json.dumps({"value": maxhordelen}).encode())
|
||||
@@ -4856,7 +4929,11 @@ Change Mode<br>
|
||||
response_body = (json.dumps({"logprobs":logprobsdict}).encode())
|
||||
|
||||
elif clean_path.endswith('/v1/models') or clean_path=='/models':
|
||||
mlist = [{"id":friendlymodelname,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}]
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
|
||||
mlist = [{"id":modelNameToReturn,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}]
|
||||
if args.routermode:
|
||||
alist = get_current_admindir_list()
|
||||
for itm in alist:
|
||||
@@ -4873,20 +4950,25 @@ Change Mode<br>
|
||||
response_body = (json.dumps([]).encode())
|
||||
|
||||
elif clean_path.endswith('/sdapi/v1/sd-models'):
|
||||
if friendlysdmodelname=="inactive" or fullsdmodelpath=="":
|
||||
if autoswapmode and imageName is not None:
|
||||
response_body = (json.dumps([{"title":imageName,"model_name":imageName,"hash":"8888888888","sha256":"8888888888888888888888888888888888888888888888888888888888888888","filename":imageName,"config": None}]).encode())
|
||||
elif friendlysdmodelname=="inactive" or fullsdmodelpath=="":
|
||||
response_body = (json.dumps([]).encode())
|
||||
else:
|
||||
response_body = (json.dumps([{"title":friendlysdmodelname,"model_name":friendlysdmodelname,"hash":"8888888888","sha256":"8888888888888888888888888888888888888888888888888888888888888888","filename":fullsdmodelpath,"config": None}]).encode())
|
||||
elif clean_path.endswith('/sdapi/v1/options'):
|
||||
response_body = (json.dumps({"samples_format":"png","sd_model_checkpoint":friendlysdmodelname}).encode())
|
||||
modelNameToReturn = friendlysdmodelname
|
||||
if autoswapmode and imageName is not None:
|
||||
modelNameToReturn = imageName
|
||||
response_body = (json.dumps({"samples_format":"png","sd_model_checkpoint":modelNameToReturn}).encode())
|
||||
elif clean_path.endswith('/sdapi/v1/samplers'):
|
||||
if friendlysdmodelname=="inactive" or fullsdmodelpath=="":
|
||||
if (friendlysdmodelname=="inactive" or fullsdmodelpath=="") and not(autoswapmode and imageName is not None):
|
||||
response_body = (json.dumps([]).encode())
|
||||
else:
|
||||
response_body = (json.dumps([{"name":"Euler","aliases":["k_euler"],"options":{}},{"name":"Euler a","aliases":["k_euler_a","k_euler_ancestral"],"options":{}},{"name":"Heun","aliases":["k_heun"],"options":{}},{"name":"DPM2","aliases":["k_dpm_2"],"options":{}},{"name":"DPM++ 2M","aliases":["k_dpmpp_2m"],"options":{}},{"name":"DDIM","aliases":["ddim"],"options":{}},{"name":"LCM","aliases":["k_lcm"],"options":{}},{"name":"Res 2s","aliases":["k_res_2s"],"options":{}},{"name":"Res Multistep","aliases":["k_res_multistep"],"options":{}},
|
||||
{"name":"Default","aliases":["default"],"options":{}}]).encode())
|
||||
elif clean_path.endswith('/sdapi/v1/schedulers'):
|
||||
if friendlysdmodelname=="inactive" or fullsdmodelpath=="":
|
||||
if (friendlysdmodelname=="inactive" or fullsdmodelpath=="") and not(autoswapmode and imageName is not None):
|
||||
response_body = (json.dumps([]).encode())
|
||||
else:
|
||||
response_body = (json.dumps([{"name":name,"label":name} for name in sd_get_available_schedulers()]).encode())
|
||||
@@ -4923,7 +5005,10 @@ Change Mode<br>
|
||||
response_body = (json.dumps({"temperature":0.75,"speed":1,"length_penalty":1,"repetition_penalty":1,"top_p":1,"top_k":4,"enable_text_splitting":True,"stream_chunk_size":100}).encode()) #some random voices for them to enjoy
|
||||
|
||||
elif clean_path.endswith('/api/tags') or clean_path.endswith('/api/ps'): #ollama compatible
|
||||
response_body = (json.dumps({"models":[{"name":"koboldcpp","model":f"{friendlymodelname}:latest","modified_at":"2024-07-19T15:26:55.6122841+08:00","expires_at": "2055-06-04T19:06:25.5433636+08:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}},{"name":"koboldcpp","model":friendlymodelname,"modified_at":"2025-01-01T01:00:00.0000000+00:00","expires_at": "2069-01-01T01:00:00.0000000+00:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}}]}).encode())
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
response_body = (json.dumps({"models":[{"name":"koboldcpp","model":f"{modelNameToReturn}:latest","modified_at":"2024-07-19T15:26:55.6122841+08:00","expires_at": "2055-06-04T19:06:25.5433636+08:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}},{"name":"koboldcpp","model":modelNameToReturn,"modified_at":"2025-01-01T01:00:00.0000000+00:00","expires_at": "2069-01-01T01:00:00.0000000+00:00","size":394998579,"size_vram":394998579,"digest":"b5dc5e784f2a3ee1582373093acf69a2f4e2ac1710b253a001712b86a61f88bb","details":{"parent_model":"","format":"gguf","family":"koboldcpp","families":["koboldcpp"],"parameter_size":"128M","quantization_level":"Q4_0"}}]}).encode())
|
||||
elif clean_path.endswith('/api/version'): #ollama compatible, NOT the kcpp version
|
||||
response_body = (json.dumps({"version":"0.7.0"}).encode())
|
||||
elif clean_path=='/ping':
|
||||
@@ -4933,9 +5018,14 @@ Change Mode<br>
|
||||
elif clean_path=='/system_stats':
|
||||
response_body = (json.dumps({"system":{"os":"posix","ram_total":12345678900,"ram_free":12345678900,"comfyui_version":"v0.3.4-3-g7126ecf","python_version":"3.10.12","pytorch_version":"2.5.1","embedded_python":False,"argv":[]},"devices":[{"name":"koboldcpp","type":"cuda","index":0,"vram_total":12345678900,"vram_free":12345678900,"torch_vram_total":12345678900,"torch_vram_free":12345678900}]}).encode())
|
||||
elif clean_path=='/object_info':
|
||||
response_body = (json.dumps({"KSampler":{"input":{"required":{"model":["MODEL",{"tooltip":""}],"seed":["INT",{"default":0,"min":0,"max":512,"tooltip":""}],"steps":["INT",{"default":20,"min":1,"max":512,"tooltip":""}],"cfg":["FLOAT",{"default":8.0,"min":0.0,"max":100.0,"step":0.1,"round":0.01,"tooltip":"512"}],"sampler_name":[["euler"],{"tooltip":""}],"scheduler":[["normal"],{"tooltip":""}],"positive":["CONDITIONING",{"tooltip":""}],"negative":["CONDITIONING",{"tooltip":""}],"latent_image":["LATENT",{"tooltip":""}],"denoise":["FLOAT",{"default":1.0,"min":0.0,"max":1.0,"step":0.01,"tooltip":""}]}},"input_order":{"required":["model","seed","steps","cfg","sampler_name","scheduler","positive","negative","latent_image","denoise"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"KSampler","display_name":"KSampler","description":"KSampler","python_module":"nodes","category":"sampling","output_node":False,"output_tooltips":[""]},"CheckpointLoaderSimple":{"input":{"required":{"ckpt_name":[[friendlysdmodelname],{"tooltip":""}]}},"input_order":{"required":["ckpt_name"]},"output":["MODEL","CLIP","VAE"],"output_is_list":[False,False,False],"output_name":["MODEL","CLIP","VAE"],"name":"CheckpointLoaderSimple","display_name":"Load","description":"","python_module":"nodes","category":"loaders","output_node":False,"output_tooltips":["","",""]},"CLIPTextEncode":{"input":{"required":{"text":["STRING",{"multiline":True,"dynamicPrompts":True,"tooltip":""}],"clip":["CLIP",{"tooltip":""}]}},"input_order":{"required":["text","clip"]},"output":["CONDITIONING"],"output_is_list":[False],"output_name":["CONDITIONING"],"name":"CLIPTextEncode","display_name":"CLIP","description":"","python_module":"nodes","category":"conditioning","output_node":False,"output_tooltips":[""]},"CLIPSetLastLayer":{"input":{"required":{"clip":["CLIP"],"stop_at_clip_layer":["INT",{"default":-1,"min":-24,"max":-1,"step":1}]}},"input_order":{"required":["clip","stop_at_clip_layer"]},"output":["CLIP"],"output_is_list":[False],"output_name":["CLIP"],"name":"CLIPSetLastLayer","display_name":"CLIPSLL","description":"","python_module":"nodes","category":"conditioning","output_node":False},"VAEDecode":{"input":{"required":{"samples":["LATENT",{"tooltip":""}],"vae":["VAE",{"tooltip":""}]}},"input_order":{"required":["samples","vae"]},"output":["IMAGE"],"output_is_list":[False],"output_name":["IMAGE"],"name":"VAEDecode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]},"VAEEncode":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"]}},"input_order":{"required":["pixels","vae"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False},"VAEEncodeForInpaint":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"],"mask":["MASK"],"grow_mask_by":["INT",{"default":6,"min":0,"max":64,"step":1}]}},"input_order":{"required":["pixels","vae","mask","grow_mask_by"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncodeForInpaint","display_name":"VAE","description":"","python_module":"nodes","category":"latent/inpaint","output_node":False},"VAELoader":{"input":{"required":{"vae_name":[["kcpp_vae"]]}},"input_order":{"required":["vae_name"]},"output":["VAE"],"output_is_list":[False],"output_name":["VAE"],"name":"VAELoader","display_name":"Load VAE","description":"","python_module":"nodes","category":"loaders","output_node":False},"EmptyLatentImage":{"input":{"required":{"width":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"height":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"batch_size":["INT",{"default":1,"min":1,"max":1,"tooltip":""}]}},"input_order":{"required":["width","height","batch_size"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"EmptyLatentImage","display_name":"Empty Latent Image","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]}}).encode())
|
||||
modelNameToReturn = friendlysdmodelname
|
||||
if autoswapmode and imageName is not None:
|
||||
modelNameToReturn = imageName
|
||||
response_body = (json.dumps({"KSampler":{"input":{"required":{"model":["MODEL",{"tooltip":""}],"seed":["INT",{"default":0,"min":0,"max":512,"tooltip":""}],"steps":["INT",{"default":20,"min":1,"max":512,"tooltip":""}],"cfg":["FLOAT",{"default":8.0,"min":0.0,"max":100.0,"step":0.1,"round":0.01,"tooltip":"512"}],"sampler_name":[["euler"],{"tooltip":""}],"scheduler":[["normal"],{"tooltip":""}],"positive":["CONDITIONING",{"tooltip":""}],"negative":["CONDITIONING",{"tooltip":""}],"latent_image":["LATENT",{"tooltip":""}],"denoise":["FLOAT",{"default":1.0,"min":0.0,"max":1.0,"step":0.01,"tooltip":""}]}},"input_order":{"required":["model","seed","steps","cfg","sampler_name","scheduler","positive","negative","latent_image","denoise"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"KSampler","display_name":"KSampler","description":"KSampler","python_module":"nodes","category":"sampling","output_node":False,"output_tooltips":[""]},"CheckpointLoaderSimple":{"input":{"required":{"ckpt_name":[[modelNameToReturn],{"tooltip":""}]}},"input_order":{"required":["ckpt_name"]},"output":["MODEL","CLIP","VAE"],"output_is_list":[False,False,False],"output_name":["MODEL","CLIP","VAE"],"name":"CheckpointLoaderSimple","display_name":"Load","description":"","python_module":"nodes","category":"loaders","output_node":False,"output_tooltips":["","",""]},"CLIPTextEncode":{"input":{"required":{"text":["STRING",{"multiline":True,"dynamicPrompts":True,"tooltip":""}],"clip":["CLIP",{"tooltip":""}]}},"input_order":{"required":["text","clip"]},"output":["CONDITIONING"],"output_is_list":[False],"output_name":["CONDITIONING"],"name":"CLIPTextEncode","display_name":"CLIP","description":"","python_module":"nodes","category":"conditioning","output_node":False,"output_tooltips":[""]},"CLIPSetLastLayer":{"input":{"required":{"clip":["CLIP"],"stop_at_clip_layer":["INT",{"default":-1,"min":-24,"max":-1,"step":1}]}},"input_order":{"required":["clip","stop_at_clip_layer"]},"output":["CLIP"],"output_is_list":[False],"output_name":["CLIP"],"name":"CLIPSetLastLayer","display_name":"CLIPSLL","description":"","python_module":"nodes","category":"conditioning","output_node":False},"VAEDecode":{"input":{"required":{"samples":["LATENT",{"tooltip":""}],"vae":["VAE",{"tooltip":""}]}},"input_order":{"required":["samples","vae"]},"output":["IMAGE"],"output_is_list":[False],"output_name":["IMAGE"],"name":"VAEDecode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]},"VAEEncode":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"]}},"input_order":{"required":["pixels","vae"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncode","display_name":"VAE","description":"","python_module":"nodes","category":"latent","output_node":False},"VAEEncodeForInpaint":{"input":{"required":{"pixels":["IMAGE"],"vae":["VAE"],"mask":["MASK"],"grow_mask_by":["INT",{"default":6,"min":0,"max":64,"step":1}]}},"input_order":{"required":["pixels","vae","mask","grow_mask_by"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"VAEEncodeForInpaint","display_name":"VAE","description":"","python_module":"nodes","category":"latent/inpaint","output_node":False},"VAELoader":{"input":{"required":{"vae_name":[["kcpp_vae"]]}},"input_order":{"required":["vae_name"]},"output":["VAE"],"output_is_list":[False],"output_name":["VAE"],"name":"VAELoader","display_name":"Load VAE","description":"","python_module":"nodes","category":"loaders","output_node":False},"EmptyLatentImage":{"input":{"required":{"width":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"height":["INT",{"default":512,"min":16,"max":16384,"step":8,"tooltip":""}],"batch_size":["INT",{"default":1,"min":1,"max":1,"tooltip":""}]}},"input_order":{"required":["width","height","batch_size"]},"output":["LATENT"],"output_is_list":[False],"output_name":["LATENT"],"name":"EmptyLatentImage","display_name":"Empty Latent Image","description":"","python_module":"nodes","category":"latent","output_node":False,"output_tooltips":[""]}}).encode())
|
||||
elif clean_path.endswith('/api/models/checkpoints') or clean_path.endswith('/models/checkpoints'): #emulate comfyui, duplication is redundant but added for clarity
|
||||
if friendlysdmodelname=="inactive" or fullsdmodelpath=="":
|
||||
if autoswapmode and imageName is not None:
|
||||
response_body = (json.dumps([imageName]).encode())
|
||||
elif friendlysdmodelname=="inactive" or fullsdmodelpath=="":
|
||||
response_body = (json.dumps([]).encode())
|
||||
else:
|
||||
response_body = (json.dumps([friendlysdmodelname]).encode())
|
||||
@@ -4945,8 +5035,11 @@ Change Mode<br>
|
||||
content_type = 'image/png'
|
||||
response_body = lastgeneratedcomfyimg
|
||||
elif clean_path=='/history' or clean_path=='/api/history' or clean_path.startswith('/api/history/') or clean_path.startswith('/history/'): #emulate comfyui
|
||||
modelNameToReturn = friendlysdmodelname
|
||||
if autoswapmode and imageName is not None:
|
||||
modelNameToReturn = imageName
|
||||
imgdone = (False if lastgeneratedcomfyimg==b'' else True)
|
||||
response_body = (json.dumps({"12345678-0000-0000-0000-000000000001":{"prompt":[0,"12345678-0000-0000-0000-000000000001",{"3":{"class_type":"KSampler","inputs":{"cfg":5.0,"denoise":1.0,"latent_image":["5",0],"model":["4",0],"negative":["7",0],"positive":["6",0],"sampler_name":"euler","scheduler":"normal","seed":1,"steps":20}},"4":{"class_type":"CheckpointLoaderSimple","inputs":{"ckpt_name":friendlysdmodelname}},"5":{"class_type":"EmptyLatentImage","inputs":{"batch_size":1,"height":512,"width":512}},"6":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":"prompt"}},"7":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":""}},"8":{"class_type":"VAEDecode","inputs":{"samples":["3",0],"vae":["4",2]}},"9":{"class_type":"SaveImage","inputs":{"filename_prefix":"kliteimg","images":["8",0]}}},{},["9"]],"outputs":{"9":{"images":[{"filename":"kliteimg_00001_.png","subfolder":"","type":"output"}]}},"status":{"status_str":"success","completed":imgdone,"messages":[["execution_start",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_cached",{"nodes":[],"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_success",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}]]},"meta":{"9":{"node_id":"9","display_node":"9","parent_node":None,"real_node_id":"9"}}}}).encode())
|
||||
response_body = (json.dumps({"12345678-0000-0000-0000-000000000001":{"prompt":[0,"12345678-0000-0000-0000-000000000001",{"3":{"class_type":"KSampler","inputs":{"cfg":5.0,"denoise":1.0,"latent_image":["5",0],"model":["4",0],"negative":["7",0],"positive":["6",0],"sampler_name":"euler","scheduler":"normal","seed":1,"steps":20}},"4":{"class_type":"CheckpointLoaderSimple","inputs":{"ckpt_name":modelNameToReturn}},"5":{"class_type":"EmptyLatentImage","inputs":{"batch_size":1,"height":512,"width":512}},"6":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":"prompt"}},"7":{"class_type":"CLIPTextEncode","inputs":{"clip":["4",1],"text":""}},"8":{"class_type":"VAEDecode","inputs":{"samples":["3",0],"vae":["4",2]}},"9":{"class_type":"SaveImage","inputs":{"filename_prefix":"kliteimg","images":["8",0]}}},{},["9"]],"outputs":{"9":{"images":[{"filename":"kliteimg_00001_.png","subfolder":"","type":"output"}]}},"status":{"status_str":"success","completed":imgdone,"messages":[["execution_start",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_cached",{"nodes":[],"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}],["execution_success",{"prompt_id":"12345678-0000-0000-0000-000000000001","timestamp":1}]]},"meta":{"9":{"node_id":"9","display_node":"9","parent_node":None,"real_node_id":"9"}}}}).encode())
|
||||
elif clean_path=='/ws' and ('Upgrade' in self.headers and self.headers['Upgrade'].lower() == 'websocket' and
|
||||
'Sec-WebSocket-Key' in self.headers):
|
||||
ws_key = self.headers['Sec-WebSocket-Key']
|
||||
@@ -4976,16 +5069,22 @@ Change Mode<br>
|
||||
response_body = (json.dumps({"version":"0.2","software":{"name":"KoboldCpp","version":KcppVersion,"repository":"https://github.com/LostRuins/koboldcpp","homepage":"https://github.com/LostRuins/koboldcpp","logo":"https://raw.githubusercontent.com/LostRuins/koboldcpp/refs/heads/concedo/niko.ico"},"api":{"koboldai":{"name":"KoboldAI API","rel_url":"/api","documentation":"https://lite.koboldai.net/koboldcpp_api","version":KcppVersion},"openai":{"name":"OpenAI API","rel_url ":"/v1","documentation":"https://openai.com/documentation/api","version":KcppVersion}}}).encode())
|
||||
|
||||
elif clean_path=="/props":
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
mmprojOverride = False
|
||||
if autoswapmode and mmprojName is not None:
|
||||
mmprojOverride = True
|
||||
response_body = (json.dumps({
|
||||
"chat_template": cached_chat_template,
|
||||
"id": 0,
|
||||
"id_task": -1,
|
||||
"total_slots": 1,
|
||||
"modalities": {
|
||||
"vision": has_vision_support,
|
||||
"vision": mmprojOverride or has_vision_support,
|
||||
"audio": has_audio_support
|
||||
},
|
||||
"model_path": friendlymodelname,
|
||||
"model_path": modelNameToReturn,
|
||||
"n_ctx": maxctx,
|
||||
"default_generation_settings": {
|
||||
"n_ctx": maxctx,
|
||||
@@ -5067,6 +5166,7 @@ Change Mode<br>
|
||||
|
||||
def do_POST(self):
|
||||
global modelbusy, requestsinqueue, currentusergenkey, totalgens, pendingabortkey, lastuploadedcomfyimg, lastgeneratedcomfyimg, multiplayer_turn_major, multiplayer_turn_minor, multiplayer_story_data_compressed, multiplayer_dataformat, multiplayer_lastactive, net_save_slots, has_vision_support, savestate_limit, mcp_lock
|
||||
global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName
|
||||
contlenstr = self.headers['content-length']
|
||||
content_length = 0
|
||||
body = None
|
||||
@@ -5620,7 +5720,10 @@ Change Mode<br>
|
||||
elif self.path.endswith('/v1/chat/completions') or self.path=='/chat/completions':
|
||||
api_format = 4
|
||||
elif self.path.endswith('/sdapi/v1/interrogate'):
|
||||
if not has_vision_support:
|
||||
mmprojOverride = False
|
||||
if autoswapmode and mmprojName is not None:
|
||||
mmprojOverride = True
|
||||
if not mmprojOverride and not has_vision_support:
|
||||
self.send_response(503)
|
||||
self.end_headers(content_type='application/json')
|
||||
self.wfile.write(json.dumps({"detail": {
|
||||
@@ -5747,6 +5850,9 @@ Change Mode<br>
|
||||
gendat = asyncio.run(self.handle_request(genparams, api_format, sse_stream_flag))
|
||||
|
||||
try:
|
||||
modelNameToReturn = friendlymodelname
|
||||
if autoswapmode and textName is not None:
|
||||
modelNameToReturn = textName
|
||||
# Headers are already sent when streaming
|
||||
if (api_format == 6 or api_format == 7) and genparams.get('stream', True):
|
||||
#ollama fake streaming
|
||||
@@ -5758,7 +5864,7 @@ Change Mode<br>
|
||||
if api_format == 6:
|
||||
bodytxt = gendat.get("response","") # extract and erase the AI response from the sync payload.
|
||||
gendat["response"] = ""
|
||||
pl = {"model":friendlymodelname,"created_at":str(datetime.now(timezone.utc).isoformat()),"response":bodytxt,"done":False}
|
||||
pl = {"model":modelNameToReturn,"created_at":str(datetime.now(timezone.utc).isoformat()),"response":bodytxt,"done":False}
|
||||
self.wfile.write(f'{json.dumps(pl)}\n'.encode())
|
||||
self.wfile.flush()
|
||||
time.sleep(0.05) #short delay
|
||||
@@ -5768,7 +5874,7 @@ Change Mode<br>
|
||||
else:
|
||||
bodytxt = gendat.get("message",{}).get("content","") # extract and erase the AI response from the sync payload.
|
||||
gendat["message"] = {"role":"assistant","content":""}
|
||||
pl = {"model":friendlymodelname,"created_at":str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":bodytxt},"done":False}
|
||||
pl = {"model":modelNameToReturn,"created_at":str(datetime.now(timezone.utc).isoformat()),"message":{"role":"assistant","content":bodytxt},"done":False}
|
||||
self.wfile.write(f'{json.dumps(pl)}\n'.encode())
|
||||
self.wfile.flush()
|
||||
time.sleep(0.05) #short delay
|
||||
@@ -5801,7 +5907,7 @@ Change Mode<br>
|
||||
"id": "koboldcpp",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": friendlymodelname,
|
||||
"model": modelNameToReturn,
|
||||
"choices": [{"index": 0, "finish_reason": None, "delta": {"role": "assistant"}}]
|
||||
})
|
||||
self.wfile.write(f"data: {chunk_role}\n\n".encode())
|
||||
@@ -5813,7 +5919,7 @@ Change Mode<br>
|
||||
"id": "koboldcpp",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": friendlymodelname,
|
||||
"model": modelNameToReturn,
|
||||
"choices": [{"index": 0, "finish_reason": None, "delta": {"content": content_text}}]
|
||||
})
|
||||
self.wfile.write(f"data: {chunk_content}\n\n".encode())
|
||||
@@ -5835,7 +5941,7 @@ Change Mode<br>
|
||||
"id": "koboldcpp",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": friendlymodelname,
|
||||
"model": modelNameToReturn,
|
||||
"choices": [{"index": 0, "finish_reason": None, "delta": {"tool_calls": [tc_meta]}}]
|
||||
})
|
||||
self.wfile.write(f"data: {chunk_meta}\n\n".encode())
|
||||
@@ -5852,7 +5958,7 @@ Change Mode<br>
|
||||
"id": "koboldcpp",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": friendlymodelname,
|
||||
"model": modelNameToReturn,
|
||||
"choices": [{"index": 0, "finish_reason": None, "delta": {"tool_calls": [tc_args]}}]
|
||||
})
|
||||
self.wfile.write(f"data: {chunk_args}\n\n".encode())
|
||||
@@ -5863,7 +5969,7 @@ Change Mode<br>
|
||||
"id": "koboldcpp",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": friendlymodelname,
|
||||
"model": modelNameToReturn,
|
||||
"choices": [{"index": 0, "finish_reason": "tool_calls", "delta": {}}]
|
||||
})
|
||||
self.wfile.write(f"data: {chunk_final}\n\n".encode())
|
||||
@@ -5980,6 +6086,9 @@ Change Mode<br>
|
||||
return
|
||||
elif is_embeddings:
|
||||
try:
|
||||
modelNameToReturn = friendlyembeddingsmodelname
|
||||
if autoswapmode and embedName is not None:
|
||||
modelNameToReturn = embedName
|
||||
gendat = embeddings_generate(genparams)
|
||||
outdatas = []
|
||||
odidx = 0
|
||||
@@ -5991,7 +6100,7 @@ Change Mode<br>
|
||||
else:
|
||||
outdatas.append({"object":"embedding","index":odidx,"embedding":od})
|
||||
odidx += 1
|
||||
genresp = (json.dumps({"object":"list","data":outdatas,"model":friendlyembeddingsmodelname,"usage":{"prompt_tokens":gendat["count"],"total_tokens":gendat["count"]}}).encode())
|
||||
genresp = (json.dumps({"object":"list","data":outdatas,"model":modelNameToReturn,"usage":{"prompt_tokens":gendat["count"],"total_tokens":gendat["count"]}}).encode())
|
||||
self.send_response(200)
|
||||
self.send_header('content-length', str(len(genresp)))
|
||||
self.end_headers(content_type='application/json')
|
||||
@@ -6660,6 +6769,7 @@ def show_gui():
|
||||
admin_password_var = ctk.StringVar()
|
||||
singleinstance_var = ctk.IntVar(value=0)
|
||||
router_mode_var = ctk.IntVar(value=0)
|
||||
autoswap_mode_var = ctk.IntVar(value=0)
|
||||
admin_unload_timeout_var = ctk.StringVar(value=str(0))
|
||||
|
||||
nozenity_var = ctk.IntVar(value=0)
|
||||
@@ -7486,12 +7596,19 @@ def show_gui():
|
||||
router_mode_box.grid()
|
||||
else:
|
||||
router_mode_box.grid_remove()
|
||||
def togglerouter(a,b,c):
|
||||
if router_mode_var.get()==1:
|
||||
autoswap_mode_box.grid()
|
||||
else:
|
||||
autoswap_mode_box.grid_remove()
|
||||
|
||||
makecheckbox(admin_tab, "Enable Model Administration", admin_var, 1, 0, command=toggleadmin,tooltiptxt="Enable a admin server, allowing you to remotely relaunch and swap models and configs.")
|
||||
makelabelentry(admin_tab, "Admin Password:" , admin_password_var, 3, 150,padx=(120),singleline=True,tooltip="Require a password to access admin functions. You are strongly advised to use one for publically accessible instances!")
|
||||
makefileentry(admin_tab, "Config Directory (Required):", "Select directory containing .gguf or .kcpps files to relaunch from", admin_dir_var, 5, width=280, dialog_type=2, tooltiptxt="Specify a directory to look for .kcpps configs in, which can be used to swap models.")
|
||||
makelabelentry(admin_tab, "Auto Unload Timeout:" , admin_unload_timeout_var, 7, 70,padx=(150),singleline=True,tooltip="Set an idle timeout in seconds after which KoboldCpp will automatically unload the current model.")
|
||||
makecheckbox(admin_tab, "SingleInstance Mode", singleinstance_var, 10, 0,tooltiptxt="Allows this server to be shut down by another KoboldCpp instance with singleinstance starting on the same port.")
|
||||
router_mode_box = makecheckbox(admin_tab, "Router Mode", router_mode_var, 15, 0,tooltiptxt="Router mode uses a reverse proxy router, allowing you to easily hotswap models and configs within a single request. Requires admin mode.")
|
||||
makecheckbox(admin_tab, "SingleInstance Mode", singleinstance_var, 9, 0,tooltiptxt="Allows this server to be shut down by another KoboldCpp instance with singleinstance starting on the same port.")
|
||||
router_mode_box = makecheckbox(admin_tab, "Router Mode", router_mode_var, 11, 0, command=togglerouter, tooltiptxt="Router mode uses a reverse proxy router, allowing you to easily hotswap models and configs within a single request. Requires admin mode.")
|
||||
autoswap_mode_box = makecheckbox(admin_tab, "Autoswap Mode", autoswap_mode_var, 13, 0,tooltiptxt="Autoswap mode builds on router mode to allow switching of model types within the same config automatically. Requires admin mode and router mode. All models desired must be defined within the same config.")
|
||||
|
||||
def kcpp_export_template():
|
||||
nonlocal kcpp_exporting_template
|
||||
@@ -7811,6 +7928,7 @@ def show_gui():
|
||||
args.adminpassword = admin_password_var.get()
|
||||
args.singleinstance = (singleinstance_var.get()==1)
|
||||
args.routermode = router_mode_var.get()==1
|
||||
args.autoswapmode = autoswap_mode_var.get()==1
|
||||
args.adminunloadtimeout = (0 if admin_unload_timeout_var.get()=="" else int(admin_unload_timeout_var.get()))
|
||||
args.showgui = False #prevent showgui from leaking into configs, its cli only
|
||||
|
||||
@@ -8069,6 +8187,7 @@ def show_gui():
|
||||
|
||||
admin_var.set(dict["admin"] if ("admin" in dict) else 0)
|
||||
router_mode_var.set(dict["routermode"] if ("routermode" in dict) else 0)
|
||||
autoswap_mode_var.set(dict["autoswapmode"] if ("autoswapmode" in dict) else 0)
|
||||
admin_dir_var.set(dict["admindir"] if ("admindir" in dict and dict["admindir"]) else "")
|
||||
admin_password_var.set(dict["adminpassword"] if ("adminpassword" in dict and dict["adminpassword"]) else "")
|
||||
admin_unload_timeout_var.set(dict["adminunloadtimeout"] if ("adminunloadtimeout" in dict and dict["adminunloadtimeout"]) else 0)
|
||||
@@ -9102,6 +9221,10 @@ def main(launch_args, default_args):
|
||||
sslvalid = True
|
||||
|
||||
args.proxy_port = None #normally unused
|
||||
if args.autoswapmode:
|
||||
if not args.routermode:
|
||||
print("\nWARNING: Autoswap mode requires router, enabling router...")
|
||||
args.routermode = True
|
||||
if args.routermode:
|
||||
if not args.admin:
|
||||
print("\nWARNING: Router mode requires admin, enabling admin...")
|
||||
@@ -9146,7 +9269,7 @@ def main(launch_args, default_args):
|
||||
input()
|
||||
else: # manager command queue for admin mode
|
||||
with multiprocessing.Manager() as mp_manager:
|
||||
global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model"})
|
||||
global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_config_target":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "swapReqType": None, "autoswapmode": False})
|
||||
|
||||
if args.remotetunnel and not args.prompt and not args.benchmark and not args.cli:
|
||||
setuptunnel(global_memory, True if args.sdmodel else False)
|
||||
@@ -9179,6 +9302,7 @@ def main(launch_args, default_args):
|
||||
kcpp_instance.start()
|
||||
global_memory["restart_target"] = ""
|
||||
global_memory["restart_override_config_target"] = ""
|
||||
global_memory["swapReqType"] = None
|
||||
time.sleep(3)
|
||||
else:
|
||||
break # kill the program
|
||||
@@ -9191,10 +9315,16 @@ def main(launch_args, default_args):
|
||||
curtime = datetime.now()
|
||||
elapsedtime = curtime - last_active
|
||||
time_since_last_active = elapsedtime.total_seconds()
|
||||
if time_since_last_active > args.adminunloadtimeout and global_memory["current_model"]!="unload_model":
|
||||
print(f"[Unload Timeout] Inactive for over {time_since_last_active}s, unloading models...")
|
||||
restart_target = "unload_model"
|
||||
global_memory["triggered_sleeping"] = True
|
||||
if time_since_last_active > args.adminunloadtimeout:
|
||||
if args.autoswapmode:
|
||||
if global_memory["swapReqType"] is not None and global_memory["swapReqType"] != "nomodel":
|
||||
print(f"[Unload Timeout] Inactive for over {time_since_last_active}s, unloading models via autoswap...")
|
||||
global_memory["swapReqType"] = "nomodel"
|
||||
global_memory["triggered_sleeping"] = True
|
||||
elif global_memory["current_model"]!="unload_model":
|
||||
print(f"[Unload Timeout] Inactive for over {time_since_last_active}s, unloading models...")
|
||||
restart_target = "unload_model"
|
||||
global_memory["triggered_sleeping"] = True
|
||||
if restart_target!="":
|
||||
overridetxt = ("" if not restart_override_config_target else f" with override config {restart_override_config_target}")
|
||||
print(f"Reloading new model/config: {restart_target}{overridetxt}")
|
||||
@@ -9235,6 +9365,7 @@ def main(launch_args, default_args):
|
||||
args.model_param = targetfilepath
|
||||
else:
|
||||
reload_new_config(targetfilepath,defaultargs)
|
||||
global_memory["autoswapmode"] = args.autoswapmode
|
||||
kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": False})
|
||||
kcpp_instance.daemon = True
|
||||
kcpp_instance.start()
|
||||
@@ -9341,6 +9472,27 @@ def mk_lora_info(imgloras, multipliers, mock_filesystem=False):
|
||||
preloaded_table.append(lora_entry)
|
||||
return preloaded_table, lora_path_map, lora_name_map
|
||||
|
||||
def disableSwappedFieldsInConfig(args, swapReqType):
|
||||
print(f"Swapping to type: {swapReqType}")
|
||||
if swapReqType != "text":
|
||||
for e in ["model", "model_param", "lora", "mmproj"]:
|
||||
setattr(args, e, "")
|
||||
if swapReqType != "stt":
|
||||
for e in ["whispermodel"]:
|
||||
setattr(args, e, "")
|
||||
if swapReqType != "tts":
|
||||
for e in ["ttsmodel", "ttswavtokenizer"]:
|
||||
setattr(args, e, "")
|
||||
if swapReqType != "embed":
|
||||
for e in ["embeddingsmodel"]:
|
||||
setattr(args, e, "")
|
||||
if swapReqType != "music":
|
||||
for e in ["musicllm", "musicembeddings", "musicdiffusion", "musicvae"]:
|
||||
setattr(args, e, "")
|
||||
if swapReqType != "image":
|
||||
for e in ["sdmodel", "sdt5xxl", "sdclip1", "sdclip2", "sdphotomaker", "sdupscaler", "sdvae", "sdlora"]:
|
||||
setattr(args, e, "")
|
||||
|
||||
|
||||
def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
||||
global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz, start_time, exitcounter, global_memory, using_gui_launcher
|
||||
@@ -9356,6 +9508,56 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
||||
if args.model_param and (args.prompt and not args.cli) and not args.benchmark and not (args.debugmode >= 1):
|
||||
suppress_stdout()
|
||||
|
||||
|
||||
global autoswapmode, textName, sttName, ttsName, embedName, musicName, imageName, mmprojName
|
||||
autoswapmode = False
|
||||
textName = None
|
||||
mmprojName = None
|
||||
sttName = None
|
||||
ttsName = None
|
||||
embedName = None
|
||||
musicName = None
|
||||
imageName = None
|
||||
if args.autoswapmode is not None and args.autoswapmode:
|
||||
autoswapmode = True
|
||||
global_memory["autoswapmode"] = True
|
||||
if args.model_param and args.model_param!="":
|
||||
tempName = os.path.basename(os.path.abspath(args.model_param))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
textName = "koboldcpp/" + sanitize_string(tempName)
|
||||
if args.mmproj and args.mmproj!="": # multimodal vision and audio support is assumed to work with mmproj - this may be incorrect!
|
||||
tempName = os.path.basename(os.path.abspath(args.mmproj))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
mmprojName = sanitize_string(tempName)
|
||||
if args.whispermodel and args.whispermodel!="":
|
||||
tempName = os.path.basename(os.path.abspath(args.whispermodel))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
sttName = sanitize_string(tempName)
|
||||
if args.ttsmodel and args.ttsmodel!="":
|
||||
tempName = os.path.basename(os.path.abspath(args.ttsmodel))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
ttsName = sanitize_string(tempName)
|
||||
if args.embeddingsmodel and args.embeddingsmodel!="":
|
||||
tempName = os.path.basename(os.path.abspath(args.embeddingsmodel))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
embedName = sanitize_string(tempName)
|
||||
if args.musicdiffusion and args.musicdiffusion!="":
|
||||
tempName = os.path.basename(os.path.abspath(args.musicdiffusion))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
musicName = sanitize_string(tempName)
|
||||
if args.sdmodel and args.sdmodel!="":
|
||||
tempName = os.path.basename(os.path.abspath(args.sdmodel))
|
||||
tempName = os.path.splitext(tempName)[0]
|
||||
imageName = sanitize_string(tempName)
|
||||
if global_memory["swapReqType"] is not None:
|
||||
disableSwappedFieldsInConfig(args, global_memory["swapReqType"])
|
||||
else:
|
||||
global_memory["swapReqType"] = "nomodel"
|
||||
setattr(args, "nomodel", True)
|
||||
disableSwappedFieldsInConfig(args, "nomodel")
|
||||
else:
|
||||
global_memory["autoswapmode"] = False
|
||||
|
||||
if args.model_param and (args.benchmark or args.prompt or args.cli):
|
||||
start_server = False
|
||||
|
||||
@@ -9997,7 +10199,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
||||
print("Could not find Embedded MusicUI.")
|
||||
|
||||
# load all TTS audio files
|
||||
if args.ttsmodel:
|
||||
if args.ttsmodel or ttsName is not None:
|
||||
try:
|
||||
global voicebank, voicelist
|
||||
voicebank = {}
|
||||
@@ -10428,6 +10630,7 @@ if __name__ == '__main__':
|
||||
admingroup.add_argument("--admindir", metavar=('[directory]'), help="Specify a directory to look for .kcpps configs in, which can be used to swap models.", default="")
|
||||
admingroup.add_argument("--adminunloadtimeout", help="Set an idle timeout in seconds after which KoboldCpp will automatically unload the current model.", type=int, default=0)
|
||||
admingroup.add_argument("--routermode", help="Router mode uses a reverse proxy router, allowing you to easily hotswap models and configs within a single request. Requires admin mode.", action='store_true')
|
||||
admingroup.add_argument("--autoswapmode", help="Autoswap mode builds on router mode to allow switching of model types within the same config automatically. Requires admin mode and router mode. All models desired must be defined within the same config.", action='store_true')
|
||||
|
||||
deprecatedgroup = parser.add_argument_group('Deprecated Commands, DO NOT USE!')
|
||||
deprecatedgroup.add_argument("--hordeconfig", help=argparse.SUPPRESS, nargs='+')
|
||||
|
||||
Reference in New Issue
Block a user