From 96ec87127aebd0de9f567c74f8de0522638e6b9b Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Tue, 21 Apr 2026 21:46:13 +0800 Subject: [PATCH] updated colab, handle connection dropping during prompt processing --- colab.ipynb | 11 ++++++++--- gpttype_adapter.cpp | 2 +- koboldcpp.py | 47 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/colab.ipynb b/colab.ipynb index 5464b1c59..4aa243089 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -52,21 +52,21 @@ "#@markdown You will need to restart KoboldCpp with the play button when you make changes to these settings.\n", "\n", "#@markdown ### Select a quick preset template (optional)\n", - "Template = \"Tiefighter 13B (General)\" # @param [\"Tiefighter 13B (General)\",\"Estopia 13B (General)\",\"Qwen3 VL 8B (Vision)\",\"Cydonia 24B (Roleplay)\",\"Broken Tutu 24B (Roleplay)\",\"Dans PersonalityEngine 24B (Roleplay)\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"Mistral Small 24B (Corporate)\",\"GPT OSS 20B (Corporate)\",\"Gemma3 12B (Corporate)\",\"GLM4 9b chat (Corporate)\",\"Z-Image Turbo (Image)\",\"Flux Klein 4B (Image)\"]\n", + "Template = \"Gemma4 E4B Heretic (General)\" # @param [\"Gemma4 E4B Heretic (General)\", \"Tiefighter 13B (General)\",\"Estopia 13B (General)\",\"Qwen3 VL 8B (Vision)\",\"Cydonia 24B (Roleplay)\",\"Broken Tutu 24B (Roleplay)\",\"Dans PersonalityEngine 24B (Roleplay)\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"Mistral Small 24B (Corporate)\",\"GPT OSS 20B (Corporate)\",\"Gemma3 12B (Corporate)\",\"GLM4 9b chat (Corporate)\",\"Z-Image Turbo (Image)\",\"Flux Klein 4B (Image)\"]\n", "\n", "\n", "#@markdown ### Or, choose manually by selecting checkboxes and dropdown below (Overwrites above templates!)\n", "\n", "#@markdown
\n", "LoadTextModel = False #@param {type:\"boolean\"}\n", - "Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", + "Model = \"https://huggingface.co/mradermacher/gemma-4-E4B-it-heretic-GGUF/resolve/main/gemma-4-E4B-it-heretic.Q6_K.gguf\" #@param [\"https://huggingface.co/mradermacher/gemma-4-E4B-it-heretic-GGUF/resolve/main/gemma-4-E4B-it-heretic.Q6_K.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", "MdCommand = \"\" #@markdown
\n", "Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n", "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n", "\n", "#@markdown
\n", "LoadVisionMMProjector = False #@param {type:\"boolean\"}\n", - "Mmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\" #@param [\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n", + "Mmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\" #@param [\"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n", "VCommand = \"\"\n", "\n", "#@markdown
\n", @@ -127,6 +127,11 @@ " ImgClip1 = \"\"\n", " ImgVAE = \"\"\n", " Mmproj = \"\"\n", + " if Template == \"Gemma4 E4B Heretic (General)\":\n", + " Customized = True\n", + " Model = \"https://huggingface.co/mradermacher/gemma-4-E4B-it-heretic-GGUF/resolve/main/gemma-4-E4B-it-heretic.Q6_K.gguf\"\n", + " CustomCtxSize = \"16384\"\n", + " CustomMmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\"\n", " if Template == \"Tiefighter 13B (General)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n", diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 522dd39dd..463aaaa53 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5384,7 +5384,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) { printf("\n(Draft Results - Success:%d, Failure:%d)",draft_successes,draft_failures); } - if(check_slowness && generated_tps<2.0f) + if(check_slowness && generated_tps<2.0f && real_n_generated>1) { check_slowness = false; if(!is_quiet) diff --git a/koboldcpp.py b/koboldcpp.py index d6c721a91..c5be3c645 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -5188,17 +5188,45 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): self.close_connection = True await asyncio.sleep(0.05) + async def monitor_connection(self): #Poll the socket to detect client disconnection during prompt processing + import select + loop = asyncio.get_event_loop() + def check_connection_closed(): + try: + sock = self.connection + readable, _, exceptional = select.select([sock], [], [sock], 0) + if exceptional: + return True + if readable: + data = sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) + if len(data) == 0: + return True + return False + except (OSError, Exception): + return True # Treat any error as disconnected + while True: + try: + await asyncio.sleep(0.5) + disconnected = await loop.run_in_executor(None, check_connection_closed) + if disconnected: + if args.debugmode: + print("\nClient disconnected unexpectedly, aborting...") + handle.abort_generate() + return + except Exception: + return async def handle_request(self, genparams, api_format, stream_flag): tasks = [] genparams["oai_uniqueid"] = random.randint(100000, 999999) + monitor_task = None try: if stream_flag: tasks.append(self.handle_sse_stream(genparams, api_format)) - generate_task = asyncio.create_task(self.generate_text(genparams, api_format, stream_flag)) tasks.append(generate_task) - + if stream_flag: + monitor_task = asyncio.create_task(self.monitor_connection()) await asyncio.gather(*tasks) generate_result = generate_task.result() return generate_result @@ -5209,6 +5237,13 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): await asyncio.sleep(0.2) #short delay except Exception as e: print(e) + finally: + if monitor_task and not monitor_task.done(): + monitor_task.cancel() + try: + await monitor_task + except asyncio.CancelledError: + pass def get_multiplayer_idle_state(self,userid): if modelbusy.locked(): @@ -9378,7 +9413,7 @@ def convert_invalid_args(args): dict["sdloramult"] = sanitize_lora_multipliers(dict["sdloramult"]) return args -def setuptunnel(global_memory, has_sd): +def setuptunnel(global_memory, has_sd, has_music): # This script will help setup a cloudflared tunnel for accessing KoboldCpp over the internet # It should work out of the box on both linux and windows try: @@ -9432,6 +9467,8 @@ def setuptunnel(global_memory, has_sd): print(f"Your remote llama.cpp secondary WebUI at {tunneloutput}/lcpp/") if has_sd: print(f"StableUI is available at {tunneloutput}/sdui/") + if has_music: + print(f"MusicUI is available at {tunneloutput}/musicui/") print("======\n") print(f"Your remote tunnel is ready, please connect to {tunneloutput}", flush=True) if global_memory: @@ -10002,7 +10039,7 @@ def main(launch_args, default_args): if not args.admin: #run in single process mode if args.remotetunnel and not args.prompt and not args.benchmark and not args.cli: - setuptunnel(global_memory, True if args.sdmodel else False) + setuptunnel(global_memory, True if args.sdmodel else False, True if (args.musicdiffusion or args.musicllm or args.ttsmodel) else False) kcpp_main_process(args,global_memory,using_gui_launcher) if global_memory["input_to_exit"]: print("===") @@ -10013,7 +10050,7 @@ def main(launch_args, default_args): global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_base_config":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "base_config":"", "swapReqType": None, "autoswapmode": False}) if args.remotetunnel and not args.prompt and not args.benchmark and not args.cli: - setuptunnel(global_memory, True if args.sdmodel else False) + setuptunnel(global_memory, True if args.sdmodel else False, True if (args.musicdiffusion or args.musicllm or args.ttsmodel) else False) # invoke the main koboldcpp process original_args = copy.deepcopy(args)