diff --git a/colab.ipynb b/colab.ipynb
index 5464b1c59..4aa243089 100644
--- a/colab.ipynb
+++ b/colab.ipynb
@@ -52,21 +52,21 @@
"#@markdown You will need to restart KoboldCpp with the play button when you make changes to these settings.\n",
"\n",
"#@markdown ### Select a quick preset template (optional)\n",
- "Template = \"Tiefighter 13B (General)\" # @param [\"Tiefighter 13B (General)\",\"Estopia 13B (General)\",\"Qwen3 VL 8B (Vision)\",\"Cydonia 24B (Roleplay)\",\"Broken Tutu 24B (Roleplay)\",\"Dans PersonalityEngine 24B (Roleplay)\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"Mistral Small 24B (Corporate)\",\"GPT OSS 20B (Corporate)\",\"Gemma3 12B (Corporate)\",\"GLM4 9b chat (Corporate)\",\"Z-Image Turbo (Image)\",\"Flux Klein 4B (Image)\"]\n",
+ "Template = \"Gemma4 E4B Heretic (General)\" # @param [\"Gemma4 E4B Heretic (General)\", \"Tiefighter 13B (General)\",\"Estopia 13B (General)\",\"Qwen3 VL 8B (Vision)\",\"Cydonia 24B (Roleplay)\",\"Broken Tutu 24B (Roleplay)\",\"Dans PersonalityEngine 24B (Roleplay)\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"Mistral Small 24B (Corporate)\",\"GPT OSS 20B (Corporate)\",\"Gemma3 12B (Corporate)\",\"GLM4 9b chat (Corporate)\",\"Z-Image Turbo (Image)\",\"Flux Klein 4B (Image)\"]\n",
"\n",
"\n",
"#@markdown ### Or, choose manually by selecting checkboxes and dropdown below (Overwrites above templates!)\n",
"\n",
"#@markdown
\n",
"LoadTextModel = False #@param {type:\"boolean\"}\n",
- "Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n",
+ "Model = \"https://huggingface.co/mradermacher/gemma-4-E4B-it-heretic-GGUF/resolve/main/gemma-4-E4B-it-heretic.Q6_K.gguf\" #@param [\"https://huggingface.co/mradermacher/gemma-4-E4B-it-heretic-GGUF/resolve/main/gemma-4-E4B-it-heretic.Q6_K.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n",
"MdCommand = \"\" #@markdown
\n",
"Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n",
"ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n",
"\n",
"#@markdown
\n",
"LoadVisionMMProjector = False #@param {type:\"boolean\"}\n",
- "Mmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\" #@param [\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n",
+ "Mmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\" #@param [\"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n",
"VCommand = \"\"\n",
"\n",
"#@markdown
\n",
@@ -127,6 +127,11 @@
" ImgClip1 = \"\"\n",
" ImgVAE = \"\"\n",
" Mmproj = \"\"\n",
+ " if Template == \"Gemma4 E4B Heretic (General)\":\n",
+ " Customized = True\n",
+ " Model = \"https://huggingface.co/mradermacher/gemma-4-E4B-it-heretic-GGUF/resolve/main/gemma-4-E4B-it-heretic.Q6_K.gguf\"\n",
+ " CustomCtxSize = \"16384\"\n",
+ " CustomMmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\"\n",
" if Template == \"Tiefighter 13B (General)\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n",
diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp
index 522dd39dd..463aaaa53 100644
--- a/gpttype_adapter.cpp
+++ b/gpttype_adapter.cpp
@@ -5384,7 +5384,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
{
printf("\n(Draft Results - Success:%d, Failure:%d)",draft_successes,draft_failures);
}
- if(check_slowness && generated_tps<2.0f)
+ if(check_slowness && generated_tps<2.0f && real_n_generated>1)
{
check_slowness = false;
if(!is_quiet)
diff --git a/koboldcpp.py b/koboldcpp.py
index d6c721a91..c5be3c645 100755
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -5188,17 +5188,45 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.close_connection = True
await asyncio.sleep(0.05)
+ async def monitor_connection(self): #Poll the socket to detect client disconnection during prompt processing
+ import select
+ loop = asyncio.get_event_loop()
+ def check_connection_closed():
+ try:
+ sock = self.connection
+ readable, _, exceptional = select.select([sock], [], [sock], 0)
+ if exceptional:
+ return True
+ if readable:
+ data = sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT)
+ if len(data) == 0:
+ return True
+ return False
+ except (OSError, Exception):
+ return True # Treat any error as disconnected
+ while True:
+ try:
+ await asyncio.sleep(0.5)
+ disconnected = await loop.run_in_executor(None, check_connection_closed)
+ if disconnected:
+ if args.debugmode:
+ print("\nClient disconnected unexpectedly, aborting...")
+ handle.abort_generate()
+ return
+ except Exception:
+ return
async def handle_request(self, genparams, api_format, stream_flag):
tasks = []
genparams["oai_uniqueid"] = random.randint(100000, 999999)
+ monitor_task = None
try:
if stream_flag:
tasks.append(self.handle_sse_stream(genparams, api_format))
-
generate_task = asyncio.create_task(self.generate_text(genparams, api_format, stream_flag))
tasks.append(generate_task)
-
+ if stream_flag:
+ monitor_task = asyncio.create_task(self.monitor_connection())
await asyncio.gather(*tasks)
generate_result = generate_task.result()
return generate_result
@@ -5209,6 +5237,13 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
await asyncio.sleep(0.2) #short delay
except Exception as e:
print(e)
+ finally:
+ if monitor_task and not monitor_task.done():
+ monitor_task.cancel()
+ try:
+ await monitor_task
+ except asyncio.CancelledError:
+ pass
def get_multiplayer_idle_state(self,userid):
if modelbusy.locked():
@@ -9378,7 +9413,7 @@ def convert_invalid_args(args):
dict["sdloramult"] = sanitize_lora_multipliers(dict["sdloramult"])
return args
-def setuptunnel(global_memory, has_sd):
+def setuptunnel(global_memory, has_sd, has_music):
# This script will help setup a cloudflared tunnel for accessing KoboldCpp over the internet
# It should work out of the box on both linux and windows
try:
@@ -9432,6 +9467,8 @@ def setuptunnel(global_memory, has_sd):
print(f"Your remote llama.cpp secondary WebUI at {tunneloutput}/lcpp/")
if has_sd:
print(f"StableUI is available at {tunneloutput}/sdui/")
+ if has_music:
+ print(f"MusicUI is available at {tunneloutput}/musicui/")
print("======\n")
print(f"Your remote tunnel is ready, please connect to {tunneloutput}", flush=True)
if global_memory:
@@ -10002,7 +10039,7 @@ def main(launch_args, default_args):
if not args.admin: #run in single process mode
if args.remotetunnel and not args.prompt and not args.benchmark and not args.cli:
- setuptunnel(global_memory, True if args.sdmodel else False)
+ setuptunnel(global_memory, True if args.sdmodel else False, True if (args.musicdiffusion or args.musicllm or args.ttsmodel) else False)
kcpp_main_process(args,global_memory,using_gui_launcher)
if global_memory["input_to_exit"]:
print("===")
@@ -10013,7 +10050,7 @@ def main(launch_args, default_args):
global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_base_config":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "base_config":"", "swapReqType": None, "autoswapmode": False})
if args.remotetunnel and not args.prompt and not args.benchmark and not args.cli:
- setuptunnel(global_memory, True if args.sdmodel else False)
+ setuptunnel(global_memory, True if args.sdmodel else False, True if (args.musicdiffusion or args.musicllm or args.ttsmodel) else False)
# invoke the main koboldcpp process
original_args = copy.deepcopy(args)