From 3b9385a627b8bdc66e775bae057c5adf7b7e5f7c Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sun, 15 Mar 2026 00:38:29 +0800 Subject: [PATCH] updated colab, wip model router --- colab.ipynb | 112 ++++++++++++++++++++++++++++++++------------------- koboldcpp.py | 39 +++++++++++------- 2 files changed, 96 insertions(+), 55 deletions(-) diff --git a/colab.ipynb b/colab.ipynb index 046453796..de988f8ac 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -50,54 +50,79 @@ "source": [ "# @title v-- Enter your model below and then click this to start Koboldcpp {\"display-mode\":\"form\"}\n", "#@markdown You will need to restart KoboldCpp with the play button when you make changes to these settings.\n", - "Model = \"Tiefighter 13B\" # @param [\"=== KoboldAI Models ===\",\"Tiefighter 13B\",\"Estopia 13B\",\"Bookadventures 8B (Writing)\",\"=== Roleplay Models ===\",\"Cydonia 24B\",\"Broken Tutu 24B\",\"Dans PersonalityEngine 24B\",\"==== Adventure Models ====\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"=== Image Recognition Models ===\",\"Qwen 3 VL\",\"=== Corporate Models ===\",\"Mistral Small 24B\",\"GPT OSS 20B\",\"Gemma 3n E4B it\",\"GLM4 9b chat\",\"=== Classic Models ===\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", - "Layers = 99 #@param [99]{allow-input: true}\n", - "ContextSize = \"Default\" #@param [\"Default\",\"4096\",\"8192\"] {allow-input: true}\n", - "FlashAttention = True #@param {type:\"boolean\"}\n", - "Multiplayer = False #@param {type:\"boolean\"}\n", - "DeleteExistingModels = True #@param {type:\"boolean\"}\n", - "AlwaysUseManualSettings = False #@param {type:\"boolean\"}\n", - "FACommand = \"\"\n", - "MPCommand = \"\"\n", + "\n", + "#@markdown ### Select a quick preset template (optional)\n", + "Template = \"Tiefighter 13B (General)\" # @param [\"None (Manual Choice)\",\"Tiefighter 13B (General)\",\"Estopia 13B (General)\",\"Qwen 3 VL (Vision)\",\"Bookadventures 8B (Writing)\",\"Cydonia 24B (Roleplay)\",\"Broken Tutu 24B (Roleplay)\",\"Dans PersonalityEngine 24B (Roleplay)\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"Mistral Small 24B (Corporate)\",\"GPT OSS 20B (Corporate)\",\"Gemma 3n E4B it (Corporate)\",\"GLM4 9b chat (Corporate)\"]\n", + "\n", + "\n", + "#@markdown ### Or choose manually (Overwrites above template!)\n", + "\n", + "#@markdown
\n", + "LoadTextModel = False #@param {type:\"boolean\"}\n", + "Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", + "MdCommand = \"\" #@markdown
\n", + "Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n", + "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n", + "\n", "#@markdown
\n", "LoadVisionMMProjector = False #@param {type:\"boolean\"}\n", - "Mmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\" #@param [\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n", + "Mmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\" #@param [\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n", "VCommand = \"\"\n", + "\n", "#@markdown
\n", "LoadImgModel = False #@param {type:\"boolean\"}\n", - "ImgModel = \"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\"]{allow-input: true}\n", + "ImgModel = \"https://huggingface.co/koboldcpp/imgmodel/resolve/main/picx_real_q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/picx_real_q4_0.gguf\",\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\"]{allow-input: true}\n", "SCommand = \"\"\n", + "\n", "#@markdown
\n", "LoadSpeechModel = False #@param {type:\"boolean\"}\n", "SpeechModel = \"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\" #@param [\"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\"]{allow-input: true}\n", "WCommand = \"\"\n", + "\n", "#@markdown
\n", "LoadTTSModel = False #@param {type:\"boolean\"}\n", - "TTSModel = \"https://huggingface.co/koboldcpp/tts/resolve/main/OuteTTS-0.2-500M-Q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/OuteTTS-0.2-500M-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf\"]{allow-input: true}\n", - "WavTokModel = \"https://huggingface.co/koboldcpp/tts/resolve/main/WavTokenizer-Large-75-Q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/WavTokenizer-Large-75-Q4_0.gguf\"]{allow-input: true}\n", + "TTSModel = \"https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/OuteTTS-0.2-500M-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf\"]{allow-input: true}\n", + "WavTokModel = \"\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/WavTokenizer-Large-75-Q4_0.gguf\"]{allow-input: true}\n", "TTSCommand = \"\"\n", + "\n", "#@markdown
\n", "LoadEmbeddingsModel = False #@param {type:\"boolean\"}\n", "EmbeddingsModel = \"https://huggingface.co/yixuan-chia/snowflake-arctic-embed-s-GGUF/resolve/main/snowflake-arctic-embed-s-Q4_0.gguf\" #@param [\"https://huggingface.co/yixuan-chia/snowflake-arctic-embed-s-GGUF/resolve/main/snowflake-arctic-embed-s-Q4_0.gguf\"]{allow-input: true}\n", "ECommand = \"\"\n", + "\n", + "#@markdown
\n", + "#@markdown Currently only AceStep 1.5 is supported, it will used if picked\n", + "LoadMusicModel = False #@param {type:\"boolean\"}\n", + "MusicModel = \"--musiclowvram --musicllm https://huggingface.co/koboldcpp/music/resolve/main/acestep-5Hz-lm-1.7B-BF16.gguf --musicembeddings https://huggingface.co/koboldcpp/music/resolve/main/Ace-Qwen3-Embedding-0.6B-BF16.gguf --musicdiffusion https://huggingface.co/koboldcpp/music/resolve/main/acestep-v15-sftturbo50-Q8_0.gguf --musicvae https://huggingface.co/koboldcpp/music/resolve/main/ace-vae-BF16.gguf\"\n", + "MuCommand = \"\"\n", + "\n", + "#@markdown
\n", + "#@markdown Deletes all models, forcing a fresh redownload when you restart colab.\n", + "DeleteExistingModels = True #@param {type:\"boolean\"}\n", + "\n", + "#@markdown
\n", + "#@markdown Enables multiplayer sessions, simply load the link and click join multiplayer.\n", + "Multiplayer = False #@param {type:\"boolean\"}\n", + "MPCommand = \"\"\n", + "\n", "#@markdown
\n", "#@markdown This enables saving stories directly to your google drive. You will have to grant permissions, and then you can access the saves from the \"KoboldCpp Server Storage\" option.\n", "AllowSaveToGoogleDrive = False #@param {type:\"boolean\"}\n", "SavGdriveCommand = \"\"\n", + "\n", "#@markdown
\n", "#@markdown Only select the following box if regular cloudflare tunnel fails to work. It will generate an inferior localtunnel tunnel, which you can use after entering a password.\n", "MakeLocalTunnelFallback = False #@param {type:\"boolean\"}\n", "\n", "CustomCtxSize = \"\"\n", - "CustomFlashAttention = True\n", "CustomMmproj = \"\"\n", "Customized = False\n", - "if Model == \"Tiefighter 13B\":\n", + "if Model == \"Tiefighter 13B (General)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n", " CustomCtxSize = \"4096\"\n", " CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n", - "if Model == \"Estopia 13B\":\n", + "if Model == \"Estopia 13B (General)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\"\n", " CustomCtxSize = \"4096\"\n", @@ -106,15 +131,15 @@ " Customized = True\n", " Model = \"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\"\n", " CustomCtxSize = \"65536\"\n", - "if Model == \"Cydonia 24B\":\n", + "if Model == \"Cydonia 24B (Roleplay)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\"\n", " CustomCtxSize = \"10240\"\n", - "if Model == \"Broken Tutu 24B\":\n", + "if Model == \"Broken Tutu 24B (Roleplay)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\"\n", " CustomCtxSize = \"10240\"\n", - "if Model == \"Dans PersonalityEngine 24B\":\n", + "if Model == \"Dans PersonalityEngine 24B (Roleplay)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\"\n", " CustomCtxSize = \"10240\"\n", @@ -126,38 +151,41 @@ " Customized = True\n", " Model = \"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\"\n", " CustomCtxSize = \"49152\"\n", - "if Model == \"Qwen 3 VL\":\n", + "if Model == \"Qwen 3 VL (Vision)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\"\n", " CustomMmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\"\n", " CustomCtxSize = \"32768\"\n", - "if Model == \"Mistral Small 24B\":\n", + "if Model == \"Mistral Small 24B (Corporate)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\"\n", " CustomCtxSize = \"10240\"\n", - "if Model == \"GPT OSS 20B\":\n", + "if Model == \"GPT OSS 20B (Corporate)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\"\n", " CustomCtxSize = \"32768\"\n", - "if Model == \"Gemma 3n E4B it\":\n", + "if Model == \"Gemma 3n E4B it (Corporate)\":\n", " Customized = True\n", - " Model = \"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf?download=true\"\n", - " CustomCtxSize = \"65536\"\n", - "if Model == \"GLM4 9b chat\":\n", + " Model = \"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\"\n", + " CustomCtxSize = \"49152\"\n", + "if Model == \"GLM4 9b chat (Corporate)\":\n", " Customized = True\n", - " Model = \"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf?download=true\"\n", - " CustomCtxSize = \"65536\"\n", + " Model = \"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\"\n", + " CustomCtxSize = \"49152\"\n", "\n", - "if Customized and not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel, LoadVisionMMProjector)):\n", - " print(\"Overriding model settings to the optimal ones for your chosen model\")\n", - " FlashAttention = CustomFlashAttention\n", + "if not Customized or Model==\"\" or Model==\"None (Manual Choice)\" or any((LoadTextModel, LoadVisionMMProjector, LoadImgModel, LoadSpeechModel,LoadTTSModel, LoadEmbeddingsModel)):\n", + " print(\"Manual override detected, using manual selection...\")\n", + "elif Customized:\n", + " print(\"Loading a quick preset template...\")\n", + " LoadTextModel = True\n", " if CustomMmproj!=\"\":\n", " Mmproj = CustomMmproj\n", " LoadVisionMMProjector = True\n", - " if ContextSize==\"Default\" and CustomCtxSize!=\"\":\n", + " if CustomCtxSize!=\"\":\n", " ContextSize = CustomCtxSize\n", - "if ContextSize==\"Default\":\n", - " ContextSize = \"4096\"\n", + "\n", + "if Layers==\"Auto\":\n", + " Layers = -1\n", "\n", "import os, glob\n", "if not os.path.isfile(\"/opt/bin/nvidia-smi\"):\n", @@ -187,10 +215,14 @@ " print(\"Please ensure you grant Google Drive permissions and try again.\")\n", "\n", "%cd /content\n", + "if Model and LoadTextModel and Model!=\"None (Manual Choice)\":\n", + " MdCommand = f\"--model {Model}\"\n", + "else:\n", + " MdCommand = \"\"\n", "if Mmproj and LoadVisionMMProjector:\n", " VCommand = f\"--mmproj {Mmproj}\"\n", "else:\n", - " SCommand = \"\"\n", + " VCommand = \"\"\n", "if ImgModel and LoadImgModel:\n", " SCommand = f\"--sdmodel {ImgModel} --sdthreads 4 --sdquant --sdclamped\"\n", "else:\n", @@ -209,17 +241,15 @@ " ECommand = f\"--embeddingsmodel {EmbeddingsModel}\"\n", "else:\n", " ECommand = \"\"\n", - "if FlashAttention:\n", - " FACommand = \"--flashattention\"\n", + "if LoadMusicModel:\n", + " MuCommand = f\"{MusicModel}\"\n", "else:\n", - " FACommand = \"\"\n", + " MuCommand = \"\"\n", "if Multiplayer:\n", " MPCommand = \"--multiplayer\"\n", "else:\n", " MPCommand = \"\"\n", "\n", - "ModelCommand = f\"--model {Model}\"\n", - "\n", "if DeleteExistingModels:\n", " print(\"Deleting all cached models to redownload...\")\n", " patterns = ['*.gguf', '*.bin', '*.ggml']\n", @@ -251,7 +281,7 @@ " print(f\"Please open the above link, and input the password '{ltpw}'\\nYour KoboldCpp will start shortly...\")\n", " print(\"=================\")\n", " !sleep 10\n", - "!./koboldcpp_linux $ModelCommand --usecuda 0 mmq --chatcompletionsadapter AutoGuess --multiuser --gpulayers $Layers --contextsize $ContextSize --websearch --quiet --remotetunnel $FACommand $MPCommand $VCommand $SCommand $WCommand $TTSCommand $ECommand $SavGdriveCommand\n" + "!./koboldcpp_linux $MdCommand --usecuda 0 mmq --chatcompletionsadapter AutoGuess --multiuser --gpulayers $Layers --contextsize $ContextSize --websearch --quiet --remotetunnel $MPCommand $VCommand $SCommand $WCommand $TTSCommand $ECommand $MuCommand $SavGdriveCommand\n" ] } ], diff --git a/koboldcpp.py b/koboldcpp.py index 171b4d7bc..858f42df9 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -1173,6 +1173,24 @@ def get_capabilities(): admin_type = (2 if args.admin and args.admindir and args.adminpassword else (1 if args.admin and args.admindir else 0)) return {"result":"KoboldCpp", "version":KcppVersion, "protected":has_password, "llm":has_llm, "txt2img":has_txt2img,"vision":has_vision_support,"audio":has_audio_support,"transcribe":has_whisper,"multiplayer":has_multiplayer,"websearch":has_search,"tts":has_tts, "embeddings":has_embeddings, "music":has_music, "savedata":(savedata_obj is not None), "admin": admin_type, "guidance": has_guidance, "jinja": has_jinja, "mcp":has_mcp} +def get_current_admindir_list(): + opts = [] + if args.admin and args.admindir: + dirpath = os.path.abspath(args.admindir) + valid_exts = (".kcpps", ".kcppt", ".gguf") + for entry in sorted(os.listdir(dirpath)): # Scan top-level directory + full_path = os.path.join(dirpath, entry) + if os.path.isfile(full_path) and entry.endswith(valid_exts): # If toplevel file + opts.append(entry) + elif os.path.isdir(full_path): #if dir, scan up to 1 level deep + for subentry in sorted(os.listdir(full_path)): + sub_full_path = os.path.join(full_path, subentry) + if os.path.isfile(sub_full_path) and subentry.endswith(valid_exts): + rel_path = os.path.join(entry, subentry) + opts.append(rel_path) + opts.append("unload_model") + return opts + def dump_gguf_metadata(file_path): #if you're gonna copy this into your own project at least credit concedo chunk_size = 1024*1024*12 # read first 12mb of file try: @@ -4253,19 +4271,7 @@ Change Mode
elif clean_path.endswith(('/api/admin/list_options')): # used by admin to get info about a kcpp instance opts = [] if args.admin and args.admindir and os.path.exists(args.admindir) and self.check_header_password(args.adminpassword): - dirpath = os.path.abspath(args.admindir) - valid_exts = (".kcpps", ".kcppt", ".gguf") - for entry in sorted(os.listdir(dirpath)): # Scan top-level directory - full_path = os.path.join(dirpath, entry) - if os.path.isfile(full_path) and entry.endswith(valid_exts): # If toplevel file - opts.append(entry) - elif os.path.isdir(full_path): #if dir, scan up to 1 level deep - for subentry in sorted(os.listdir(full_path)): - sub_full_path = os.path.join(full_path, subentry) - if os.path.isfile(sub_full_path) and subentry.endswith(valid_exts): - rel_path = os.path.join(entry, subentry) - opts.append(rel_path) - opts.append("unload_model") + opts = get_current_admindir_list() response_body = (json.dumps(opts).encode()) elif clean_path.endswith(('/api/extra/perf')): @@ -4334,7 +4340,12 @@ Change Mode
response_body = (json.dumps({"logprobs":logprobsdict}).encode()) elif clean_path.endswith('/v1/models') or clean_path=='/models': - response_body = (json.dumps({"object":"list","data":[{"id":friendlymodelname,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}]}).encode()) + mlist = [{"id":friendlymodelname,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}] + if args.routermode: + alist = get_current_admindir_list() + for itm in alist: + mlist.append({"id":itm,"object":"model","created":int(time.time()),"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}) + response_body = (json.dumps({"object":"list","data":mlist}).encode()) elif clean_path.endswith('/sdapi/v1/loras'): response_body = (json.dumps([{'name': name, 'path': path} for _, name, path, multiplier in imglorainfo if multiplier == 0.])).encode()