mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-04-09 10:31:45 +02:00
360 lines
23 KiB
Plaintext
360 lines
23 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "view-in-github"
|
||
},
|
||
"source": [
|
||
"<a href=\"https://colab.research.google.com/github/LostRuins/koboldcpp/blob/concedo/colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "2FCn5tmpn3UV"
|
||
},
|
||
"source": [
|
||
"## Welcome to the Official KoboldCpp Colab Notebook\n",
|
||
"It's really easy to get started. Just press the two **Play** buttons below, and then connect to the **Cloudflare URL** shown at the end.\n",
|
||
"You can select a model from the dropdown, or enter a **custom URL** to a GGUF model (Example: `https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf`)\n",
|
||
"\n",
|
||
"Named models in the Model selector automatically use the optimal context settings as long as you do not load additional models such as an image generation model.\n",
|
||
"\n",
|
||
"**Keep this page open and occationally check for captcha's so that your AI is not shut down**"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"id": "QNaj3u0jn3UW"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"#@title v-- Tap this if you play on Mobile { display-mode: \"form\" }\n",
|
||
"%%html\n",
|
||
"<b>Ensure this audio is playing to prevent colab from shutting down, then start KoboldCpp below</b><br/>\n",
|
||
"<audio autoplay=\"\" src=\"https://raw.githubusercontent.com/KoboldAI/KoboldAI-Client/main/colab/silence.m4a\" loop controls>"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"cellView": "form",
|
||
"id": "uJS9i_Dltv8Y"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# @title <b>v-- Enter your model below and then click this to start Koboldcpp</b> {\"display-mode\":\"form\"}\n",
|
||
"#@markdown You will need to restart KoboldCpp with the play button when you make changes to these settings.\n",
|
||
"\n",
|
||
"#@markdown ### Select a quick preset template (optional)\n",
|
||
"Template = \"Tiefighter 13B (General)\" # @param [\"Tiefighter 13B (General)\",\"Estopia 13B (General)\",\"Qwen3 VL 8B (Vision)\",\"Cydonia 24B (Roleplay)\",\"Broken Tutu 24B (Roleplay)\",\"Dans PersonalityEngine 24B (Roleplay)\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"Mistral Small 24B (Corporate)\",\"GPT OSS 20B (Corporate)\",\"Gemma3 12B (Corporate)\",\"GLM4 9b chat (Corporate)\",\"Z-Image Turbo (Image)\",\"Flux Klein 4B (Image)\"]\n",
|
||
"\n",
|
||
"\n",
|
||
"#@markdown ### Or, choose manually by <u>selecting checkboxes and dropdown</u> below (Overwrites above templates!)\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"LoadTextModel = False #@param {type:\"boolean\"}\n",
|
||
"Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n",
|
||
"MdCommand = \"\" #@markdown <br>\n",
|
||
"Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n",
|
||
"ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"LoadVisionMMProjector = False #@param {type:\"boolean\"}\n",
|
||
"Mmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\" #@param [\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n",
|
||
"VCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"LoadImgModel = False #@param {type:\"boolean\"}\n",
|
||
"ImgModel = \"https://huggingface.co/koboldcpp/imgmodel/resolve/main/picx_real_q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/picx_real_q4_0.gguf\",\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\"]{allow-input: true}\n",
|
||
"#@markdown <br>\n",
|
||
"ImgClip1 = \"\" #@param [\"\"]{allow-input: true}\n",
|
||
"ImgVAE = \"\" #@param [\"\"]{allow-input: true}\n",
|
||
"SCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"LoadSpeechModel = False #@param {type:\"boolean\"}\n",
|
||
"SpeechModel = \"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\" #@param [\"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\"]{allow-input: true}\n",
|
||
"WCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"LoadTTSModel = False #@param {type:\"boolean\"}\n",
|
||
"TTSModel = \"https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/OuteTTS-0.2-500M-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf\"]{allow-input: true}\n",
|
||
"WavTokModel = \"\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/WavTokenizer-Large-75-Q4_0.gguf\"]{allow-input: true}\n",
|
||
"TTSCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"LoadEmbeddingsModel = False #@param {type:\"boolean\"}\n",
|
||
"EmbeddingsModel = \"https://huggingface.co/yixuan-chia/snowflake-arctic-embed-s-GGUF/resolve/main/snowflake-arctic-embed-s-Q4_0.gguf\" #@param [\"https://huggingface.co/yixuan-chia/snowflake-arctic-embed-s-GGUF/resolve/main/snowflake-arctic-embed-s-Q4_0.gguf\"]{allow-input: true}\n",
|
||
"ECommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"#@markdown Currently only AceStep 1.5 is supported, will use it if picked\n",
|
||
"LoadMusicModel = False #@param {type:\"boolean\"}\n",
|
||
"MusicModel = \"--musiclowvram --musicllm https://huggingface.co/koboldcpp/music/resolve/main/acestep-5Hz-lm-1.7B-BF16.gguf --musicembeddings https://huggingface.co/koboldcpp/music/resolve/main/Ace-Qwen3-Embedding-0.6B-BF16.gguf --musicdiffusion https://huggingface.co/koboldcpp/music/resolve/main/acestep-v15-sftturbo50-Q8_0.gguf --musicvae https://huggingface.co/koboldcpp/music/resolve/main/ace-vae-BF16.gguf\"\n",
|
||
"MuCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"#@markdown Deletes all models, forcing a fresh redownload when you restart colab.\n",
|
||
"DeleteExistingModels = True #@param {type:\"boolean\"}\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"#@markdown Enables multiplayer sessions, simply load the link and click join multiplayer.\n",
|
||
"Multiplayer = False #@param {type:\"boolean\"}\n",
|
||
"MPCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"#@markdown This enables saving stories directly to your google drive. You will have to grant permissions, and then you can access the saves from the \"KoboldCpp Server Storage\" option.\n",
|
||
"AllowSaveToGoogleDrive = False #@param {type:\"boolean\"}\n",
|
||
"SavGdriveCommand = \"\"\n",
|
||
"\n",
|
||
"#@markdown <hr>\n",
|
||
"#@markdown Only select the following box if regular cloudflare tunnel fails to work. It will generate an inferior localtunnel tunnel, which you can use after entering a password.\n",
|
||
"MakeLocalTunnelFallback = False #@param {type:\"boolean\"}\n",
|
||
"\n",
|
||
"CustomCtxSize = \"\"\n",
|
||
"CustomMmproj = \"\"\n",
|
||
"Customized = False\n",
|
||
"has_checked_box = any((LoadTextModel, LoadVisionMMProjector, LoadImgModel, LoadSpeechModel,LoadTTSModel, LoadEmbeddingsModel))\n",
|
||
"if not has_checked_box:\n",
|
||
" Model = \"\"\n",
|
||
" ImgModel = \"\"\n",
|
||
" ImgClip1 = \"\"\n",
|
||
" ImgVAE = \"\"\n",
|
||
" Mmproj = \"\"\n",
|
||
" if Template == \"Tiefighter 13B (General)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"4096\"\n",
|
||
" CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
|
||
" if Template == \"Estopia 13B (General)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"4096\"\n",
|
||
" CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
|
||
" if Template == \"Cydonia 24B (Roleplay)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"10240\"\n",
|
||
" if Template == \"Broken Tutu 24B (Roleplay)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"10240\"\n",
|
||
" if Template == \"Dans PersonalityEngine 24B (Roleplay)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"10240\"\n",
|
||
" if Template == \"Harbinger 24B (Instruct)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"10240\"\n",
|
||
" if Template == \"Muse 12B (Instruct)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"49152\"\n",
|
||
" if Template == \"Qwen3 VL 8B (Vision)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\"\n",
|
||
" CustomMmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\"\n",
|
||
" CustomCtxSize = \"32768\"\n",
|
||
" if Template == \"Mistral Small 24B (Corporate)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"10240\"\n",
|
||
" if Template == \"GPT OSS 20B (Corporate)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\"\n",
|
||
" CustomCtxSize = \"32768\"\n",
|
||
" if Template == \"Gemma3 12B (Corporate)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\"\n",
|
||
" CustomCtxSize = \"16384\"\n",
|
||
" if Template == \"GLM4 9b chat (Corporate)\":\n",
|
||
" Customized = True\n",
|
||
" Model = \"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\"\n",
|
||
" CustomCtxSize = \"49152\"\n",
|
||
" if Template == \"Z-Image Turbo (Image)\":\n",
|
||
" ImgModel = \"https://huggingface.co/leejet/Z-Image-Turbo-GGUF/resolve/main/z_image_turbo-Q4_0.gguf\"\n",
|
||
" ImgClip1 = \"https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_S.gguf\"\n",
|
||
" ImgVAE = \"https://huggingface.co/camenduru/FLUX.1-dev/resolve/main/ae.safetensors\"\n",
|
||
" Customized = True\n",
|
||
" if Template == \"Flux Klein 4B (Image)\":\n",
|
||
" ImgModel = \"https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/resolve/main/flux-2-klein-4b-Q4_0.gguf\"\n",
|
||
" ImgClip1 = \"https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/main/Qwen3-4B-Q4_K_M.gguf\"\n",
|
||
" ImgVAE = \"https://huggingface.co/koboldcpp/GGUFDumps/resolve/main/flux2vae.safetensors\"\n",
|
||
" Customized = True\n",
|
||
"\n",
|
||
"if not Customized or Template==\"\" or has_checked_box:\n",
|
||
" print(\"== Manual CHECKBOX OVERRIDE was detected, using manual selections ==\")\n",
|
||
" print(f\"LoadTextModel: {LoadTextModel}\")\n",
|
||
" print(f\"LoadVisionMMProjector: {LoadVisionMMProjector}\")\n",
|
||
" print(f\"LoadImgModel: {LoadImgModel}\")\n",
|
||
" print(f\"LoadSpeechModel: {LoadSpeechModel}\")\n",
|
||
" print(f\"LoadTTSModel: {LoadTTSModel}\")\n",
|
||
" print(f\"LoadEmbeddingsModel: {LoadEmbeddingsModel}\")\n",
|
||
"elif Customized:\n",
|
||
" print(\"== No checkbox override selected, loading a quick preset template! ==\")\n",
|
||
" print(f\"CHOSEN PRSET: {Template}\")\n",
|
||
" print(\"If you want to load your own model selection, MAKE SURE you check one of the loader checkboxes...\")\n",
|
||
" if Model:\n",
|
||
" LoadTextModel = True\n",
|
||
" Layers = 999\n",
|
||
" if ImgModel:\n",
|
||
" LoadImgModel = True\n",
|
||
" if CustomMmproj!=\"\":\n",
|
||
" Mmproj = CustomMmproj\n",
|
||
" LoadVisionMMProjector = True\n",
|
||
" if CustomCtxSize!=\"\":\n",
|
||
" ContextSize = CustomCtxSize\n",
|
||
"\n",
|
||
"if Layers==\"Auto\":\n",
|
||
" Layers = -1\n",
|
||
"\n",
|
||
"print(\"=== STARTING THE LOAD ===\")\n",
|
||
"if LoadTextModel:\n",
|
||
" print(f\"[Text Model] {Model}\")\n",
|
||
"if LoadVisionMMProjector:\n",
|
||
" print(f\"[Vision MMProj] {Mmproj}\")\n",
|
||
"if LoadImgModel:\n",
|
||
" print(f\"[Image Model] {ImgModel}\")\n",
|
||
" if ImgClip1:\n",
|
||
" print(f\" └─ Clip: {ImgClip1}\")\n",
|
||
" if ImgVAE:\n",
|
||
" print(f\" └─ VAE: {ImgVAE}\")\n",
|
||
"if LoadSpeechModel:\n",
|
||
" print(f\"[Speech Model] {SpeechModel}\")\n",
|
||
"if LoadTTSModel:\n",
|
||
" print(f\"[TTS Model] {TTSModel}\")\n",
|
||
" if WavTokModel:\n",
|
||
" print(f\" └─ WavTokenizer: {WavTokModel}\")\n",
|
||
"if LoadEmbeddingsModel:\n",
|
||
" print(f\"[Embeddings Model] {EmbeddingsModel}\")\n",
|
||
"\n",
|
||
"import os, glob\n",
|
||
"if not os.path.isfile(\"/opt/bin/nvidia-smi\"):\n",
|
||
" raise RuntimeError(\"⚠️Colab did not give you a GPU due to usage limits, this can take a few hours before they let you back in. Check out https://lite.koboldai.net for a free alternative (that does not provide an API link but can load KoboldAI saves and chat cards) or subscribe to Colab Pro for immediate access.⚠️\")\n",
|
||
"\n",
|
||
"if AllowSaveToGoogleDrive:\n",
|
||
" print(\"Attempting to request access to save to your google drive...\")\n",
|
||
" try:\n",
|
||
" from google.colab import drive\n",
|
||
" import os, json\n",
|
||
" drive.mount('/content/drive', force_remount=True)\n",
|
||
" if not os.path.exists(\"/content/drive/MyDrive\"):\n",
|
||
" raise RuntimeError(\"Google Drive mount failed. Please grant permissions and try again.\")\n",
|
||
" kcppdir = '/content/drive/MyDrive/koboldcpp_data'\n",
|
||
" os.makedirs(kcppdir, exist_ok=True)\n",
|
||
" savedatapath = os.path.join(kcppdir, \"koboldcpp_save_db.jsondb\")\n",
|
||
" if not os.path.exists(savedatapath):\n",
|
||
" settings_data = {}\n",
|
||
" with open(savedatapath, \"w\") as json_file:\n",
|
||
" json.dump(settings_data, json_file, indent=4)\n",
|
||
" print(f\"Created new koboldcpp_save_db.jsondb at {savedatapath}\")\n",
|
||
" else:\n",
|
||
" print(f\"Loading saved data at {savedatapath}\")\n",
|
||
" SavGdriveCommand = f\" --savedatafile {savedatapath}\"\n",
|
||
" except Exception as e:\n",
|
||
" print(f\"⚠️ Error: {e}\")\n",
|
||
" print(\"Please ensure you grant Google Drive permissions and try again.\")\n",
|
||
"\n",
|
||
"%cd /content\n",
|
||
"if Model and LoadTextModel:\n",
|
||
" MdCommand = f\"--model {Model}\"\n",
|
||
"else:\n",
|
||
" MdCommand = \"\"\n",
|
||
"if Mmproj and LoadVisionMMProjector:\n",
|
||
" VCommand = f\"--mmproj {Mmproj}\"\n",
|
||
"else:\n",
|
||
" VCommand = \"\"\n",
|
||
"if ImgModel and LoadImgModel:\n",
|
||
" SCommand = f\"--sdmodel {ImgModel} --sdthreads 4 --sdquant --sdclamped\"\n",
|
||
" if ImgClip1 and ImgClip1!=\"\":\n",
|
||
" SCommand += f\" --sdclip1 {ImgClip1} --sdclipgpu\"\n",
|
||
" if ImgVAE and ImgVAE!=\"\":\n",
|
||
" SCommand += f\" --sdvae {ImgVAE}\"\n",
|
||
"else:\n",
|
||
" SCommand = \"\"\n",
|
||
"if SpeechModel and LoadSpeechModel:\n",
|
||
" WCommand = f\"--whispermodel {SpeechModel}\"\n",
|
||
"else:\n",
|
||
" WCommand = \"\"\n",
|
||
"if TTSModel and WavTokModel and LoadTTSModel:\n",
|
||
" TTSCommand = f\"--ttsmodel {TTSModel} --ttswavtokenizer {WavTokModel} --ttsgpu\"\n",
|
||
"elif TTSModel and LoadTTSModel:\n",
|
||
" TTSCommand = f\"--ttsmodel {TTSModel} --ttsgpu\"\n",
|
||
"else:\n",
|
||
" TTSCommand = \"\"\n",
|
||
"if EmbeddingsModel and LoadEmbeddingsModel:\n",
|
||
" ECommand = f\"--embeddingsmodel {EmbeddingsModel}\"\n",
|
||
"else:\n",
|
||
" ECommand = \"\"\n",
|
||
"if LoadMusicModel:\n",
|
||
" MuCommand = f\"{MusicModel}\"\n",
|
||
"else:\n",
|
||
" MuCommand = \"\"\n",
|
||
"if Multiplayer:\n",
|
||
" MPCommand = \"--multiplayer\"\n",
|
||
"else:\n",
|
||
" MPCommand = \"\"\n",
|
||
"\n",
|
||
"if DeleteExistingModels:\n",
|
||
" print(\"Deleting all cached models to redownload...\")\n",
|
||
" patterns = ['*.gguf', '*.bin', '*.ggml']\n",
|
||
" for pattern in patterns:\n",
|
||
" for file_path in glob.glob(pattern):\n",
|
||
" try:\n",
|
||
" os.remove(file_path)\n",
|
||
" print(f\"Deleted: {file_path}\")\n",
|
||
" except Exception as e:\n",
|
||
" print(f\"Failed to delete {file_path}: {e}\")\n",
|
||
"\n",
|
||
"\n",
|
||
"!echo Downloading KoboldCpp, please wait...\n",
|
||
"!wget -O dlfile.tmp https://kcpplinux.concedo.workers.dev && mv dlfile.tmp koboldcpp_linux || (echo \"Primary failed, trying mirror...\" && wget -O dlfile.tmp https://github.com/LostRuins/koboldcpp/releases/latest/download/koboldcpp-linux-x64 && mv dlfile.tmp koboldcpp_linux)\n",
|
||
"!test -f koboldcpp_linux && echo Download Successful || echo Download Failed\n",
|
||
"!chmod +x ./koboldcpp_linux\n",
|
||
"!apt update\n",
|
||
"!apt install aria2 -y\n",
|
||
"\n",
|
||
"if MakeLocalTunnelFallback:\n",
|
||
" import urllib\n",
|
||
" print(\"Trying to use LocalTunnel as a fallback tunnel (not so good)...\")\n",
|
||
" ltpw = urllib.request.urlopen('https://loca.lt/mytunnelpassword').read().decode('utf8').strip(\"\\n\")\n",
|
||
" !nohup npx --yes localtunnel --port 5001 > lt.log 2>&1 &\n",
|
||
" !sleep 8\n",
|
||
" print(\"=================\")\n",
|
||
" print(\"(LocalTunnel Results)\")\n",
|
||
" !cat lt.log\n",
|
||
" print(f\"Please open the above link, and input the password '{ltpw}'\\nYour KoboldCpp will start shortly...\")\n",
|
||
" print(\"=================\")\n",
|
||
" !sleep 10\n",
|
||
"!./koboldcpp_linux $MdCommand --usecuda 0 mmq --chatcompletionsadapter AutoGuess --multiuser --flashattention --gpulayers $Layers --contextsize $ContextSize --websearch --quiet --remotetunnel $MPCommand $VCommand $SCommand $WCommand $TTSCommand $ECommand $MuCommand $SavGdriveCommand\n"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"accelerator": "GPU",
|
||
"colab": {
|
||
"cell_execution_strategy": "setup",
|
||
"gpuType": "T4",
|
||
"include_colab_link": true,
|
||
"private_outputs": true,
|
||
"provenance": []
|
||
},
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"name": "python"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 0
|
||
}
|