diff --git a/README.md b/README.md index 484cebc52..f2ca60734 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,12 @@ KoboldCpp is an easy-to-use AI text-generation software for GGML and GGUF models - LLM text generation (Supports all GGML and GGUF models, backwards compatibility with ALL past models) - Image Generation (Stable Diffusion 1.5, SDXL, SD3, Flux) - Speech-To-Text (Voice Recognition) via Whisper -- Text-To-Speech (Voice Generation) via OuteTTS +- Text-To-Speech (Voice Generation) via OuteTTS, Kokoro, Parler and Dia - Provides many compatible APIs endpoints for many popular webservices (KoboldCppApi OpenAiApi OllamaApi A1111ForgeApi ComfyUiApi WhisperTranscribeApi XttsApi OpenAiSpeechApi) - Bundled KoboldAI Lite UI with editing tools, save formats, memory, world info, author's note, characters, scenarios. - Includes multiple modes (chat, adventure, instruct, storywriter) and UI Themes (aesthetic roleplay, classic writer, corporate assistant, messsenger) - Supports loading Tavern Character Cards, importing many different data formats from various sites, reading or exporting JSON savefiles and persistent stories. -- Many other features including new samplers, regex support, websearch, RAG via TextDB and more. +- Many other features including new samplers, regex support, websearch, RAG via TextDB, image recognition/vision and more. - Ready-to-use binaries for Windows, MacOS, Linux. Runs directly with Colab, Docker, also supports other platforms if self-compiled (like Android (via Termux) and Raspberry PI). - [Need help finding a model? Read this!](https://github.com/LostRuins/koboldcpp/wiki#getting-an-ai-model-file) diff --git a/klite.embd b/klite.embd index c6a94113a..a790f23f9 100644 --- a/klite.embd +++ b/klite.embd @@ -3238,6 +3238,7 @@ Current version indicated by LITEVER below. var custom_claude_model = ""; var uses_cors_proxy = false; //we start off attempting a direct connection. switch to proxy if that fails var synchro_polled_response = null; + var synchro_polled_respimg = null; //sometimes a LLM response can also include an image. var last_stop_reason = ""; //update stop reason if known var synchro_pending_stream = ""; //used for storing incomplete streaming text var streaming_was_thinking = false; //used as a switch to determine when thinking ends, to wrap output in tags @@ -5560,10 +5561,13 @@ Current version indicated by LITEVER below. if(x.ok) { return x; - }else{ - throw new Error('Error occurred while SSE streaming: ' + (x.statusText)); - return null; } + return x.text().then(errorBody => { + throw new Error(`Error occurred while SSE streaming: ${x.statusText} - ${errorBody}`); + }).catch(bodyReadError => { + throw new Error(`${bodyReadError}`); + }); + }) .then(resp => { resp.body @@ -5704,8 +5708,13 @@ Current version indicated by LITEVER below. synchro_polled_response = data.candidates[0].output; }else if (custom_gemini_key != "" && data.candidates != null && data.candidates.length>0 && data.candidates[0].content && data.candidates[0].content.parts != null && data.candidates[0].content.parts.length>0) { synchro_polled_response = ""; + synchro_polled_respimg = null; for(let x=0;x return reqblocking = False + #handle rate limiting + ratelimiter = int(args.ratelimit) + if ratelimiter > 0: + client_ip = self.client_address[0] + lastdone = ratelimitlookup.get(client_ip, datetime.min) + diff = (datetime.now() - lastdone).total_seconds() + if diff < ratelimiter: + self.send_response(503) + self.end_headers(content_type='application/json') + self.wfile.write(json.dumps({"detail": { + "msg": f"You are sending requests too quickly. Please try again in {int(ratelimiter-diff)} seconds.", + "type": "service_unavailable", + }}).encode()) + return + ratelimitlookup[client_ip] = datetime.now() muint = int(args.multiuser) if muint<=0 and ((args.whispermodel and args.whispermodel!="") or (args.sdmodel and args.sdmodel!="") or (args.ttsmodel and args.ttsmodel!="") or (args.embeddingsmodel and args.embeddingsmodel!="")): muint = 2 # this prevents errors when using voice/img together with text @@ -4619,6 +4635,7 @@ def show_gui(): ssl_key_var = ctk.StringVar() password_var = ctk.StringVar() maxrequestsize_var = ctk.StringVar(value=str(32)) + ratelimit_var = ctk.StringVar(value=str(0)) sd_model_var = ctk.StringVar() sd_lora_var = ctk.StringVar() @@ -5344,6 +5361,7 @@ def show_gui(): makelabelentry(network_tab, "Password: ", password_var, 10, 200,tooltip="Enter a password required to use this instance.\nThis key will be required for all text endpoints.\nImage endpoints are not secured.") makelabelentry(network_tab, "Max Req. Size (MB):", maxrequestsize_var, row=20, width=50, tooltip="Specify a max request payload size. Any requests to the server larger than this size will be dropped. Do not change if unsure.") + makelabelentry(network_tab, "IP Rate Limiter (s):", ratelimit_var, row=22, width=50, tooltip="Rate limits each IP to allow a new request once per X seconds. Do not change if unsure.") # Horde Tab @@ -5632,6 +5650,7 @@ def show_gui(): args.multiplayer = (multiplayer_var.get()==1) args.websearch = (websearch_var.get()==1) args.maxrequestsize = int(maxrequestsize_var.get()) if maxrequestsize_var.get()!="" else 32 + args.ratelimit = int(ratelimit_var.get()) if ratelimit_var.get()!="" else 0 if usehorde_var.get() != 0: args.hordemodelname = horde_name_var.get() @@ -5876,6 +5895,8 @@ def show_gui(): usehorde_var.set(1 if ("hordekey" in dict and dict["hordekey"]) else 0) if "maxrequestsize" in dict and dict["maxrequestsize"]: maxrequestsize_var.set(dict["maxrequestsize"]) + if "ratelimit" in dict and dict["ratelimit"]: + ratelimit_var.set(dict["ratelimit"]) sd_model_var.set(dict["sdmodel"] if ("sdmodel" in dict and dict["sdmodel"]) else "") sd_clamped_var.set(int(dict["sdclamped"]) if ("sdclamped" in dict and dict["sdclamped"]) else 0) @@ -7650,6 +7671,7 @@ if __name__ == '__main__': advparser.add_argument("--draftgpulayers","--gpu-layers-draft","--n-gpu-layers-draft","-ngld", metavar=('[layers]'), help="How many layers to offload to GPU for the draft model (default=full offload)", type=int, default=999) advparser.add_argument("--draftgpusplit", help="GPU layer distribution ratio for draft model (default=same as main). Only works if multi-GPUs selected for MAIN model and tensor_split is set!", metavar=('[Ratios]'), type=float, nargs='+') advparser.add_argument("--password", metavar=('[API key]'), help="Enter a password required to use this instance. This key will be required for all text endpoints. Image endpoints are not secured.", default=None) + advparser.add_argument("--ratelimit", metavar=('[seconds]'), help="If enabled, rate limit generative request by IP address. Each IP can only send a new request once per X seconds.", type=int, default=0) advparser.add_argument("--ignoremissing", help="Ignores all missing non-essential files, just skipping them instead.", action='store_true') advparser.add_argument("--chatcompletionsadapter", metavar=('[filename]'), help="Select an optional ChatCompletions Adapter JSON file to force custom instruct tags.", default="AutoGuess") advparser.add_argument("--flashattention","--flash-attn","-fa", help="Enables flash attention.", action='store_true')