From 4ec8a9c57b923dcbc9f4c9ea7ba4091f300924fa Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:12:12 +0800 Subject: [PATCH] expose stop reason in generation --- expose.h | 1 + gpttype_adapter.cpp | 5 ++ klite.embd | 198 +++++++++++++++++++++++++++++++------------- koboldcpp.py | 65 +++++++++------ 4 files changed, 184 insertions(+), 85 deletions(-) diff --git a/expose.h b/expose.h index 35d9d7336..fa064b96e 100644 --- a/expose.h +++ b/expose.h @@ -97,6 +97,7 @@ struct generation_inputs struct generation_outputs { int status = -1; + int stopreason = stop_reason::INVALID; const char * text; //response will now be stored in c++ allocated memory }; struct token_count_outputs diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 1eae5e8c1..1adf603a1 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -1584,6 +1584,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) printf("\nWarning: KCPP text generation not initialized!\n"); output.text = nullptr; output.status = 0; + output.stopreason = stop_reason::INVALID; generation_finished = true; return output; } @@ -2125,6 +2126,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) fprintf(stderr, "\nFailed to predict at %d! Check your context buffer sizes!\n",n_past); output.text = nullptr; output.status = 0; + output.stopreason = stop_reason::INVALID; generation_finished = true; return output; } @@ -2334,6 +2336,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) fprintf(stderr, "\nFailed to eval llava image at %d!\n",n_past); output.text = nullptr; output.status = 0; + output.stopreason = stop_reason::INVALID; generation_finished = true; return output; } @@ -2344,6 +2347,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) fprintf(stderr, "\nLLAVA image tokens mismatch at %d! (%d vs %d tokens)\n",n_past,llavatokenscounted,llavatokensevaled); output.text = nullptr; output.status = 0; + output.stopreason = stop_reason::INVALID; generation_finished = true; return output; } @@ -2381,6 +2385,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) printf("\nCtxLimit: %d/%d, Process:%.2fs (%.1fms/T = %.2fT/s), Generate:%.2fs (%.1fms/T = %.2fT/s), Total:%.2fs (%.2fT/s)",(int)current_context_tokens.size(),(int)nctx, time1, pt1, ts1, time2, pt2, ts2, (time1 + time2), tokens_per_second); fflush(stdout); output.status = 1; + output.stopreason = last_stop_reason; generation_finished = true; last_eval_time = pt2; last_process_time = pt1; diff --git a/klite.embd b/klite.embd index 356460670..02c279ff9 100644 --- a/klite.embd +++ b/klite.embd @@ -7,7 +7,7 @@ Just copy this single static HTML file anywhere and open it in a browser, or fro Please go to https://github.com/LostRuins/lite.koboldai.net for updates on Kobold Lite. If you are submitting a pull request for Lite, PLEASE use the above repo, not the KoboldCpp one. Kobold Lite is under the AGPL v3.0 License unless otherwise exempted. Please do not remove this line. -Current version: 134 +Current version: 135 -Concedo --> @@ -1396,11 +1396,19 @@ Current version: 134 .widelbtn { - font-size: 12px; - height: 24px; - padding: 5px; - margin: 2px; - font-weight: bolder; + font-size: 12px; + height: 24px; + padding: 5px; + margin: 2px; + font-weight: bolder; + } + .wiarrowbtn + { + font-size: 12px; + height: 18px; + padding: 2px; + margin: 0px 1px 0px 1px; + font-weight: bolder; } .wiinputkeycol { @@ -3065,6 +3073,9 @@ Current version: 134 if (custom_kobold_endpoint != "" && data && data.results != null && data.results.length > 0) { synchro_streaming_response += data.results[0].text; synchro_streaming_tokens_left -= tokens_per_tick; + if (data.results[0].finish_reason == "stop") { + last_stop_reason = "stop"; + } //handle some early stopping criterias if (localsettings.opmode == 3) //stop on selfname found @@ -3213,6 +3224,10 @@ Current version: 134 for (let event of chunk) { if (event.event === 'message') { synchro_pending_stream += event.data.token; + if(event.data.finish_reason=="stop") + { + last_stop_reason = "stop"; + } } } } @@ -3614,6 +3629,7 @@ Current version: 134 var custom_claude_model = ""; var uses_cors_proxy = false; //we start off attempting a direct connection. switch to proxy if that fails var synchro_polled_response = null; + var last_stop_reason = ""; //update stop reason if known var synchro_pending_stream = ""; //used for token pseduo streaming for kobold api only var waiting_for_autosummary = false; var italics_regex = new RegExp(/\*(\S[^*]+\S)\*/g); //the fallback regex @@ -3633,8 +3649,8 @@ Current version: 134 var welcome = ""; var personal_notes = ""; var logitbiasdict = {}; - var regexreplace_pattern = []; - var regexreplace_replacement = []; + var regexreplace_data = []; + const num_regex_rows = 4; var localsettings = { my_api_key: "0000000000", //put here so it can be saved and loaded in persistent mode @@ -5084,8 +5100,7 @@ Current version: 134 new_save_storyobj.wiinsertlocation = wi_insertlocation; new_save_storyobj.personal_notes = personal_notes; new_save_storyobj.logitbiasdict = JSON.parse(JSON.stringify(logitbiasdict)); - new_save_storyobj.regexreplace_pattern = JSON.parse(JSON.stringify(regexreplace_pattern)); - new_save_storyobj.regexreplace_replacement = JSON.parse(JSON.stringify(regexreplace_replacement)); + new_save_storyobj.regexreplace_data = JSON.parse(JSON.stringify(regexreplace_data)); if (export_settings) { new_save_storyobj.savedsettings = JSON.parse(JSON.stringify(localsettings)); @@ -5257,8 +5272,7 @@ Current version: 134 let old_current_wi = current_wi; let old_extrastopseq = extrastopseq; let old_notes = personal_notes; - let old_regexreplace_pattern = regexreplace_pattern; - let old_regexreplace_replacement = regexreplace_replacement; + let old_regexreplace_data = regexreplace_data; //determine if oldui file or newui file format restart_new_game(false); @@ -5329,11 +5343,18 @@ Current version: 134 if (storyobj.personal_notes) { personal_notes = storyobj.personal_notes; } - if (storyobj.regexreplace_pattern) { - regexreplace_pattern = storyobj.regexreplace_pattern; + //todo: remove temporary backwards compatibility for regex + if (storyobj.regexreplace_pattern && storyobj.regexreplace_replacement) { + let pat = storyobj.regexreplace_pattern; + let rep = storyobj.regexreplace_replacement; + let ll = Math.min(pat.length,rep.length) + for(let i=0;i0) + { + for(let i=0;i0) + if(regexreplace_data && regexreplace_data.length>0) { - for(let i=0;i + Pattern ?The regex pattern to match against any incoming text. Leave blank to disable. + Replacement ?The string to replace matches with. Capture groups are allowed (e.g. $1). To remove all matches, leave this blank. + Both Ways ?If enabled, regex applies for both inputs and outputs, otherwise output only. + `; + let regextable = document.getElementById("regex_replace_table"); + + for(let i=0;i + + + + + `; + } + + regextable.innerHTML = regextablehtml; + + for(let i=0;i 0 && idx < current_wi.length) { + const temp = current_wi[idx - 1]; + current_wi[idx - 1] = current_wi[idx]; + current_wi[idx] = temp; + } + update_wi(); + } + + function down_wi(idx) { + save_wi(); + var ce = current_wi[idx]; + if (idx >= 0 && idx+1 < current_wi.length) { + const temp = current_wi[idx + 1]; + current_wi[idx + 1] = current_wi[idx]; + current_wi[idx] = temp; + } + update_wi(); + } + function add_wi() { save_wi(); var ne = { @@ -13263,8 +13356,10 @@ Current version: 134 let probarr = [100,90,75,50,25,10,5,1]; - selectionhtml += `` + - `` + + selectionhtml += `` + +`` + +`` + +`` + ` ` + ` @@ -15256,25 +15351,10 @@ Current version: 134

Custom Regex Replace ?Allows transforming incoming text with up to 3 regex patterns, modifying all matches. Replacements will be applied in sequence.
+ class="helptext">Allows transforming incoming text with regex patterns, modifying all matches. Replacements will be applied in sequence.
- - - - - - - - - - - - - - - - - +
Pattern ?The regex pattern to match against any incoming text. Leave blank to disable.Replacement ?The string to replace matches with. Capture groups are allowed (e.g. $1). To remove all matches, leave this blank.
+
diff --git a/koboldcpp.py b/koboldcpp.py index e2754362e..aadf9a8a8 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -95,6 +95,7 @@ class generation_inputs(ctypes.Structure): class generation_outputs(ctypes.Structure): _fields_ = [("status", ctypes.c_int), + ("stopreason", ctypes.c_int), ("text", ctypes.c_char_p)] class sd_load_model_inputs(ctypes.Structure): @@ -493,7 +494,7 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512 if pendingabortkey!="" and pendingabortkey==genkey: print(f"\nDeferred Abort for GenKey: {pendingabortkey}") pendingabortkey = "" - return "" + return {"text":"","status":-1,"stopreason":-1} else: ret = handle.generate(inputs) outstr = "" @@ -504,7 +505,7 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512 sindex = outstr.find(trim_str) if sindex != -1 and trim_str!="": outstr = outstr[:sindex] - return outstr + return {"text":outstr,"status":ret.status,"stopreason":ret.stopreason} def sd_load_model(model_filename): @@ -656,6 +657,7 @@ nocertify = False start_time = time.time() last_req_time = time.time() last_non_horde_req_time = time.time() +currfinishreason = "null" def transform_genparams(genparams, api_format): #alias all nonstandard alternative names for rep pen. @@ -765,8 +767,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): async def generate_text(self, genparams, api_format, stream_flag): from datetime import datetime - global friendlymodelname, chatcompl_adapter + global friendlymodelname, chatcompl_adapter, currfinishreason is_quiet = args.quiet + currfinishreason = "null" def run_blocking(): #api format 1=basic,2=kai,3=oai,4=oai-chat @@ -812,13 +815,16 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): render_special=genparams.get('render_special', False), ) - recvtxt = "" + genout = {"text":"","status":-1,"stopreason":-1} if stream_flag: loop = asyncio.get_event_loop() executor = ThreadPoolExecutor() - recvtxt = await loop.run_in_executor(executor, run_blocking) + genout = await loop.run_in_executor(executor, run_blocking) else: - recvtxt = run_blocking() + genout = run_blocking() + + recvtxt = genout['text'] + currfinishreason = ("length" if (genout['stopreason']!=1) else "stop") #flag instance as non-idle for a while washordereq = genparams.get('genkey', '').startswith('HORDEREQ_') @@ -834,15 +840,15 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): elif api_format==3: res = {"id": "cmpl-1", "object": "text_completion", "created": 1, "model": friendlymodelname, "usage": {"prompt_tokens": 100,"completion_tokens": 100,"total_tokens": 200}, - "choices": [{"text": recvtxt, "index": 0, "finish_reason": "length"}]} + "choices": [{"text": recvtxt, "index": 0, "finish_reason": currfinishreason}]} elif api_format==4: res = {"id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": friendlymodelname, "usage": {"prompt_tokens": 100,"completion_tokens": 100,"total_tokens": 200}, - "choices": [{"index": 0, "message":{"role": "assistant", "content": recvtxt,}, "finish_reason": "length"}]} + "choices": [{"index": 0, "message":{"role": "assistant", "content": recvtxt,}, "finish_reason": currfinishreason}]} elif api_format==5: res = {"caption": end_trim_to_sentence(recvtxt)} else: - res = {"results": [{"text": recvtxt}]} + res = {"results": [{"text": recvtxt, "finish_reason":currfinishreason}]} try: return res @@ -863,7 +869,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): self.wfile.flush() async def handle_sse_stream(self, genparams, api_format): - global friendlymodelname + global friendlymodelname, currfinishreason self.send_response(200) self.send_header("cache-control", "no-cache") self.send_header("connection", "keep-alive") @@ -877,6 +883,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): tokenReserve = "" #keeps fully formed tokens that we cannot send out yet while True: streamDone = handle.has_finished() #exit next loop on done + if streamDone: + sr = handle.get_last_stop_reason() + currfinishreason = ("length" if (sr!=1) else "stop") tokenStr = "" streamcount = handle.get_stream_count() while current_token < streamcount: @@ -893,32 +902,33 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): incomplete_token_buffer.clear() tokenStr += tokenSeg - if tokenStr!="": + if tokenStr!="" or streamDone: sseq = genparams.get('stop_sequence', []) trimstop = genparams.get('trim_stop', False) if trimstop and not streamDone and string_contains_sequence_substring(tokenStr,sseq): tokenReserve += tokenStr await asyncio.sleep(async_sleep_short) #if a stop sequence could trigger soon, do not send output else: - tokenStr = tokenReserve + tokenStr - tokenReserve = "" - - #apply trimming if needed - if trimstop: - for trim_str in sseq: - sindex = tokenStr.find(trim_str) - if sindex != -1 and trim_str!="": - tokenStr = tokenStr[:sindex] - if tokenStr!="": + tokenStr = tokenReserve + tokenStr + tokenReserve = "" + + #apply trimming if needed + if trimstop: + for trim_str in sseq: + sindex = tokenStr.find(trim_str) + if sindex != -1 and trim_str!="": + tokenStr = tokenStr[:sindex] + + if tokenStr!="" or streamDone: if api_format == 4: # if oai chat, set format to expected openai streaming response - event_str = json.dumps({"id":"koboldcpp","object":"chat.completion.chunk","created":1,"model":friendlymodelname,"choices":[{"index":0,"finish_reason":"length","delta":{'role':'assistant','content':tokenStr}}]}) + event_str = json.dumps({"id":"koboldcpp","object":"chat.completion.chunk","created":1,"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":{'role':'assistant','content':tokenStr}}]}) await self.send_oai_sse_event(event_str) elif api_format == 3: # non chat completions - event_str = json.dumps({"id":"koboldcpp","object":"text_completion","created":1,"model":friendlymodelname,"choices":[{"index":0,"finish_reason":"length","text":tokenStr}]}) + event_str = json.dumps({"id":"koboldcpp","object":"text_completion","created":1,"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]}) await self.send_oai_sse_event(event_str) else: - event_str = json.dumps({"token": tokenStr}) + event_str = json.dumps({"token": tokenStr, "finish_reason":currfinishreason}) await self.send_kai_sse_event(event_str) tokenStr = "" else: @@ -3159,7 +3169,8 @@ def main(launch_args,start_server=True): benchprompt = "11111111" for i in range(0,10): #generate massive prompt benchprompt += benchprompt - result = generate(benchprompt,memory="",images=[],max_length=benchlen,max_context_length=benchmaxctx,temperature=0.1,top_k=1,rep_pen=1,use_default_badwordsids=True) + genout = generate(benchprompt,memory="",images=[],max_length=benchlen,max_context_length=benchmaxctx,temperature=0.1,top_k=1,rep_pen=1,use_default_badwordsids=True) + result = genout['text'] result = (result[:5] if len(result)>5 else "") resultok = (result=="11111") t_pp = float(handle.get_last_process_time())*float(benchmaxctx-benchlen)*0.001 @@ -3212,7 +3223,9 @@ def run_in_queue(launch_args, input_queue, output_queue): data = input_queue.get() if data['command'] == 'generate': (args, kwargs) = data['data'] - output_queue.put({'command': 'generated text', 'data': generate(*args, **kwargs)}) + genout = generate(*args, **kwargs) + result = genout['text'] + output_queue.put({'command': 'generated text', 'data': result}) time.sleep(0.2) def start_in_seperate_process(launch_args):