diff --git a/koboldcpp.py b/koboldcpp.py index 953e2ed46..db1cb0292 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -3077,9 +3077,19 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): await self.send_kai_sse_event(event_str) tokenStr = "" # now the final finish reason can be sent alone if api_format == 4: # if oai chat, set format to expected openai streaming response + if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs + lastlogprobs = handle.last_logprobs() + logprobsdict = parse_last_logprobs(lastlogprobs) + addonstr = json.dumps({"id":"koboldcpp","object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"delta":{'role':'assistant','content':''},"logprobs":logprobsdict}]}) + await self.send_oai_sse_event(addonstr) event_str = json.dumps({"id":"koboldcpp","object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":{'role':'assistant','content':tokenStr}}]}) await self.send_oai_sse_event(event_str) elif api_format == 3: # non chat completions + if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs + lastlogprobs = handle.last_logprobs() + logprobsdict = parse_last_logprobs(lastlogprobs) + addonstr = json.dumps({"id":"koboldcpp","object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"text":"","logprobs":logprobsdict}]}) + await self.send_oai_sse_event(addonstr) event_str = json.dumps({"id":"koboldcpp","object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]}) await self.send_oai_sse_event(event_str) else: