Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

send logprobs in streaming for oai

Browse Source
This commit is contained in:
Concedo
2025-10-21 18:23:56 +08:00
parent 82137e1c89
commit 7446e03851
1 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+10
View File
@@ -3077,9 +3077,19 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
await self.send_kai_sse_event(event_str)
tokenStr = "" # now the final finish reason can be sent alone
if api_format == 4: # if oai chat, set format to expected openai streaming response
if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs
lastlogprobs = handle.last_logprobs()
logprobsdict = parse_last_logprobs(lastlogprobs)
addonstr = json.dumps({"id":"koboldcpp","object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"delta":{'role':'assistant','content':''},"logprobs":logprobsdict}]})
await self.send_oai_sse_event(addonstr)
event_str = json.dumps({"id":"koboldcpp","object":"chat.completion.chunk","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"delta":{'role':'assistant','content':tokenStr}}]})
await self.send_oai_sse_event(event_str)
elif api_format == 3: # non chat completions
if streamDone and ("logprobs" in genparams and genparams["logprobs"]): # this is a hack that sends an extra message containing ALL the logprobs
lastlogprobs = handle.last_logprobs()
logprobsdict = parse_last_logprobs(lastlogprobs)
addonstr = json.dumps({"id":"koboldcpp","object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":None,"text":"","logprobs":logprobsdict}]})
await self.send_oai_sse_event(addonstr)
event_str = json.dumps({"id":"koboldcpp","object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]})
await self.send_oai_sse_event(event_str)
else: