Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

crude openai responses streaming

Browse Source
This commit is contained in:
Concedo
2026-03-29 00:16:30 +08:00
parent 94b266a6b0
commit 5ad9e3ee31
1 changed files with 96 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+96 -34
View File
@@ -4043,6 +4043,41 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
print(f"File Upload Process Error: {e}")
return result
def prepare_basic_responses_body(self,resp_id,genparams):
global friendlymodelname
ret = {
"id": resp_id,
"object": "response",
"created_at": int(time.time()),
"completed_at": None,
"incomplete_details": None,
"previous_response_id": None,
"truncation": "disabled",
"parallel_tool_calls": False,
"text": {"format": {"type": "text"},"verbosity": "medium"},
"instructions": genparams.get('instructions', None),
"model": friendlymodelname,
"error": None,
"metadata": {},
"tools": genparams.get('tools', []),
"tool_choice": "auto",
"background": False,
"service_tier": "default",
"safety_identifier": None,
"prompt_cache_key": None,
"max_tool_calls": None,
"store": False,
"top_p": genparams.get("top_p", 0.92),
"max_output_tokens":genparams.get("max_length", None),
"presence_penalty": genparams.get("presence_penalty", 0),
"frequency_penalty": genparams.get("frequency_penalty", 0),
"top_logprobs": 0,
"temperature": genparams.get("temperature", 1),
"reasoning": {"effort": None, "summary": None},
"usage": None
}
return ret
async def generate_text(self, genparams, api_format, stream_flag):
global friendlymodelname, chatcompl_adapter, currfinishreason
currfinishreason = None
@@ -4147,39 +4182,11 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
if tool_calls and len(tool_calls) > 0: # Add function call items if tool calls exist
for tc in tool_calls:
output_items.append({"type": "function_call", "id": tc.get("id", ""), "call_id": tc.get("id", ""), "name": tc.get("function", {}).get("name", ""), "arguments": tc.get("function", {}).get("arguments", "{}"), "status": "completed"})
res = {
"id": resp_id,
"object": "response",
"created_at": int(time.time()),
"completed_at": int(time.time()),
"incomplete_details": None,
"previous_response_id": None,
"truncation": "disabled",
"parallel_tool_calls": False,
"text": {"format": {"type": "text"},"verbosity": "medium"},
"instructions": genparams.get('instructions', None),
"model": friendlymodelname,
"status": "completed" if currfinishreason != "error" else "failed",
"error": None,
"metadata": {},
"tools": genparams.get('tools', []),
"tool_choice": "auto",
"background": False,
"service_tier": "default",
"safety_identifier": None,
"prompt_cache_key": None,
"max_tool_calls": None,
"store": False,
"output": output_items,
"top_p": genparams.get("top_p", 0.92),
"max_output_tokens":genparams.get("max_length", None),
"presence_penalty": genparams.get("presence_penalty", 0),
"frequency_penalty": genparams.get("frequency_penalty", 0),
"top_logprobs": 0,
"temperature": genparams.get("temperature", 1),
"reasoning": {"effort": None, "summary": None},
"usage": {"input_tokens": prompttokens, "output_tokens": comptokens, "total_tokens": prompttokens + comptokens, "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}}
}
res = self.prepare_basic_responses_body(resp_id,genparams)
res["completed_at"] = int(time.time())
res["status"] = "completed" if currfinishreason != "error" else "failed"
res["output"] = output_items
res["usage"] = {"input_tokens": prompttokens, "output_tokens": comptokens, "total_tokens": prompttokens + comptokens, "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}}
else: #kcpp format
res = {"results": [{"text": recvtxt, "tool_calls": tool_calls, "finish_reason": currfinishreason, "logprobs":logprobsdict, "prompt_tokens": prompttokens, "completion_tokens": comptokens}]}
@@ -4195,6 +4202,10 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.wfile.write(f'data: {data}\n\n'.encode())
self.wfile.flush()
async def send_oai_responses_sse_event(self, eventname, data):
self.wfile.write(f'event: {eventname}\ndata: {data}\n\n'.encode())
self.wfile.flush()
async def send_kai_sse_event(self, data):
self.wfile.write('event: message\n'.encode())
self.wfile.write(f'data: {data}\n\n'.encode())
@@ -4220,6 +4231,8 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
encap_first_loop = True
thinkpairs = [{"start":"<|channel|>analysis<|message|>","end":"<|start|>assistant<|channel|>final<|message|>"},
{"start":"<think>","end":"</think>"}]
responses_first_loop = True
rseq_num = 0
current_token = 0
prompttokens = 0
incomplete_token_buffer = bytearray()
@@ -4329,6 +4342,53 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
await self.send_oai_sse_event(addonstr)
event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]})
await self.send_oai_sse_event(event_str)
elif api_format == 8: #oai-responses
resp_id = f"resp-A{genparams.get('oai_uniqueid', 1)}"
item_id = f"msg_0{genparams.get('oai_uniqueid', 1)}"
# Send response.created once at the start (only on first iteration)
if responses_first_loop:
res = self.prepare_basic_responses_body(resp_id, genparams)
res["status"] = "in_progress"
res["output"] = []
created_event = json.dumps({"type": "response.created", "response": res, "sequence_number":rseq_num})
rseq_num += 1
await self.send_oai_responses_sse_event("response.created",created_event)
# response.output_item.added
item_added = json.dumps({"type": "response.output_item.added", "output_index": 0, "sequence_number":rseq_num, "item": { "type": "message", "id": item_id, "status": "in_progress", "role": "assistant", "content": []}})
rseq_num += 1
await self.send_oai_responses_sse_event("response.output_item.added",item_added)
# content_part.added
part_added = json.dumps({"type": "response.content_part.added", "item_id": item_id, "output_index": 0, "sequence_number":rseq_num, "content_index": 0, "part": {"type": "output_text", "text": "", "annotations": []}})
rseq_num += 1
await self.send_oai_responses_sse_event("response.content_part.added",part_added)
responses_first_loop = False
if tokenStr != "" or streamDone:
if tokenStr != "":
delta_event = json.dumps({"type": "response.output_text.delta", "item_id": item_id, "output_index": 0, "sequence_number":rseq_num, "logprobs":[], "content_index": 0, "delta": tokenStr})
rseq_num += 1
await self.send_oai_responses_sse_event("response.output_text.delta",delta_event)
if streamDone:
# content_part.done, reply full text
await asyncio.sleep(async_sleep_short)
finaltxt = handle.get_pending_output().decode("UTF-8", "ignore")
await asyncio.sleep(async_sleep_short)
done_event = json.dumps({"type": "response.output_text.done", "item_id": item_id, "output_index": 0, "sequence_number":rseq_num, "content_index": 0, "text": finaltxt})
rseq_num += 1
await self.send_oai_responses_sse_event("response.output_text.done",done_event)
# response.output_item.done
item_done = json.dumps({"type": "response.output_item.done", "output_index": 0, "sequence_number":rseq_num, "item": { "type": "message", "id": item_id, "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": finaltxt, "annotations": [], "logprobs": []}]}})
rseq_num += 1
await self.send_oai_responses_sse_event("response.output_item.done",item_done)
usage_pp = handle.get_last_input_count()
usage_gen = current_token
res = self.prepare_basic_responses_body(resp_id,genparams)
res["completed_at"] = int(time.time())
res["status"] = "completed" if currfinishreason != "error" else "failed"
res["output"] = [{"type": "message", "id": item_id, "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": finaltxt, "annotations": [], "logprobs": []}]}]
res["usage"] = {"input_tokens": usage_pp,"input_tokens_details":{"cached_tokens":0}, "output_tokens": usage_gen, "output_tokens_details":{"reasoning_tokens":0}, "total_tokens": usage_pp + usage_gen}
completed_event = json.dumps({"type": "response.completed", "response": res, "sequence_number":rseq_num})
rseq_num += 1
await self.send_oai_responses_sse_event("response.completed",completed_event)
else:
event_str = json.dumps({"token": tokenStr, "finish_reason":currfinishreason})
await self.send_kai_sse_event(event_str)
@@ -5608,7 +5668,7 @@ Change Mode<br>
if api_format > 0: #text gen
# Check if streaming chat completions, if so, set stream mode to true
if (api_format == 4 or api_format == 3) and "stream" in genparams and genparams["stream"]:
if (api_format == 4 or api_format == 3 or api_format == 8) and "stream" in genparams and genparams["stream"]:
sse_stream_flag = True
gendat = asyncio.run(self.handle_request(genparams, api_format, sse_stream_flag))
@@ -8378,6 +8438,8 @@ def convert_invalid_args(args):
dict["sdclip2"] = dict["sdclipg"]
if "jinja_tools" in dict and dict["jinja_tools"]:
dict["jinja"] = True
if "jinja_kwargs" in dict and dict["jinja_kwargs"]:
dict["jinja"] = True
if "sdgendefaults" in dict and "gendefaults" not in dict:
dict["gendefaults"] = dict["sdgendefaults"]
if "flashattention" in dict and "noflashattention" not in dict: