From 5ad9e3ee318e158c30194c330de1fa8f170e261b Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Sun, 29 Mar 2026 00:16:30 +0800
Subject: [PATCH] crude openai responses streaming
---
koboldcpp.py | 130 +++++++++++++++++++++++++++++++++++++--------------
1 file changed, 96 insertions(+), 34 deletions(-)
diff --git a/koboldcpp.py b/koboldcpp.py
index 7e9ccd0af..3566517a9 100755
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -4043,6 +4043,41 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
print(f"File Upload Process Error: {e}")
return result
+ def prepare_basic_responses_body(self,resp_id,genparams):
+ global friendlymodelname
+ ret = {
+ "id": resp_id,
+ "object": "response",
+ "created_at": int(time.time()),
+ "completed_at": None,
+ "incomplete_details": None,
+ "previous_response_id": None,
+ "truncation": "disabled",
+ "parallel_tool_calls": False,
+ "text": {"format": {"type": "text"},"verbosity": "medium"},
+ "instructions": genparams.get('instructions', None),
+ "model": friendlymodelname,
+ "error": None,
+ "metadata": {},
+ "tools": genparams.get('tools', []),
+ "tool_choice": "auto",
+ "background": False,
+ "service_tier": "default",
+ "safety_identifier": None,
+ "prompt_cache_key": None,
+ "max_tool_calls": None,
+ "store": False,
+ "top_p": genparams.get("top_p", 0.92),
+ "max_output_tokens":genparams.get("max_length", None),
+ "presence_penalty": genparams.get("presence_penalty", 0),
+ "frequency_penalty": genparams.get("frequency_penalty", 0),
+ "top_logprobs": 0,
+ "temperature": genparams.get("temperature", 1),
+ "reasoning": {"effort": None, "summary": None},
+ "usage": None
+ }
+ return ret
+
async def generate_text(self, genparams, api_format, stream_flag):
global friendlymodelname, chatcompl_adapter, currfinishreason
currfinishreason = None
@@ -4147,39 +4182,11 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
if tool_calls and len(tool_calls) > 0: # Add function call items if tool calls exist
for tc in tool_calls:
output_items.append({"type": "function_call", "id": tc.get("id", ""), "call_id": tc.get("id", ""), "name": tc.get("function", {}).get("name", ""), "arguments": tc.get("function", {}).get("arguments", "{}"), "status": "completed"})
- res = {
- "id": resp_id,
- "object": "response",
- "created_at": int(time.time()),
- "completed_at": int(time.time()),
- "incomplete_details": None,
- "previous_response_id": None,
- "truncation": "disabled",
- "parallel_tool_calls": False,
- "text": {"format": {"type": "text"},"verbosity": "medium"},
- "instructions": genparams.get('instructions', None),
- "model": friendlymodelname,
- "status": "completed" if currfinishreason != "error" else "failed",
- "error": None,
- "metadata": {},
- "tools": genparams.get('tools', []),
- "tool_choice": "auto",
- "background": False,
- "service_tier": "default",
- "safety_identifier": None,
- "prompt_cache_key": None,
- "max_tool_calls": None,
- "store": False,
- "output": output_items,
- "top_p": genparams.get("top_p", 0.92),
- "max_output_tokens":genparams.get("max_length", None),
- "presence_penalty": genparams.get("presence_penalty", 0),
- "frequency_penalty": genparams.get("frequency_penalty", 0),
- "top_logprobs": 0,
- "temperature": genparams.get("temperature", 1),
- "reasoning": {"effort": None, "summary": None},
- "usage": {"input_tokens": prompttokens, "output_tokens": comptokens, "total_tokens": prompttokens + comptokens, "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}}
- }
+ res = self.prepare_basic_responses_body(resp_id,genparams)
+ res["completed_at"] = int(time.time())
+ res["status"] = "completed" if currfinishreason != "error" else "failed"
+ res["output"] = output_items
+ res["usage"] = {"input_tokens": prompttokens, "output_tokens": comptokens, "total_tokens": prompttokens + comptokens, "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}}
else: #kcpp format
res = {"results": [{"text": recvtxt, "tool_calls": tool_calls, "finish_reason": currfinishreason, "logprobs":logprobsdict, "prompt_tokens": prompttokens, "completion_tokens": comptokens}]}
@@ -4195,6 +4202,10 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.wfile.write(f'data: {data}\n\n'.encode())
self.wfile.flush()
+ async def send_oai_responses_sse_event(self, eventname, data):
+ self.wfile.write(f'event: {eventname}\ndata: {data}\n\n'.encode())
+ self.wfile.flush()
+
async def send_kai_sse_event(self, data):
self.wfile.write('event: message\n'.encode())
self.wfile.write(f'data: {data}\n\n'.encode())
@@ -4220,6 +4231,8 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
encap_first_loop = True
thinkpairs = [{"start":"<|channel|>analysis<|message|>","end":"<|start|>assistant<|channel|>final<|message|>"},
{"start":"","end":""}]
+ responses_first_loop = True
+ rseq_num = 0
current_token = 0
prompttokens = 0
incomplete_token_buffer = bytearray()
@@ -4329,6 +4342,53 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler):
await self.send_oai_sse_event(addonstr)
event_str = json.dumps({"id":cmpl_id,"object":"text_completion","created":int(time.time()),"model":friendlymodelname,"choices":[{"index":0,"finish_reason":currfinishreason,"text":tokenStr}]})
await self.send_oai_sse_event(event_str)
+ elif api_format == 8: #oai-responses
+ resp_id = f"resp-A{genparams.get('oai_uniqueid', 1)}"
+ item_id = f"msg_0{genparams.get('oai_uniqueid', 1)}"
+ # Send response.created once at the start (only on first iteration)
+ if responses_first_loop:
+ res = self.prepare_basic_responses_body(resp_id, genparams)
+ res["status"] = "in_progress"
+ res["output"] = []
+ created_event = json.dumps({"type": "response.created", "response": res, "sequence_number":rseq_num})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.created",created_event)
+ # response.output_item.added
+ item_added = json.dumps({"type": "response.output_item.added", "output_index": 0, "sequence_number":rseq_num, "item": { "type": "message", "id": item_id, "status": "in_progress", "role": "assistant", "content": []}})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.output_item.added",item_added)
+ # content_part.added
+ part_added = json.dumps({"type": "response.content_part.added", "item_id": item_id, "output_index": 0, "sequence_number":rseq_num, "content_index": 0, "part": {"type": "output_text", "text": "", "annotations": []}})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.content_part.added",part_added)
+ responses_first_loop = False
+ if tokenStr != "" or streamDone:
+ if tokenStr != "":
+ delta_event = json.dumps({"type": "response.output_text.delta", "item_id": item_id, "output_index": 0, "sequence_number":rseq_num, "logprobs":[], "content_index": 0, "delta": tokenStr})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.output_text.delta",delta_event)
+ if streamDone:
+ # content_part.done, reply full text
+ await asyncio.sleep(async_sleep_short)
+ finaltxt = handle.get_pending_output().decode("UTF-8", "ignore")
+ await asyncio.sleep(async_sleep_short)
+ done_event = json.dumps({"type": "response.output_text.done", "item_id": item_id, "output_index": 0, "sequence_number":rseq_num, "content_index": 0, "text": finaltxt})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.output_text.done",done_event)
+ # response.output_item.done
+ item_done = json.dumps({"type": "response.output_item.done", "output_index": 0, "sequence_number":rseq_num, "item": { "type": "message", "id": item_id, "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": finaltxt, "annotations": [], "logprobs": []}]}})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.output_item.done",item_done)
+ usage_pp = handle.get_last_input_count()
+ usage_gen = current_token
+ res = self.prepare_basic_responses_body(resp_id,genparams)
+ res["completed_at"] = int(time.time())
+ res["status"] = "completed" if currfinishreason != "error" else "failed"
+ res["output"] = [{"type": "message", "id": item_id, "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": finaltxt, "annotations": [], "logprobs": []}]}]
+ res["usage"] = {"input_tokens": usage_pp,"input_tokens_details":{"cached_tokens":0}, "output_tokens": usage_gen, "output_tokens_details":{"reasoning_tokens":0}, "total_tokens": usage_pp + usage_gen}
+ completed_event = json.dumps({"type": "response.completed", "response": res, "sequence_number":rseq_num})
+ rseq_num += 1
+ await self.send_oai_responses_sse_event("response.completed",completed_event)
else:
event_str = json.dumps({"token": tokenStr, "finish_reason":currfinishreason})
await self.send_kai_sse_event(event_str)
@@ -5608,7 +5668,7 @@ Change Mode
if api_format > 0: #text gen
# Check if streaming chat completions, if so, set stream mode to true
- if (api_format == 4 or api_format == 3) and "stream" in genparams and genparams["stream"]:
+ if (api_format == 4 or api_format == 3 or api_format == 8) and "stream" in genparams and genparams["stream"]:
sse_stream_flag = True
gendat = asyncio.run(self.handle_request(genparams, api_format, sse_stream_flag))
@@ -8378,6 +8438,8 @@ def convert_invalid_args(args):
dict["sdclip2"] = dict["sdclipg"]
if "jinja_tools" in dict and dict["jinja_tools"]:
dict["jinja"] = True
+ if "jinja_kwargs" in dict and dict["jinja_kwargs"]:
+ dict["jinja"] = True
if "sdgendefaults" in dict and "gendefaults" not in dict:
dict["gendefaults"] = dict["sdgendefaults"]
if "flashattention" in dict and "noflashattention" not in dict: