diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index d664545e2..d475dc56f 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5410,28 +5410,37 @@ generation_outputs gpttype_generate(const generation_inputs inputs) std::string chat_template = ""; if (file_format == FileFormat::GGUF_GENERIC) { chat_template = gpttype_get_chat_template(); - if (file_format_meta.model_architecture == llm_arch::LLM_ARCH_GEMMA4) { - TokenizeString("<|channel>thought",thinking_start_sequence,file_format,false); - TokenizeString("",thinking_end_sequence,file_format,false); - TokenizeString("\n(Reasoning Budget Exceeded)\n",thinking_end_phrase_toksleft,file_format,false); - //sanity check, start is 2 tokens and end is 1 - if(thinking_start_sequence.size()!=2 || thinking_end_sequence.size()!=1) - { - thinking_start_sequence.clear(); - thinking_end_sequence.clear(); - thinking_end_phrase_toksleft.clear(); - } - } else { - TokenizeString("",thinking_start_sequence,file_format,false); - TokenizeString("",thinking_end_sequence,file_format,false); - TokenizeString("\n(Reasoning Budget Exceeded)\n",thinking_end_phrase_toksleft,file_format,false); - //sanity check, start is 1 tokens and end is 1 - if(thinking_start_sequence.size()!=1 || thinking_end_sequence.size()!=1) - { - thinking_start_sequence.clear(); - thinking_end_sequence.clear(); - thinking_end_phrase_toksleft.clear(); - } + + std::string start = ""; + std::string end = ""; + std::string budget_exceeded = "\n(Reasoning budget exceeded)\nTime to respond now.\n"; + size_t expected_start_tokens = 1; + size_t expected_end_tokens = 1; + + switch (file_format_meta.model_architecture) { + case llm_arch::LLM_ARCH_GEMMA4: + start = "<|channel>thought"; + end = ""; + budget_exceeded = "\n(Reasoning budget exceeded)\nTime to respond now.\n"; + expected_start_tokens = 2; + break; + case llm_arch::LLM_ARCH_SEED_OSS: + start = ""; + end = ""; + budget_exceeded = "\n(Reasoning budget exceeded)\nThe current thinking budget is 0, so I will directly start answering the question.\nTime to respond now.\n"; + break; + default: + break; + } + + TokenizeString(start, thinking_start_sequence, file_format, false); + TokenizeString(end, thinking_end_sequence, file_format, false); + TokenizeString(budget_exceeded, thinking_end_phrase_toksleft, file_format, false); + if (thinking_start_sequence.size() != expected_start_tokens || thinking_end_sequence.size() != expected_end_tokens) + { + thinking_start_sequence.clear(); + thinking_end_sequence.clear(); + thinking_end_phrase_toksleft.clear(); } } diff --git a/koboldcpp.py b/koboldcpp.py index 4b38a5391..cbf44780b 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -182,15 +182,17 @@ default_rpc_port = 5551 thinkformats = [{"start":"<|channel|>analysis<|message|>","end":"<|start|>assistant<|channel|>final<|message|>"}, {"start":"","end":""}, {"start":"","end":""}, + {"start":"<|START_THINKING|>","end":"<|END_THINKING|>"}, {"start":"<|channel>thought","end":""}] -tool_call_pairs = [ #third element is whether its stream-handleable - ("", "", True), - ("", "", True), - ("<|tool_call_begin|>", "<|tool_call_end|>", True), - ("<|tool▁call▁begin|>", "<|tool▁call▁end|>", True), - ("", "", True), - ("<|tool_call>", "", True), - ("<|end|><|start|>assistant<|channel|>commentary to=", "", False), +tool_call_pairs = [ #third element is optional str to match in chat template before we use this pair, fourth element is whether its stream-handleable + ("", "", None, True), + ("", "", None, True), + ("<|tool_call_begin|>", "<|tool_call_end|>", None, True), + ("<|tool▁call▁begin|>", "<|tool▁call▁end|>", None, True), + ("", "", None, True), + ("<|tool_call>", "", None, True), + ("<|end|><|start|>assistant<|channel|>commentary to=", "", None, False), + ("<|tool_call_start|>", "<|tool_call_end|>", None, True), ] deprecated_keys = { "hordeconfig", @@ -3441,6 +3443,7 @@ def coerce_tool_argtypes(tool_calls: list, tool_list: list) -> list: return result def toolcall_to_normalized_json(text,start_tag,end_tag): #convert weird formats into standard tool call json + global cached_chat_template text = text.strip() def parse_qwen35(text: str) -> str: fn_match = re.search(r"", text) @@ -3552,6 +3555,28 @@ def toolcall_to_normalized_json(text,start_tag,end_tag): #convert weird formats return text return json.dumps({"name": fn_name, "arguments": args}) + def parse_lfm25(text: str) -> str: + text = text.replace('<|tool_call_start|>', '') + text = text.replace('<|tool_call_end|>', '') + text = text.strip() + try: + import ast + node = ast.parse(text, mode="eval").body + calls = node.elts if isinstance(node, ast.List) else [node] + results = [] + for call in calls: + if not isinstance(call, ast.Call) or not isinstance(call.func, ast.Name): + return text + args = { + kw.arg: ast.literal_eval(kw.value) + for kw in call.keywords + if kw.arg is not None + } + results.append({"name": call.func.id, "arguments": args}) + return json.dumps(results if len(results) > 1 else results[0]) + except Exception: + return text + # gemma4 takes precedence, since it can contain valid json fragments if end_tag=="": return parse_gemma4(text) @@ -3561,6 +3586,9 @@ def toolcall_to_normalized_json(text,start_tag,end_tag): #convert weird formats if check_ok and len(check_ok)>0: return text #is valid JSON or parsable + if start_tag=="<|tool_call_start|>" and end_tag=="<|tool_call_end|>" and cached_chat_template and "CONTINUE_FINAL_MESSAGE_TAG" in cached_chat_template: + return parse_lfm25(text) + if "" in text and "" in text: # handle glm with args return parse_glm(text) @@ -3591,7 +3619,9 @@ def repack_toolcall_tags(text: str, original_tools:list): text = re.sub(pattern, '', text, flags=re.DOTALL) text = text.strip() found = False - for start, end, streamhandled in tool_call_pairs: + for start, end, required_match_txt, streamhandled in tool_call_pairs: + if required_match_txt and cached_chat_template and required_match_txt not in cached_chat_template: + continue pattern="" if end: pattern = re.escape(start) + r"(.*?)" + re.escape(end) @@ -3621,8 +3651,14 @@ def format_jinja(messages_orig, tools, chat_template_kwargs=None): print(f"Warning: Jinja template raised an exception: {msg}") return "" global cached_chat_template + from jinja2.ext import Extension + class IgnoreGenerationTags(Extension): + tags = {"generation"} + def parse(self, parser): + parser.stream.skip(1) + return parser.parse_statements( ("name:endgeneration",), drop_needle=True) from jinja2.sandbox import ImmutableSandboxedEnvironment - jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) + jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, extensions=[IgnoreGenerationTags]) # sanitize messages to remove none types messages = json.loads(json.dumps(messages_orig)) for m in messages: @@ -5245,7 +5281,9 @@ class KcppServerRequestHandler(http.server.SimpleHTTPRequestHandler): # if tools, do not send anything else - OAI tool calls will be handled with fakestreaming! # only exception is if we know the exact toolcall tag to segment! tool_segment_tag = "" - for start, end, streamhandled in tool_call_pairs: + for start, end, required_match_txt, streamhandled in tool_call_pairs: + if required_match_txt and cached_chat_template and required_match_txt not in cached_chat_template: + continue if streamhandled and cached_chat_template and start in cached_chat_template: tool_segment_tag = start break diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index f221764df..4c4047d17 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -3052,6 +3052,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { attr = LLAMA_TOKEN_ATTR_USER_DEFINED; } + //kcpp hack render these + if (t.first == "<|tool_call_start|>" || t.first == "<|tool_call_end|>") { + LLAMA_LOG_WARN("%s: setting token '%s' (%d) attribute to USER_DEFINED (%u), old attributes: %u\n", + __func__, t.first.c_str(), t.second, LLAMA_TOKEN_ATTR_USER_DEFINED, attr); + + attr = LLAMA_TOKEN_ATTR_USER_DEFINED; + } + if (t.first == "[THINK]" || t.first == "[/THINK]" || t.first == "" || t.first == "" || t.first == "[CALL_ID]" || t.first == "[TOOL_CONTENT]" || t.first == "[TOOL_CALLS]" || t.first == "[ARGS]") { LLAMA_LOG_WARN("%s: setting token '%s' (%d) attribute to USER_DEFINED (%u), old attributes: %u\n",