diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index d7e117e4d..ed6568335 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -12,6 +12,8 @@ using json = common_json; +namespace autoparser { + // Helper to iterate over tools/functions static void foreach_function(const json & tools, const std::function & fn) { for (const auto & tool : tools) { @@ -22,8 +24,6 @@ static void foreach_function(const json & tools, const std::functionhas_explicit_template; } -// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list -// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call -static bool is_lfm2_template(const std::string & src) { - return src.find("<|tool_list_start|>") != std::string::npos && - src.find("<|tool_list_end|>") != std::string::npos; -} - common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) { common_chat_prompt_preset asr_preset; asr_preset.system = ""; @@ -914,42 +923,12 @@ common_reasoning_format common_reasoning_format_from_name(const std::string & fo throw std::runtime_error("Unknown reasoning format: " + format); } -// static void foreach_function(const json & tools, const std::function & fn) { -// for (const auto & tool : tools) { -// if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { -// LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str()); -// continue; -// } -// fn(tool); -// } -// } - -static void foreach_parameter(const json & function, - const std::function & fn) { - if (!function.contains("parameters") || !function.at("parameters").is_object()) { - return; - } - const auto & params = function.at("parameters"); - if (!params.contains("properties") || !params.at("properties").is_object()) { - return; - } - const auto & props = params.at("properties"); - std::set required; - if (params.contains("required") && params.at("required").is_array()) { - required = params.at("required").get>(); - } - for (const auto & [name, prop] : props.items()) { - bool is_required = (required.find(name) != required.end()); - fn(name, prop, is_required); - } -} - -static std::string common_chat_template_direct_apply_impl( +std::string common_chat_template_direct_apply_impl( const common_chat_template & tmpl, const autoparser::generation_params & inputs, - const std::optional & messages_override = std::nullopt, - const std::optional & tools_override = std::nullopt, - const std::optional & additional_context = std::nullopt) { + const std::optional & messages_override, + const std::optional & tools_override, + const std::optional & additional_context) { jinja::context ctx(tmpl.source()); // messages_override is already built for this template, do not touch its content parts @@ -1013,12 +992,12 @@ std::string common_chat_template_direct_apply( return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt); } -static std::string common_chat_template_generation_prompt_impl( +std::string common_chat_template_generation_prompt_impl( const common_chat_template & tmpl, const autoparser::generation_params & inputs, - const std::optional & messages_override = std::nullopt, - const std::optional & tools_override = std::nullopt, - const std::optional & additional_context = std::nullopt) { + const std::optional & messages_override, + const std::optional & tools_override, + const std::optional & additional_context) { autoparser::generation_params params = inputs; params.add_generation_prompt = false; @@ -1041,1938 +1020,6 @@ std::string common_chat_template_generation_prompt( return common_chat_template_generation_prompt_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt); } -static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja - auto adjusted_messages = json::array(); - for (const auto & msg : inputs.messages) { - auto role = msg.value("role", ""); - if (role != "system" && role != "assistant") { - // Only adjust system and assistant messages. Interestingly, the system message may contain thinking. - adjusted_messages.push_back(msg); - continue; - } - - auto content = json::array(); - - // If message contains `reasoning_content`, add it as a block of type `thinking` - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - content.push_back({ - { "type", "thinking" }, - { "thinking", msg.at("reasoning_content").get() }, - }); - } - - // If message contains `content`, add it as a block of type `text` - if (msg.contains("content")) { - if (msg.at("content").is_string()) { - content.push_back({ - { "type", "text" }, - { "text", msg.at("content").get() }, - }); - } else if (msg.at("content").is_array()) { - auto blocks = msg.at("content"); - content.insert(blocks); - } - } - - auto adjusted = msg; - adjusted["content"] = content; - adjusted.erase("reasoning_content"); - adjusted_messages.push_back(adjusted); - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = true; - - data.supports_thinking = true; - data.thinking_start_tag = "[THINK]"; - data.thinking_end_tags = {"[/THINK]"}; - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.preserved_tokens = { - "[THINK]", - "[/THINK]", - "[TOOL_CALLS]", - "[ARGS]", - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "[THINK]" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "[/THINK]" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.eps(); - auto reasoning = - extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps(); - - // Response format parser - if (has_response_format) { - // Ministral wants to emit json surrounded by code fences - return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```"); - } - - // Tool call parser - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - tool_choice |= - p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") + - p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); - }); - - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls)); - - return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls); - } - - // Content only parser - include_grammar = false; - return generation_prompt + (reasoning << p.content(p.rest())); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" } - }; - } - - return data; -} - -static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - const std::string GEN_PREFIX = "<|im_start|>assistant\n"; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - - auto supports_reasoning = tmpl.source().find("") != std::string::npos; - - data.supports_thinking = supports_reasoning; - data.preserved_tokens = { - "", - "", - }; - - auto is_qwen3_coder = !supports_reasoning; - - if (supports_reasoning) { - data.thinking_start_tag = ""; - // Support both and as reasoning end sequences. - // ", "" }; - data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); - } - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PREFIX; - if (supports_reasoning) { - data.generation_prompt += "\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n\n\n"; - } - } - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - std::vector tool_call_starts = { "" }; - - if (is_qwen3_coder) { - // Match complete opener for Qwen3-Coder models that occasionally omit the - // starting . The model may hallucinate a tool name, but it is preferable over - // constraining on - foreach_function(inputs.tools, [&](const json & tool) { - const std::string name = tool.at("function").at("name"); - tool_call_starts.push_back(""); - }); - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PREFIX); - - auto reasoning = p.eps(); - if (supports_reasoning && extract_reasoning) { - reasoning = p.optional("" + p.space() + - p.reasoning(p.until_one_of({ "", "" })) + - (p.literal("") | p.peek(p.literal("")))); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); - } - - // Tool call parser - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto arg_close = p.tool_arg_close(p.literal("\n\n")); - auto arg_string = p.rule("xml-arg-string", - p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(parameters); - - std::vector required_args; - std::vector optional_args; - - foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { - auto rule_name = "tool-" + name + "-arg-" + param_name; - - auto arg_open = p.tool_arg_open("\n"); - - auto arg_value = schema_info.resolves_to_string(param_schema) ? - arg_string : - p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; - - auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); - - (is_required ? required_args : optional_args).push_back(arg_rule); - }); - - // Accept required arguments in any order, as Qwen does not always adhere to the - // order provided. - auto args = p.permute("tool-" + name + "-args", required_args); - if (!optional_args.empty()) { - args = args + p.zero_or_more(p.choice(optional_args)); - } - - auto func = p.tool(p.tool_open("\n") + - p.tool_args(args) + - p.tool_close(p.literal("\n"))); - - tool_choice |= p.rule("tool-" + name, func); - }); - - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - - auto tool_call_body = tool_choice + "" + p.space(); - auto tool_call = p.rule("tool-call", "\n" + tool_call_body); - - // Qwen3-Coder models may occasionally omit the token. - auto tool_call_first = is_qwen3_coder ? - p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body) : - tool_call; - - auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); - - return generation_prompt + - (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); - } - - // Content only parser - return generation_prompt + (reasoning << p.content(p.rest())); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - if (data.grammar_lazy) { - for (const auto & start : tool_call_starts) { - data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); - } - } - } - - return data; -} - -static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - // Copy reasoning to the "thinking" field as expected by the gpt-oss template - auto adjusted_messages = json::array(); - for (auto msg : inputs.messages) { - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - msg["thinking"] = msg.at("reasoning_content"); - if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { - msg.erase("content"); - } - } - adjusted_messages.push_back(msg); - } - - auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - - // Check if we need to replace the return token with end token during - // inference and without generation prompt. For more details see: - // https://github.com/ggml-org/llama.cpp/issues/15417 - if (inputs.is_inference && !inputs.add_generation_prompt) { - static constexpr std::string_view return_token = "<|return|>"; - static constexpr std::string_view end_token = "<|end|>"; - if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) { - prompt.replace(pos, return_token.length(), end_token); - } - } - - data.prompt = prompt; - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, - { COMMON_CHAT_ROLE_USER, "<|start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, - { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, - }; - - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - data.thinking_start_tag = "<|channel|>analysis<|message|>"; - data.thinking_end_tags = {"<|end|>"}; - - // These special tokens are required to parse properly, so we include them - // even if parse_tool_calls is false. - data.preserved_tokens = { - "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", - }; - - // Adjust prompt for continuation - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); - auto end = p.rule("end", p.literal("<|end|>")); - auto content = p.rule("message-content", p.until("<|end|>")); - auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); - auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); - - // Occasionally, gpt-oss-20b will prefix channels with this commentary - auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); - auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>"); - - if (extract_reasoning) { - p.rule("analysis", start_analysis + p.reasoning(content) + end); - } else { - p.rule("analysis", p.content(start_analysis + content + end)); - } - - auto analysis = p.ref("analysis"); - auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); - auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content)); - - // Consume any unsolicited tool calls, e.g. builtin functions - auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); - - auto any = p.rule("any", preamble | analysis); - - if (has_response_format) { - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); - auto response_format = p.rule("response-format", - p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); - - return p.zero_or_more(start + analysis) + start + response_format; - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto tool_choice = p.choice(); - - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & params = function.at("parameters"); - - auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); - auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params)); - - // recipient in role header - // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS - auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args); - - // recipient in channel header - // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS - auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args); - - tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel); - }); - - auto tool_call = p.trigger_rule("tool-call", tool_choice); - - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + any) + start + tool_call; - } - - return p.zero_or_more(start + any) + start + (tool_call | final_msg); - } - - return p.zero_or_more(start + any) + start + (final_msg | unsolicited); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" } - }; - } - - return data; -} - -static common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - - if (inputs.add_generation_prompt && string_ends_with(data.prompt, "\n")) { - // This may happen if the model generates content + tool_call, the - // template does not add the model's next turn and confuses the model - // from emitting its proper reasoning token sequence. - data.generation_prompt = "<|turn>model\n"; - data.prompt += data.generation_prompt; - } - - data.message_delimiters = { - { COMMON_CHAT_ROLE_USER, "<|turn>user" }, - { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, - }; - - data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; - data.supports_thinking = true; - data.thinking_start_tag = "<|channel>thought"; - data.thinking_end_tags = {""}; - - data.preserved_tokens = { - "<|channel>", - "", - "<|tool_call>", - "", - "<|turn>", - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = string_ends_with(data.prompt, "\n") ? "<|turn>model\n" : ""; - data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.optional(p.literal("<|turn>model\n"))); - - if (extract_reasoning) { - p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("")) + p.literal("")); - } else { - p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("") + p.literal(""))); - } - - auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), ""); - auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>")); - - if (has_response_format) { - auto response_format = p.literal("```json") << - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) << - p.literal("```"); - return start + p.optional(thought) + response_format; - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Gemma4 tool calling syntax - // Rules should match traversal logic in gemma4_to_json() - p.rule("gemma4-string-content", p.until("<|\"|>")); - p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>")); - p.rule("gemma4-bool", p.json_bool()); - p.rule("gemma4-null", p.json_null()); - p.rule("gemma4-number", p.json_number()); - p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":")); - p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")); - p.rule("gemma4-dict", [&]() { - auto ws = p.space(); - auto member = p.ref("gemma4-dict-kv"); - auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); - return p.sequence({ - p.literal("{"), ws, - p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) - }); - }); - p.rule("gemma4-array", [&]() { - auto ws = p.space(); - auto value = p.ref("gemma4-value"); - auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); - return p.sequence({ - p.literal("["), ws, - p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) - }); - }); - p.rule("gemma4-value", [&]() { - return p.choice({ - p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), - p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") - }); - }); - - auto tool_choice = p.choice(); - - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules - // const auto & params = function.at("parameters"); - - tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({ - p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))), - p.tool_args(p.ref("gemma4-dict")), - }))); - }); - - auto tool_call = p.trigger_rule("tool-call", p.repeat( - "<|tool_call>call:" + tool_choice + "", - /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0, - /* max = */ inputs.parallel_tool_calls ? -1 : 1 - )); - - auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>")); - auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "", "<|tool_call>"}))); - auto message = p.rule("message", thought + content); - return start + p.zero_or_more(message) + scan_to_toolcall + tool_call; - } - - // Gemma 4 may emit an extra <|channel>thought\n at the end of the content. It may - // also emit a single trailing token. Consume all complete reasoning blocks and - // then stop at the first unmatched token. - auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", ""}))); - auto message = p.rule("message", thought + content); - return start + p.one_or_more(message); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" }, - }; - } - - return data; -} - -// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content} -static common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.preserved_tokens = { - ">>>all", - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content(); - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - // Functionary v3.2 format: - // - Normal content: >>>all\n{content} - // - Tool calls: >>>function_name\n{json_args} - // Generation prompt ends with ">>>" so model outputs recipient immediately - - // Build content parser for >>>all\n{content} - // When tools are present, content stops before the next ">>>" (tool call) - // When no tools, content goes until end - auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>")); - auto content_until_end = p.literal("all\n") + p.content(p.rest()); - auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>"); - - // If no tools or tool_choice is NONE, just parse content - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - // When no tools, just match the prefix and capture everything after - return generation_prompt + content_until_end + p.end(); - } - - // Build tool call parsers for each available function - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - // Tool format: >>>function_name\n{json_args} - auto tool_parser = p.tool( - p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) + - p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) - ); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto content_only = content_until_end; - auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice)); - auto content_and_tools = content_until_tool + tools_only; - - auto ret = p.eps(); - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - if (inputs.parallel_tool_calls) { - ret = p.choice({ content_and_tools, tools_only }) + p.end(); - } else { - ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end(); - } - } else if (inputs.parallel_tool_calls) { - ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end(); - } else { - auto content_and_tool = content_until_tool + tool_choice; - ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end(); - } - return generation_prompt + ret; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - // Grammar trigger for when the model starts outputting a tool call - // (after the initial ">>>" in the generation prompt but recipient other than "all") - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" } - }; - } - - return data; -} - -// Kimi K2 Thinking - uses unique tool call ID format: functions.: -// The ID contains both the function name and an incrementing counter -static common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { - "<|tool_calls_section_begin|>", - "<|tool_calls_section_end|>", - "<|tool_call_begin|>", - "<|tool_call_argument_begin|>", - "<|tool_call_end|>", - "", - "", - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>"; - const std::string SECTION_END = "<|tool_calls_section_end|>"; - const std::string CALL_BEGIN = "<|tool_call_begin|>"; - const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>"; - const std::string CALL_END = "<|tool_call_end|>"; - - const std::string THINK_START = ""; - const std::string THINK_END = ""; - const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>"; - - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END}; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - // Kimi K2 Thinking format: - // - Reasoning: {reasoning} - // - Content: text after reasoning - // - Tool calls section: - // <|tool_calls_section_begin|> - // <|tool_call_begin|>functions.:<|tool_call_argument_begin|>{json_args}<|tool_call_end|> - // ... - // <|tool_calls_section_end|> - // The ID format is: functions.: where counter is 0, 1, 2, ... - - // Tool call markers - auto end = p.end(); - - // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny. - // For example, it can call tools at the end of reasoning without closing reasoning... - auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning( - p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) + - p.optional(p.literal(THINK_END))) : p.eps(); - auto generation_prompt = p.literal(GEN_PROMPT); - - - // Content only parser (no tools) - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - - // Build tool call parsers for each available function - // The ID format is: functions.: - // We need to match: functions.: - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - // Match: functions.: - // Capture the full call id (functions.:) using tool_id tag - auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1)); - auto tool_parser = p.tool( - p.tool_open(tool_id + p.literal(ARGS_BEGIN)) + - p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) + - p.tool_close(p.optional((p.literal(CALL_END)))) - ); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|> - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - // Use trigger_rule so grammar generator knows where to start generating rules - auto tool_calls = p.rule("tool-calls", - p.optional(p.literal(SECTION_BEGIN)) + - p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) + - p.optional(p.literal(SECTION_END))) - ); - - auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN })); - - return generation_prompt + reasoning + content_before_tools + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" } - }; - } - - return data; -} - -// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable -// (except dotted names and JSON literals true/false/null). -// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional reasoning. -// tool_list_tokens preserves LFM2 system tool-list markers. -static common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs, - bool tool_list_tokens) { - common_chat_params data; - - const std::string TOOL_CALL_START = "<|tool_call_start|>"; - const std::string TOOL_CALL_END = "<|tool_call_end|>"; - const std::string TOOL_LIST_START = "<|tool_list_start|>"; - const std::string TOOL_LIST_END = "<|tool_list_end|>"; - const std::string THINK_START = ""; - const std::string THINK_END = ""; - const std::string GEN_PROMPT = "<|im_start|>assistant\n"; - - // Copy reasoning to the "thinking" field the template expects - auto adjusted_messages = json::array(); - for (auto msg : inputs.messages) { - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - msg["thinking"] = msg.at("reasoning_content"); - } - adjusted_messages.push_back(msg); - } - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END }; - if (tool_list_tokens) { - data.preserved_tokens.push_back(TOOL_LIST_START); - data.preserved_tokens.push_back(TOOL_LIST_END); - } - - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END}; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - // Gate by reasoning format and whether the template supports - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE && - tmpl.source().find(THINK_START) != std::string::npos; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); - - auto reasoning = p.eps(); - if (extract_reasoning) { - reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); - } - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - if (has_response_format) { - auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); - return generation_prompt + reasoning + response_format + end; - } - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - auto tool_calls = p.rule("tool-calls", - p.trigger_rule("tool-call", - p.literal(TOOL_CALL_START) + - p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) + - p.literal(TOOL_CALL_END) - ) - ); - - auto content = p.content(p.until(TOOL_CALL_START)); - - return generation_prompt + reasoning + content + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START } - }; - } - - return data; -} - -static common_chat_params common_chat_params_init_gigachat_v3( - const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = false; - data.preserved_tokens = { - "<|message_sep|>\n\n", - "<|role_sep|>\n", - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content(); - data.prompt += data.generation_prompt; - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n"; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto ret = p.eps(); - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Build a choice of all available tools - auto tool_choice = p.choice(); - for (const auto & tool : inputs.tools) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\""); - auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); - - auto tool_open = p.tool_open(p.literal("{") << tool_name); - - tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}"); - } - - // Define the tool call structure - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = 1; // parallel toolcalls are not supported - auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice); - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls)); - - ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls; - } else { - // Content only parser - include_grammar = false; - ret = p.content(p.rest()); - } - - return p.literal("assistant<|role_sep|>\n") + ret; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix} - }; - } - return data; -} - -// The DeepSeek V4 reference implementation renders consecutive tool results into a single -// user block, ordered by the tool call order of the preceding assistant message (matched -// by tool call id) rather than by the order they appear in the conversation. -static json deepseek_v4_sort_tool_results(const json & messages) { - json adjusted = messages; - std::map call_order; - - for (size_t i = 0; i < adjusted.size();) { - const auto & msg = adjusted[i]; - const auto role = msg.value("role", ""); - - if (role == "assistant" && msg.contains("tool_calls") && - msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { - call_order.clear(); - const auto & tool_calls = msg.at("tool_calls"); - for (size_t idx = 0; idx < tool_calls.size(); idx++) { - auto id = tool_calls[idx].value("id", ""); - if (!id.empty()) { - call_order[id] = idx; - } - } - i++; - continue; - } - - if (role != "user" && role != "tool") { - i++; - continue; - } - - // collect a maximal run of user/tool messages - they render into one user block - std::vector tool_positions; - size_t run_end = i; - for (; run_end < adjusted.size(); run_end++) { - const auto r = adjusted[run_end].value("role", ""); - if (r == "tool") { - tool_positions.push_back(run_end); - } else if (r != "user") { - break; - } - } - - if (tool_positions.size() > 1 && !call_order.empty()) { - std::vector results; - results.reserve(tool_positions.size()); - for (auto pos : tool_positions) { - results.push_back(adjusted[pos]); - } - std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) { - const auto order = [&](const json & m) { - auto it = call_order.find(m.value("tool_call_id", "")); - return it == call_order.end() ? (size_t) 0 : it->second; - }; - return order(a) < order(b); - }); - for (size_t k = 0; k < tool_positions.size(); k++) { - adjusted[tool_positions[k]] = std::move(results[k]); - } - } - - i = run_end; - } - - return adjusted; -} - -static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls" - // instead of "function_calls", renders tool results in tool call order and its - // non-thinking generation prompt ends with a bare instead of an empty - // pair. - const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos; - - std::optional adjusted_messages; - if (is_v4) { - adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - std::optional additional_context; - if (is_v4 && has_response_format) { - additional_context = json{ { "response_format", inputs.json_schema } }; - } - - const std::string DSML = "｜DSML｜"; - const std::string THINK_START = ""; - const std::string THINK_END = ""; - const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls"; - const std::string FC_START = "<" + DSML + TC_BLOCK + ">"; - const std::string FC_END = ""; - const std::string INVOKE_START = "<" + DSML + "invoke"; - const std::string INVOKE_END = ""; - const std::string PARAM_START = "<" + DSML + "parameter"; - const std::string PARAM_END = ""; - const std::string GEN_PROMPT = "<｜Assistant｜>"; - const std::string TC_SEPARATOR = "\n\n"; - - data.prompt = common_chat_template_direct_apply_impl( - tmpl, inputs, adjusted_messages, std::nullopt, additional_context); - data.generation_prompt = common_chat_template_generation_prompt_impl( - tmpl, inputs, adjusted_messages, std::nullopt, additional_context); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END, FC_START}; - data.preserved_tokens = { - DSML, - THINK_START, - THINK_END, - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - if (is_v4 && msg.reasoning_content.empty()) { - data.generation_prompt = GEN_PROMPT + THINK_END; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += msg.render_content(); - } - } else { - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - } - - data.prompt += data.generation_prompt; - } - - bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); - - // build tool call section first since we might need it in reasoning - auto tool_choice = p.choice(); - if (has_tool_calls) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - - std::set required; - if (params.contains("required")) { - required = params.at("required").get>(); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - std::vector required_parsers; - std::vector optional_parsers; - for (const auto & [param_name, param_schema] : props.items()) { - bool is_required = required.find(param_name) != required.end(); - bool is_string = schema_info.resolves_to_string(param_schema); - - auto arg = p.tool_arg( - p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + - p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + - (is_string ? - p.tool_arg_string_value(p.until(PARAM_END)) : - p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", - param_schema, false))) + - p.tool_arg_close(p.literal(PARAM_END))); - - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { - required_parsers.push_back(named_arg); - } else { - optional_parsers.push_back(named_arg); - } - } - - common_peg_parser args_seq = p.eps(); - for (size_t i = 0; i < required_parsers.size(); i++) { - if (i > 0) { - args_seq = args_seq + p.space(); - } - args_seq = args_seq + required_parsers[i]; - } - - if (!optional_parsers.empty()) { - common_peg_parser any_opt = p.choice(); - for (const auto & opt : optional_parsers) { - any_opt |= opt; - } - args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); - } - - common_peg_parser invoke_body = args_seq; - auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + - p.tool_name(p.literal(name)) + p.literal("\">\n")) + - invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); - - tool_choice |= p.rule("tool-" + name, func_parser); - }); - } - - common_peg_parser tool_calls = p.eps(); - if (inputs.parallel_tool_calls) { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + - p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); - } else { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); - } - - auto reasoning = p.eps(); - auto reasoning_with_tc = p.eps(); - auto obligatory_tool_calls = tool_calls; - bool allow_reasoning_with_tc = false; - - if (!require_tools) { - tool_calls = p.optional(tool_calls); - } - - if (extract_reasoning && inputs.enable_thinking) { - reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); - reasoning_with_tc = THINK_START + - p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + - p.space() + obligatory_tool_calls; - allow_reasoning_with_tc = true; - } else if (extract_reasoning) { - // Thinking disabled but reasoning extraction requested: the generation prompt - // contains an empty pair (V3.2) or a bare (V4) that - // must still be consumed. - reasoning = is_v4 - ? p.optional(p.literal(THINK_END)) - : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); - } - - if (has_response_format) { - auto response_format = p.rule("response-format", - p.literal("```json") + p.space() + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.space() + p.literal("```")); - return generation_prompt + reasoning + response_format + end; - } - - if (!has_tool_calls) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - - auto content_before_tools = p.negate(p.literal(THINK_START)) + - p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + - p.space(); - return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : - generation_prompt + reasoning + content_before_tools + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && !require_tools; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, - }; - } - - return data; -} - -// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros: -// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|> -// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|> -// the generation prompt already opens the think (or response) section, so the -// section opener is optional here - same as Kimi K2 Thinking -static common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - const std::string SEP = "<|sep|>"; - const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>"; - const std::string THINK_START = "<|open|>think<|sep|>"; - const std::string THINK_END = "<|close|>think<|sep|>"; - const std::string RESP_START = "<|open|>response<|sep|>"; - const std::string RESP_END = "<|close|>response<|sep|>"; - const std::string TOOLS_START = "<|open|>tools<|sep|>"; - const std::string TOOLS_END = "<|close|>tools<|sep|>"; - const std::string CALL_START = "<|open|>call tool=\""; - const std::string CALL_END = "<|close|>call<|sep|>"; - const std::string ARG_START = "<|open|>argument key=\""; - const std::string ARG_END = "<|close|>argument<|sep|>"; - const std::string MSG_END = "<|close|>message<|sep|>"; - const std::string EOM_TOKEN = "<|end_of_msg|>"; - - // only the markers are special tokens. tag names ("think", "response", ...) are - // normal tokens and must not be preserved, or prose with those words is broken - data.preserved_tokens = { - "<|open|>", - "<|close|>", - "<|sep|>", - "<|end_of_msg|>", - }; - - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = { THINK_END }; - - // per-role message-start delimiters. user/assistant messages only have the role - // attribute, so the full opener is used. system and tool messages have more - // attributes, so those delimiters stop after the closing quote of the role - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" }, - { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" }, - { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + RESP_START + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto end = p.end(); - - auto start = p.optional(p.literal(MSG_START)); - - // the think section is always consumed, even with reasoning extraction off: - // the generation prompt ends with open_tag('think'), so it is always present. - // reasoning stops at its own closer, or at the response opener if the model - // skips the closer - auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) : - p.content(p.until_one_of({ THINK_END, RESP_START })); - - auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body + - p.optional(p.literal(THINK_END))); - - // content runs to the response closer, or to the next section if truncated - auto response = p.optional(p.literal(RESP_START)) + - p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) + - p.optional(p.literal(RESP_END)); - - // the EOG token after the message closer reaches the parser as text, - // so it must be consumed or the parse stays incomplete - auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN)); - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return start + reasoning + response + trailer + end; - } - - auto tool_choices = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const json schema = function.contains("parameters") ? function.at("parameters") : json::object(); - - // arguments come one tag per key, with the JSON type in a type="..." - // attribute. the type is taken from the tool schema instead, as it tells - // us if the value is JSON or a literal string - auto args = p.eps(); - if (schema.contains("properties") && !schema.at("properties").empty()) { - auto arg_choices = p.choice(); - for (const auto & prop : schema.at("properties").items()) { - const std::string & key = prop.key(); - - std::string type = "string"; - if (prop.value().is_object() && prop.value().contains("type") && - prop.value().at("type").is_string()) { - type = prop.value().at("type").get(); - } - - auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) : - p.tool_arg_value(p.until(ARG_END)); - - // skip the trailing type="..." attribute: anything up to <|sep|> - arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key, - p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) + - p.tool_arg_name(p.literal(key)) + p.literal("\"") + - p.until(SEP) + p.literal(SEP) + value + - p.tool_arg_close(p.literal(ARG_END)))); - } - args = p.zero_or_more(arg_choices); - } - - // skip the trailing index="N" attribute the same way - auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") + - p.until(SEP) + p.literal(SEP)) + - p.tool_args(args) + p.tool_close(p.literal(CALL_END))); - - tool_choices |= p.rule("kimi-k3-tool-" + name, call); - }); - - // all calls go inside one tools section, then the message is closed. the - // message closer is part of the trigger rule, or else the lazy grammar - // rejects it once tool calls have started - auto tools_section = - p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) + - p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) + - p.optional(p.literal(EOM_TOKEN))); - - auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section : - p.optional(tools_section); - - return start + reasoning + response + tools + trailer + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - if (function.contains("parameters")) { - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - } - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START }, - }; - } - - return data; -} - -// Cohere2 MoE (a.k.a. "North Code") parser. -// -// The assistant turn is fully marker-wrapped: -// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> -// <|START_THINKING|>{reasoning}<|END_THINKING|> -// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|> -// OR tool calls: <|START_ACTION|>[ -// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... -// ]<|END_ACTION|> -// <|END_OF_TURN_TOKEN|> -// -// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is -// the template default), so the model's output continues from *inside* the thinking block. The -// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix -// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself, -// regardless of whether they came from the generation prompt or the generated text. -static common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - const std::string TURN_START = "<|START_OF_TURN_TOKEN|>"; - const std::string TURN_END = "<|END_OF_TURN_TOKEN|>"; - const std::string CHATBOT = "<|CHATBOT_TOKEN|>"; - const std::string USER = "<|USER_TOKEN|>"; - const std::string SYSTEM = "<|SYSTEM_TOKEN|>"; - const std::string THINK_START = "<|START_THINKING|>"; - const std::string THINK_END = "<|END_THINKING|>"; - const std::string TEXT_START = "<|START_TEXT|>"; - const std::string TEXT_END = "<|END_TEXT|>"; - const std::string ACTION_START = "<|START_ACTION|>"; - const std::string ACTION_END = "<|END_ACTION|>"; - const std::string RESULT_START = "<|START_TOOL_RESULT|>"; - const std::string RESULT_END = "<|END_TOOL_RESULT|>"; - - // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker. - const std::string GEN_PREFIX = TURN_START + CHATBOT; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END}; - data.preserved_tokens = { - TURN_START, TURN_END, CHATBOT, USER, SYSTEM, - THINK_START, THINK_END, - TEXT_START, TEXT_END, - ACTION_START, ACTION_END, - RESULT_START, RESULT_END, - }; - - // Declare per-role message delimiters. Tool results are rendered with the - // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before - // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, - { COMMON_CHAT_ROLE_USER, TURN_START + USER }, - { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, - { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + TEXT_START + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PREFIX); - auto end = p.end(); - - // The thinking block is always present (the generation prompt forces <|START_THINKING|>). - // When extracting reasoning, capture its body; otherwise keep the whole block (markers - // included) inline as content, matching reasoning_format=NONE conventions. - common_peg_parser reasoning = p.eps(); - if (extract_reasoning) { - reasoning = p.optional(p.literal(THINK_START) + - p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) + - p.optional(p.literal(THINK_END))); - } else { - reasoning = p.optional(p.content(p.literal(THINK_START) + - p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) + - p.optional(p.literal(THINK_END)))); - } - - auto text_content = has_response_format - ? p.literal(TEXT_START) + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.optional(p.literal(TEXT_END)) - : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END)); - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end; - } - - auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - - // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|> - auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls, - /* force_tool_calls = */ true, - /* name_key = */ "tool_name", - /* args_key = */ "parameters", - /* array_wrapped = */ true, - /* function_is_key = */ false, - /* call_id_key = */ "", - /* gen_call_id_key = */ "tool_call_id", - /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" }); - - // Content and tool calls are mutually exclusive in this format. - common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content }); - - return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START } - }; - } - - return data; -} - -static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3; - data.supports_thinking = true; - data.thinking_start_tag = ""; - data.thinking_end_tags = {""}; - - // M3 prefixes every tool tag with the namespace token "]<]minimax[>["; - // params use the parameter name as the tag (...). - const std::string NS = "]<]minimax[>["; - const std::string THINK_START = ""; - const std::string THINK_END = ""; - const std::string FC_START = NS + ""; - const std::string FC_END = NS + ""; - const std::string INVOKE_END = NS + ""; - - data.preserved_tokens = { - NS, - "", - "", - THINK_START, - THINK_END, - }; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" }, - { COMMON_CHAT_ROLE_USER, "]~b]user" }, - { COMMON_CHAT_ROLE_TOOL, "]~b]tool" }, - { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" }, - { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - const std::string GEN_PROMPT = data.generation_prompt; - - using mm3 = common_chat_peg_minimax_m3_mapper; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START); - auto end = p.end(); - - auto reasoning = p.eps(); - if (extract_reasoning) { - auto block = inputs.enable_thinking - ? p.literal(THINK_START) + p.space() + - p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END) - : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END); - - // A turn without reasoning is prefixed with a bare , written either by the - // generation prompt (thinking_mode = "disabled") or by the model itself. - reasoning = p.optional(p.choice({ block, p.literal(THINK_END) })); - } - - if (has_response_format) { - auto response_format = p.rule("response-format", - p.literal("```json") + p.space() + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.space() + p.literal("```")); - return generation_prompt + reasoning + response_format + end; - } - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - - auto alternatives_of = [](const json & schema) -> std::optional { - for (const auto * keyword : { "oneOf", "anyOf" }) { - if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) { - return schema.at(keyword); - } - } - return std::nullopt; - }; - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - // The template expands argument values recursively in XML (see the to_xml() macro) - std::function value_of; - std::function members_of; - - auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) { - const std::string close = NS + ""; - return p.rule(rule_name, - p.tool_arg( - p.tool_arg_open( - p.literal(NS + "<") + - p.tool_arg_name(p.literal(tag)) + - p.literal(">")) + - value_of(schema, rule_name, close))); - }; - - value_of = [&](const json & schema, - const std::string & rule_name, - const std::string & close) -> common_peg_parser { - auto close_tag = p.tool_arg_close(p.literal(close)); - - // A string accepts anything, so a union with a string alternative is a string - if (schema_info.resolves_to_string(schema)) { - return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); - } - - if (auto alternatives = alternatives_of(schema)) { - std::vector choices; - - size_t index = 0; - for (const auto & alternative : *alternatives) { - const std::string alt_name = rule_name + "-" + std::to_string(index++); - - // There is a risk that this breaks streaming deltas, but that's a risk we - // assume to provide tool arg streaming. - choices.push_back(value_of(alternative, alt_name, close)); - } - - return p.choice(choices); - } - - const std::string type = schema.contains("type") && schema.at("type").is_string() - ? schema.at("type").get() - : ""; - - if (type == "object" && schema.contains("properties")) { - return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag; - } - - if (type == "array" && schema.contains("items")) { - const std::string item_close = NS + ""; - auto item = p.rule(rule_name + "-item", - p.tag(mm3::TOOL_ARG_ITEM, - p.literal(NS + "") + - value_of(schema.at("items"), rule_name + "-item", item_close))); - return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; - } - - return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag; - }; - - // Required properties in schema order, then any number of optional ones in any order. - members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser { - const auto & props = schema.at("properties"); - - std::set required; - if (schema.contains("required")) { - required = schema.at("required").get>(); - } - - std::vector required_elements; - std::vector optional_elements; - for (const auto & [key, key_schema] : props.items()) { - auto element = element_of(key, key_schema, rule_prefix + "-" + key); - if (required.find(key) != required.end()) { - required_elements.push_back(element); - } else { - optional_elements.push_back(element); - } - } - - common_peg_parser members = p.eps(); - for (size_t i = 0; i < required_elements.size(); i++) { - if (i > 0) { - members = members + p.space(); - } - members = members + required_elements[i]; - } - - if (!optional_elements.empty()) { - common_peg_parser any_optional = p.choice(); - for (const auto & element : optional_elements) { - any_optional |= element; - } - members = members + p.repeat(p.space() + any_optional, 0, -1); - } - - return members; - }; - - common_peg_parser invoke_body = - params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps(); - - auto func_parser = p.tool( - p.tool_open(p.literal(NS + "")) + - p.space() + invoke_body + p.space() + - p.tool_close(p.literal(INVOKE_END))); - - tool_choice |= p.rule("tool-" + name, func_parser); - }); - - auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - - common_peg_parser tool_calls = p.eps(); - if (inputs.parallel_tool_calls) { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + - p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); - } else { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); - } - - if (!require_tools) { - tool_calls = p.optional(tool_calls); - } - - auto content_before_tools = p.content(p.until(FC_START)); - return generation_prompt + reasoning + content_before_tools + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, - }; - } - - return data; -} - namespace workaround { static void map_developer_role_to_system(json & messages) { @@ -3012,150 +1059,6 @@ static void requires_non_null_content(json & messages) { } } -// Gemma4 uses a custom tool_responses field instead of role:tool messages. -// -// This will transform a sequence of messages: -// assistant(tool_call+) -> tool+ -> assistant(content) -// -// Into a single assistant message containing a tool_responses field: -// assistant(content + tool_call + tool_responses) -// -// This is necessary for the Gemma4 chat template to properly format the prompt. -// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 -struct gemma4_model_turn_builder { - json & messages; - size_t pos; - json tool_calls = json::array(); - json tool_responses = json::array(); - json content; - json reasoning_content; - - gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {} - - void collect() { - // Collect the first assistant message - auto & msg = messages[pos]; - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - // According to the prompt formatting guide, we need to preserve reasoning_content - // between function calls. The current chat templates do not support this, but we will do it anyway. - reasoning_content = msg.at("reasoning_content"); - } - for (auto & tc : msg.at("tool_calls")) { - tool_calls.push_back(tc); - } - pos++; - - // Collect tool call results - while (pos < messages.size() && messages[pos].value("role", "") == "tool") { - collect_result(messages[pos]); - pos++; - } - - // Check if the next assistant message is the final message - if (pos < messages.size() && messages[pos].value("role", "") == "assistant") { - auto & next = messages[pos]; - if (!has_tool_calls(next) && has_content(next)) { - content = next.at("content"); - pos++; - } - } - } - - void collect_result(const json & curr) { - json response; - if (curr.contains("content")) { - const auto & content = curr.at("content"); - if (content.is_string()) { - // Try to parse the content as JSON; fall back to raw string - try { - response = json::parse(content.get()); - } catch (...) { - response = content; - } - } else { - response = content; - } - } - - std::string name; - - // Match name with corresponding tool call - size_t idx = tool_responses.size(); - if (idx < tool_calls.size()) { - auto & tc = tool_calls[idx]; - if (tc.contains("function")) { - name = tc.at("function").value("name", ""); - } - } - - // Fallback to the tool call id - if (name.empty()) { - name = curr.value("tool_call_id", ""); - } - - tool_responses.push_back({{"name", name}, {"response", response}}); - } - - json build() { - collect(); - - json msg = { - {"role", "assistant"}, - {"tool_calls", tool_calls}, - }; - if (!tool_responses.empty()) { - msg["tool_responses"] = tool_responses; - } - if (!content.is_null()) { - msg["content"] = content; - } - if (!reasoning_content.is_null()) { - msg["reasoning_content"] = reasoning_content; - } - return msg; - } - - static bool has_content(const json & msg) { - if (!msg.contains("content") || msg.at("content").is_null()) { - return false; - } - const auto & content = msg.at("content"); - if (content.is_string() && !content.get().empty()) { - return true; - } - if (content.is_array() && !content.empty()) { - return true; - } - return false; - } - - static bool has_tool_calls(const json & msg) { - return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty(); - } -}; - -static void convert_tool_responses_gemma4(json & messages) { - json result = json::array(); - size_t i = 0; - - while (i < messages.size()) { - auto & msg = messages[i]; - - if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") || - !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) { - result.push_back(msg); - i++; - continue; - } - - gemma4_model_turn_builder builder(messages, i); - result.push_back(builder.build()); - i = builder.pos; - } - - messages = result; -} - static void func_args_not_string(json & messages) { GGML_ASSERT(messages.is_array()); for (auto & message : messages) { @@ -3195,296 +1098,6 @@ static void trim_all_content(std::vector & messages) { } -// MiniCPM5 format: -// - Reasoning: {reasoning} (optional) -// - Tool calls: value -static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { - "", - "", - "", - "", - }; - - data.thinking_start_tag = ""; - data.thinking_end_tags = {""}; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|im_start|>assistant\n\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n\n\n" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal("<|im_start|>assistant\n"); - - auto reasoning = p.eps(); - if (extract_reasoning) { - reasoning = ("" << p.reasoning(p.until("")) << "") + p.space(); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // CDATA lets a value carry characters that would otherwise close the tag (e.g. - // ); capture the inner text only, excluding the CDATA markers. - auto string_value = p.choice({ - p.literal("")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("")), - p.negate(p.literal("")) + p.tool_arg_close(p.literal("")), "") - }); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) - ) + p.tool_arg_close(p.literal("")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser - ); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal("")) - << p.tool_args(args) - << p.tool_close(p.literal(""))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); - - auto content = p.content(p.until("assistant to=<|message|>{content}{END}" where END is -// <|eom|> (more messages follow) or <|eot|> (end of turn): -// - chain-of-thought: to=self, terminated by <|eom|> -// - final answer: to=user, terminated by <|eot|> -// The generation prompt is just "<|start|>assistant"; the model emits its own -// " to=...<|message|>". -static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = "<|start|>assistant"; - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - data.preserved_tokens = { - "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", - // ATEM tool-call markup emitted on " to=" turns. - "", "", - "", "", - }; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, - { COMMON_CHAT_ROLE_USER, "<|start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, - { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" }, - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - // Constrained grammar whenever tools are offered. - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); - - if (!extract_reasoning && !include_grammar) { - return start + p.content(p.rest()); - } - - if (extract_reasoning) { - p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); - } else { - p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); - } - auto analysis = p.ref("analysis"); - - auto recipient = p.optional(p.literal(" to=user")); - auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + - p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto string_value = p.ac( - p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), - ""); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)) - + p.tool_arg_close(p.literal("")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal(" to=") + p.until("<|message|>") + - p.literal("<|message|>") + p.space() + - p.literal("") + p.space()) - << p.tool_args(args) - << p.tool_close(p.literal("") + p.space() + p.literal(""))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto tool_calls = inputs.parallel_tool_calls - ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) - : p.trigger_rule("tool-call", tool_choice); - - - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + analysis) + start + tool_calls; - } - auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); - return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); - } - - return p.zero_or_more(start + analysis) + start + final_msg; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, - "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, - }; - } - - return data; -} - static json common_chat_extra_context() { json ctx = json::object(); std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cpp new file mode 100644 index 000000000..46a2a01ba --- /dev/null +++ b/common/parsers/cohere2moe.cpp @@ -0,0 +1,150 @@ +#include "parsers.h" + +// Cohere2 MoE (a.k.a. "North Code") parser. +// +// The assistant turn is fully marker-wrapped: +// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> +// <|START_THINKING|>{reasoning}<|END_THINKING|> +// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|> +// OR tool calls: <|START_ACTION|>[ +// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... +// ]<|END_ACTION|> +// <|END_OF_TURN_TOKEN|> +// +// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is +// the template default), so the model's output continues from *inside* the thinking block. The +// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix +// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself, +// regardless of whether they came from the generation prompt or the generated text. +common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string TURN_START = "<|START_OF_TURN_TOKEN|>"; + const std::string TURN_END = "<|END_OF_TURN_TOKEN|>"; + const std::string CHATBOT = "<|CHATBOT_TOKEN|>"; + const std::string USER = "<|USER_TOKEN|>"; + const std::string SYSTEM = "<|SYSTEM_TOKEN|>"; + const std::string THINK_START = "<|START_THINKING|>"; + const std::string THINK_END = "<|END_THINKING|>"; + const std::string TEXT_START = "<|START_TEXT|>"; + const std::string TEXT_END = "<|END_TEXT|>"; + const std::string ACTION_START = "<|START_ACTION|>"; + const std::string ACTION_END = "<|END_ACTION|>"; + const std::string RESULT_START = "<|START_TOOL_RESULT|>"; + const std::string RESULT_END = "<|END_TOOL_RESULT|>"; + + // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker. + const std::string GEN_PREFIX = TURN_START + CHATBOT; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END}; + data.preserved_tokens = { + TURN_START, TURN_END, CHATBOT, USER, SYSTEM, + THINK_START, THINK_END, + TEXT_START, TEXT_END, + ACTION_START, ACTION_END, + RESULT_START, RESULT_END, + }; + + // Declare per-role message delimiters. Tool results are rendered with the + // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before + // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, + { COMMON_CHAT_ROLE_USER, TURN_START + USER }, + { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, + { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + TEXT_START + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + auto end = p.end(); + + // The thinking block is always present (the generation prompt forces <|START_THINKING|>). + // When extracting reasoning, capture its body; otherwise keep the whole block (markers + // included) inline as content, matching reasoning_format=NONE conventions. + common_peg_parser reasoning = p.eps(); + if (extract_reasoning) { + reasoning = p.optional(p.literal(THINK_START) + + p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) + + p.optional(p.literal(THINK_END))); + } else { + reasoning = p.optional(p.content(p.literal(THINK_START) + + p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) + + p.optional(p.literal(THINK_END)))); + } + + auto text_content = has_response_format + ? p.literal(TEXT_START) + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.optional(p.literal(TEXT_END)) + : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END)); + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end; + } + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|> + auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls, + /* force_tool_calls = */ true, + /* name_key = */ "tool_name", + /* args_key = */ "parameters", + /* array_wrapped = */ true, + /* function_is_key = */ false, + /* call_id_key = */ "", + /* gen_call_id_key = */ "tool_call_id", + /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" }); + + // Content and tool calls are mutually exclusive in this format. + common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content }); + + return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START } + }; + } + + return data; +} diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp new file mode 100644 index 000000000..5e2581727 --- /dev/null +++ b/common/parsers/deepseek.cpp @@ -0,0 +1,287 @@ +#include "parsers.h" + +// The DeepSeek V4 reference implementation renders consecutive tool results into a single +// user block, ordered by the tool call order of the preceding assistant message (matched +// by tool call id) rather than by the order they appear in the conversation. +static json deepseek_v4_sort_tool_results(const json & messages) { + json adjusted = messages; + std::map call_order; + + for (size_t i = 0; i < adjusted.size();) { + const auto & msg = adjusted[i]; + const auto role = msg.value("role", ""); + + if (role == "assistant" && msg.contains("tool_calls") && + msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { + call_order.clear(); + const auto & tool_calls = msg.at("tool_calls"); + for (size_t idx = 0; idx < tool_calls.size(); idx++) { + auto id = tool_calls[idx].value("id", ""); + if (!id.empty()) { + call_order[id] = idx; + } + } + i++; + continue; + } + + if (role != "user" && role != "tool") { + i++; + continue; + } + + // collect a maximal run of user/tool messages - they render into one user block + std::vector tool_positions; + size_t run_end = i; + for (; run_end < adjusted.size(); run_end++) { + const auto r = adjusted[run_end].value("role", ""); + if (r == "tool") { + tool_positions.push_back(run_end); + } else if (r != "user") { + break; + } + } + + if (tool_positions.size() > 1 && !call_order.empty()) { + std::vector results; + results.reserve(tool_positions.size()); + for (auto pos : tool_positions) { + results.push_back(adjusted[pos]); + } + std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) { + const auto order = [&](const json & m) { + auto it = call_order.find(m.value("tool_call_id", "")); + return it == call_order.end() ? (size_t) 0 : it->second; + }; + return order(a) < order(b); + }); + for (size_t k = 0; k < tool_positions.size(); k++) { + adjusted[tool_positions[k]] = std::move(results[k]); + } + } + + i = run_end; + } + + return adjusted; +} + +common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls" + // instead of "function_calls", renders tool results in tool call order and its + // non-thinking generation prompt ends with a bare instead of an empty + // pair. + const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos; + + std::optional adjusted_messages; + if (is_v4) { + adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + std::optional additional_context; + if (is_v4 && has_response_format) { + additional_context = json{ { "response_format", inputs.json_schema } }; + } + + const std::string DSML = "｜DSML｜"; + const std::string THINK_START = ""; + const std::string THINK_END = ""; + const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls"; + const std::string FC_START = "<" + DSML + TC_BLOCK + ">"; + const std::string FC_END = ""; + const std::string INVOKE_START = "<" + DSML + "invoke"; + const std::string INVOKE_END = ""; + const std::string PARAM_START = "<" + DSML + "parameter"; + const std::string PARAM_END = ""; + const std::string GEN_PROMPT = "<｜Assistant｜>"; + const std::string TC_SEPARATOR = "\n\n"; + + data.prompt = common_chat_template_direct_apply_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.generation_prompt = common_chat_template_generation_prompt_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END, FC_START}; + data.preserved_tokens = { + DSML, + THINK_START, + THINK_END, + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + if (is_v4 && msg.reasoning_content.empty()) { + data.generation_prompt = GEN_PROMPT + THINK_END; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + } else { + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + } + + data.prompt += data.generation_prompt; + } + + bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PROMPT); + auto end = p.end(); + + // build tool call section first since we might need it in reasoning + auto tool_choice = p.choice(); + if (has_tool_calls) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + const auto & props = params.contains("properties") ? params.at("properties") : json::object(); + + std::set required; + if (params.contains("required")) { + required = params.at("required").get>(); + } + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + std::vector required_parsers; + std::vector optional_parsers; + for (const auto & [param_name, param_schema] : props.items()) { + bool is_required = required.find(param_name) != required.end(); + bool is_string = schema_info.resolves_to_string(param_schema); + + auto arg = p.tool_arg( + p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + + p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + + (is_string ? + p.tool_arg_string_value(p.until(PARAM_END)) : + p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", + param_schema, false))) + + p.tool_arg_close(p.literal(PARAM_END))); + + auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); + if (is_required) { + required_parsers.push_back(named_arg); + } else { + optional_parsers.push_back(named_arg); + } + } + + common_peg_parser args_seq = p.eps(); + for (size_t i = 0; i < required_parsers.size(); i++) { + if (i > 0) { + args_seq = args_seq + p.space(); + } + args_seq = args_seq + required_parsers[i]; + } + + if (!optional_parsers.empty()) { + common_peg_parser any_opt = p.choice(); + for (const auto & opt : optional_parsers) { + any_opt |= opt; + } + args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); + } + + common_peg_parser invoke_body = args_seq; + auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + + p.tool_name(p.literal(name)) + p.literal("\">\n")) + + invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + } + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } + + auto reasoning = p.eps(); + auto reasoning_with_tc = p.eps(); + auto obligatory_tool_calls = tool_calls; + bool allow_reasoning_with_tc = false; + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + + if (extract_reasoning && inputs.enable_thinking) { + reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + reasoning_with_tc = THINK_START + + p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + + p.space() + obligatory_tool_calls; + allow_reasoning_with_tc = true; + } else if (extract_reasoning) { + // Thinking disabled but reasoning extraction requested: the generation prompt + // contains an empty pair (V3.2) or a bare (V4) that + // must still be consumed. + reasoning = is_v4 + ? p.optional(p.literal(THINK_END)) + : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); + } + + if (has_response_format) { + auto response_format = p.rule("response-format", + p.literal("```json") + p.space() + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.space() + p.literal("```")); + return generation_prompt + reasoning + response_format + end; + } + + if (!has_tool_calls) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + auto content_before_tools = p.negate(p.literal(THINK_START)) + + p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + + p.space(); + return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : + generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && !require_tools; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, + }; + } + + return data; +} diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cpp new file mode 100644 index 000000000..349b8065a --- /dev/null +++ b/common/parsers/functionary-v3-2.cpp @@ -0,0 +1,101 @@ +#include "parsers.h" + +// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content} +common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.preserved_tokens = { + ">>>all", + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content(); + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + // Functionary v3.2 format: + // - Normal content: >>>all\n{content} + // - Tool calls: >>>function_name\n{json_args} + // Generation prompt ends with ">>>" so model outputs recipient immediately + + // Build content parser for >>>all\n{content} + // When tools are present, content stops before the next ">>>" (tool call) + // When no tools, content goes until end + auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>")); + auto content_until_end = p.literal("all\n") + p.content(p.rest()); + auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>"); + + // If no tools or tool_choice is NONE, just parse content + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + // When no tools, just match the prefix and capture everything after + return generation_prompt + content_until_end + p.end(); + } + + // Build tool call parsers for each available function + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto & schema = function.at("parameters"); + + // Tool format: >>>function_name\n{json_args} + auto tool_parser = p.tool( + p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) + + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) + ); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto content_only = content_until_end; + auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice)); + auto content_and_tools = content_until_tool + tools_only; + + auto ret = p.eps(); + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + if (inputs.parallel_tool_calls) { + ret = p.choice({ content_and_tools, tools_only }) + p.end(); + } else { + ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end(); + } + } else if (inputs.parallel_tool_calls) { + ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end(); + } else { + auto content_and_tool = content_until_tool + tool_choice; + ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end(); + } + return generation_prompt + ret; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + // Grammar trigger for when the model starts outputting a tool call + // (after the initial ">>>" in the generation prompt but recipient other than "all") + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" } + }; + } + + return data; +} diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp new file mode 100644 index 000000000..041523acb --- /dev/null +++ b/common/parsers/gemma4.cpp @@ -0,0 +1,312 @@ +#include "parsers.h" + +namespace workaround { + +// Gemma4 uses a custom tool_responses field instead of role:tool messages. +// +// This will transform a sequence of messages: +// assistant(tool_call+) -> tool+ -> assistant(content) +// +// Into a single assistant message containing a tool_responses field: +// assistant(content + tool_call + tool_responses) +// +// This is necessary for the Gemma4 chat template to properly format the prompt. +// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 +struct gemma4_model_turn_builder { + json & messages; + size_t pos; + json tool_calls = json::array(); + json tool_responses = json::array(); + json content; + json reasoning_content; + + gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {} + + void collect() { + // Collect the first assistant message + auto & msg = messages[pos]; + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + // According to the prompt formatting guide, we need to preserve reasoning_content + // between function calls. The current chat templates do not support this, but we will do it anyway. + reasoning_content = msg.at("reasoning_content"); + } + for (auto & tc : msg.at("tool_calls")) { + tool_calls.push_back(tc); + } + pos++; + + // Collect tool call results + while (pos < messages.size() && messages[pos].value("role", "") == "tool") { + collect_result(messages[pos]); + pos++; + } + + // Check if the next assistant message is the final message + if (pos < messages.size() && messages[pos].value("role", "") == "assistant") { + auto & next = messages[pos]; + if (!has_tool_calls(next) && has_content(next)) { + content = next.at("content"); + pos++; + } + } + } + + void collect_result(const json & curr) { + json response; + if (curr.contains("content")) { + const auto & content = curr.at("content"); + if (content.is_string()) { + // Try to parse the content as JSON; fall back to raw string + try { + response = json::parse(content.get()); + } catch (...) { + response = content; + } + } else { + response = content; + } + } + + std::string name; + + // Match name with corresponding tool call + size_t idx = tool_responses.size(); + if (idx < tool_calls.size()) { + auto & tc = tool_calls[idx]; + if (tc.contains("function")) { + name = tc.at("function").value("name", ""); + } + } + + // Fallback to the tool call id + if (name.empty()) { + name = curr.value("tool_call_id", ""); + } + + tool_responses.push_back({{"name", name}, {"response", response}}); + } + + json build() { + collect(); + + json msg = { + {"role", "assistant"}, + {"tool_calls", tool_calls}, + }; + if (!tool_responses.empty()) { + msg["tool_responses"] = tool_responses; + } + if (!content.is_null()) { + msg["content"] = content; + } + if (!reasoning_content.is_null()) { + msg["reasoning_content"] = reasoning_content; + } + return msg; + } + + static bool has_content(const json & msg) { + if (!msg.contains("content") || msg.at("content").is_null()) { + return false; + } + const auto & content = msg.at("content"); + if (content.is_string() && !content.get().empty()) { + return true; + } + if (content.is_array() && !content.empty()) { + return true; + } + return false; + } + + static bool has_tool_calls(const json & msg) { + return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty(); + } +}; + +void convert_tool_responses_gemma4(json & messages) { + json result = json::array(); + size_t i = 0; + + while (i < messages.size()) { + auto & msg = messages[i]; + + if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") || + !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) { + result.push_back(msg); + i++; + continue; + } + + gemma4_model_turn_builder builder(messages, i); + result.push_back(builder.build()); + i = builder.pos; + } + + messages = result; +} + +} + +common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + + if (inputs.add_generation_prompt && string_ends_with(data.prompt, "\n")) { + // This may happen if the model generates content + tool_call, the + // template does not add the model's next turn and confuses the model + // from emitting its proper reasoning token sequence. + data.generation_prompt = "<|turn>model\n"; + data.prompt += data.generation_prompt; + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_USER, "<|turn>user" }, + { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, + }; + + data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; + data.supports_thinking = true; + data.thinking_start_tag = "<|channel>thought"; + data.thinking_end_tags = {""}; + + data.preserved_tokens = { + "<|channel>", + "", + "<|tool_call>", + "", + "<|turn>", + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = string_ends_with(data.prompt, "\n") ? "<|turn>model\n" : ""; + data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.optional(p.literal("<|turn>model\n"))); + + if (extract_reasoning) { + p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("")) + p.literal("")); + } else { + p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("") + p.literal(""))); + } + + auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), ""); + auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>")); + + if (has_response_format) { + auto response_format = p.literal("```json") << + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) << + p.literal("```"); + return start + p.optional(thought) + response_format; + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // Gemma4 tool calling syntax + // Rules should match traversal logic in gemma4_to_json() + p.rule("gemma4-string-content", p.until("<|\"|>")); + p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>")); + p.rule("gemma4-bool", p.json_bool()); + p.rule("gemma4-null", p.json_null()); + p.rule("gemma4-number", p.json_number()); + p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":")); + p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")); + p.rule("gemma4-dict", [&]() { + auto ws = p.space(); + auto member = p.ref("gemma4-dict-kv"); + auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); + return p.sequence({ + p.literal("{"), ws, + p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) + }); + }); + p.rule("gemma4-array", [&]() { + auto ws = p.space(); + auto value = p.ref("gemma4-value"); + auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); + return p.sequence({ + p.literal("["), ws, + p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) + }); + }); + p.rule("gemma4-value", [&]() { + return p.choice({ + p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), + p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") + }); + }); + + auto tool_choice = p.choice(); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules + // const auto & params = function.at("parameters"); + + tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({ + p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))), + p.tool_args(p.ref("gemma4-dict")), + }))); + }); + + auto tool_call = p.trigger_rule("tool-call", p.repeat( + "<|tool_call>call:" + tool_choice + "", + /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0, + /* max = */ inputs.parallel_tool_calls ? -1 : 1 + )); + + auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>")); + auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "", "<|tool_call>"}))); + auto message = p.rule("message", thought + content); + return start + p.zero_or_more(message) + scan_to_toolcall + tool_call; + } + + // Gemma 4 may emit an extra <|channel>thought\n at the end of the content. It may + // also emit a single trailing token. Consume all complete reasoning blocks and + // then stop at the first unmatched token. + auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", ""}))); + auto message = p.rule("message", thought + content); + return start + p.one_or_more(message); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" }, + }; + } + + return data; +} diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cpp new file mode 100644 index 000000000..41da5554a --- /dev/null +++ b/common/parsers/gigachat-v3.cpp @@ -0,0 +1,81 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_gigachat_v3( + const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = false; + data.preserved_tokens = { + "<|message_sep|>\n\n", + "<|role_sep|>\n", + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content(); + data.prompt += data.generation_prompt; + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n"; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto ret = p.eps(); + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // Build a choice of all available tools + auto tool_choice = p.choice(); + for (const auto & tool : inputs.tools) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto & schema = function.at("parameters"); + + auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\""); + auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); + + auto tool_open = p.tool_open(p.literal("{") << tool_name); + + tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}"); + } + + // Define the tool call structure + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = 1; // parallel toolcalls are not supported + auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice); + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls)); + + ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls; + } else { + // Content only parser + include_grammar = false; + ret = p.content(p.rest()); + } + + return p.literal("assistant<|role_sep|>\n") + ret; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix} + }; + } + return data; +} diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp new file mode 100644 index 000000000..d7dbfbfb5 --- /dev/null +++ b/common/parsers/gpt-oss.cpp @@ -0,0 +1,167 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + // Copy reasoning to the "thinking" field as expected by the gpt-oss template + auto adjusted_messages = json::array(); + for (auto msg : inputs.messages) { + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + msg["thinking"] = msg.at("reasoning_content"); + if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { + msg.erase("content"); + } + } + adjusted_messages.push_back(msg); + } + + auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); + + // Check if we need to replace the return token with end token during + // inference and without generation prompt. For more details see: + // https://github.com/ggml-org/llama.cpp/issues/15417 + if (inputs.is_inference && !inputs.add_generation_prompt) { + static constexpr std::string_view return_token = "<|return|>"; + static constexpr std::string_view end_token = "<|end|>"; + if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) { + prompt.replace(pos, return_token.length(), end_token); + } + } + + data.prompt = prompt; + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, + }; + + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + data.thinking_start_tag = "<|channel|>analysis<|message|>"; + data.thinking_end_tags = {"<|end|>"}; + + // These special tokens are required to parse properly, so we include them + // even if parse_tool_calls is false. + data.preserved_tokens = { + "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", + }; + + // Adjust prompt for continuation + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.literal("<|start|>assistant")); + auto end = p.rule("end", p.literal("<|end|>")); + auto content = p.rule("message-content", p.until("<|end|>")); + auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); + auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); + + // Occasionally, gpt-oss-20b will prefix channels with this commentary + auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); + auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>"); + + if (extract_reasoning) { + p.rule("analysis", start_analysis + p.reasoning(content) + end); + } else { + p.rule("analysis", p.content(start_analysis + content + end)); + } + + auto analysis = p.ref("analysis"); + auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); + auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content)); + + // Consume any unsolicited tool calls, e.g. builtin functions + auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); + + auto any = p.rule("any", preamble | analysis); + + if (has_response_format) { + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); + auto response_format = p.rule("response-format", + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); + + return p.zero_or_more(start + analysis) + start + response_format; + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto tool_choice = p.choice(); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto & params = function.at("parameters"); + + auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); + auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params)); + + // recipient in role header + // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS + auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args); + + // recipient in channel header + // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS + auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args); + + tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel); + }); + + auto tool_call = p.trigger_rule("tool-call", tool_choice); + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return p.zero_or_more(start + any) + start + tool_call; + } + + return p.zero_or_more(start + any) + start + (tool_call | final_msg); + } + + return p.zero_or_more(start + any) + start + (final_msg | unsolicited); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" } + }; + } + + return data; +} diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cpp new file mode 100644 index 000000000..57f6bfdcb --- /dev/null +++ b/common/parsers/kimi-k2.cpp @@ -0,0 +1,133 @@ +#include "parsers.h" + +// Kimi K2 Thinking - uses unique tool call ID format: functions.: +// The ID contains both the function name and an incrementing counter +common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { + "<|tool_calls_section_begin|>", + "<|tool_calls_section_end|>", + "<|tool_call_begin|>", + "<|tool_call_argument_begin|>", + "<|tool_call_end|>", + "", + "", + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>"; + const std::string SECTION_END = "<|tool_calls_section_end|>"; + const std::string CALL_BEGIN = "<|tool_call_begin|>"; + const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>"; + const std::string CALL_END = "<|tool_call_end|>"; + + const std::string THINK_START = ""; + const std::string THINK_END = ""; + const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>"; + + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END}; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + // Kimi K2 Thinking format: + // - Reasoning: {reasoning} + // - Content: text after reasoning + // - Tool calls section: + // <|tool_calls_section_begin|> + // <|tool_call_begin|>functions.:<|tool_call_argument_begin|>{json_args}<|tool_call_end|> + // ... + // <|tool_calls_section_end|> + // The ID format is: functions.: where counter is 0, 1, 2, ... + + // Tool call markers + auto end = p.end(); + + // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny. + // For example, it can call tools at the end of reasoning without closing reasoning... + auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning( + p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) + + p.optional(p.literal(THINK_END))) : p.eps(); + auto generation_prompt = p.literal(GEN_PROMPT); + + + // Content only parser (no tools) + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + // Build tool call parsers for each available function + // The ID format is: functions.: + // We need to match: functions.: + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto & schema = function.at("parameters"); + + // Match: functions.: + // Capture the full call id (functions.:) using tool_id tag + auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1)); + auto tool_parser = p.tool( + p.tool_open(tool_id + p.literal(ARGS_BEGIN)) + + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) + + p.tool_close(p.optional((p.literal(CALL_END)))) + ); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|> + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + // Use trigger_rule so grammar generator knows where to start generating rules + auto tool_calls = p.rule("tool-calls", + p.optional(p.literal(SECTION_BEGIN)) + + p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) + + p.optional(p.literal(SECTION_END))) + ); + + auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN })); + + return generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" } + }; + } + + return data; +} diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cpp new file mode 100644 index 000000000..56a49903f --- /dev/null +++ b/common/parsers/kimi-k3.cpp @@ -0,0 +1,174 @@ +#include "parsers.h" + +// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros: +// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|> +// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|> +// the generation prompt already opens the think (or response) section, so the +// section opener is optional here - same as Kimi K2 Thinking +common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + const std::string SEP = "<|sep|>"; + const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>"; + const std::string THINK_START = "<|open|>think<|sep|>"; + const std::string THINK_END = "<|close|>think<|sep|>"; + const std::string RESP_START = "<|open|>response<|sep|>"; + const std::string RESP_END = "<|close|>response<|sep|>"; + const std::string TOOLS_START = "<|open|>tools<|sep|>"; + const std::string TOOLS_END = "<|close|>tools<|sep|>"; + const std::string CALL_START = "<|open|>call tool=\""; + const std::string CALL_END = "<|close|>call<|sep|>"; + const std::string ARG_START = "<|open|>argument key=\""; + const std::string ARG_END = "<|close|>argument<|sep|>"; + const std::string MSG_END = "<|close|>message<|sep|>"; + const std::string EOM_TOKEN = "<|end_of_msg|>"; + + // only the markers are special tokens. tag names ("think", "response", ...) are + // normal tokens and must not be preserved, or prose with those words is broken + data.preserved_tokens = { + "<|open|>", + "<|close|>", + "<|sep|>", + "<|end_of_msg|>", + }; + + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = { THINK_END }; + + // per-role message-start delimiters. user/assistant messages only have the role + // attribute, so the full opener is used. system and tool messages have more + // attributes, so those delimiters stop after the closing quote of the role + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" }, + { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" }, + { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + RESP_START + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto end = p.end(); + + auto start = p.optional(p.literal(MSG_START)); + + // the think section is always consumed, even with reasoning extraction off: + // the generation prompt ends with open_tag('think'), so it is always present. + // reasoning stops at its own closer, or at the response opener if the model + // skips the closer + auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) : + p.content(p.until_one_of({ THINK_END, RESP_START })); + + auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body + + p.optional(p.literal(THINK_END))); + + // content runs to the response closer, or to the next section if truncated + auto response = p.optional(p.literal(RESP_START)) + + p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) + + p.optional(p.literal(RESP_END)); + + // the EOG token after the message closer reaches the parser as text, + // so it must be consumed or the parse stays incomplete + auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN)); + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return start + reasoning + response + trailer + end; + } + + auto tool_choices = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const json schema = function.contains("parameters") ? function.at("parameters") : json::object(); + + // arguments come one tag per key, with the JSON type in a type="..." + // attribute. the type is taken from the tool schema instead, as it tells + // us if the value is JSON or a literal string + auto args = p.eps(); + if (schema.contains("properties") && !schema.at("properties").empty()) { + auto arg_choices = p.choice(); + for (const auto & prop : schema.at("properties").items()) { + const std::string & key = prop.key(); + + std::string type = "string"; + if (prop.value().is_object() && prop.value().contains("type") && + prop.value().at("type").is_string()) { + type = prop.value().at("type").get(); + } + + auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) : + p.tool_arg_value(p.until(ARG_END)); + + // skip the trailing type="..." attribute: anything up to <|sep|> + arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key, + p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) + + p.tool_arg_name(p.literal(key)) + p.literal("\"") + + p.until(SEP) + p.literal(SEP) + value + + p.tool_arg_close(p.literal(ARG_END)))); + } + args = p.zero_or_more(arg_choices); + } + + // skip the trailing index="N" attribute the same way + auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") + + p.until(SEP) + p.literal(SEP)) + + p.tool_args(args) + p.tool_close(p.literal(CALL_END))); + + tool_choices |= p.rule("kimi-k3-tool-" + name, call); + }); + + // all calls go inside one tools section, then the message is closed. the + // message closer is part of the trigger rule, or else the lazy grammar + // rejects it once tool calls have started + auto tools_section = + p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) + + p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) + + p.optional(p.literal(EOM_TOKEN))); + + auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section : + p.optional(tools_section); + + return start + reasoning + response + tools + trailer + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + if (function.contains("parameters")) { + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + } + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START }, + }; + } + + return data; +} diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cpp new file mode 100644 index 000000000..4514f908b --- /dev/null +++ b/common/parsers/lfm2.cpp @@ -0,0 +1,119 @@ +#include "parsers.h" + +// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list +// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call +bool is_lfm2_template(const std::string & src) { + return src.find("<|tool_list_start|>") != std::string::npos && + src.find("<|tool_list_end|>") != std::string::npos; +} + +// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable +// (except dotted names and JSON literals true/false/null). +// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional reasoning. +// tool_list_tokens preserves LFM2 system tool-list markers. +common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs, + bool tool_list_tokens) { + common_chat_params data; + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + const std::string TOOL_LIST_START = "<|tool_list_start|>"; + const std::string TOOL_LIST_END = "<|tool_list_end|>"; + const std::string THINK_START = ""; + const std::string THINK_END = ""; + const std::string GEN_PROMPT = "<|im_start|>assistant\n"; + + // Copy reasoning to the "thinking" field the template expects + auto adjusted_messages = json::array(); + for (auto msg : inputs.messages) { + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + msg["thinking"] = msg.at("reasoning_content"); + } + adjusted_messages.push_back(msg); + } + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END }; + if (tool_list_tokens) { + data.preserved_tokens.push_back(TOOL_LIST_START); + data.preserved_tokens.push_back(TOOL_LIST_END); + } + + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END}; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + // Gate by reasoning format and whether the template supports + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE && + tmpl.source().find(THINK_START) != std::string::npos; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PROMPT); + auto end = p.end(); + + auto reasoning = p.eps(); + if (extract_reasoning) { + reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + } + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + if (has_response_format) { + auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); + return generation_prompt + reasoning + response_format + end; + } + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + auto tool_calls = p.rule("tool-calls", + p.trigger_rule("tool-call", + p.literal(TOOL_CALL_START) + + p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) + + p.literal(TOOL_CALL_END) + ) + ); + + auto content = p.content(p.until(TOOL_CALL_START)); + + return generation_prompt + reasoning + content + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START } + }; + } + + return data; +} diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp new file mode 100644 index 000000000..e6e0abf06 --- /dev/null +++ b/common/parsers/minicpm5.cpp @@ -0,0 +1,144 @@ +#include "parsers.h" + +// MiniCPM5 format: +// - Reasoning: {reasoning} (optional) +// - Tool calls: value +common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { + "", + "", + "", + "", + }; + + data.thinking_start_tag = ""; + data.thinking_end_tags = {""}; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|im_start|>assistant\n\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal("<|im_start|>assistant\n"); + + auto reasoning = p.eps(); + if (extract_reasoning) { + reasoning = ("" << p.reasoning(p.until("")) << "") + p.space(); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // CDATA lets a value carry characters that would otherwise close the tag (e.g. + // ); capture the inner text only, excluding the CDATA markers. + auto string_value = p.choice({ + p.literal("")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("")), + p.negate(p.literal("")) + p.tool_arg_close(p.literal("")), "") + }); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto args = p.eps(); + if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + auto arg_choice = p.choice(); + for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { + auto value_parser = p.eps(); + if (schema_info.resolves_to_string(prop_schema)) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) + ) + p.tool_arg_close(p.literal("")); + } + + auto arg_rule = p.tool_arg( + p.tool_arg_open(p.literal("")) + + value_parser + ); + + arg_choice |= arg_rule; + } + args = p.zero_or_more(arg_choice + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal("")) + << p.tool_args(args) + << p.tool_close(p.literal(""))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); + + auto content = p.content(p.until(""}; + + // M3 prefixes every tool tag with the namespace token "]<]minimax[>["; + // params use the parameter name as the tag (...). + const std::string NS = "]<]minimax[>["; + const std::string THINK_START = ""; + const std::string THINK_END = ""; + const std::string FC_START = NS + ""; + const std::string FC_END = NS + ""; + const std::string INVOKE_END = NS + ""; + + data.preserved_tokens = { + NS, + "", + "", + THINK_START, + THINK_END, + }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" }, + { COMMON_CHAT_ROLE_USER, "]~b]user" }, + { COMMON_CHAT_ROLE_TOOL, "]~b]tool" }, + { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + const std::string GEN_PROMPT = data.generation_prompt; + + using mm3 = common_chat_peg_minimax_m3_mapper; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START); + auto end = p.end(); + + auto reasoning = p.eps(); + if (extract_reasoning) { + auto block = inputs.enable_thinking + ? p.literal(THINK_START) + p.space() + + p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END) + : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END); + + // A turn without reasoning is prefixed with a bare , written either by the + // generation prompt (thinking_mode = "disabled") or by the model itself. + reasoning = p.optional(p.choice({ block, p.literal(THINK_END) })); + } + + if (has_response_format) { + auto response_format = p.rule("response-format", + p.literal("```json") + p.space() + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.space() + p.literal("```")); + return generation_prompt + reasoning + response_format + end; + } + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + auto alternatives_of = [](const json & schema) -> std::optional { + for (const auto * keyword : { "oneOf", "anyOf" }) { + if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) { + return schema.at(keyword); + } + } + return std::nullopt; + }; + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + // The template expands argument values recursively in XML (see the to_xml() macro) + std::function value_of; + std::function members_of; + + auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) { + const std::string close = NS + ""; + return p.rule(rule_name, + p.tool_arg( + p.tool_arg_open( + p.literal(NS + "<") + + p.tool_arg_name(p.literal(tag)) + + p.literal(">")) + + value_of(schema, rule_name, close))); + }; + + value_of = [&](const json & schema, + const std::string & rule_name, + const std::string & close) -> common_peg_parser { + auto close_tag = p.tool_arg_close(p.literal(close)); + + // A string accepts anything, so a union with a string alternative is a string + if (schema_info.resolves_to_string(schema)) { + return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); + } + + if (auto alternatives = alternatives_of(schema)) { + std::vector choices; + + size_t index = 0; + for (const auto & alternative : *alternatives) { + const std::string alt_name = rule_name + "-" + std::to_string(index++); + + // There is a risk that this breaks streaming deltas, but that's a risk we + // assume to provide tool arg streaming. + choices.push_back(value_of(alternative, alt_name, close)); + } + + return p.choice(choices); + } + + const std::string type = schema.contains("type") && schema.at("type").is_string() + ? schema.at("type").get() + : ""; + + if (type == "object" && schema.contains("properties")) { + return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag; + } + + if (type == "array" && schema.contains("items")) { + const std::string item_close = NS + ""; + auto item = p.rule(rule_name + "-item", + p.tag(mm3::TOOL_ARG_ITEM, + p.literal(NS + "") + + value_of(schema.at("items"), rule_name + "-item", item_close))); + return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; + } + + return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag; + }; + + // Required properties in schema order, then any number of optional ones in any order. + members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser { + const auto & props = schema.at("properties"); + + std::set required; + if (schema.contains("required")) { + required = schema.at("required").get>(); + } + + std::vector required_elements; + std::vector optional_elements; + for (const auto & [key, key_schema] : props.items()) { + auto element = element_of(key, key_schema, rule_prefix + "-" + key); + if (required.find(key) != required.end()) { + required_elements.push_back(element); + } else { + optional_elements.push_back(element); + } + } + + common_peg_parser members = p.eps(); + for (size_t i = 0; i < required_elements.size(); i++) { + if (i > 0) { + members = members + p.space(); + } + members = members + required_elements[i]; + } + + if (!optional_elements.empty()) { + common_peg_parser any_optional = p.choice(); + for (const auto & element : optional_elements) { + any_optional |= element; + } + members = members + p.repeat(p.space() + any_optional, 0, -1); + } + + return members; + }; + + common_peg_parser invoke_body = + params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps(); + + auto func_parser = p.tool( + p.tool_open(p.literal(NS + "")) + + p.space() + invoke_body + p.space() + + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + + auto content_before_tools = p.content(p.until(FC_START)); + return generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, + }; + } + + return data; +} diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cpp new file mode 100644 index 000000000..075f14dbc --- /dev/null +++ b/common/parsers/ministral3.cpp @@ -0,0 +1,135 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja + auto adjusted_messages = json::array(); + for (const auto & msg : inputs.messages) { + auto role = msg.value("role", ""); + if (role != "system" && role != "assistant") { + // Only adjust system and assistant messages. Interestingly, the system message may contain thinking. + adjusted_messages.push_back(msg); + continue; + } + + auto content = json::array(); + + // If message contains `reasoning_content`, add it as a block of type `thinking` + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + content.push_back({ + { "type", "thinking" }, + { "thinking", msg.at("reasoning_content").get() }, + }); + } + + // If message contains `content`, add it as a block of type `text` + if (msg.contains("content")) { + if (msg.at("content").is_string()) { + content.push_back({ + { "type", "text" }, + { "text", msg.at("content").get() }, + }); + } else if (msg.at("content").is_array()) { + auto blocks = msg.at("content"); + content.insert(blocks); + } + } + + auto adjusted = msg; + adjusted["content"] = content; + adjusted.erase("reasoning_content"); + adjusted_messages.push_back(adjusted); + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = true; + + data.supports_thinking = true; + data.thinking_start_tag = "[THINK]"; + data.thinking_end_tags = {"[/THINK]"}; + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.preserved_tokens = { + "[THINK]", + "[/THINK]", + "[TOOL_CALLS]", + "[ARGS]", + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "[THINK]" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "[/THINK]" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.eps(); + auto reasoning = + extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps(); + + // Response format parser + if (has_response_format) { + // Ministral wants to emit json surrounded by code fences + return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```"); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto & schema = function.at("parameters"); + + tool_choice |= + p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") + + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls)); + + return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls); + } + + // Content only parser + include_grammar = false; + return generation_prompt + (reasoning << p.content(p.rest())); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" } + }; + } + + return data; +} diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cpp new file mode 100644 index 000000000..7f4dfcd51 --- /dev/null +++ b/common/parsers/muse-glimmer.cpp @@ -0,0 +1,148 @@ +#include "parsers.h" + +// An assistant turn is rendered as one or more messages, each +// "<|start|>assistant to=<|message|>{content}{END}" where END is +// <|eom|> (more messages follow) or <|eot|> (end of turn): +// - chain-of-thought: to=self, terminated by <|eom|> +// - final answer: to=user, terminated by <|eot|> +// The generation prompt is just "<|start|>assistant"; the model emits its own +// " to=...<|message|>". +common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = "<|start|>assistant"; + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + data.preserved_tokens = { + "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", + // ATEM tool-call markup emitted on " to=" turns. + "", "", + "", "", + }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" }, + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + // Constrained grammar whenever tools are offered. + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.literal("<|start|>assistant")); + + if (!extract_reasoning && !include_grammar) { + return start + p.content(p.rest()); + } + + if (extract_reasoning) { + p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); + } else { + p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); + } + auto analysis = p.ref("analysis"); + + auto recipient = p.optional(p.literal(" to=user")); + auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + + p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto string_value = p.ac( + p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), + ""); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto args = p.eps(); + if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + auto arg_choice = p.choice(); + for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { + auto value_parser = p.eps(); + if (schema_info.resolves_to_string(prop_schema)) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)) + + p.tool_arg_close(p.literal("")); + } + + auto arg_rule = p.tool_arg( + p.tool_arg_open(p.literal("")) + + value_parser); + + arg_choice |= arg_rule; + } + args = p.zero_or_more(arg_choice + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal(" to=") + p.until("<|message|>") + + p.literal("<|message|>") + p.space() + + p.literal("") + p.space()) + << p.tool_args(args) + << p.tool_close(p.literal("") + p.space() + p.literal(""))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto tool_calls = inputs.parallel_tool_calls + ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) + : p.trigger_rule("tool-call", tool_choice); + + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return p.zero_or_more(start + analysis) + start + tool_calls; + } + auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); + return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); + } + + return p.zero_or_more(start + analysis) + start + final_msg; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, + "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, + }; + } + + return data; +} diff --git a/common/parsers/parsers.cpp b/common/parsers/parsers.cpp new file mode 100644 index 000000000..0a4d5cfbb --- /dev/null +++ b/common/parsers/parsers.cpp @@ -0,0 +1,34 @@ +#include "parsers.h" + +#include "log.h" + +#include + +void foreach_function(const json & tools, const std::function & fn) { + for (const auto & tool : tools) { + if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { + LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str()); + continue; + } + fn(tool); + } +} + +void foreach_parameter(const json & function, const std::function & fn) { + if (!function.contains("parameters") || !function.at("parameters").is_object()) { + return; + } + const auto & params = function.at("parameters"); + if (!params.contains("properties") || !params.at("properties").is_object()) { + return; + } + const auto & props = params.at("properties"); + std::set required; + if (params.contains("required") && params.at("required").is_array()) { + required = params.at("required").get>(); + } + for (const auto & [name, prop] : props.items()) { + bool is_required = (required.find(name) != required.end()); + fn(name, prop, is_required); + } +} diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h new file mode 100644 index 000000000..7898f0007 --- /dev/null +++ b/common/parsers/parsers.h @@ -0,0 +1,77 @@ +#pragma once + +#include "chat.h" +#include "chat-auto-parser.h" +#include "chat-auto-parser-helpers.h" +#include "chat-peg-parser.h" +#include "common.h" +#include "ggml.h" +#include "json-schema-to-grammar.h" +#include "json.h" + +#include +#include +#include +#include +#include + +using json = common_json; + +// iterate over the function tools of an OpenAI-style tools array +void foreach_function(const json & tools, const std::function & fn); + +// iterate over the parameters of a function tool, flagging the ones listed as required +void foreach_parameter(const json & function, const std::function & fn); + +// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten +std::string common_chat_template_direct_apply_impl( + const common_chat_template & tmpl, + const autoparser::generation_params & inputs, + const std::optional & messages_override = std::nullopt, + const std::optional & tools_override = std::nullopt, + const std::optional & additional_context = std::nullopt); + +// the suffix a template appends when add_generation_prompt is set +std::string common_chat_template_generation_prompt_impl( + const common_chat_template & tmpl, + const autoparser::generation_params & inputs, + const std::optional & messages_override = std::nullopt, + const std::optional & tools_override = std::nullopt, + const std::optional & additional_context = std::nullopt); + +bool is_lfm2_template(const std::string & src); + +namespace workaround { + +void convert_tool_responses_gemma4(json & messages); + +} + +common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_gigachat_v3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +// tool_list_tokens preserves the LFM2 system tool-list markers; LFM2.5 renders without them +common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, const autoparser::generation_params & inputs, bool tool_list_tokens); + +common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs); diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp new file mode 100644 index 000000000..8a1e52137 --- /dev/null +++ b/common/parsers/qwen3-coder.cpp @@ -0,0 +1,181 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string GEN_PREFIX = "<|im_start|>assistant\n"; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + + auto supports_reasoning = tmpl.source().find("") != std::string::npos; + + data.supports_thinking = supports_reasoning; + data.preserved_tokens = { + "", + "", + }; + + auto is_qwen3_coder = !supports_reasoning; + + if (supports_reasoning) { + data.thinking_start_tag = ""; + // Support both and as reasoning end sequences. + // ", "" }; + data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX; + if (supports_reasoning) { + data.generation_prompt += "\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n"; + } + } + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + std::vector tool_call_starts = { "" }; + + if (is_qwen3_coder) { + // Match complete opener for Qwen3-Coder models that occasionally omit the + // starting . The model may hallucinate a tool name, but it is preferable over + // constraining on + foreach_function(inputs.tools, [&](const json & tool) { + const std::string name = tool.at("function").at("name"); + tool_call_starts.push_back(""); + }); + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + + auto reasoning = p.eps(); + if (supports_reasoning && extract_reasoning) { + reasoning = p.optional("" + p.space() + + p.reasoning(p.until_one_of({ "", "" })) + + (p.literal("") | p.peek(p.literal("")))); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto arg_close = p.tool_arg_close(p.literal("\n\n")); + auto arg_string = p.rule("xml-arg-string", + p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(parameters); + + std::vector required_args; + std::vector optional_args; + + foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { + auto rule_name = "tool-" + name + "-arg-" + param_name; + + auto arg_open = p.tool_arg_open("\n"); + + auto arg_value = schema_info.resolves_to_string(param_schema) ? + arg_string : + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; + + auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); + + (is_required ? required_args : optional_args).push_back(arg_rule); + }); + + // Accept required arguments in any order, as Qwen does not always adhere to the + // order provided. + auto args = p.permute("tool-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto func = p.tool(p.tool_open("\n") + + p.tool_args(args) + + p.tool_close(p.literal("\n"))); + + tool_choice |= p.rule("tool-" + name, func); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + auto tool_call_body = tool_choice + "" + p.space(); + auto tool_call = p.rule("tool-call", "\n" + tool_call_body); + + // Qwen3-Coder models may occasionally omit the token. + auto tool_call_first = is_qwen3_coder ? + p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body) : + tool_call; + + auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); + + return generation_prompt + + (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); + } + + // Content only parser + return generation_prompt + (reasoning << p.content(p.rest())); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + if (data.grammar_lazy) { + for (const auto & start : tool_call_starts) { + data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); + } + } + } + + return data; +} diff --git a/common/parsers/sources.cmake b/common/parsers/sources.cmake new file mode 100644 index 000000000..9d7fb0992 --- /dev/null +++ b/common/parsers/sources.cmake @@ -0,0 +1,20 @@ +# Specialized chat template parsers, listed explicitly so that adding or removing one re-runs CMake instead of leaving an incremental build stale. + +set(LLAMA_CHAT_PARSERS_SOURCES + ${CMAKE_CURRENT_LIST_DIR}/parsers.cpp + ${CMAKE_CURRENT_LIST_DIR}/parsers.h + ${CMAKE_CURRENT_LIST_DIR}/cohere2moe.cpp + ${CMAKE_CURRENT_LIST_DIR}/deepseek.cpp + ${CMAKE_CURRENT_LIST_DIR}/functionary-v3-2.cpp + ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp + ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp + ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp + ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp + ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp + ${CMAKE_CURRENT_LIST_DIR}/lfm2.cpp + ${CMAKE_CURRENT_LIST_DIR}/minicpm5.cpp + ${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp + ${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp + ${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp + ${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp +) diff --git a/embd_res/klite.embd b/embd_res/klite.embd index 7d82e8bae..ba3ad2bb4 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -26001,7 +26001,7 @@ Current version indicated by LITEVER below. element.innerHTML = pend; }); } else { - render_gametext(false); + render_gametext(false,false); } handle_autoscroll(false); @@ -26083,62 +26083,102 @@ Current version indicated by LITEVER below. } - var autoscrollEnabled = true; - var suppressScrollEvent = false; - var autoscrollTimeout = null; + const autoscrollStates = new Map(); let autoscrollRAF = null; - var lastScrollTop = 0; function handle_autoscroll(alwaysscroll = true) { if (!localsettings.autoscroll) return; - // Cancel pending frame to avoid stacking work - if (autoscrollRAF) cancelAnimationFrame(autoscrollRAF); + // Explicit navigation resets following now, so subsequent user input can + // still override it before the frame runs. Streaming never resets it. + if (alwaysscroll) { + for (let state of autoscrollStates.values()) state.following = true; + } + if (autoscrollRAF !== null) return; autoscrollRAF = requestAnimationFrame(() => { - const boxes = [ - document.getElementById("gametext"), - document.getElementById("chat_msg_body"), - document.getElementById("corpostylemain") - ].filter(Boolean); - - if (!boxes.length) return; - - // ---- READ PHASE (no writes) ---- - const threshold = autoscrollEnabled ? 195 : 70; - const scrollData = boxes.map(el => { - const maxScrollTop = el.scrollHeight - el.clientHeight + 10; - const isAtBottom = - el.scrollHeight - el.scrollTop - el.clientHeight <= threshold; - - return { el, maxScrollTop, shouldScroll: alwaysscroll || isAtBottom }; - }); - - // ---- WRITE PHASE (no reads) ---- - suppressScrollEvent = true; - for (let item of scrollData) { - if (item.shouldScroll) { - item.el.scrollTop = item.maxScrollTop; + autoscrollRAF = null; + if (!localsettings.autoscroll) return; + const scrollData = []; + for (let [element, state] of autoscrollStates) { + if (!element.clientHeight) continue; // Ignore hidden layouts. + if (state.following && !state.pointerDown) { + scrollData.push({element, state, top: Math.max(0, element.scrollHeight - element.clientHeight)}); } } - suppressScrollEvent = false; + for (let {element, state, top} of scrollData) { + // Scroll events arrive asynchronously. Record the position we write + // instead of briefly suppressing events around the assignment. + state.lastScrollTop = top; + element.scrollTop = top; + } }); } function attach_scroll_detection(element) { - element.addEventListener("scroll", () => { - const currentScrollTop = element.scrollTop; - const delta = currentScrollTop - lastScrollTop; - lastScrollTop = currentScrollTop; - if (suppressScrollEvent) { return; } - if (delta>=0) { return; } - autoscrollEnabled = false; - if (autoscrollTimeout) { - clearTimeout(autoscrollTimeout); + if (!element || autoscrollStates.has(element)) return; + const state = {following: true, lastScrollTop: element.scrollTop, pointerDown: false}; + autoscrollStates.set(element, state); + const pause = () => { state.following = false; }; + + // Stop before the next render, even if the browser has not dispatched + // the corresponding scroll event yet. Leave native scrolling untouched. + element.addEventListener("wheel", event => { + if (event.deltaY < 0 && !event.ctrlKey) pause(); + }, {passive: true}); + let touchY = null; + element.addEventListener("touchstart", event => { + touchY = event.touches.length === 1 ? event.touches[0].clientY : null; + }, {passive: true}); + element.addEventListener("touchmove", event => { + if (touchY !== null && event.touches.length === 1) { + const nextY = event.touches[0].clientY; + if (nextY > touchY) pause(); + touchY = nextY; } - autoscrollTimeout = setTimeout(() => { - autoscrollEnabled = true; - }, 180); + }, {passive: true}); + element.addEventListener("keydown", event => { + if (event.defaultPrevented || event.target.closest("input, textarea, select, [contenteditable=true]")) return; + if (["ArrowUp", "PageUp", "Home"].includes(event.key) || (event.key === " " && event.shiftKey)) pause(); }); + + const onScroll = () => { + if (!element.clientHeight) return; + const maxTop = Math.max(0, element.scrollHeight - element.clientHeight); + // Clamp Safari rubber-band overscroll; allow only rounding tolerance. + const top = Math.max(0, Math.min(element.scrollTop, maxTop)); + const previousTop = state.lastScrollTop; + state.lastScrollTop = top; + // Reflow and scroll anchoring can also move scrollTop upward. Only + // mouse/pen interaction makes that a manual scroll; wheel, touch and + // keyboard intent is handled directly above. + if (state.pointerDown && top < Math.min(previousTop, maxTop) - 1) { + pause(); + } else if (top > previousTop && maxTop - top <= 1) { + state.following = true; + } + }; + element.addEventListener("scroll", onScroll, {passive: true}); + + // Let scrollbar dragging and selection scrolling finish before writing + // scrollTop. Listen outside the element too, since a drag can end there. + element.addEventListener("pointerdown", event => { + if (event.pointerType !== "touch") state.pointerDown = true; + }, {passive: true}); + const releasePointer = () => { + if (!state.pointerDown) return; + onScroll(); + state.pointerDown = false; + handle_autoscroll(false); + }; + document.addEventListener("pointerup", releasePointer, {passive: true}); + document.addEventListener("pointercancel", releasePointer, {passive: true}); + + // Keep a followed view at the bottom after reflow or late-loading media. + if (typeof ResizeObserver !== "undefined") { + const observer = new ResizeObserver(() => handle_autoscroll(false)); + observer.observe(element); + } + element.addEventListener("load", () => handle_autoscroll(false), true); } function render_gametext(save=true, force_scroll=true)