mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-10 06:49:12 +02:00
Merge commit '895c045fd104ced72132160245edcd6a86e50ba0' into concedo_experimental
# Conflicts: # common/CMakeLists.txt # common/chat.cpp
This commit is contained in:
Concedo
@@ -12,6 +12,8 @@
|
||||
|
||||
using json = common_json;
|
||||
|
||||
namespace autoparser {
|
||||
|
||||
// Helper to iterate over tools/functions
|
||||
static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
|
||||
for (const auto & tool : tools) {
|
||||
@@ -22,8 +24,6 @@ static void foreach_function(const json & tools, const std::function<void(const
|
||||
}
|
||||
}
|
||||
|
||||
namespace autoparser {
|
||||
|
||||
parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :
|
||||
p(p),
|
||||
inputs(inputs),
|
||||
|
||||
+24
-2411
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,150 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Cohere2 MoE (a.k.a. "North Code") parser.
|
||||
//
|
||||
// The assistant turn is fully marker-wrapped:
|
||||
// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
|
||||
// <|START_THINKING|>{reasoning}<|END_THINKING|>
|
||||
// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|>
|
||||
// OR tool calls: <|START_ACTION|>[
|
||||
// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...
|
||||
// ]<|END_ACTION|>
|
||||
// <|END_OF_TURN_TOKEN|>
|
||||
//
|
||||
// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is
|
||||
// the template default), so the model's output continues from *inside* the thinking block. The
|
||||
// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix
|
||||
// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,
|
||||
// regardless of whether they came from the generation prompt or the generated text.
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string TURN_START = "<|START_OF_TURN_TOKEN|>";
|
||||
const std::string TURN_END = "<|END_OF_TURN_TOKEN|>";
|
||||
const std::string CHATBOT = "<|CHATBOT_TOKEN|>";
|
||||
const std::string USER = "<|USER_TOKEN|>";
|
||||
const std::string SYSTEM = "<|SYSTEM_TOKEN|>";
|
||||
const std::string THINK_START = "<|START_THINKING|>";
|
||||
const std::string THINK_END = "<|END_THINKING|>";
|
||||
const std::string TEXT_START = "<|START_TEXT|>";
|
||||
const std::string TEXT_END = "<|END_TEXT|>";
|
||||
const std::string ACTION_START = "<|START_ACTION|>";
|
||||
const std::string ACTION_END = "<|END_ACTION|>";
|
||||
const std::string RESULT_START = "<|START_TOOL_RESULT|>";
|
||||
const std::string RESULT_END = "<|END_TOOL_RESULT|>";
|
||||
|
||||
// Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.
|
||||
const std::string GEN_PREFIX = TURN_START + CHATBOT;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END};
|
||||
data.preserved_tokens = {
|
||||
TURN_START, TURN_END, CHATBOT, USER, SYSTEM,
|
||||
THINK_START, THINK_END,
|
||||
TEXT_START, TEXT_END,
|
||||
ACTION_START, ACTION_END,
|
||||
RESULT_START, RESULT_END,
|
||||
};
|
||||
|
||||
// Declare per-role message delimiters. Tool results are rendered with the
|
||||
// system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before
|
||||
// the plain "system" one (it is a strict superset, and the role split tries delimiters in order).
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX },
|
||||
{ COMMON_CHAT_ROLE_USER, TURN_START + USER },
|
||||
{ COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + TEXT_START + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
auto end = p.end();
|
||||
|
||||
// The thinking block is always present (the generation prompt forces <|START_THINKING|>).
|
||||
// When extracting reasoning, capture its body; otherwise keep the whole block (markers
|
||||
// included) inline as content, matching reasoning_format=NONE conventions.
|
||||
common_peg_parser reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
reasoning = p.optional(p.literal(THINK_START) +
|
||||
p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +
|
||||
p.optional(p.literal(THINK_END)));
|
||||
} else {
|
||||
reasoning = p.optional(p.content(p.literal(THINK_START) +
|
||||
p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +
|
||||
p.optional(p.literal(THINK_END))));
|
||||
}
|
||||
|
||||
auto text_content = has_response_format
|
||||
? p.literal(TEXT_START) +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.optional(p.literal(TEXT_END))
|
||||
: p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;
|
||||
}
|
||||
|
||||
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
|
||||
// <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>
|
||||
auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,
|
||||
/* force_tool_calls = */ true,
|
||||
/* name_key = */ "tool_name",
|
||||
/* args_key = */ "parameters",
|
||||
/* array_wrapped = */ true,
|
||||
/* function_is_key = */ false,
|
||||
/* call_id_key = */ "",
|
||||
/* gen_call_id_key = */ "tool_call_id",
|
||||
/* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });
|
||||
|
||||
// Content and tool calls are mutually exclusive in this format.
|
||||
common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });
|
||||
|
||||
return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// The DeepSeek V4 reference implementation renders consecutive tool results into a single
|
||||
// user block, ordered by the tool call order of the preceding assistant message (matched
|
||||
// by tool call id) rather than by the order they appear in the conversation.
|
||||
static json deepseek_v4_sort_tool_results(const json & messages) {
|
||||
json adjusted = messages;
|
||||
std::map<std::string, size_t> call_order;
|
||||
|
||||
for (size_t i = 0; i < adjusted.size();) {
|
||||
const auto & msg = adjusted[i];
|
||||
const auto role = msg.value("role", "");
|
||||
|
||||
if (role == "assistant" && msg.contains("tool_calls") &&
|
||||
msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
|
||||
call_order.clear();
|
||||
const auto & tool_calls = msg.at("tool_calls");
|
||||
for (size_t idx = 0; idx < tool_calls.size(); idx++) {
|
||||
auto id = tool_calls[idx].value("id", "");
|
||||
if (!id.empty()) {
|
||||
call_order[id] = idx;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (role != "user" && role != "tool") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// collect a maximal run of user/tool messages - they render into one user block
|
||||
std::vector<size_t> tool_positions;
|
||||
size_t run_end = i;
|
||||
for (; run_end < adjusted.size(); run_end++) {
|
||||
const auto r = adjusted[run_end].value("role", "");
|
||||
if (r == "tool") {
|
||||
tool_positions.push_back(run_end);
|
||||
} else if (r != "user") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (tool_positions.size() > 1 && !call_order.empty()) {
|
||||
std::vector<json> results;
|
||||
results.reserve(tool_positions.size());
|
||||
for (auto pos : tool_positions) {
|
||||
results.push_back(adjusted[pos]);
|
||||
}
|
||||
std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
|
||||
const auto order = [&](const json & m) {
|
||||
auto it = call_order.find(m.value("tool_call_id", ""));
|
||||
return it == call_order.end() ? (size_t) 0 : it->second;
|
||||
};
|
||||
return order(a) < order(b);
|
||||
});
|
||||
for (size_t k = 0; k < tool_positions.size(); k++) {
|
||||
adjusted[tool_positions[k]] = std::move(results[k]);
|
||||
}
|
||||
}
|
||||
|
||||
i = run_end;
|
||||
}
|
||||
|
||||
return adjusted;
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"
|
||||
// instead of "function_calls", renders tool results in tool call order and its
|
||||
// non-thinking generation prompt ends with a bare </think> instead of an empty
|
||||
// <think></think> pair.
|
||||
const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
|
||||
|
||||
std::optional<json> adjusted_messages;
|
||||
if (is_v4) {
|
||||
adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
std::optional<json> additional_context;
|
||||
if (is_v4 && has_response_format) {
|
||||
additional_context = json{ { "response_format", inputs.json_schema } };
|
||||
}
|
||||
|
||||
const std::string DSML = "｜DSML｜";
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";
|
||||
const std::string FC_START = "<" + DSML + TC_BLOCK + ">";
|
||||
const std::string FC_END = "</" + DSML + TC_BLOCK + ">";
|
||||
const std::string INVOKE_START = "<" + DSML + "invoke";
|
||||
const std::string INVOKE_END = "</" + DSML + "invoke>";
|
||||
const std::string PARAM_START = "<" + DSML + "parameter";
|
||||
const std::string PARAM_END = "</" + DSML + "parameter>";
|
||||
const std::string GEN_PROMPT = "<｜Assistant｜>";
|
||||
const std::string TC_SEPARATOR = "\n\n";
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(
|
||||
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(
|
||||
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END, FC_START};
|
||||
data.preserved_tokens = {
|
||||
DSML,
|
||||
THINK_START,
|
||||
THINK_END,
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
if (is_v4 && msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = GEN_PROMPT + THINK_END;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
} else {
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
auto end = p.end();
|
||||
|
||||
// build tool call section first since we might need it in reasoning
|
||||
auto tool_choice = p.choice();
|
||||
if (has_tool_calls) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
const auto & props = params.contains("properties") ? params.at("properties") : json::object();
|
||||
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required")) {
|
||||
required = params.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
for (const auto & [param_name, param_schema] : props.items()) {
|
||||
bool is_required = required.find(param_name) != required.end();
|
||||
bool is_string = schema_info.resolves_to_string(param_schema);
|
||||
|
||||
auto arg = p.tool_arg(
|
||||
p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) +
|
||||
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
|
||||
(is_string ?
|
||||
p.tool_arg_string_value(p.until(PARAM_END)) :
|
||||
p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema",
|
||||
param_schema, false))) +
|
||||
p.tool_arg_close(p.literal(PARAM_END)));
|
||||
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
||||
if (is_required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
}
|
||||
|
||||
common_peg_parser args_seq = p.eps();
|
||||
for (size_t i = 0; i < required_parsers.size(); i++) {
|
||||
if (i > 0) {
|
||||
args_seq = args_seq + p.space();
|
||||
}
|
||||
args_seq = args_seq + required_parsers[i];
|
||||
}
|
||||
|
||||
if (!optional_parsers.empty()) {
|
||||
common_peg_parser any_opt = p.choice();
|
||||
for (const auto & opt : optional_parsers) {
|
||||
any_opt |= opt;
|
||||
}
|
||||
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
|
||||
}
|
||||
|
||||
common_peg_parser invoke_body = args_seq;
|
||||
auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +
|
||||
p.tool_name(p.literal(name)) + p.literal("\">\n")) +
|
||||
invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, func_parser);
|
||||
});
|
||||
}
|
||||
|
||||
common_peg_parser tool_calls = p.eps();
|
||||
if (inputs.parallel_tool_calls) {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice +
|
||||
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
||||
} else {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
||||
}
|
||||
|
||||
auto reasoning = p.eps();
|
||||
auto reasoning_with_tc = p.eps();
|
||||
auto obligatory_tool_calls = tool_calls;
|
||||
bool allow_reasoning_with_tc = false;
|
||||
|
||||
if (!require_tools) {
|
||||
tool_calls = p.optional(tool_calls);
|
||||
}
|
||||
|
||||
if (extract_reasoning && inputs.enable_thinking) {
|
||||
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
|
||||
reasoning_with_tc = THINK_START +
|
||||
p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
|
||||
p.space() + obligatory_tool_calls;
|
||||
allow_reasoning_with_tc = true;
|
||||
} else if (extract_reasoning) {
|
||||
// Thinking disabled but reasoning extraction requested: the generation prompt
|
||||
// contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
|
||||
// must still be consumed.
|
||||
reasoning = is_v4
|
||||
? p.optional(p.literal(THINK_END))
|
||||
: p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
|
||||
}
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("```json") + p.space() +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.space() + p.literal("```"));
|
||||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
|
||||
if (!has_tool_calls) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
auto content_before_tools = p.negate(p.literal(THINK_START)) +
|
||||
p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
|
||||
p.space();
|
||||
return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
|
||||
generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && !require_tools;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content}
|
||||
common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.preserved_tokens = {
|
||||
">>>all",
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content();
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
// Functionary v3.2 format:
|
||||
// - Normal content: >>>all\n{content}
|
||||
// - Tool calls: >>>function_name\n{json_args}
|
||||
// Generation prompt ends with ">>>" so model outputs recipient immediately
|
||||
|
||||
// Build content parser for >>>all\n{content}
|
||||
// When tools are present, content stops before the next ">>>" (tool call)
|
||||
// When no tools, content goes until end
|
||||
auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>"));
|
||||
auto content_until_end = p.literal("all\n") + p.content(p.rest());
|
||||
auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>");
|
||||
|
||||
// If no tools or tool_choice is NONE, just parse content
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// When no tools, just match the prefix and capture everything after
|
||||
return generation_prompt + content_until_end + p.end();
|
||||
}
|
||||
|
||||
// Build tool call parsers for each available function
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
|
||||
// Tool format: >>>function_name\n{json_args}
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) +
|
||||
p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))
|
||||
);
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_parser);
|
||||
});
|
||||
|
||||
auto content_only = content_until_end;
|
||||
auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice));
|
||||
auto content_and_tools = content_until_tool + tools_only;
|
||||
|
||||
auto ret = p.eps();
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
if (inputs.parallel_tool_calls) {
|
||||
ret = p.choice({ content_and_tools, tools_only }) + p.end();
|
||||
} else {
|
||||
ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end();
|
||||
}
|
||||
} else if (inputs.parallel_tool_calls) {
|
||||
ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end();
|
||||
} else {
|
||||
auto content_and_tool = content_until_tool + tool_choice;
|
||||
ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end();
|
||||
}
|
||||
return generation_prompt + ret;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
// Grammar trigger for when the model starts outputting a tool call
|
||||
// (after the initial ">>>" in the generation prompt but recipient other than "all")
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
#include "parsers.h"
|
||||
|
||||
namespace workaround {
|
||||
|
||||
// Gemma4 uses a custom tool_responses field instead of role:tool messages.
|
||||
//
|
||||
// This will transform a sequence of messages:
|
||||
// assistant(tool_call+) -> tool+ -> assistant(content)
|
||||
//
|
||||
// Into a single assistant message containing a tool_responses field:
|
||||
// assistant(content + tool_call + tool_responses)
|
||||
//
|
||||
// This is necessary for the Gemma4 chat template to properly format the prompt.
|
||||
// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4
|
||||
struct gemma4_model_turn_builder {
|
||||
json & messages;
|
||||
size_t pos;
|
||||
json tool_calls = json::array();
|
||||
json tool_responses = json::array();
|
||||
json content;
|
||||
json reasoning_content;
|
||||
|
||||
gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}
|
||||
|
||||
void collect() {
|
||||
// Collect the first assistant message
|
||||
auto & msg = messages[pos];
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
// According to the prompt formatting guide, we need to preserve reasoning_content
|
||||
// between function calls. The current chat templates do not support this, but we will do it anyway.
|
||||
reasoning_content = msg.at("reasoning_content");
|
||||
}
|
||||
for (auto & tc : msg.at("tool_calls")) {
|
||||
tool_calls.push_back(tc);
|
||||
}
|
||||
pos++;
|
||||
|
||||
// Collect tool call results
|
||||
while (pos < messages.size() && messages[pos].value("role", "") == "tool") {
|
||||
collect_result(messages[pos]);
|
||||
pos++;
|
||||
}
|
||||
|
||||
// Check if the next assistant message is the final message
|
||||
if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {
|
||||
auto & next = messages[pos];
|
||||
if (!has_tool_calls(next) && has_content(next)) {
|
||||
content = next.at("content");
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void collect_result(const json & curr) {
|
||||
json response;
|
||||
if (curr.contains("content")) {
|
||||
const auto & content = curr.at("content");
|
||||
if (content.is_string()) {
|
||||
// Try to parse the content as JSON; fall back to raw string
|
||||
try {
|
||||
response = json::parse(content.get<std::string>());
|
||||
} catch (...) {
|
||||
response = content;
|
||||
}
|
||||
} else {
|
||||
response = content;
|
||||
}
|
||||
}
|
||||
|
||||
std::string name;
|
||||
|
||||
// Match name with corresponding tool call
|
||||
size_t idx = tool_responses.size();
|
||||
if (idx < tool_calls.size()) {
|
||||
auto & tc = tool_calls[idx];
|
||||
if (tc.contains("function")) {
|
||||
name = tc.at("function").value("name", "");
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to the tool call id
|
||||
if (name.empty()) {
|
||||
name = curr.value("tool_call_id", "");
|
||||
}
|
||||
|
||||
tool_responses.push_back({{"name", name}, {"response", response}});
|
||||
}
|
||||
|
||||
json build() {
|
||||
collect();
|
||||
|
||||
json msg = {
|
||||
{"role", "assistant"},
|
||||
{"tool_calls", tool_calls},
|
||||
};
|
||||
if (!tool_responses.empty()) {
|
||||
msg["tool_responses"] = tool_responses;
|
||||
}
|
||||
if (!content.is_null()) {
|
||||
msg["content"] = content;
|
||||
}
|
||||
if (!reasoning_content.is_null()) {
|
||||
msg["reasoning_content"] = reasoning_content;
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
|
||||
static bool has_content(const json & msg) {
|
||||
if (!msg.contains("content") || msg.at("content").is_null()) {
|
||||
return false;
|
||||
}
|
||||
const auto & content = msg.at("content");
|
||||
if (content.is_string() && !content.get<std::string>().empty()) {
|
||||
return true;
|
||||
}
|
||||
if (content.is_array() && !content.empty()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool has_tool_calls(const json & msg) {
|
||||
return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();
|
||||
}
|
||||
};
|
||||
|
||||
void convert_tool_responses_gemma4(json & messages) {
|
||||
json result = json::array();
|
||||
size_t i = 0;
|
||||
|
||||
while (i < messages.size()) {
|
||||
auto & msg = messages[i];
|
||||
|
||||
if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||
|
||||
!msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {
|
||||
result.push_back(msg);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
gemma4_model_turn_builder builder(messages, i);
|
||||
result.push_back(builder.build());
|
||||
i = builder.pos;
|
||||
}
|
||||
|
||||
messages = result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
|
||||
if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {
|
||||
// This may happen if the model generates content + tool_call, the
|
||||
// template does not add the model's next turn and confuses the model
|
||||
// from emitting its proper reasoning token sequence.
|
||||
data.generation_prompt = "<|turn>model\n";
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_USER, "<|turn>user" },
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" },
|
||||
};
|
||||
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<|channel>thought";
|
||||
data.thinking_end_tags = {"<channel|>"};
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|channel>",
|
||||
"<channel|>",
|
||||
"<|tool_call>",
|
||||
"<tool_call|>",
|
||||
"<|turn>",
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : "";
|
||||
data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<channel|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));
|
||||
} else {
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));
|
||||
}
|
||||
|
||||
auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");
|
||||
auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.literal("```json") <<
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<
|
||||
p.literal("```");
|
||||
return start + p.optional(thought) + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// Gemma4 tool calling syntax
|
||||
// Rules should match traversal logic in gemma4_to_json()
|
||||
p.rule("gemma4-string-content", p.until("<|\"|>"));
|
||||
p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));
|
||||
p.rule("gemma4-bool", p.json_bool());
|
||||
p.rule("gemma4-null", p.json_null());
|
||||
p.rule("gemma4-number", p.json_number());
|
||||
p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));
|
||||
p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));
|
||||
p.rule("gemma4-dict", [&]() {
|
||||
auto ws = p.space();
|
||||
auto member = p.ref("gemma4-dict-kv");
|
||||
auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
|
||||
return p.sequence({
|
||||
p.literal("{"), ws,
|
||||
p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
|
||||
});
|
||||
});
|
||||
p.rule("gemma4-array", [&]() {
|
||||
auto ws = p.space();
|
||||
auto value = p.ref("gemma4-value");
|
||||
auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
|
||||
return p.sequence({
|
||||
p.literal("["), ws,
|
||||
p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
|
||||
});
|
||||
});
|
||||
p.rule("gemma4-value", [&]() {
|
||||
return p.choice({
|
||||
p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
|
||||
p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
|
||||
});
|
||||
});
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
// TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules
|
||||
// const auto & params = function.at("parameters");
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({
|
||||
p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),
|
||||
p.tool_args(p.ref("gemma4-dict")),
|
||||
})));
|
||||
});
|
||||
|
||||
auto tool_call = p.trigger_rule("tool-call", p.repeat(
|
||||
"<|tool_call>call:" + tool_choice + "<tool_call|>",
|
||||
/* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,
|
||||
/* max = */ inputs.parallel_tool_calls ? -1 : 1
|
||||
));
|
||||
|
||||
auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));
|
||||
auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));
|
||||
auto message = p.rule("message", thought + content);
|
||||
return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;
|
||||
}
|
||||
|
||||
// Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may
|
||||
// also emit a single trailing <channel|> token. Consume all complete reasoning blocks and
|
||||
// then stop at the first unmatched <channel|> token.
|
||||
auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));
|
||||
auto message = p.rule("message", thought + content);
|
||||
return start + p.one_or_more(message);
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_gigachat_v3(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = false;
|
||||
data.preserved_tokens = {
|
||||
"<|message_sep|>\n\n",
|
||||
"<|role_sep|>\n",
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content();
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto ret = p.eps();
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// Build a choice of all available tools
|
||||
auto tool_choice = p.choice();
|
||||
for (const auto & tool : inputs.tools) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
|
||||
auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\"");
|
||||
auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));
|
||||
|
||||
auto tool_open = p.tool_open(p.literal("{") << tool_name);
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}");
|
||||
}
|
||||
|
||||
// Define the tool call structure
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto max_calls = 1; // parallel toolcalls are not supported
|
||||
auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice);
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls));
|
||||
|
||||
ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls;
|
||||
} else {
|
||||
// Content only parser
|
||||
include_grammar = false;
|
||||
ret = p.content(p.rest());
|
||||
}
|
||||
|
||||
return p.literal("assistant<|role_sep|>\n") + ret;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix}
|
||||
};
|
||||
}
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// Copy reasoning to the "thinking" field as expected by the gpt-oss template
|
||||
auto adjusted_messages = json::array();
|
||||
for (auto msg : inputs.messages) {
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
msg["thinking"] = msg.at("reasoning_content");
|
||||
if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
|
||||
msg.erase("content");
|
||||
}
|
||||
}
|
||||
adjusted_messages.push_back(msg);
|
||||
}
|
||||
|
||||
auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
|
||||
|
||||
// Check if we need to replace the return token with end token during
|
||||
// inference and without generation prompt. For more details see:
|
||||
// https://github.com/ggml-org/llama.cpp/issues/15417
|
||||
if (inputs.is_inference && !inputs.add_generation_prompt) {
|
||||
static constexpr std::string_view return_token = "<|return|>";
|
||||
static constexpr std::string_view end_token = "<|end|>";
|
||||
if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) {
|
||||
prompt.replace(pos, return_token.length(), end_token);
|
||||
}
|
||||
}
|
||||
|
||||
data.prompt = prompt;
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>functions" },
|
||||
};
|
||||
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.thinking_start_tag = "<|channel|>analysis<|message|>";
|
||||
data.thinking_end_tags = {"<|end|>"};
|
||||
|
||||
// These special tokens are required to parse properly, so we include them
|
||||
// even if parse_tool_calls is false.
|
||||
data.preserved_tokens = {
|
||||
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
|
||||
};
|
||||
|
||||
// Adjust prompt for continuation
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
auto end = p.rule("end", p.literal("<|end|>"));
|
||||
auto content = p.rule("message-content", p.until("<|end|>"));
|
||||
auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
|
||||
auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
|
||||
|
||||
// Occasionally, gpt-oss-20b will prefix channels with this commentary
|
||||
auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
|
||||
auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", start_analysis + p.reasoning(content) + end);
|
||||
} else {
|
||||
p.rule("analysis", p.content(start_analysis + content + end));
|
||||
}
|
||||
|
||||
auto analysis = p.ref("analysis");
|
||||
auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
|
||||
auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));
|
||||
|
||||
// Consume any unsolicited tool calls, e.g. builtin functions
|
||||
auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
|
||||
|
||||
auto any = p.rule("any", preamble | analysis);
|
||||
|
||||
if (has_response_format) {
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & params = function.at("parameters");
|
||||
|
||||
auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params));
|
||||
|
||||
// recipient in role header
|
||||
// <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS
|
||||
auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);
|
||||
|
||||
// recipient in channel header
|
||||
// <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS
|
||||
auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel);
|
||||
});
|
||||
|
||||
auto tool_call = p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + any) + start + tool_call;
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (tool_call | final_msg);
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (final_msg | unsolicited);
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>
|
||||
// The ID contains both the function name and an incrementing counter
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.preserved_tokens = {
|
||||
"<|tool_calls_section_begin|>",
|
||||
"<|tool_calls_section_end|>",
|
||||
"<|tool_call_begin|>",
|
||||
"<|tool_call_argument_begin|>",
|
||||
"<|tool_call_end|>",
|
||||
"<think>",
|
||||
"</think>",
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>";
|
||||
const std::string SECTION_END = "<|tool_calls_section_end|>";
|
||||
const std::string CALL_BEGIN = "<|tool_call_begin|>";
|
||||
const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>";
|
||||
const std::string CALL_END = "<|tool_call_end|>";
|
||||
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>";
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
// Kimi K2 Thinking format:
|
||||
// - Reasoning: <think>{reasoning}</think>
|
||||
// - Content: text after reasoning
|
||||
// - Tool calls section:
|
||||
// <|tool_calls_section_begin|>
|
||||
// <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|>
|
||||
// ...
|
||||
// <|tool_calls_section_end|>
|
||||
// The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ...
|
||||
|
||||
// Tool call markers
|
||||
auto end = p.end();
|
||||
|
||||
// Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny.
|
||||
// For example, it can call tools at the end of reasoning without closing reasoning...
|
||||
auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning(
|
||||
p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) +
|
||||
p.optional(p.literal(THINK_END))) : p.eps();
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
|
||||
|
||||
// Content only parser (no tools)
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
// Build tool call parsers for each available function
|
||||
// The ID format is: functions.<name>:<index>
|
||||
// We need to match: functions.<name>:<digits>
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
|
||||
// Match: functions.<name>:<digits>
|
||||
// Capture the full call id (functions.<name>:<digits>) using tool_id tag
|
||||
auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1));
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(tool_id + p.literal(ARGS_BEGIN)) +
|
||||
p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) +
|
||||
p.tool_close(p.optional((p.literal(CALL_END))))
|
||||
);
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_parser);
|
||||
});
|
||||
|
||||
// Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|>
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
|
||||
// Use trigger_rule so grammar generator knows where to start generating rules
|
||||
auto tool_calls = p.rule("tool-calls",
|
||||
p.optional(p.literal(SECTION_BEGIN)) +
|
||||
p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) +
|
||||
p.optional(p.literal(SECTION_END)))
|
||||
);
|
||||
|
||||
auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN }));
|
||||
|
||||
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros:
|
||||
// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|>
|
||||
// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|>
|
||||
// the generation prompt already opens the think (or response) section, so the
|
||||
// section opener is optional here - same as Kimi K2 Thinking
|
||||
common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
const std::string SEP = "<|sep|>";
|
||||
const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>";
|
||||
const std::string THINK_START = "<|open|>think<|sep|>";
|
||||
const std::string THINK_END = "<|close|>think<|sep|>";
|
||||
const std::string RESP_START = "<|open|>response<|sep|>";
|
||||
const std::string RESP_END = "<|close|>response<|sep|>";
|
||||
const std::string TOOLS_START = "<|open|>tools<|sep|>";
|
||||
const std::string TOOLS_END = "<|close|>tools<|sep|>";
|
||||
const std::string CALL_START = "<|open|>call tool=\"";
|
||||
const std::string CALL_END = "<|close|>call<|sep|>";
|
||||
const std::string ARG_START = "<|open|>argument key=\"";
|
||||
const std::string ARG_END = "<|close|>argument<|sep|>";
|
||||
const std::string MSG_END = "<|close|>message<|sep|>";
|
||||
const std::string EOM_TOKEN = "<|end_of_msg|>";
|
||||
|
||||
// only the markers are special tokens. tag names ("think", "response", ...) are
|
||||
// normal tokens and must not be preserved, or prose with those words is broken
|
||||
data.preserved_tokens = {
|
||||
"<|open|>",
|
||||
"<|close|>",
|
||||
"<|sep|>",
|
||||
"<|end_of_msg|>",
|
||||
};
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = { THINK_END };
|
||||
|
||||
// per-role message-start delimiters. user/assistant messages only have the role
|
||||
// attribute, so the full opener is used. system and tool messages have more
|
||||
// attributes, so those delimiters stop after the closing quote of the role
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + RESP_START + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto end = p.end();
|
||||
|
||||
auto start = p.optional(p.literal(MSG_START));
|
||||
|
||||
// the think section is always consumed, even with reasoning extraction off:
|
||||
// the generation prompt ends with open_tag('think'), so it is always present.
|
||||
// reasoning stops at its own closer, or at the response opener if the model
|
||||
// skips the closer
|
||||
auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) :
|
||||
p.content(p.until_one_of({ THINK_END, RESP_START }));
|
||||
|
||||
auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body +
|
||||
p.optional(p.literal(THINK_END)));
|
||||
|
||||
// content runs to the response closer, or to the next section if truncated
|
||||
auto response = p.optional(p.literal(RESP_START)) +
|
||||
p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) +
|
||||
p.optional(p.literal(RESP_END));
|
||||
|
||||
// the EOG token after the message closer reaches the parser as text,
|
||||
// so it must be consumed or the parse stays incomplete
|
||||
auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN));
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return start + reasoning + response + trailer + end;
|
||||
}
|
||||
|
||||
auto tool_choices = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const json schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
// arguments come one tag per key, with the JSON type in a type="..."
|
||||
// attribute. the type is taken from the tool schema instead, as it tells
|
||||
// us if the value is JSON or a literal string
|
||||
auto args = p.eps();
|
||||
if (schema.contains("properties") && !schema.at("properties").empty()) {
|
||||
auto arg_choices = p.choice();
|
||||
for (const auto & prop : schema.at("properties").items()) {
|
||||
const std::string & key = prop.key();
|
||||
|
||||
std::string type = "string";
|
||||
if (prop.value().is_object() && prop.value().contains("type") &&
|
||||
prop.value().at("type").is_string()) {
|
||||
type = prop.value().at("type").get<std::string>();
|
||||
}
|
||||
|
||||
auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) :
|
||||
p.tool_arg_value(p.until(ARG_END));
|
||||
|
||||
// skip the trailing type="..." attribute: anything up to <|sep|>
|
||||
arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key,
|
||||
p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) +
|
||||
p.tool_arg_name(p.literal(key)) + p.literal("\"") +
|
||||
p.until(SEP) + p.literal(SEP) + value +
|
||||
p.tool_arg_close(p.literal(ARG_END))));
|
||||
}
|
||||
args = p.zero_or_more(arg_choices);
|
||||
}
|
||||
|
||||
// skip the trailing index="N" attribute the same way
|
||||
auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") +
|
||||
p.until(SEP) + p.literal(SEP)) +
|
||||
p.tool_args(args) + p.tool_close(p.literal(CALL_END)));
|
||||
|
||||
tool_choices |= p.rule("kimi-k3-tool-" + name, call);
|
||||
});
|
||||
|
||||
// all calls go inside one tools section, then the message is closed. the
|
||||
// message closer is part of the trigger rule, or else the lazy grammar
|
||||
// rejects it once tool calls have started
|
||||
auto tools_section =
|
||||
p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) +
|
||||
p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) +
|
||||
p.optional(p.literal(EOM_TOKEN)));
|
||||
|
||||
auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :
|
||||
p.optional(tools_section);
|
||||
|
||||
return start + reasoning + response + tools + trailer + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
if (function.contains("parameters")) {
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list
|
||||
// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call
|
||||
bool is_lfm2_template(const std::string & src) {
|
||||
return src.find("<|tool_list_start|>") != std::string::npos &&
|
||||
src.find("<|tool_list_end|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable
|
||||
// (except dotted names and JSON literals true/false/null).
|
||||
// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning.
|
||||
// tool_list_tokens preserves LFM2 system tool-list markers.
|
||||
common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
bool tool_list_tokens) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string TOOL_CALL_START = "<|tool_call_start|>";
|
||||
const std::string TOOL_CALL_END = "<|tool_call_end|>";
|
||||
const std::string TOOL_LIST_START = "<|tool_list_start|>";
|
||||
const std::string TOOL_LIST_END = "<|tool_list_end|>";
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string GEN_PROMPT = "<|im_start|>assistant\n";
|
||||
|
||||
// Copy reasoning to the "thinking" field the template expects
|
||||
auto adjusted_messages = json::array();
|
||||
for (auto msg : inputs.messages) {
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
msg["thinking"] = msg.at("reasoning_content");
|
||||
}
|
||||
adjusted_messages.push_back(msg);
|
||||
}
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END };
|
||||
if (tool_list_tokens) {
|
||||
data.preserved_tokens.push_back(TOOL_LIST_START);
|
||||
data.preserved_tokens.push_back(TOOL_LIST_END);
|
||||
}
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
// Gate by reasoning format and whether the template supports <think>
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE &&
|
||||
tmpl.source().find(THINK_START) != std::string::npos;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
auto end = p.end();
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
|
||||
}
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
if (has_response_format) {
|
||||
auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
|
||||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
auto tool_calls = p.rule("tool-calls",
|
||||
p.trigger_rule("tool-call",
|
||||
p.literal(TOOL_CALL_START) +
|
||||
p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) +
|
||||
p.literal(TOOL_CALL_END)
|
||||
)
|
||||
);
|
||||
|
||||
auto content = p.content(p.until(TOOL_CALL_START));
|
||||
|
||||
return generation_prompt + reasoning + content + tool_calls + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// MiniCPM5 format:
|
||||
// - Reasoning: <think>{reasoning}</think> (optional)
|
||||
// - Tool calls: <function name="foo"><param name="bar">value</param></function>
|
||||
common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.preserved_tokens = {
|
||||
"<function",
|
||||
"<param",
|
||||
"</function>",
|
||||
"</param>",
|
||||
"<think>",
|
||||
"</think>",
|
||||
};
|
||||
|
||||
data.thinking_start_tag = "<think>";
|
||||
data.thinking_end_tags = {"</think>"};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|im_start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "\n</think>\n\n" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal("<|im_start|>assistant\n");
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space();
|
||||
}
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// CDATA lets a value carry characters that would otherwise close the tag (e.g.
|
||||
// </param>); capture the inner text only, excluding the CDATA markers.
|
||||
auto string_value = p.choice({
|
||||
p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),
|
||||
p.negate(p.literal("< {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto args = p.eps();
|
||||
if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
auto arg_choice = p.choice();
|
||||
for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {
|
||||
auto value_parser = p.eps();
|
||||
if (schema_info.resolves_to_string(prop_schema)) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)
|
||||
) + p.tool_arg_close(p.literal("</param>"));
|
||||
}
|
||||
|
||||
auto arg_rule = p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +
|
||||
value_parser
|
||||
);
|
||||
|
||||
arg_choice |= arg_rule;
|
||||
}
|
||||
args = p.zero_or_more(arg_choice + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">"))
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</function>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_parser);
|
||||
});
|
||||
|
||||
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
|
||||
auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));
|
||||
|
||||
auto content = p.content(p.until("<function"));
|
||||
|
||||
return generation_prompt + reasoning + content + tool_calls + p.end();
|
||||
}
|
||||
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + p.end();
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<mm:think>";
|
||||
data.thinking_end_tags = {"</mm:think>"};
|
||||
|
||||
// M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
|
||||
// params use the parameter name as the tag (<file_path>...</file_path>).
|
||||
const std::string NS = "]<]minimax[>[";
|
||||
const std::string THINK_START = "<mm:think>";
|
||||
const std::string THINK_END = "</mm:think>";
|
||||
const std::string FC_START = NS + "<tool_call>";
|
||||
const std::string FC_END = NS + "</tool_call>";
|
||||
const std::string INVOKE_END = NS + "</invoke>";
|
||||
|
||||
data.preserved_tokens = {
|
||||
NS,
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
THINK_START,
|
||||
THINK_END,
|
||||
};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" },
|
||||
{ COMMON_CHAT_ROLE_USER, "]~b]user" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "]~b]tool" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "]~b]system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
const std::string GEN_PROMPT = data.generation_prompt;
|
||||
|
||||
using mm3 = common_chat_peg_minimax_m3_mapper;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
|
||||
auto end = p.end();
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
auto block = inputs.enable_thinking
|
||||
? p.literal(THINK_START) + p.space() +
|
||||
p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)
|
||||
: p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);
|
||||
|
||||
// A turn without reasoning is prefixed with a bare </mm:think>, written either by the
|
||||
// generation prompt (thinking_mode = "disabled") or by the model itself.
|
||||
reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));
|
||||
}
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("```json") + p.space() +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.space() + p.literal("```"));
|
||||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
auto alternatives_of = [](const json & schema) -> std::optional<json> {
|
||||
for (const auto * keyword : { "oneOf", "anyOf" }) {
|
||||
if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) {
|
||||
return schema.at(keyword);
|
||||
}
|
||||
}
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
// The template expands argument values recursively in XML (see the to_xml() macro)
|
||||
std::function<common_peg_parser(const json &, const std::string &, const std::string &)> value_of;
|
||||
std::function<common_peg_parser(const json &, const std::string &)> members_of;
|
||||
|
||||
auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) {
|
||||
const std::string close = NS + "</" + tag + ">";
|
||||
return p.rule(rule_name,
|
||||
p.tool_arg(
|
||||
p.tool_arg_open(
|
||||
p.literal(NS + "<") +
|
||||
p.tool_arg_name(p.literal(tag)) +
|
||||
p.literal(">")) +
|
||||
value_of(schema, rule_name, close)));
|
||||
};
|
||||
|
||||
value_of = [&](const json & schema,
|
||||
const std::string & rule_name,
|
||||
const std::string & close) -> common_peg_parser {
|
||||
auto close_tag = p.tool_arg_close(p.literal(close));
|
||||
|
||||
// A string accepts anything, so a union with a string alternative is a string
|
||||
if (schema_info.resolves_to_string(schema)) {
|
||||
return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
|
||||
}
|
||||
|
||||
if (auto alternatives = alternatives_of(schema)) {
|
||||
std::vector<common_peg_parser> choices;
|
||||
|
||||
size_t index = 0;
|
||||
for (const auto & alternative : *alternatives) {
|
||||
const std::string alt_name = rule_name + "-" + std::to_string(index++);
|
||||
|
||||
// There is a risk that this breaks streaming deltas, but that's a risk we
|
||||
// assume to provide tool arg streaming.
|
||||
choices.push_back(value_of(alternative, alt_name, close));
|
||||
}
|
||||
|
||||
return p.choice(choices);
|
||||
}
|
||||
|
||||
const std::string type = schema.contains("type") && schema.at("type").is_string()
|
||||
? schema.at("type").get<std::string>()
|
||||
: "";
|
||||
|
||||
if (type == "object" && schema.contains("properties")) {
|
||||
return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag;
|
||||
}
|
||||
|
||||
if (type == "array" && schema.contains("items")) {
|
||||
const std::string item_close = NS + "</item>";
|
||||
auto item = p.rule(rule_name + "-item",
|
||||
p.tag(mm3::TOOL_ARG_ITEM,
|
||||
p.literal(NS + "<item>") +
|
||||
value_of(schema.at("items"), rule_name + "-item", item_close)));
|
||||
return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
|
||||
}
|
||||
|
||||
return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag;
|
||||
};
|
||||
|
||||
// Required properties in schema order, then any number of optional ones in any order.
|
||||
members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser {
|
||||
const auto & props = schema.at("properties");
|
||||
|
||||
std::set<std::string> required;
|
||||
if (schema.contains("required")) {
|
||||
required = schema.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
|
||||
std::vector<common_peg_parser> required_elements;
|
||||
std::vector<common_peg_parser> optional_elements;
|
||||
for (const auto & [key, key_schema] : props.items()) {
|
||||
auto element = element_of(key, key_schema, rule_prefix + "-" + key);
|
||||
if (required.find(key) != required.end()) {
|
||||
required_elements.push_back(element);
|
||||
} else {
|
||||
optional_elements.push_back(element);
|
||||
}
|
||||
}
|
||||
|
||||
common_peg_parser members = p.eps();
|
||||
for (size_t i = 0; i < required_elements.size(); i++) {
|
||||
if (i > 0) {
|
||||
members = members + p.space();
|
||||
}
|
||||
members = members + required_elements[i];
|
||||
}
|
||||
|
||||
if (!optional_elements.empty()) {
|
||||
common_peg_parser any_optional = p.choice();
|
||||
for (const auto & element : optional_elements) {
|
||||
any_optional |= element;
|
||||
}
|
||||
members = members + p.repeat(p.space() + any_optional, 0, -1);
|
||||
}
|
||||
|
||||
return members;
|
||||
};
|
||||
|
||||
common_peg_parser invoke_body =
|
||||
params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps();
|
||||
|
||||
auto func_parser = p.tool(
|
||||
p.tool_open(p.literal(NS + "<invoke name=\"") +
|
||||
p.tool_name(p.literal(name)) + p.literal("\">")) +
|
||||
p.space() + invoke_body + p.space() +
|
||||
p.tool_close(p.literal(INVOKE_END)));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, func_parser);
|
||||
});
|
||||
|
||||
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
|
||||
common_peg_parser tool_calls = p.eps();
|
||||
if (inputs.parallel_tool_calls) {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice +
|
||||
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
||||
} else {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
||||
}
|
||||
|
||||
if (!require_tools) {
|
||||
tool_calls = p.optional(tool_calls);
|
||||
}
|
||||
|
||||
auto content_before_tools = p.content(p.until(FC_START));
|
||||
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja
|
||||
auto adjusted_messages = json::array();
|
||||
for (const auto & msg : inputs.messages) {
|
||||
auto role = msg.value("role", "");
|
||||
if (role != "system" && role != "assistant") {
|
||||
// Only adjust system and assistant messages. Interestingly, the system message may contain thinking.
|
||||
adjusted_messages.push_back(msg);
|
||||
continue;
|
||||
}
|
||||
|
||||
auto content = json::array();
|
||||
|
||||
// If message contains `reasoning_content`, add it as a block of type `thinking`
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
content.push_back({
|
||||
{ "type", "thinking" },
|
||||
{ "thinking", msg.at("reasoning_content").get<std::string>() },
|
||||
});
|
||||
}
|
||||
|
||||
// If message contains `content`, add it as a block of type `text`
|
||||
if (msg.contains("content")) {
|
||||
if (msg.at("content").is_string()) {
|
||||
content.push_back({
|
||||
{ "type", "text" },
|
||||
{ "text", msg.at("content").get<std::string>() },
|
||||
});
|
||||
} else if (msg.at("content").is_array()) {
|
||||
auto blocks = msg.at("content");
|
||||
content.insert(blocks);
|
||||
}
|
||||
}
|
||||
|
||||
auto adjusted = msg;
|
||||
adjusted["content"] = content;
|
||||
adjusted.erase("reasoning_content");
|
||||
adjusted_messages.push_back(adjusted);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = true;
|
||||
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "[THINK]";
|
||||
data.thinking_end_tags = {"[/THINK]"};
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.preserved_tokens = {
|
||||
"[THINK]",
|
||||
"[/THINK]",
|
||||
"[TOOL_CALLS]",
|
||||
"[ARGS]",
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "[THINK]" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "[/THINK]" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.eps();
|
||||
auto reasoning =
|
||||
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
// Ministral wants to emit json surrounded by code fences
|
||||
return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");
|
||||
}
|
||||
|
||||
// Tool call parser
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
|
||||
tool_choice |=
|
||||
p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +
|
||||
p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));
|
||||
});
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
|
||||
auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));
|
||||
|
||||
return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
include_grammar = false;
|
||||
return generation_prompt + (reasoning << p.content(p.rest()));
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// An assistant turn is rendered as one or more messages, each
|
||||
// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
|
||||
// <|eom|> (more messages follow) or <|eot|> (end of turn):
|
||||
// - chain-of-thought: to=self, terminated by <|eom|>
|
||||
// - final answer: to=user, terminated by <|eot|>
|
||||
// The generation prompt is just "<|start|>assistant"; the model emits its own
|
||||
// " to=...<|message|>".
|
||||
common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = "<|start|>assistant";
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
|
||||
// ATEM tool-call markup emitted on " to=<tool>" turns.
|
||||
"<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
|
||||
"</atem:invoke>", "</atem:function_calls>",
|
||||
};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
// Constrained grammar whenever tools are offered.
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
|
||||
if (!extract_reasoning && !include_grammar) {
|
||||
return start + p.content(p.rest());
|
||||
}
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
} else {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
}
|
||||
auto analysis = p.ref("analysis");
|
||||
|
||||
auto recipient = p.optional(p.literal(" to=user"));
|
||||
auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
|
||||
p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
"</atem:parameter>");
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto args = p.eps();
|
||||
if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
auto arg_choice = p.choice();
|
||||
for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {
|
||||
auto value_parser = p.eps();
|
||||
if (schema_info.resolves_to_string(prop_schema)) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false))
|
||||
+ p.tool_arg_close(p.literal("</atem:parameter>"));
|
||||
}
|
||||
|
||||
auto arg_rule = p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +
|
||||
value_parser);
|
||||
|
||||
arg_choice |= arg_rule;
|
||||
}
|
||||
args = p.zero_or_more(arg_choice + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal(" to=") + p.until("<|message|>") +
|
||||
p.literal("<|message|><atem:function_calls>") + p.space() +
|
||||
p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_parser);
|
||||
});
|
||||
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
|
||||
: p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + analysis) + start + tool_calls;
|
||||
}
|
||||
auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
|
||||
return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + final_msg;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
|
||||
"<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
#include "parsers.h"
|
||||
|
||||
#include "log.h"
|
||||
|
||||
#include <set>
|
||||
|
||||
void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
|
||||
for (const auto & tool : tools) {
|
||||
if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
|
||||
LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());
|
||||
continue;
|
||||
}
|
||||
fn(tool);
|
||||
}
|
||||
}
|
||||
|
||||
void foreach_parameter(const json & function, const std::function<void(const std::string &, const json &, bool)> & fn) {
|
||||
if (!function.contains("parameters") || !function.at("parameters").is_object()) {
|
||||
return;
|
||||
}
|
||||
const auto & params = function.at("parameters");
|
||||
if (!params.contains("properties") || !params.at("properties").is_object()) {
|
||||
return;
|
||||
}
|
||||
const auto & props = params.at("properties");
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required") && params.at("required").is_array()) {
|
||||
required = params.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
for (const auto & [name, prop] : props.items()) {
|
||||
bool is_required = (required.find(name) != required.end());
|
||||
fn(name, prop, is_required);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
#pragma once
|
||||
|
||||
#include "chat.h"
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using json = common_json;
|
||||
|
||||
// iterate over the function tools of an OpenAI-style tools array
|
||||
void foreach_function(const json & tools, const std::function<void(const json &)> & fn);
|
||||
|
||||
// iterate over the parameters of a function tool, flagging the ones listed as required
|
||||
void foreach_parameter(const json & function, const std::function<void(const std::string &, const json &, bool)> & fn);
|
||||
|
||||
// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten
|
||||
std::string common_chat_template_direct_apply_impl(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
const std::optional<json> & messages_override = std::nullopt,
|
||||
const std::optional<json> & tools_override = std::nullopt,
|
||||
const std::optional<json> & additional_context = std::nullopt);
|
||||
|
||||
// the suffix a template appends when add_generation_prompt is set
|
||||
std::string common_chat_template_generation_prompt_impl(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
const std::optional<json> & messages_override = std::nullopt,
|
||||
const std::optional<json> & tools_override = std::nullopt,
|
||||
const std::optional<json> & additional_context = std::nullopt);
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
namespace workaround {
|
||||
|
||||
void convert_tool_responses_gemma4(json & messages);
|
||||
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_gigachat_v3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
// tool_list_tokens preserves the LFM2 system tool-list markers; LFM2.5 renders without them
|
||||
common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, const autoparser::generation_params & inputs, bool tool_list_tokens);
|
||||
|
||||
common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
@@ -0,0 +1,181 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string GEN_PREFIX = "<|im_start|>assistant\n";
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
|
||||
auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
|
||||
|
||||
data.supports_thinking = supports_reasoning;
|
||||
data.preserved_tokens = {
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
};
|
||||
|
||||
auto is_qwen3_coder = !supports_reasoning;
|
||||
|
||||
if (supports_reasoning) {
|
||||
data.thinking_start_tag = "<think>";
|
||||
// Support both </think> and <tool_call> as reasoning end sequences.
|
||||
// <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
|
||||
data.thinking_end_tags = { "</think>", "<tool_call>" };
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
|
||||
}
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash
|
||||
{ COMMON_CHAT_ROLE_USER, "<|im_start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PREFIX;
|
||||
if (supports_reasoning) {
|
||||
data.generation_prompt += "<think>\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "\n</think>\n\n";
|
||||
}
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
std::vector<std::string> tool_call_starts = { "<tool_call>" };
|
||||
|
||||
if (is_qwen3_coder) {
|
||||
// Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
|
||||
// starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
|
||||
// constraining on <function which may occur in valid content generation, e.g. #include <functional>
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const std::string name = tool.at("function").at("name");
|
||||
tool_call_starts.push_back("<function=" + name + ">");
|
||||
});
|
||||
}
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (supports_reasoning && extract_reasoning) {
|
||||
reasoning = p.optional("<think>" + p.space() +
|
||||
p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
|
||||
(p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
|
||||
}
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
|
||||
}
|
||||
|
||||
// Tool call parser
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));
|
||||
auto arg_string = p.rule("xml-arg-string",
|
||||
p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(parameters);
|
||||
|
||||
std::vector<common_peg_parser> required_args;
|
||||
std::vector<common_peg_parser> optional_args;
|
||||
|
||||
foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {
|
||||
auto rule_name = "tool-" + name + "-arg-" + param_name;
|
||||
|
||||
auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");
|
||||
|
||||
auto arg_value = schema_info.resolves_to_string(param_schema) ?
|
||||
arg_string :
|
||||
p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;
|
||||
|
||||
auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
|
||||
|
||||
(is_required ? required_args : optional_args).push_back(arg_rule);
|
||||
});
|
||||
|
||||
// Accept required arguments in any order, as Qwen does not always adhere to the
|
||||
// order provided.
|
||||
auto args = p.permute("tool-" + name + "-args", required_args);
|
||||
if (!optional_args.empty()) {
|
||||
args = args + p.zero_or_more(p.choice(optional_args));
|
||||
}
|
||||
|
||||
auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
|
||||
p.tool_args(args) +
|
||||
p.tool_close(p.literal("</function>\n")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, func);
|
||||
});
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
|
||||
auto tool_call_body = tool_choice + "</tool_call>" + p.space();
|
||||
auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
|
||||
|
||||
// Qwen3-Coder models may occasionally omit the <tool_call> token.
|
||||
auto tool_call_first = is_qwen3_coder ?
|
||||
p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :
|
||||
tool_call;
|
||||
|
||||
auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
|
||||
|
||||
return generation_prompt +
|
||||
(reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
return generation_prompt + (reasoning << p.content(p.rest()));
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
for (const auto & start : tool_call_starts) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
# Specialized chat template parsers, listed explicitly so that adding or removing one re-runs CMake instead of leaving an incremental build stale.
|
||||
|
||||
set(LLAMA_CHAT_PARSERS_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/parsers.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/parsers.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/cohere2moe.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/deepseek.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/functionary-v3-2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/lfm2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/minicpm5.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp
|
||||
)
|
||||
+83
-43
@@ -26001,7 +26001,7 @@ Current version indicated by LITEVER below.
|
||||
element.innerHTML = pend;
|
||||
});
|
||||
} else {
|
||||
render_gametext(false);
|
||||
render_gametext(false,false);
|
||||
}
|
||||
|
||||
handle_autoscroll(false);
|
||||
@@ -26083,62 +26083,102 @@ Current version indicated by LITEVER below.
|
||||
|
||||
}
|
||||
|
||||
var autoscrollEnabled = true;
|
||||
var suppressScrollEvent = false;
|
||||
var autoscrollTimeout = null;
|
||||
const autoscrollStates = new Map();
|
||||
let autoscrollRAF = null;
|
||||
var lastScrollTop = 0;
|
||||
function handle_autoscroll(alwaysscroll = true) {
|
||||
if (!localsettings.autoscroll) return;
|
||||
|
||||
// Cancel pending frame to avoid stacking work
|
||||
if (autoscrollRAF) cancelAnimationFrame(autoscrollRAF);
|
||||
// Explicit navigation resets following now, so subsequent user input can
|
||||
// still override it before the frame runs. Streaming never resets it.
|
||||
if (alwaysscroll) {
|
||||
for (let state of autoscrollStates.values()) state.following = true;
|
||||
}
|
||||
if (autoscrollRAF !== null) return;
|
||||
|
||||
autoscrollRAF = requestAnimationFrame(() => {
|
||||
const boxes = [
|
||||
document.getElementById("gametext"),
|
||||
document.getElementById("chat_msg_body"),
|
||||
document.getElementById("corpostylemain")
|
||||
].filter(Boolean);
|
||||
|
||||
if (!boxes.length) return;
|
||||
|
||||
// ---- READ PHASE (no writes) ----
|
||||
const threshold = autoscrollEnabled ? 195 : 70;
|
||||
const scrollData = boxes.map(el => {
|
||||
const maxScrollTop = el.scrollHeight - el.clientHeight + 10;
|
||||
const isAtBottom =
|
||||
el.scrollHeight - el.scrollTop - el.clientHeight <= threshold;
|
||||
|
||||
return { el, maxScrollTop, shouldScroll: alwaysscroll || isAtBottom };
|
||||
});
|
||||
|
||||
// ---- WRITE PHASE (no reads) ----
|
||||
suppressScrollEvent = true;
|
||||
for (let item of scrollData) {
|
||||
if (item.shouldScroll) {
|
||||
item.el.scrollTop = item.maxScrollTop;
|
||||
autoscrollRAF = null;
|
||||
if (!localsettings.autoscroll) return;
|
||||
const scrollData = [];
|
||||
for (let [element, state] of autoscrollStates) {
|
||||
if (!element.clientHeight) continue; // Ignore hidden layouts.
|
||||
if (state.following && !state.pointerDown) {
|
||||
scrollData.push({element, state, top: Math.max(0, element.scrollHeight - element.clientHeight)});
|
||||
}
|
||||
}
|
||||
suppressScrollEvent = false;
|
||||
for (let {element, state, top} of scrollData) {
|
||||
// Scroll events arrive asynchronously. Record the position we write
|
||||
// instead of briefly suppressing events around the assignment.
|
||||
state.lastScrollTop = top;
|
||||
element.scrollTop = top;
|
||||
}
|
||||
});
|
||||
}
|
||||
function attach_scroll_detection(element)
|
||||
{
|
||||
element.addEventListener("scroll", () => {
|
||||
const currentScrollTop = element.scrollTop;
|
||||
const delta = currentScrollTop - lastScrollTop;
|
||||
lastScrollTop = currentScrollTop;
|
||||
if (suppressScrollEvent) { return; }
|
||||
if (delta>=0) { return; }
|
||||
autoscrollEnabled = false;
|
||||
if (autoscrollTimeout) {
|
||||
clearTimeout(autoscrollTimeout);
|
||||
if (!element || autoscrollStates.has(element)) return;
|
||||
const state = {following: true, lastScrollTop: element.scrollTop, pointerDown: false};
|
||||
autoscrollStates.set(element, state);
|
||||
const pause = () => { state.following = false; };
|
||||
|
||||
// Stop before the next render, even if the browser has not dispatched
|
||||
// the corresponding scroll event yet. Leave native scrolling untouched.
|
||||
element.addEventListener("wheel", event => {
|
||||
if (event.deltaY < 0 && !event.ctrlKey) pause();
|
||||
}, {passive: true});
|
||||
let touchY = null;
|
||||
element.addEventListener("touchstart", event => {
|
||||
touchY = event.touches.length === 1 ? event.touches[0].clientY : null;
|
||||
}, {passive: true});
|
||||
element.addEventListener("touchmove", event => {
|
||||
if (touchY !== null && event.touches.length === 1) {
|
||||
const nextY = event.touches[0].clientY;
|
||||
if (nextY > touchY) pause();
|
||||
touchY = nextY;
|
||||
}
|
||||
autoscrollTimeout = setTimeout(() => {
|
||||
autoscrollEnabled = true;
|
||||
}, 180);
|
||||
}, {passive: true});
|
||||
element.addEventListener("keydown", event => {
|
||||
if (event.defaultPrevented || event.target.closest("input, textarea, select, [contenteditable=true]")) return;
|
||||
if (["ArrowUp", "PageUp", "Home"].includes(event.key) || (event.key === " " && event.shiftKey)) pause();
|
||||
});
|
||||
|
||||
const onScroll = () => {
|
||||
if (!element.clientHeight) return;
|
||||
const maxTop = Math.max(0, element.scrollHeight - element.clientHeight);
|
||||
// Clamp Safari rubber-band overscroll; allow only rounding tolerance.
|
||||
const top = Math.max(0, Math.min(element.scrollTop, maxTop));
|
||||
const previousTop = state.lastScrollTop;
|
||||
state.lastScrollTop = top;
|
||||
// Reflow and scroll anchoring can also move scrollTop upward. Only
|
||||
// mouse/pen interaction makes that a manual scroll; wheel, touch and
|
||||
// keyboard intent is handled directly above.
|
||||
if (state.pointerDown && top < Math.min(previousTop, maxTop) - 1) {
|
||||
pause();
|
||||
} else if (top > previousTop && maxTop - top <= 1) {
|
||||
state.following = true;
|
||||
}
|
||||
};
|
||||
element.addEventListener("scroll", onScroll, {passive: true});
|
||||
|
||||
// Let scrollbar dragging and selection scrolling finish before writing
|
||||
// scrollTop. Listen outside the element too, since a drag can end there.
|
||||
element.addEventListener("pointerdown", event => {
|
||||
if (event.pointerType !== "touch") state.pointerDown = true;
|
||||
}, {passive: true});
|
||||
const releasePointer = () => {
|
||||
if (!state.pointerDown) return;
|
||||
onScroll();
|
||||
state.pointerDown = false;
|
||||
handle_autoscroll(false);
|
||||
};
|
||||
document.addEventListener("pointerup", releasePointer, {passive: true});
|
||||
document.addEventListener("pointercancel", releasePointer, {passive: true});
|
||||
|
||||
// Keep a followed view at the bottom after reflow or late-loading media.
|
||||
if (typeof ResizeObserver !== "undefined") {
|
||||
const observer = new ResizeObserver(() => handle_autoscroll(false));
|
||||
observer.observe(element);
|
||||
}
|
||||
element.addEventListener("load", () => handle_autoscroll(false), true);
|
||||
}
|
||||
|
||||
function render_gametext(save=true, force_scroll=true)
|
||||
|
||||
Reference in New Issue
Block a user