Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-25 22:21:03 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
llama.cpp/tests/test-chat-auto-parser.cpp
T
Xuan-Son Nguyen 4a08fa2970 test: move tools/parser to tests (#27548)
2026-08-23 18:38:51 +02:00

2708 lines
122 KiB
C++
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#include "chat-auto-parser-helpers.h"
#include "chat-auto-parser.h"
#include "chat-peg-parser.h"
#include "chat.h"
#include "gguf.h"
#include "jinja/runtime.h"
#include "log.h"
#include "peg-parser.h"
#include "testing.h"
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <optional>
#include <sstream>
#include <string>
using namespace autoparser;
static void test_calculate_diff_split_basic(testing & t);
static void test_calculate_diff_split_identical(testing & t);
static void test_calculate_diff_split_common_prefix(testing & t);
static void test_calculate_diff_split_common_suffix(testing & t);
static void test_calculate_diff_split_common_both(testing & t);
static void test_calculate_diff_split_empty_cases(testing & t);
static void test_calculate_diff_split_no_common(testing & t);
static void test_calculate_diff_split_single_char(testing & t);
static void test_calculate_diff_split_overlaps(testing & t);
static void test_calculate_diff_split_tag_boundaries(testing & t);
static void test_calculate_diff_split_generation_prompt(testing & t);
static void test_calculate_diff_split(testing & t);
static void test_until_common_prefix_basic(testing & t);
static void test_until_common_prefix(testing & t);
static void test_after_common_suffix_basic(testing & t);
static void test_after_common_suffix(testing & t);
static void test_analyze_tool_call_pure_json(testing & t);
static void test_analyze_tool_call_function_name_markers(testing & t);
static void test_analyze_tool_call_full_markers(testing & t);
static void test_analyze_tool_call_edge_cases(testing & t);
static void test_compare_variants_basic(testing & t);
static void test_compare_variants_messages_modifier(testing & t);
static void test_compare_variants_tools_modifier(testing & t);
static void test_compare_variants_both_modifiers(testing & t);
static void test_compare_variants_template_failure(testing & t);
static void test_compare_variants_identity(testing & t);
static void test_compare_variants(testing & t);
// Seed-OSS template tool calling analysis tests
static void test_seed_oss_tool_analysis(testing & t);
static void test_seed_oss_tool_presence(testing & t);
static void test_seed_oss_call_count(testing & t);
static void test_seed_oss_function_names(testing & t);
static void test_seed_oss_argument_count(testing & t);
static void test_seed_oss_args_presence(testing & t);
static void test_seed_oss_tool_with_reasoning(testing & t);
// Nemotron template analysis tests
static void test_nemotron_analysis(testing & t);
static void test_nemotron_reasoning_detection(testing & t);
static void test_nemotron_tool_format(testing & t);
static void test_laguna_analysis(testing & t);
static void test_laguna_reasoning_detection(testing & t);
static void test_laguna_tool_format(testing & t);
static void test_laguna_s_analysis(testing & t);
static void test_laguna_s_reasoning_detection(testing & t);
static void test_laguna_s_tool_format(testing & t);
static void test_laguna_s_preserve_reasoning(testing & t);
static void test_laguna_xs2_analysis(testing & t);
static void test_laguna_xs2_reasoning_detection(testing & t);
static void test_laguna_xs2_tool_format(testing & t);
// CohereForAI template analysis tests
static void test_cohere_reasoning_detection(testing & t);
static void test_cohere_analysis(testing & t);
// SmolLM3 template analysis tests
static void test_smollm3_analysis(testing & t);
// Marker separation
static void test_marker_separation(testing & t);
// standard_json_tools format tests
static void test_standard_json_tools_formats(testing & t);
static void test_standard_json_tools_openai(testing & t);
static void test_standard_json_tools_cohere(testing & t);
static void test_standard_json_tools_function_key(testing & t);
// normalize_quotes_to_json tests
static void test_normalize_quotes_to_json(testing & t);
static void test_normalize_quotes_with_embedded_quotes(testing & t);
// TAG_WITH_TAGGED argument parsing tests
static void test_tagged_args_with_embedded_quotes(testing & t);
static void test_bailing_v3_tool_format(testing & t);
static void test_role_markers_all_templates(testing & t);
static json build_tools_definition();
//
// debug mode: analyze a single template and dump the generated parser and grammar
//
enum class output_mode {
ANALYSIS, // Only output analysis results (default)
TEMPLATE, // Only output rendered template
BOTH // Output both
};
enum class input_message_type {
NONE, // Don't render any message scenarios (only analysis)
CONTENT_ONLY, // Simple assistant message with content
REASONING_CONTENT, // Message with reasoning_content + content
TOOL_CALL_ONLY, // Message with tool_calls only
CONTENT_TOOL_CALL, // Message with content + tool_calls
REASONING_TOOL_CALL, // Message with reasoning_content + tool_calls
CONTENT_FAKE_TOOL_CALL, // Message with content but no actual tool_calls (for testing)
ALL // Render all scenarios
};
struct debug_options {
std::string template_path;
bool with_tools = true;
bool generation_prompt = true;
bool enable_reasoning = true;
bool debug_jinja = false;
bool force_tool_call = false;
bool parallel_tool_calls = true;
output_mode mode = output_mode::BOTH;
input_message_type input_message = input_message_type::NONE;
};
static std::string read_file(const std::string & path) {
std::ifstream fin(path, std::ios::binary);
if (!fin.is_open()) {
throw std::runtime_error("Could not open file: " + path);
}
std::ostringstream buf;
buf << fin.rdbuf();
return buf.str();
}
static std::string read_gguf_chat_template(const std::string & path) {
struct gguf_init_params params = { /*no_alloc =*/true, // We only need metadata, not tensor data
/*ctx=*/nullptr };
struct gguf_context * ctx = gguf_init_from_file(path.c_str(), params);
if (ctx == nullptr) {
throw std::runtime_error("Could not open GGUF file: " + path);
}
const char * key = "tokenizer.chat_template";
int64_t key_id = gguf_find_key(ctx, key);
if (key_id == -1) {
gguf_free(ctx);
throw std::runtime_error("GGUF file does not contain chat template key: " + std::string(key));
}
const char * template_str = gguf_get_val_str(ctx, key_id);
if (template_str == nullptr) {
gguf_free(ctx);
throw std::runtime_error("GGUF file contains chat template key but value is null");
}
std::string result = template_str;
gguf_free(ctx);
return result;
}
static void print_usage(const char * program_name) {
LOG_ERR("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n");
LOG_ERR("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name);
LOG_ERR(" %s <template_or_gguf_path> [options] debug a single template\n", program_name);
LOG_ERR("\nDebug mode options:\n");
LOG_ERR(" --no-tools Disable tool definitions\n");
LOG_ERR(" --force-tool-call Set tool calls to forced\n");
LOG_ERR(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n");
LOG_ERR(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n");
LOG_ERR(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n");
LOG_ERR(" --output=MODE Output mode: analysis, template, both (default: both)\n");
LOG_ERR(" --debug-jinja Enable Jinja fine-grained debug\n");
LOG_ERR(" --input-message=TYPE Message type to render:\n");
LOG_ERR(" content_only, reasoning_content, tool_call_only,\n");
LOG_ERR(" content_tool_call, reasoning_tool_call,\n");
LOG_ERR(" content_fake_tool_call, all\n");
LOG_ERR("\nExamples:\n");
LOG_ERR(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name);
LOG_ERR(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name);
}
static bool parse_bool_option(const std::string & value) {
return value == "1" || value == "true" || value == "yes";
}
static bool parse_debug_options(int argc, char ** argv, debug_options & opts) {
opts.template_path = argv[1];
for (int i = 2; i < argc; ++i) {
std::string arg = argv[i];
if (arg == "--force-tool-call") {
opts.force_tool_call = true;
} else if (arg == "--debug-jinja") {
opts.debug_jinja = true;
} else if (arg == "--no-tools") {
opts.with_tools = false;
} else if (arg.rfind("--parallel-tool-calls=", 0) == 0) {
opts.parallel_tool_calls = parse_bool_option(arg.substr(22));
} else if (arg.rfind("--generation-prompt=", 0) == 0) {
opts.generation_prompt = parse_bool_option(arg.substr(20));
} else if (arg.rfind("--enable-reasoning=", 0) == 0) {
opts.enable_reasoning = parse_bool_option(arg.substr(19));
} else if (arg.rfind("--output=", 0) == 0) {
std::string mode = arg.substr(9);
if (mode == "analysis") {
opts.mode = output_mode::ANALYSIS;
} else if (mode == "template") {
opts.mode = output_mode::TEMPLATE;
} else if (mode == "both") {
opts.mode = output_mode::BOTH;
} else {
LOG_ERR("Unknown output mode: %s\n", mode.c_str());
return false;
}
} else if (arg.rfind("--input-message=", 0) == 0) {
std::string type = arg.substr(16);
if (type == "content_only") {
opts.input_message = input_message_type::CONTENT_ONLY;
} else if (type == "reasoning_content") {
opts.input_message = input_message_type::REASONING_CONTENT;
} else if (type == "tool_call_only") {
opts.input_message = input_message_type::TOOL_CALL_ONLY;
} else if (type == "content_tool_call") {
opts.input_message = input_message_type::CONTENT_TOOL_CALL;
} else if (type == "reasoning_tool_call") {
opts.input_message = input_message_type::REASONING_TOOL_CALL;
} else if (type == "content_fake_tool_call") {
opts.input_message = input_message_type::CONTENT_FAKE_TOOL_CALL;
} else if (type == "all") {
opts.input_message = input_message_type::ALL;
} else {
LOG_ERR("Unknown input message type: %s\n", type.c_str());
return false;
}
} else {
LOG_ERR("Unknown option: %s\n", arg.c_str());
print_usage(argv[0]);
return false;
}
}
return true;
}
static json build_debug_user_message() {
return json{
{ "role", "user" },
{ "content", "Hello, please help me with a task." }
};
}
static json build_content_only_message() {
return json{
{ "role", "assistant" },
{ "content", "Hello! I'm here to help you with your task." }
};
}
static json build_reasoning_content_message() {
return json{
{ "role", "assistant" },
{ "content", "Hello! I'm here to help you with your task." },
{ "reasoning_content", "The user is greeting me and asking for help. I should respond politely." }
};
}
static json build_tool_call_only_message() {
return json{
{ "role", "assistant" },
{ "content", nullptr },
{ "tool_calls",
json::array({ json{
{ "type", "function" },
{ "function", json{ { "name", "test_function_name" },
{ "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } },
{ "id", "123456789" } } }) }
};
}
static json build_content_tool_call_message() {
return json{
{ "role", "assistant" },
{ "content", "I'll help you by calling a function." },
{ "tool_calls",
json::array({ json{
{ "type", "function" },
{ "function",
json{ { "name", "test_function_name" },
{ "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) }
};
}
static json build_reasoning_tool_call_message() {
return json{
{ "role", "assistant" },
{ "content", nullptr },
{ "reasoning_content", "I need to call a function to help with this task." },
{ "tool_calls",
json::array({ json{
{ "type", "function" },
{ "function",
json{ { "name", "test_function_name" },
{ "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) }
};
}
static json build_content_fake_tool_call_message() {
// This message has content but NO tool_calls field
// It's used to test if a template renders tool definitions but not tool calls
return json{
{ "role", "assistant" },
{ "content", "I'll help you by calling a function." }
};
}
static void render_scenario(const common_chat_template & tmpl,
const std::string & scenario_name,
const json & messages,
const json & tools,
bool add_generation_prompt,
bool enable_thinking) {
LOG_ERR("\n=== Scenario: %s ===\n", scenario_name.c_str());
LOG_ERR("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false",
enable_thinking ? "true" : "false");
// When add_generation_prompt is true, add a trailing user message to trigger the prompt
json final_messages = messages;
if (add_generation_prompt && !messages.empty() && messages.back().value("role", "") == "assistant") {
final_messages.push_back(json{
{ "role", "user" },
{ "content", "Now please continue with another response." }
});
}
LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str());
try {
generation_params inputs;
inputs.messages = final_messages;
inputs.add_generation_prompt = add_generation_prompt;
inputs.extra_context["enable_thinking"] = enable_thinking;
if (!tools.is_null() && tools.is_array() && !tools.empty()) {
inputs.tools = tools;
}
std::string output = common_chat_template_direct_apply(tmpl, inputs);
LOG_ERR("\n--- Rendered Output ---\n");
LOG_ERR("%s\n", output.c_str());
LOG_ERR("--- End Output (length: %zu) ---\n", output.length());
} catch (const std::exception & e) {
LOG_ERR("Rendering failed: %s\n", e.what());
}
}
static void render_all_scenarios(const common_chat_template & tmpl,
const json & tools,
bool add_generation_prompt,
bool enable_thinking,
input_message_type message_type) {
json user_msg = build_debug_user_message();
auto render_if = [&](input_message_type type, const std::string & name, const json & assistant_msg) {
if (message_type == input_message_type::ALL || message_type == type) {
json messages = json::array({ user_msg, assistant_msg });
render_scenario(tmpl, name, messages, tools, add_generation_prompt, enable_thinking);
}
};
render_if(input_message_type::CONTENT_ONLY, "content_only", build_content_only_message());
render_if(input_message_type::REASONING_CONTENT, "reasoning_content", build_reasoning_content_message());
render_if(input_message_type::TOOL_CALL_ONLY, "tool_call_only", build_tool_call_only_message());
render_if(input_message_type::CONTENT_TOOL_CALL, "content_tool_call", build_content_tool_call_message());
render_if(input_message_type::REASONING_TOOL_CALL, "reasoning_tool_call", build_reasoning_tool_call_message());
render_if(input_message_type::CONTENT_FAKE_TOOL_CALL, "content_fake_tool_call",
build_content_fake_tool_call_message());
// Also render with add_generation_prompt=true to show the prompt ending
if (message_type == input_message_type::ALL) {
LOG_ERR("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n");
json prompt_messages = json::array({ user_msg });
render_scenario(tmpl, "generation_prompt_only", prompt_messages, tools, true, enable_thinking);
// With enable_thinking toggled
render_scenario(tmpl, "generation_prompt_thinking_disabled", prompt_messages, tools, true, false);
}
}
static generation_params prepare_debug_params(const debug_options & opts, const json & tools) {
generation_params params;
params.messages = json::array({ build_debug_user_message() });
params.reasoning_format = opts.enable_reasoning ? COMMON_REASONING_FORMAT_DEEPSEEK : COMMON_REASONING_FORMAT_NONE;
params.enable_thinking = opts.enable_reasoning;
params.add_generation_prompt = opts.generation_prompt;
if (opts.with_tools) {
params.tools = tools;
params.tool_choice = opts.force_tool_call ? COMMON_CHAT_TOOL_CHOICE_REQUIRED : COMMON_CHAT_TOOL_CHOICE_AUTO;
} else {
params.tools = json();
params.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE;
}
params.parallel_tool_calls = opts.parallel_tool_calls;
return params;
}
static int debug_single_template(const debug_options & opts) {
std::string template_source;
try {
// Check if the file is a GGUF file
if (opts.template_path.size() >= 5 &&
opts.template_path.compare(opts.template_path.size() - 5, 5, ".gguf") == 0) {
template_source = read_gguf_chat_template(opts.template_path);
} else {
template_source = read_file(opts.template_path);
}
} catch (const std::exception & e) {
LOG_ERR("Error reading template: %s\n", e.what());
return 1;
}
LOG_ERR("Analyzing template: %s\n", opts.template_path.c_str());
LOG_ERR("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false",
opts.generation_prompt ? "true" : "false", opts.enable_reasoning ? "true" : "false");
try {
common_chat_template chat_template(template_source, "", "");
json tools = opts.with_tools ? build_tools_definition() : json();
generation_params params = prepare_debug_params(opts, tools);
common_chat_params parser_data;
if (std::optional<common_chat_params> spec_tmpl =
common_chat_try_specialized_template(chat_template, template_source, params)) {
LOG_ERR("\n");
LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");
parser_data = *spec_tmpl;
} else {
// Render template scenarios if requested
if (opts.input_message != input_message_type::NONE &&
(opts.mode == output_mode::TEMPLATE || opts.mode == output_mode::BOTH)) {
LOG_ERR("\n");
LOG_ERR("================================================================================\n");
LOG_ERR(" TEMPLATE RENDERING OUTPUT\n");
LOG_ERR("================================================================================\n");
render_all_scenarios(chat_template, tools, opts.generation_prompt, opts.enable_reasoning,
opts.input_message);
}
// Output analysis if requested
if (opts.mode == output_mode::ANALYSIS || opts.mode == output_mode::BOTH) {
LOG_ERR("\n");
LOG_ERR("================================================================================\n");
LOG_ERR(" TEMPLATE ANALYSIS\n");
LOG_ERR("================================================================================\n");
struct autoparser analysis;
analysis.analyze_template(chat_template);
// Generate Parser
parser_data = peg_generator::generate_parser(chat_template, params, analysis);
}
}
if (!std::empty(parser_data.parser)) {
LOG_ERR("\n=== Generated Parser ===\n");
common_peg_arena arena;
arena.load(parser_data.parser);
LOG_ERR("%s\n", arena.dump(arena.root()).c_str());
LOG_ERR("\n=== Generated Grammar ===\n");
LOG_ERR("%s\n", parser_data.grammar.c_str());
LOG_ERR("\n=== Generated Lazy Grammar ===\n");
LOG_ERR("%d\n", parser_data.grammar_lazy);
LOG_ERR("\n=== Generated Grammar Triggers ===\n");
for (const common_grammar_trigger & cgt : parser_data.grammar_triggers) {
LOG_ERR("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str());
}
LOG_ERR("\n=== Preserved Tokens ===\n");
for (const std::string & token : parser_data.preserved_tokens) {
LOG_ERR(" '%s'\n", token.c_str());
}
}
} catch (const std::exception & e) {
LOG_ERR("Analysis failed: %s\n", e.what());
return 1;
}
return 0;
}
int main(int argc, char * argv[]) {
if (argc > 1) {
std::string arg = argv[1];
if (arg == "-h" || arg == "--help") {
common_log_set_verbosity_thold(99);
print_usage(argv[0]);
return 0;
}
// debug mode: if the first argument is an existing file, analyze that template instead of running the automated tests
if (std::filesystem::is_regular_file(arg)) {
common_log_set_verbosity_thold(99);
debug_options opts;
if (!parse_debug_options(argc, argv, opts)) {
return 1;
}
if (opts.debug_jinja || std::getenv("LLAMA_DEBUG_JINJA") != nullptr) {
jinja::enable_debug(true);
}
return debug_single_template(opts);
}
}
testing t(std::cout);
t.verbose = true;
// usage: test-chat-auto-parser [filter_regex]
if (argc > 1) {
t.set_filter(argv[1]);
}
t.test("diff_split", test_calculate_diff_split);
t.test("common_prefix", test_until_common_prefix);
t.test("common_suffix", test_after_common_suffix);
t.test("compare_variants", test_compare_variants);
t.test("segments", test_marker_separation);
t.test("seed_oss_diffs", test_seed_oss_tool_analysis);
t.test("cohere", test_cohere_analysis);
t.test("nemotron", test_nemotron_analysis);
t.test("laguna", test_laguna_analysis);
t.test("laguna-s", test_laguna_s_analysis);
t.test("laguna-xs2", test_laguna_xs2_analysis);
t.test("smollm3", test_smollm3_analysis);
t.test("standard_json_tools", test_standard_json_tools_formats);
t.test("normalize_quotes_to_json", test_normalize_quotes_to_json);
t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes);
t.test("bailing_v3", test_bailing_v3_tool_format);
t.test("role_markers_all_templates", test_role_markers_all_templates);
return t.summary();
}
static void test_marker_separation(testing & t) {
auto single_square_marker = segmentize_markers("pre_marker[marker]post_marker");
auto single_diag_marker = segmentize_markers("pre_marker<marker>post_marker");
auto paired_markers = segmentize_markers("<hello>world</hello>");
auto double_different_markers = segmentize_markers("<hello>[hello]<world>[world]");
auto in_between = segmentize_markers("im<blue>daba<dee>da[hey]");
t.test("single_square_marker", [&] (testing & t) {
t.assert_equal("first is text", segment_type::TEXT, single_square_marker[0].type);
t.assert_equal("second is marker", segment_type::MARKER, single_square_marker[1].type);
t.assert_equal("last is text", segment_type::TEXT, single_square_marker[2].type);
t.assert_equal("first is 'pre_marker'", "pre_marker", single_square_marker[0].value);
t.assert_equal("second is '[marker]'", "[marker]", single_square_marker[1].value);
t.assert_equal("last is 'post_marker'", "post_marker", single_square_marker[2].value);
});
t.test("single_diagonal_marker", [&] (testing & t) {
t.assert_equal("first is text", segment_type::TEXT, single_diag_marker[0].type);
t.assert_equal("second is marker", segment_type::MARKER, single_diag_marker[1].type);
t.assert_equal("last is text", segment_type::TEXT, single_diag_marker[2].type);
t.assert_equal("first is 'pre_marker'", "pre_marker", single_diag_marker[0].value);
t.assert_equal("second is '<marker>'", "<marker>", single_diag_marker[1].value);
t.assert_equal("last is 'post_marker'", "post_marker", single_diag_marker[2].value);
});
t.test("paired_markers", [&] (testing & t) {
t.assert_equal("first is marker", segment_type::MARKER, paired_markers[0].type);
t.assert_equal("second is text", segment_type::TEXT, paired_markers[1].type);
t.assert_equal("third is marker", segment_type::MARKER, paired_markers[2].type);
t.assert_equal("first is '<hello>'", "<hello>", paired_markers[0].value);
t.assert_equal("second is 'world'", "world", paired_markers[1].value);
t.assert_equal("third is '</hello>'", "</hello>", paired_markers[2].value);
});
t.test("double_different_markers", [&] (testing & t) {
t.assert_equal("first is marker", segment_type::MARKER, double_different_markers[0].type);
t.assert_equal("second is marker", segment_type::MARKER, double_different_markers[1].type);
t.assert_equal("third is marker", segment_type::MARKER, double_different_markers[2].type);
t.assert_equal("fourth is marker", segment_type::MARKER, double_different_markers[3].type);
t.assert_equal("first is '<hello>'", "<hello>", double_different_markers[0].value);
t.assert_equal("second is '[hello]'", "[hello]", double_different_markers[1].value);
t.assert_equal("third is '<world>'", "<world>", double_different_markers[2].value);
t.assert_equal("fourth is '[world]'", "[world]", double_different_markers[3].value);
});
t.test("in_between", [&] (testing & t) {
t.assert_equal("first is text", segment_type::TEXT, in_between[0].type);
t.assert_equal("second is marker", segment_type::MARKER, in_between[1].type);
t.assert_equal("third is text", segment_type::TEXT, in_between[2].type);
t.assert_equal("fourth is marker", segment_type::MARKER, in_between[3].type);
t.assert_equal("fifth is text", segment_type::TEXT, in_between[4].type);
t.assert_equal("sixth is marker", segment_type::MARKER, in_between[5].type);
t.assert_equal("first is 'im'", "im", in_between[0].value);
t.assert_equal("second is '<blue>'", "<blue>", in_between[1].value);
t.assert_equal("third is 'daba'", "daba", in_between[2].value);
t.assert_equal("fourth is '<dee>'", "<dee>", in_between[3].value);
t.assert_equal("fifth is 'da'", "da", in_between[4].value);
t.assert_equal("sixth is '[hey]'", "[hey]", in_between[5].value);
});
}
static void test_calculate_diff_split(testing & t) {
t.test("calculate_diff_split basic", test_calculate_diff_split_basic);
t.test("calculate_diff_split identical", test_calculate_diff_split_identical);
t.test("calculate_diff_split common prefix", test_calculate_diff_split_common_prefix);
t.test("calculate_diff_split common suffix", test_calculate_diff_split_common_suffix);
t.test("calculate_diff_split common both", test_calculate_diff_split_common_both);
t.test("calculate_diff_split empty cases", test_calculate_diff_split_empty_cases);
t.test("calculate_diff_split no common", test_calculate_diff_split_no_common);
t.test("calculate_diff_split single char", test_calculate_diff_split_single_char);
t.test("calculate_diff_split overlaps", test_calculate_diff_split_overlaps);
t.test("calculate_diff_split tag boundaries", test_calculate_diff_split_tag_boundaries);
t.test("calculate_diff_split generation prompt", test_calculate_diff_split_generation_prompt);
}
static void test_calculate_diff_split_basic(testing & t) {
diff_split result = calculate_diff_split("hello world", "hello test");
t.assert_equal("prefix should be 'hello '", "hello ", result.prefix);
t.assert_equal("left should be 'world'", "world", result.left);
t.assert_equal("right should be 'test'", "test", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("abc", "xyz");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'abc'", "abc", result.left);
t.assert_equal("right should be 'xyz'", "xyz", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("prefixA suffix", "prefixB suffix");
t.assert_equal("prefix should be 'prefix'", "prefix", result.prefix);
t.assert_equal("left should be 'A'", "A", result.left);
t.assert_equal("right should be 'B'", "B", result.right);
t.assert_equal("suffix should be ' suffix'", " suffix", result.suffix);
}
static void test_calculate_diff_split_identical(testing & t) {
diff_split result = calculate_diff_split("hello", "hello");
t.assert_equal("prefix should be 'hello'", "hello", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("", "");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("a", "a");
t.assert_equal("prefix should be 'a'", "a", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("<row><row><row><your><boat><gently>", "<row><row><row><your><boat><gently>");
t.assert_equal("prefix should be '<row><row><row><your><boat><gently>'", "<row><row><row><your><boat><gently>", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_common_prefix(testing & t) {
diff_split result = calculate_diff_split("abcdef", "abcxyz");
t.assert_equal("prefix should be 'abc'", "abc", result.prefix);
t.assert_equal("left should be 'def'", "def", result.left);
t.assert_equal("right should be 'xyz'", "xyz", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("same", "sameagain");
t.assert_equal("prefix should be 'same'", "same", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be 'again'", "again", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("test", "testing");
t.assert_equal("prefix should be 'test'", "test", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be 'ing'", "ing", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_common_suffix(testing & t) {
diff_split result = calculate_diff_split("123end", "456end");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be '123'", "123", result.left);
t.assert_equal("right should be '456'", "456", result.right);
t.assert_equal("suffix should be 'end'", "end", result.suffix);
result = calculate_diff_split("start", "end");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'start'", "start", result.left);
t.assert_equal("right should be 'end'", "end", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("abcsuffix", "xyzsuffix");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'abc'", "abc", result.left);
t.assert_equal("right should be 'xyz'", "xyz", result.right);
t.assert_equal("suffix should be 'suffix'", "suffix", result.suffix);
}
static void test_calculate_diff_split_common_both(testing & t) {
diff_split result = calculate_diff_split("helloXworld", "helloYworld");
t.assert_equal("prefix should be 'hello'", "hello", result.prefix);
t.assert_equal("left should be 'X'", "X", result.left);
t.assert_equal("right should be 'Y'", "Y", result.right);
t.assert_equal("suffix should be 'world'", "world", result.suffix);
result = calculate_diff_split("ABCmiddleXYZ", "ABCdifferentXYZ");
t.assert_equal("prefix should be 'ABC'", "ABC", result.prefix);
t.assert_equal("left should be 'middle'", "middle", result.left);
t.assert_equal("right should be 'different'", "different", result.right);
t.assert_equal("suffix should be 'XYZ'", "XYZ", result.suffix);
result = calculate_diff_split("startAend", "startBend");
t.assert_equal("prefix should be 'start'", "start", result.prefix);
t.assert_equal("left should be 'A'", "A", result.left);
t.assert_equal("right should be 'B'", "B", result.right);
t.assert_equal("suffix should be 'end'", "end", result.suffix);
// Edge case: common prefix and suffix overlap
result = calculate_diff_split("aa", "ab");
t.assert_equal("prefix should be 'a'", "a", result.prefix);
t.assert_equal("left should be 'a'", "a", result.left);
t.assert_equal("right should be 'b'", "b", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_empty_cases(testing & t) {
// Empty left, non-empty right
diff_split result = calculate_diff_split("", "hello");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be 'hello'", "hello", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Non-empty left, empty right
result = calculate_diff_split("hello", "");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'hello'", "hello", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Both empty
result = calculate_diff_split("", "");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Left single char, empty right
result = calculate_diff_split("a", "");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'a'", "a", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Empty left, right single char
result = calculate_diff_split("", "a");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be 'a'", "a", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_no_common(testing & t) {
diff_split result = calculate_diff_split("abc", "xyz");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'abc'", "abc", result.left);
t.assert_equal("right should be 'xyz'", "xyz", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("left", "right");
// The algorithm finds "t" as a common suffix since both strings end with 't'
// This is the algorithm's actual behavior - it finds maximal common suffix
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'lef'", "lef", result.left);
t.assert_equal("right should be 'righ'", "righ", result.right);
t.assert_equal("suffix should be 't'", "t", result.suffix);
result = calculate_diff_split("123", "456");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be '123'", "123", result.left);
t.assert_equal("right should be '456'", "456", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_single_char(testing & t) {
diff_split result = calculate_diff_split("a", "b");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'a'", "a", result.left);
t.assert_equal("right should be 'b'", "b", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("a", "a");
t.assert_equal("prefix should be 'a'", "a", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("a", "ab");
t.assert_equal("prefix should be 'a'", "a", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be 'b'", "b", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("ab", "a");
t.assert_equal("prefix should be 'a'", "a", result.prefix);
t.assert_equal("left should be 'b'", "b", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_overlaps(testing & t) {
// One string is substring of another
diff_split result = calculate_diff_split("test", "testing");
t.assert_equal("prefix should be 'test'", "test", result.prefix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be 'ing'", "ing", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
result = calculate_diff_split("testing", "test");
t.assert_equal("prefix should be 'test'", "test", result.prefix);
t.assert_equal("left should be 'ing'", "ing", result.left);
t.assert_equal("right should be empty", "", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Similar strings with one extra char at start
result = calculate_diff_split("Xtest", "Ytest");
// The algorithm finds "test" as a common suffix since both strings end with "test"
// This is the algorithm's actual behavior - it finds maximal common suffix
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'X'", "X", result.left);
t.assert_equal("right should be 'Y'", "Y", result.right);
t.assert_equal("suffix should be 'test'", "test", result.suffix);
// Similar strings with one extra char at end
result = calculate_diff_split("testX", "testY");
t.assert_equal("prefix should be 'test'", "test", result.prefix);
t.assert_equal("left should be 'X'", "X", result.left);
t.assert_equal("right should be 'Y'", "Y", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Strings that are reverses
result = calculate_diff_split("abc", "cba");
t.assert_equal("prefix should be empty", "", result.prefix);
t.assert_equal("left should be 'abc'", "abc", result.left);
t.assert_equal("right should be 'cba'", "cba", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
}
static void test_calculate_diff_split_tag_boundaries(testing & t) {
// Test with unclosed XML tags
diff_split result = calculate_diff_split("test<tag", "test>content");
// The fix_tag_boundaries should move incomplete tags appropriately
t.assert_true("prefix should start with 'test'", result.prefix.find("test") == 0);
t.assert_true("should handle tag boundaries", result.left != "" || result.right != "" || result.suffix != "");
// Test with unclosed brackets
result = calculate_diff_split("test[", "test]value");
t.assert_true("should handle bracket boundaries", result.left != "" || result.right != "" || result.suffix != "");
// Test with partial tags on both sides
result = calculate_diff_split("prefix<tag>", "prefix</tag>suffix");
// fix_tag_boundaries moves the incomplete '<' from prefix to left/right
t.assert_equal("prefix should be 'prefix'", "prefix", result.prefix);
t.assert_equal("left should be '<tag>'", "<tag>", result.left);
t.assert_equal("right should be '</tag>suffix'", "</tag>suffix", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Test with complex nested tags
result = calculate_diff_split("prefix<div>content</div>", "prefix<div>different</div>");
// Algorithm finds "ent</div>" as a common suffix because both strings end with it
// This is the actual algorithm behavior, though not semantically ideal
t.assert_equal("prefix should be 'prefix<div>'", "prefix<div>", result.prefix);
t.assert_equal("left should be 'cont'", "cont", result.left);
t.assert_equal("right should be 'differ'", "differ", result.right);
t.assert_equal("suffix should be 'ent</div>'", "ent</div>", result.suffix);
// Test with unclosed angle bracket
result = calculate_diff_split("Hello <world>", "Hello test");
t.assert_equal("prefix should be 'Hello '", "Hello ", result.prefix);
t.assert_true("left should contain '<world>'", result.left.find("<world>") != std::string::npos);
t.assert_equal("right should be 'test'", "test", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Test with unclosed square bracket
result = calculate_diff_split("test [array]", "test other");
t.assert_equal("prefix should be 'test '", "test ", result.prefix);
t.assert_true("left should contain '[array]'", result.left.find("[array]") != std::string::npos);
t.assert_equal("right should be 'other'", "other", result.right);
t.assert_equal("suffix should be empty", "", result.suffix);
// Test empty prefix and suffix with tags
result = calculate_diff_split("<tag>left</tag>", "<tag>righ</tag>");
t.assert_equal("prefix should be '<tag>'", "<tag>", result.prefix);
t.assert_equal("left should be 'left'", "left", result.left);
t.assert_equal("right should be 'righ'", "righ", result.right);
t.assert_equal("suffix should be '</tag>'", "</tag>", result.suffix);
{
// real case from template tests, simplified
std::string left = "PREFIX</think>Sure";
std::string right = "PREFIX<think>Lemme think</think>Sure";
result = calculate_diff_split(left, right);
t.assert_equal("prefix should be PREFIX", "PREFIX", result.prefix);
t.assert_equal("suffix should be </think>Sure", "</think>Sure", result.suffix);
t.assert_equal("left should be empty", "", result.left);
t.assert_equal("right should be <think>Lemme think", "<think>Lemme think", result.right);
}
{
// Real case: special tokens with |> boundary issue
// The suffix starts with |> which should be moved to complete <|END_RESPONSE and <|END_ACTION
std::string prefix = "SOME_PREFIX";
std::string suffix = "|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>";
std::string left_diff = "<|START_RESPONSE|>Let me help you.<|END_RESPONSE";
std::string right_diff =
"<|START_THINKING|><|END_THINKING|><|START_ACTION|>[\n"
" {\"tool_call_id\": \"0\", \"tool_name\": \"test_function_name\", "
"\"parameters\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\n"
"]<|END_ACTION";
std::string left = prefix + left_diff + suffix;
std::string right = prefix + right_diff + suffix;
result = calculate_diff_split(left, right);
t.assert_equal("special token prefix", prefix, result.prefix);
// The |> should be moved from suffix to complete the tokens
t.assert_equal("special token left", "<|START_RESPONSE|>Let me help you.<|END_RESPONSE|>", result.left);
t.assert_true("special token right ends with |>", result.right.find("<|END_ACTION|>") != std::string::npos);
t.assert_equal("special token suffix", "<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
result.suffix);
}
}
static void test_calculate_diff_split_generation_prompt(testing & t) {
// ChatML thinking template: left is a prefix of right, generation_prompt is the appended part.
// The trailing \n in left matches the trailing \n in the generation_prompt, causing
// the suffix matcher to steal it and rotate the diff result.
{
// Simplified reproduction: left ends with \n, right = left + "<|im_start|>assistant\n<think>\n"
std::string left = "<|im_start|>user\nHello<|im_end|>\n";
std::string right = left + "<|im_start|>assistant\n<think>\n";
diff_split result = calculate_diff_split(left, right);
t.assert_equal("chatml prefix", left, result.prefix);
t.assert_equal("chatml left", "", result.left);
t.assert_equal("chatml right should be generation prompt",
"<|im_start|>assistant\n<think>\n", result.right);
t.assert_equal("chatml suffix", "", result.suffix);
}
{
// More realistic: longer conversation ending with tool_response
std::string common =
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\nSearch for files<|im_end|>\n"
"<|im_start|>assistant\n<think>\nLet me search.\n</think>\n\n"
"<tool_call>\n<function=search>\n</function>\n</tool_call><|im_end|>\n"
"<|im_start|>user\n<tool_response>\nNo files found\n</tool_response><|im_end|>\n";
std::string left = common;
std::string right = common + "<|im_start|>assistant\n<think>\n";
diff_split result = calculate_diff_split(left, right);
t.assert_equal("tool_response left", "", result.left);
t.assert_equal("tool_response right should be generation prompt",
"<|im_start|>assistant\n<think>\n", result.right);
}
}
static void test_until_common_prefix(testing & t) {
t.test("until_common_prefix basic", test_until_common_prefix_basic);
}
static void test_until_common_prefix_basic(testing & t) {
// Test case from the user request
std::string result = until_common_prefix("<function name=foo><arg name=bar>", "<arg name=bar>", "<arg name=baz>");
t.assert_equal("untilCommonPrefix should return '<function name=foo>'", "<function name=foo>", result);
// Additional test cases to ensure robustness
// Test with different common prefix lengths
result = until_common_prefix("prefix<test>suffix", "<test>different", "<test>other");
t.assert_equal("should return 'prefix'", "prefix", result);
// Test when common prefix is at the start
result = until_common_prefix("<common>rest", "<common>left", "<common>right");
t.assert_equal("should return empty string when common prefix at start", "", result);
// Test when there's no common prefix
result = until_common_prefix("something", "left", "right");
t.assert_equal("should return empty string when no common prefix", "", result);
// Test with empty strings
result = until_common_prefix("test", "", "right");
t.assert_equal("should return empty string when left is empty", "", result);
// Test with longer common prefix
result = until_common_prefix("abcXYZ<shared_prefix>rest", "<shared_prefix>left", "<shared_prefix>right");
t.assert_equal("should return 'abcXYZ'", "abcXYZ", result);
}
static void test_after_common_suffix(testing & t) {
t.test("after_common_suffix basic", test_after_common_suffix_basic);
}
static void test_after_common_suffix_basic(testing & t) {
// Test case from the user request
std::string result = after_common_suffix("<function name=foo><arg name=bar>100</arg></function>",
"<arg name=bar>100</arg>",
"<arg name=baz>535</arg>");
t.assert_equal("afterCommonSuffix should return '</function>'", "</function>", result);
// Test when common suffix is at the end
result = after_common_suffix("rest<common>", "left<common>", "right<common>");
t.assert_equal("should return empty string when common suffix at end", "", result);
// Test with empty strings
result = after_common_suffix("test", "left", "");
t.assert_equal("should return empty string when right is empty", "", result);
// Test case with XML-like structure similar to the main example
result = after_common_suffix("<outer><inner>value</inner></outer>",
"<inner>value</inner>",
"<inner>different</inner>");
t.assert_equal("should return '</outer>'", "</outer>", result);
// Test with longer common suffix appearing at the end of full
result = after_common_suffix("prefix<shared>rest</shared>", "prefix<shared>left</shared>", "prefix<shared>right</shared>");
t.assert_equal("should return '' when common suffix is at end of full", "", result);
// Test with common suffix appearing in middle but not at end
result = after_common_suffix("<tag>content</tag><extra>", "<tag>value</tag>", "<tag>other</tag>");
t.assert_equal("should return '<extra>' when common suffix appears before end", "<extra>", result);
// Test with multi-character common suffix at the very end of full
result = after_common_suffix("start<middle>end</middle>", "prefix<middle>left</middle>", "prefix<middle>right</middle>");
t.assert_equal("should return '' when common suffix </middle> is at end of full", "", result);
}
static void test_compare_variants(testing & t) {
t.test("compare_variants basic", test_compare_variants_basic);
t.test("compare_variants messages modifier", test_compare_variants_messages_modifier);
t.test("compare_variants tools modifier", test_compare_variants_tools_modifier);
t.test("compare_variants both modifiers", test_compare_variants_both_modifiers);
t.test("compare_variants template failure", test_compare_variants_template_failure);
t.test("compare_variants identity", test_compare_variants_identity);
}
static void test_compare_variants_basic(testing & t) {
// Create a simple template that just echoes messages
common_chat_template tmpl("{{ messages[0]['content'] }}", "", "");
template_params params;
params.messages = json::array({
json {{"role", "user"}, {"content", "Hello"}}
});
auto modifier = [](template_params & p) {
p.messages[0]["content"] = "World";
};
auto result = ::compare_variants(tmpl, params, modifier);
if (!t.assert_true("result should have value", result.has_value())) {
return;
}
// The template might not output anything if messages is empty or format is different
// Check that we get a valid result
t.assert_true("prefix or left should have content", !result->diff.prefix.empty() || !result->diff.left.empty());
}
static void test_compare_variants_messages_modifier(testing & t) {
// Test with messages modifier only
common_chat_template tmpl("{% for message in messages %}{{ message['role'] }}:{{ message['content'] }}{% endfor %}", "", "");
template_params params;
params.messages = json::array({
json {{"role", "user"}, {"content", "A"}}
});
auto modifier = [](template_params & p) {
p.messages[0]["content"] = "B";
};
std::optional<compare_variants_result> result = ::compare_variants(tmpl, params, modifier);
if (!t.assert_true("result should have value", result.has_value())) {
return;
}
t.assert_equal("left should be 'A'", "A", result->diff.left);
t.assert_equal("right should be 'B'", "B", result->diff.right);
}
static void test_compare_variants_tools_modifier(testing & t) {
// Test with tools modifier only
common_chat_template tmpl(
"{% for tool in tools %}{{ tool['name'] }}{% endfor %}", "", "");
template_params params;
params.tools = json::array({
json {{"name", "foo"}}
});
auto modifier = [](template_params & p) {
p.tools[0]["name"] = "bar";
};
auto result = ::compare_variants(tmpl, params, modifier);
if (!t.assert_true("result should have value", result.has_value())) {
return;
}
t.assert_equal("left should be 'foo'", "foo", result->diff.left);
t.assert_equal("right should be 'bar'", "bar", result->diff.right);
}
static void test_compare_variants_both_modifiers(testing & t) {
// Test with both messages and tools modifiers using the for loop approach
common_chat_template tmpl(
"{% for message in messages %}{{ message['role'] }}:{{ message['content'] }}{% endfor %}", "", "");
template_params params;
params.messages = json::array({
json {{"role", "user"}, {"content", "A"}}
});
auto modifier = [](template_params & p) {
p.messages[0]["content"] = "B";
p.messages[0]["role"] = "newuser";
};
auto result = ::compare_variants(tmpl, params, modifier);
if (!t.assert_true("result should have value", result.has_value())) {
return;
}
t.assert_equal("left should be 'user:A'", "user:A", result->diff.left);
t.assert_equal("right should be 'newuser:B'", "newuser:B", result->diff.right);
}
static void test_compare_variants_template_failure(testing & t) {
// Test with template that causes failure during application (not construction)
// We use a valid template syntax but one that will fail during application
common_chat_template tmpl("{{ messages.cahoot()[0]['nonexistent_field'] }}", "", "");
template_params params;
params.messages = json::array({
json {{"role", "user"}, {"content", "Hello"}}
});
auto modifier = [](template_params & p) {
p.messages[0]["content"] = "World";
};
auto result = ::compare_variants(tmpl, params, modifier);
t.assert_true("result should be nullopt on template failure", !result.has_value());
}
static void test_compare_variants_identity(testing & t) {
// Test with identity modifier (no change)
common_chat_template tmpl("{{ messages[0]['content'] }}", "", "");
template_params params;
params.messages = json::array({
json {{"role", "user"}, {"content", "Hello"}}
});
// No modifier - should use identity
auto result = ::compare_variants(tmpl, params, nullptr);
if (!t.assert_true("result should have value", result.has_value())) {
return;
}
t.assert_equal("prefix should be 'Hello'", "Hello", result->diff.prefix);
t.assert_equal("left should be empty", "", result->diff.left);
t.assert_equal("right should be empty", "", result->diff.right);
t.assert_equal("suffix should be empty", "", result->diff.suffix);
}
// ============================================================================
// Seed-OSS Template Tool Calling Analysis Tests
// ============================================================================
static void test_seed_oss_tool_analysis(testing & t) {
t.test("Seed-OSS tool presence", test_seed_oss_tool_presence);
t.test("Seed-OSS call count", test_seed_oss_call_count);
t.test("Seed-OSS function names", test_seed_oss_function_names);
t.test("Seed-OSS argument count", test_seed_oss_argument_count);
t.test("Seed-OSS args presence", test_seed_oss_args_presence);
t.test("Seed-OSS tool with reasoning", test_seed_oss_tool_with_reasoning);
}
// Helper to load Seed-OSS template
static common_chat_template load_seed_oss_template(testing & t) {
std::string template_path = "models/templates/ByteDance-Seed-OSS.jinja";
std::ifstream fin(template_path, std::ios::binary);
std::ostringstream buf;
if (fin.is_open()) {
buf << fin.rdbuf();
}
std::string template_source = buf.str();
common_chat_template tmpl(template_source, "", "");
t.assert_true("Seed-OSS template loaded successfully", template_source.length() > 0);
return tmpl;
}
// Helper to build tool call JSON
static json build_tool_call(const std::string & name, const json & args, const std::string & id = "call_001") {
return json{
{"id", id},
{"type", "function"},
{"function", json{
{"name", name},
{"arguments", args}
}}
};
}
// Helper to build tools definition
static json build_tools_definition() {
json parameters_schema = json::object();
parameters_schema["type"] = "object";
parameters_schema["properties"] = json::object();
parameters_schema["properties"]["param1"] = json::object({
{"type", "string"},
{"description", "First parameter"}
});
parameters_schema["properties"]["param2"] = json::object({
{"type", "string"},
{"description", "Second parameter"}
});
parameters_schema["required"] = json::array({"param1", "param2"});
return json::array({
json{
{"type", "function"},
{"function", json{
{"name", "test_function_name"},
{"description", "A test function for debugging"},
{"parameters", parameters_schema}
}}
}
});
}
// T1: Compare with/without tool call (user, assistant)
static void test_seed_oss_tool_presence(testing & t) {
common_chat_template tmpl = load_seed_oss_template(t);
json assistant_no_tools = json{
{"role", "assistant"},
{"content", "Let me help you."}
};
json assistant_with_tools = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
})}
};
json user_msg = json{
{"role", "user"},
{"content", "Hello, please help me."}
};
template_params params_no_tools;
params_no_tools.messages = json::array({user_msg, assistant_no_tools});
params_no_tools.tools = build_tools_definition();
params_no_tools.add_generation_prompt = false;
params_no_tools.enable_thinking = true;
template_params params_with_tools;
params_with_tools.messages = json::array({user_msg, assistant_with_tools});
params_with_tools.tools = build_tools_definition();
params_with_tools.add_generation_prompt = false;
params_with_tools.enable_thinking = true;
auto result = ::compare_variants(tmpl, params_no_tools,
[&](template_params & p) {
p.messages = params_with_tools.messages;
});
if (!t.assert_true("T1 result should have value", result.has_value())) {
return;
}
const auto & diff = result->diff;
t.assert_true("T1 prefix should contain system", diff.prefix.find("system") != std::string::npos);
t.assert_true("T1 prefix should contain user", diff.prefix.find("user") != std::string::npos);
t.assert_true("T1 prefix should contain assistant", diff.prefix.find("assistant") != std::string::npos);
// Left should be the assistant content without tool
t.assert_equal("T1 left should contain 'Let me help you.'", "Let me help you.", diff.left);
// Right should contain the tool call markers
t.assert_true("T1 right should contain tool_call begin", diff.right.find("<seed:tool_call>") != std::string::npos);
t.assert_true("T1 right should contain function tag", diff.right.find("<function=test_function_name>") != std::string::npos);
t.assert_true("T1 right should contain parameter=param1", diff.right.find("<parameter=param1>") != std::string::npos);
t.assert_true("T1 right should contain parameter=param2", diff.right.find("<parameter=param2>") != std::string::npos);
t.assert_true("T1 right should contain value1", diff.right.find("value1") != std::string::npos);
t.assert_true("T1 right should contain value2", diff.right.find("value2") != std::string::npos);
t.assert_true("T1 right should contain tool_call end", diff.right.find("</seed:tool_call>") != std::string::npos);
// Suffix should be the eos token
t.assert_equal("T1 suffix should be '<seed:eos>'", "<seed:eos>", diff.suffix);
}
// T2: Compare one vs two tool calls
static void test_seed_oss_call_count(testing & t) {
common_chat_template tmpl = load_seed_oss_template(t);
json assistant_one_call = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
})}
};
json assistant_two_calls = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})),
build_tool_call("test_function_name", json::object({{"param1", "value3"}, {"param2", "value4"}}), "call_002")
})}
};
json user_msg = json{
{"role", "user"},
{"content", "Hello, please help me."}
};
template_params params_one;
params_one.messages = json::array({user_msg, assistant_one_call});
params_one.tools = build_tools_definition();
params_one.add_generation_prompt = false;
params_one.enable_thinking = true;
auto result = ::compare_variants(tmpl, params_one,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_two_calls});
});
if (!t.assert_true("T2 result should have value", result.has_value())) {
return;
}
const auto & diff = result->diff;
// Prefix should include the first tool call
t.assert_true("T2 prefix should contain first tool_call begin", diff.prefix.find("<seed:tool_call>") != std::string::npos);
t.assert_true("T2 prefix should contain first function", diff.prefix.find("<function=test_function_name>") != std::string::npos);
t.assert_true("T2 prefix should contain value1", diff.prefix.find("value1") != std::string::npos);
t.assert_true("T2 prefix should contain value2", diff.prefix.find("value2") != std::string::npos);
t.assert_true("T2 prefix should contain first tool_call end", diff.prefix.find("</seed:tool_call>") != std::string::npos);
// Left should be empty (no second tool call in variant A)
t.assert_equal("T2 left should be empty", "", diff.left);
// Right should contain the second tool call
t.assert_true("T2 right should contain second tool_call begin", diff.right.find("<seed:tool_call>") != std::string::npos);
t.assert_true("T2 right should contain second function", diff.right.find("<function=test_function_name>") != std::string::npos);
t.assert_true("T2 right should contain value3", diff.right.find("value3") != std::string::npos);
t.assert_true("T2 right should contain value4", diff.right.find("value4") != std::string::npos);
t.assert_true("T2 right should contain second tool_call end", diff.right.find("</seed:tool_call>") != std::string::npos);
// Suffix should end with the eos token
t.assert_equal("T2 suffix should end with '<seed:eos>'", "<seed:eos>", diff.suffix.substr(diff.suffix.length() - 10, 10));
}
// T3: Compare different function names
static void test_seed_oss_function_names(testing & t) {
common_chat_template tmpl = load_seed_oss_template(t);
// Build tools with two different function names
json parameters_schema = json::object();
parameters_schema["type"] = "object";
parameters_schema["properties"] = json::object();
parameters_schema["properties"]["arg1"] = json::object({
{"type", "string"},
{"description", "Argument 1"}
});
parameters_schema["required"] = json::array({"arg1"});
json tools = json::array({
json{
{"type", "function"},
{"function", json{
{"name", "func_alpha"},
{"description", "First function"},
{"parameters", parameters_schema}
}}
},
json{
{"type", "function"},
{"function", json{
{"name", "func_beta"},
{"description", "Second function"},
{"parameters", parameters_schema}
}}
}
});
json assistant_func_alpha = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("func_alpha", json::object({{"arg1", "test_value"}}))
})}
};
json assistant_func_beta = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("func_beta", json::object({{"arg1", "test_value"}}))
})}
};
json user_msg = json{
{"role", "user"},
{"content", "Hello"}
};
template_params params_alpha;
params_alpha.messages = json::array({user_msg, assistant_func_alpha});
params_alpha.tools = tools;
params_alpha.add_generation_prompt = false;
params_alpha.enable_thinking = true;
auto result = ::compare_variants(tmpl, params_alpha,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_func_beta});
});
if (!t.assert_true("T3 result should have value", result.has_value())) {
return;
}
const auto & diff = result->diff;
bool func_alpha_in_left = diff.left.find("func_alpha") != std::string::npos;
bool func_alpha_in_prefix = diff.prefix.find("func_alpha") != std::string::npos;
bool func_beta_in_right = diff.right.find("func_beta") != std::string::npos;
bool func_beta_in_prefix = diff.prefix.find("func_beta") != std::string::npos;
bool func_beta_in_suffix = diff.suffix.find("func_beta") != std::string::npos;
// Left should contain func_alpha (or be in prefix)
t.assert_true("T3 left should contain func_alpha (or prefix)", func_alpha_in_left || func_alpha_in_prefix);
// Right should contain func_beta
t.assert_true("T3 right should contain func_beta", func_beta_in_right || func_beta_in_prefix || func_beta_in_suffix);
// Both should have the same parameter value (in common parts, not in diffs)
// Since both have same args, test_value will be in prefix/suffix
t.assert_true("T3 diff should contain test_value (in prefix or suffix)",
diff.prefix.find("test_value") != std::string::npos || diff.suffix.find("test_value") != std::string::npos);
}
// T4: Compare different argument counts (zero, one, two parameters)
static void test_seed_oss_argument_count(testing & t) {
common_chat_template tmpl = load_seed_oss_template(t);
// Build tools with 0, 1, or 2 required parameters
json params_2_required = json::object();
params_2_required["type"] = "object";
params_2_required["properties"] = json::object();
params_2_required["properties"]["arg1"] = json::object({
{"type", "string"},
{"description", "Argument 1"}
});
params_2_required["properties"]["arg2"] = json::object({
{"type", "string"},
{"description", "Argument 2"}
});
params_2_required["required"] = json::array({"arg1", "arg2"});
json params_1_required = json::object();
params_1_required["type"] = "object";
params_1_required["properties"] = json::object();
params_1_required["properties"]["arg1"] = json::object({
{"type", "string"},
{"description", "Argument 1"}
});
params_1_required["required"] = json::array({"arg1"});
json tools = json::array({
json{
{"type", "function"},
{"function", json{
{"name", "test_func"},
{"description", "Test function"},
{"parameters", params_2_required}
}}
}
});
// Test: zero args vs one arg
json assistant_zero_args = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_func", json::object())
})}
};
json assistant_one_arg = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_func", json::object({{"arg1", "value1"}}))
})}
};
json assistant_two_args = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_func", json::object({{"arg1", "value1"}, {"arg2", "value2"}}))
})}
};
json user_msg = json{
{"role", "user"},
{"content", "Hello"}
};
// Test zero vs one
template_params params_zero;
params_zero.messages = json::array({user_msg, assistant_zero_args});
params_zero.tools = tools;
params_zero.add_generation_prompt = false;
params_zero.enable_thinking = true;
auto result_zero_one = ::compare_variants(tmpl, params_zero,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_one_arg});
});
if (!t.assert_true("T4 zero vs one result should have value", result_zero_one.has_value())) {
return;
}
t.assert_true("T4 zero vs one left should be empty or minimal", result_zero_one->diff.left.empty() || result_zero_one->diff.left == "");
t.assert_true("T4 zero vs one right should contain arg1", result_zero_one->diff.right.find("arg1") != std::string::npos);
// Test one vs two
template_params params_one;
params_one.messages = json::array({user_msg, assistant_one_arg});
params_one.tools = tools;
params_one.add_generation_prompt = false;
params_one.enable_thinking = true;
auto result_one_two = ::compare_variants(tmpl, params_one,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_two_args});
});
if (!t.assert_true("T4 one vs two result should have value", result_one_two.has_value())) {
return;
}
const auto & diff4 = result_one_two->diff;
t.assert_true("T4 one vs two left should contain arg1 (or prefix)",
diff4.left.find("arg1") != std::string::npos || diff4.prefix.find("arg1") != std::string::npos);
t.assert_true("T4 one vs two right should contain arg1 (or prefix)",
diff4.right.find("arg1") != std::string::npos || diff4.prefix.find("arg1") != std::string::npos);
t.assert_true("T4 one vs two right should contain arg2 (or prefix/suffix)",
diff4.right.find("arg2") != std::string::npos || diff4.prefix.find("arg2") != std::string::npos || diff4.suffix.find("arg2") != std::string::npos);
}
// T5: Compare different argument values
static void test_seed_oss_args_presence(testing & t) {
common_chat_template tmpl = load_seed_oss_template(t);
json assistant_same_arg = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}}))
})}
};
json assistant_other_arg = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param2", "value2"}}))
})}
};
json assistant_both_args = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
})}
};
json user_msg = json{
{"role", "user"},
{"content", "Hello"}
};
template_params params_same;
params_same.messages = json::array({user_msg, assistant_same_arg});
params_same.tools = build_tools_definition();
params_same.add_generation_prompt = false;
params_same.enable_thinking = true;
// Test same arg vs other arg
auto result_same_other = ::compare_variants(tmpl, params_same,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_other_arg});
});
if (!t.assert_true("T5 same vs other result should have value", result_same_other.has_value())) {
return;
}
const auto & diff5a = result_same_other->diff;
t.assert_true("T5 same vs other left should contain param1 (or prefix/suffix)",
diff5a.left.find("param1") != std::string::npos || diff5a.prefix.find("param1") != std::string::npos || diff5a.suffix.find("param1") != std::string::npos);
t.assert_true("T5 same vs other left should contain value1 (or prefix/suffix)",
diff5a.left.find("value1") != std::string::npos || diff5a.prefix.find("value1") != std::string::npos);
t.assert_true("T5 same vs other right should contain param2 (or prefix/suffix)",
diff5a.right.find("param2") != std::string::npos || diff5a.prefix.find("param2") != std::string::npos || diff5a.suffix.find("param2") != std::string::npos);
t.assert_true("T5 same vs other right should contain value2 (or prefix/suffix)",
diff5a.right.find("value2") != std::string::npos || diff5a.prefix.find("value2") != std::string::npos || diff5a.suffix.find("value2") != std::string::npos);
// Test same arg vs both args
auto result_same_both = ::compare_variants(tmpl, params_same,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_both_args});
});
if (!t.assert_true("T5 same vs both result should have value", result_same_both.has_value())) {
return;
}
const auto & diff5b = result_same_both->diff;
t.assert_true("T5 same vs both left should contain param1 (or prefix/suffix)",
diff5b.left.find("param1") != std::string::npos || diff5b.prefix.find("param1") != std::string::npos || diff5b.suffix.find("param1") != std::string::npos);
t.assert_true("T5 same vs both right should contain param1 (or prefix/suffix)",
diff5b.right.find("param1") != std::string::npos || diff5b.prefix.find("param1") != std::string::npos || diff5b.suffix.find("param1") != std::string::npos);
t.assert_true("T5 same vs both right should contain param2 (or prefix/suffix)",
diff5b.right.find("param2") != std::string::npos || diff5b.prefix.find("param2") != std::string::npos || diff5b.suffix.find("param2") != std::string::npos);
}
// T6: Tool call with vs without reasoning_content
static void test_seed_oss_tool_with_reasoning(testing & t) {
common_chat_template tmpl = load_seed_oss_template(t);
json assistant_tool_only = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
})}
};
json assistant_tool_with_reasoning = json{
{"role", "assistant"},
{"content", nullptr},
{"tool_calls", json::array({
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
})},
{"reasoning_content", "I need to call the tool first."}
};
json user_msg = json{
{"role", "user"},
{"content", "Hello, please help me."}
};
template_params params_tool_only;
params_tool_only.messages = json::array({user_msg, assistant_tool_only});
params_tool_only.tools = build_tools_definition();
params_tool_only.add_generation_prompt = false;
params_tool_only.enable_thinking = true;
auto result = ::compare_variants(tmpl, params_tool_only,
[&](template_params & p) {
p.messages = json::array({user_msg, assistant_tool_with_reasoning});
});
if (!t.assert_true("T6 result should have value", result.has_value())) {
return;
}
const auto & diff = result->diff;
// Left should be empty (no reasoning in variant A)
t.assert_equal("T6 left should be empty", "", diff.left);
// Right should contain the thinking token with reasoning content
t.assert_true("T6 right should contain think begin", diff.right.find("<seed:think>") != std::string::npos);
t.assert_true("T6 right should contain reasoning content", diff.right.find("I need to call the tool first.") != std::string::npos);
t.assert_true("T6 right should contain think end", diff.right.find("</seed:think>") != std::string::npos);
// Prefix should contain the assistant role
t.assert_true("T6 prefix should contain assistant", diff.prefix.find("assistant") != std::string::npos);
// Suffix should contain the tool call
t.assert_true("T6 suffix should contain tool_call begin", diff.suffix.find("<seed:tool_call>") != std::string::npos);
t.assert_true("T6 suffix should contain function name", diff.suffix.find("test_function_name") != std::string::npos);
t.assert_true("T6 suffix should contain eos", diff.suffix.find("<seed:eos>") != std::string::npos);
}
static common_chat_template load_template(testing & t, const std::string & template_path) {
std::ifstream fin(template_path, std::ios::binary);
std::ostringstream buf;
if (fin.is_open()) {
buf << fin.rdbuf();
}
std::string template_source = buf.str();
common_chat_template tmpl(template_source, "", "");
t.assert_true("Nemotron template loaded successfully", template_source.length() > 0);
return tmpl;
}
// ============================================================================
// Nemotron Template Analysis Tests
// ============================================================================
static common_chat_template load_nemotron_template(testing & t) {
return load_template(t, "models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja");
}
static void test_nemotron_analysis(testing & t) {
t.test("Nemotron reasoning detection", test_nemotron_reasoning_detection);
t.test("Nemotron tool format", test_nemotron_tool_format);
}
static void test_nemotron_reasoning_detection(testing & t) {
common_chat_template tmpl = load_nemotron_template(t);
// Test the comparison manually to see what's happening
json user_msg = json{ { "role", "user" }, { "content", "Hello" } };
json assistant_no_reasoning = json{
{ "role", "assistant" },
{ "content", "I can help." }
};
json assistant_with_reasoning = json{
{ "role", "assistant" },
{ "content", "I can help." },
{ "reasoning_content", "Let me think about this." }
};
template_params params;
params.messages = json::array({ user_msg, assistant_no_reasoning });
params.add_generation_prompt = false;
params.enable_thinking = true;
// Run differential analysis
struct autoparser analysis;
analysis.analyze_template(tmpl);
// Check reasoning markers
t.assert_equal("reasoning_start should be '<think>\\n'", "<think>\n", analysis.reasoning.start);
t.assert_equal("reasoning_end should be '\\n</think>\\n'", "\n</think>\n", analysis.reasoning.end);
// Check reasoning mode detection
// Nemotron uses tag-based reasoning; prefill handles the template's forced markers
t.assert_equal("reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode);
// Make sure reasoning markers don't spill over to content markers
t.assert_equal("content start should be empty", "", analysis.content.start);
t.assert_equal("content end should be empty", "", analysis.content.end);
t.assert_equal("content should be PLAIN", content_mode::PLAIN, analysis.content.mode);
}
static void test_nemotron_tool_format(testing & t) {
common_chat_template tmpl = load_nemotron_template(t);
// Run differential analysis
struct autoparser analysis;
analysis.analyze_template(tmpl);
// Check tool markers - Nemotron uses per-call wrapping (each call individually wrapped)
t.assert_equal("tool_section_start should be empty (per-call format)", "", analysis.tools.format.section_start);
t.assert_equal("tool_section_end should be empty (per-call format)", "", analysis.tools.format.section_end);
t.assert_equal("per_call_start should be '<tool_call>\\n'", "<tool_call>\n", analysis.tools.format.per_call_start);
t.assert_equal("per_call_end should be '</tool_call>'", "</tool_call>", analysis.tools.format.per_call_end);
t.assert_true("should support parallel calls", analysis.jinja_caps.supports_parallel_tool_calls);
// Check function markers
t.assert_equal("func_name_prefix should be '<function='", "<function=", analysis.tools.function.name_prefix);
t.assert_equal("func_name_suffix should be '>\\n'", ">\n", analysis.tools.function.name_suffix);
t.assert_equal("func_close should be '</function>\\n'", "</function>\n", analysis.tools.function.close);
// Check argument markers (note: markers retain trailing newlines for proper parsing)
t.assert_equal("arg_name_prefix should be '<parameter='", "<parameter=", analysis.tools.arguments.name_prefix);
t.assert_equal("arg_name_suffix should be '>\\n'", ">\n", analysis.tools.arguments.name_suffix);
t.assert_equal("arg_value_suffix should be '\\n</parameter>\\n'", "\n</parameter>\n", analysis.tools.arguments.value_suffix);
// Check format classification
t.assert_true("tool format should be TAG_WITH_TAGGED", analysis.tools.format.mode == tool_format::TAG_WITH_TAGGED);
// Verify tool support
t.assert_true("should support tools", analysis.jinja_caps.supports_tools);
}
// ============================================================================
// Laguna Template Analysis Tests
// ============================================================================
static common_chat_template load_laguna_template(testing & t) {
return load_template(t, "models/templates/poolside-Laguna-XS-2.1.jinja");
}
static void test_laguna_reasoning_detection(testing & t) {
common_chat_template tmpl = load_laguna_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
// Laguna's template renders reasoning delimiters with formatting whitespace
// ("<think>\n") that the model does not emit; the Laguna patch trims them.
t.assert_equal("reasoning_start should be '<think>'", "<think>", analysis.reasoning.start);
t.assert_equal("reasoning_end should be '</think>'", "</think>", analysis.reasoning.end);
t.assert_equal("reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode);
}
static void test_laguna_tool_format(testing & t) {
common_chat_template tmpl = load_laguna_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("arg_value_suffix should be '</arg_value>'", "</arg_value>", analysis.tools.arguments.value_suffix);
}
static void test_laguna_stop_string(testing & t) {
// The </assistant> turn terminator can be emitted as ordinary text tokens
// (not the single eot token), so it must also be a literal stop string.
common_chat_template tmpl = load_laguna_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
bool has_stop = false;
for (const auto & stop : analysis.additional_stops) {
if (stop == "</assistant>") { has_stop = true; break; }
}
t.assert_true("Laguna additional_stops contains </assistant>", has_stop);
}
static void test_laguna_analysis(testing & t) {
t.test("Laguna reasoning detection", test_laguna_reasoning_detection);
t.test("Laguna tool format", test_laguna_tool_format);
t.test("Laguna stop string", test_laguna_stop_string);
}
static common_chat_template load_laguna_s_template(testing & t) {
return load_template(t, "models/templates/poolside-Laguna-S-2.1.jinja");
}
static void test_laguna_s_reasoning_detection(testing & t) {
common_chat_template tmpl = load_laguna_s_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("Laguna-S(v8) reasoning_start should be '<think>'", "<think>", analysis.reasoning.start);
t.assert_equal("Laguna-S(v8) reasoning_end should be '</think>'", "</think>", analysis.reasoning.end);
t.assert_equal("Laguna-S(v8) reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode);
}
static void test_laguna_s_tool_format(testing & t) {
common_chat_template tmpl = load_laguna_s_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("Laguna-S(v8) arg_value_suffix should be '</arg_value>'", "</arg_value>", analysis.tools.arguments.value_suffix);
}
static void test_laguna_s_preserve_reasoning(testing & t) {
common_chat_template tmpl = load_laguna_s_template(t);
t.assert_true("Laguna-S(v8) supports preserving reasoning", tmpl.original_caps().supports_preserve_reasoning);
}
static void test_laguna_s_analysis(testing & t) {
t.test("Laguna-S(v8) reasoning detection", test_laguna_s_reasoning_detection);
t.test("Laguna-S(v8) tool format", test_laguna_s_tool_format);
t.test("Laguna-S(v8) preserve reasoning", test_laguna_s_preserve_reasoning);
}
static common_chat_template load_laguna_xs2_template(testing & t) {
return load_template(t, "models/templates/poolside-Laguna-XS.2.jinja");
}
static void test_laguna_xs2_reasoning_detection(testing & t) {
common_chat_template tmpl = load_laguna_xs2_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("Laguna-XS.2(v5) reasoning_start should be '<think>'", "<think>", analysis.reasoning.start);
t.assert_equal("Laguna-XS.2(v5) reasoning_end should be '</think>'", "</think>", analysis.reasoning.end);
t.assert_equal("Laguna-XS.2(v5) reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode);
}
static void test_laguna_xs2_tool_format(testing & t) {
common_chat_template tmpl = load_laguna_xs2_template(t);
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("Laguna-XS.2(v5) arg_value_suffix should be '</arg_value>'", "</arg_value>", analysis.tools.arguments.value_suffix);
}
static void test_laguna_xs2_analysis(testing & t) {
t.test("Laguna-XS.2(v5) reasoning detection", test_laguna_xs2_reasoning_detection);
t.test("Laguna-XS.2(v5) tool format", test_laguna_xs2_tool_format);
}
static common_chat_template load_cohere_template(testing & t) {
return load_template(t, "models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja");
}
static void test_cohere_analysis(testing & t) {
t.test("Cohere reasoning detection", test_cohere_reasoning_detection);
}
static void test_cohere_reasoning_detection(testing & t) {
common_chat_template tmpl = load_cohere_template(t);
// Run differential analysis
struct autoparser analysis;
analysis.analyze_template(tmpl);
// Check reasoning markers - Cohere uses special token format
t.assert_equal("reasoning_start should be '<|START_THINKING|>'", "<|START_THINKING|>", analysis.reasoning.start);
t.assert_equal("reasoning_end should be '<|END_THINKING|>'", "<|END_THINKING|>", analysis.reasoning.end);
// Check reasoning mode - Cohere only shows reasoning with tool calls (TOOLS_ONLY)
t.assert_equal("reasoning should be TOOLS_ONLY", reasoning_mode::TOOLS_ONLY, analysis.reasoning.mode);
// Check content markers - Cohere wraps all content with START/END_RESPONSE
t.assert_equal("content_start should be '<|START_RESPONSE|>'", "<|START_RESPONSE|>", analysis.content.start);
t.assert_equal("content_end should be '<|END_RESPONSE|>'", "<|END_RESPONSE|>", analysis.content.end);
// Content is always wrapped (both with and without tools)
t.assert_equal("content should be ALWAYS_WRAPPED", content_mode::ALWAYS_WRAPPED, analysis.content.mode);
}
static void test_tool_format_cohere(testing & t) {
common_chat_template tmpl = load_cohere_template(t);
// Run differential analysis
struct autoparser analysis;
analysis.analyze_template(tmpl);
// Check tool section markers - Cohere uses ACTION markers
t.assert_equal("tool_section_start should be '<|START_ACTION|>'", "<|START_ACTION|>", analysis.tools.format.section_start);
t.assert_equal("tool_section_end should be '<|END_ACTION|>'", "<|END_ACTION|>", analysis.tools.format.section_end);
// JSON_NATIVE format has no per-call markers
t.assert_equal("per_call_start should be empty", "", analysis.tools.format.per_call_start);
t.assert_equal("per_call_end should be empty", "", analysis.tools.format.per_call_end);
// JSON_NATIVE format has empty function markers (no XML-style markers)
t.assert_equal("func_name_prefix should be empty", "", analysis.tools.function.name_prefix);
t.assert_equal("func_name_suffix should be empty", "", analysis.tools.function.name_suffix);
t.assert_equal("func_close should be empty", "", analysis.tools.function.close);
// JSON_NATIVE format has empty args markers
t.assert_equal("args_start should be empty", "", analysis.tools.arguments.start);
t.assert_equal("args_end should be empty", "", analysis.tools.arguments.end);
// JSON_NATIVE format has empty argument markers
t.assert_equal("arg_name_prefix should be empty", "", analysis.tools.arguments.name_prefix);
t.assert_equal("arg_name_suffix should be empty", "", analysis.tools.arguments.name_suffix);
t.assert_equal("arg_value_prefix should be empty", "", analysis.tools.arguments.value_prefix);
t.assert_equal("arg_value_suffix should be empty", "", analysis.tools.arguments.value_suffix);
t.assert_equal("arg_separator should be empty", "", analysis.tools.arguments.separator);
// Check JSON field names - Cohere uses non-standard names
t.assert_equal("name_field should be 'tool_name'", "tool_name", analysis.tools.format.name_field);
t.assert_equal("args_field should be 'parameters'", "parameters", analysis.tools.format.args_field);
// This isn't a real tool call id field, i.e. with the OpenAI tool call ID format
t.assert_equal("id_field should be 'tool_call_id'", "", analysis.tools.format.id_field);
// Check format classification
t.assert_equal("tool format should be JSON_NATIVE", tool_format::JSON_NATIVE, analysis.tools.format.mode);
// Check flags
t.assert_true("should support tools", analysis.jinja_caps.supports_tools);
t.assert_true("should support parallel calls", analysis.jinja_caps.supports_parallel_tool_calls);
t.assert_true("should not require nonnull content", !analysis.content.requires_nonnull_content);
t.assert_true("tools_array_wrapped should be true", analysis.tools.format.tools_array_wrapped);
}
// ============================================================================
// SmolLM3 Template Analysis Tests
// Tests for templates that change system message when enable_thinking flips
// and prefill an empty <think></think> block in no-think mode.
// ============================================================================
static common_chat_template load_smollm3_template(testing & t) {
return load_template(t, "models/templates/HuggingFaceTB-SmolLM3-3B.jinja");
}
static void test_smollm3_reasoning_detection(testing & t);
static void test_smollm3_analysis(testing & t) {
t.test("SmolLM3 reasoning detection", test_smollm3_reasoning_detection);
}
static void test_smollm3_reasoning_detection(testing & t) {
common_chat_template tmpl = load_smollm3_template(t);
// Run differential analysis
struct autoparser analysis;
analysis.analyze_template(tmpl);
// SmolLM3 uses <think>/<think> reasoning tags.
// The template changes the entire system message when enable_thinking flips,
// so the analyzer must compare isolated generation prompts (not full outputs).
t.assert_equal("reasoning_start should be '<think>'", "<think>", analysis.reasoning.start);
t.assert_equal("reasoning_end should be '</think>'", "</think>", analysis.reasoning.end);
t.assert_equal("reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode);
// Content should remain plain (no wrappers)
t.assert_equal("content start should be empty", "", analysis.content.start);
t.assert_equal("content end should be empty", "", analysis.content.end);
t.assert_equal("content should be PLAIN", content_mode::PLAIN, analysis.content.mode);
// Preserved tokens should include the reasoning markers
bool has_think_start = std::find(analysis.preserved_tokens.begin(), analysis.preserved_tokens.end(), "<think>") != analysis.preserved_tokens.end();
bool has_think_end = std::find(analysis.preserved_tokens.begin(), analysis.preserved_tokens.end(), "</think>") != analysis.preserved_tokens.end();
t.assert_true("preserved_tokens should contain '<think>'", has_think_start);
t.assert_true("preserved_tokens should contain '</think>'", has_think_end);
}
// ============================================================================
// standard_json_tools Format Tests
// ============================================================================
// Helper to build tools definition for tests
static json build_test_tools() {
json parameters_schema = json::object();
parameters_schema["type"] = "object";
parameters_schema["properties"] = json::object();
parameters_schema["properties"]["location"] = json::object({
{"type", "string"},
{"description", "The city and state"}
});
parameters_schema["properties"]["unit"] = json::object({
{"type", "string"},
{"description", "Temperature unit"},
{"enum", json::array({"celsius", "fahrenheit"})}
});
parameters_schema["required"] = json::array({"location"});
return json::array({
json{
{"type", "function"},
{"function", json{
{"name", "get_current_weather"},
{"description", "Get the current weather in a given location"},
{"parameters", parameters_schema}
}}
}
});
}
static void test_standard_json_tools_formats(testing & t) {
t.test("OpenAI format", test_standard_json_tools_openai);
t.test("Cohere format", test_standard_json_tools_cohere);
t.test("function-as-key format", test_standard_json_tools_function_key);
}
// Test 1: OpenAI Standard Format
// {"id": "call_abc", "function": {"name": "get_weather", "arguments": {"location": "NYC"}}}
static void test_standard_json_tools_openai(testing & t) {
json tools = build_test_tools();
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto tool_call = p.standard_json_tools(
"<tool_call>", "</tool_call>", tools,
/* parallel */ true,
/* force */ false,
/* name_key */ "function.name",
/* args_key */ "function.arguments",
/* array_wrapped */ false,
/* function_is_key */ false,
/* call_id_key */ "id",
/* gen_call_id_key */ "",
/* parameters_order */ {}
);
return p.content(p.until("<tool_call>")) + p.optional(tool_call) + p.end();
});
std::string input =
"Let me check the weather."
"<tool_call>"
R"({"id": "call_abc123", "function": {"name": "get_current_weather", "arguments": {"location": "NYC"}}})"
"</tool_call>";
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
return;
}
common_chat_msg msg;
auto mapper = common_chat_peg_mapper(msg);
mapper.from_ast(ctx.ast, result);
t.assert_equal("tool calls count", 1u, msg.tool_calls.size());
if (!msg.tool_calls.empty()) {
t.assert_equal("tool name", "get_current_weather", msg.tool_calls[0].name);
t.assert_equal("tool id", "call_abc123", msg.tool_calls[0].id);
}
t.assert_true("content present", msg.content.find("Let me check the weather") != std::string::npos);
}
// Test 2: Cohere Format
// {"tool_call_id": 0, "tool_name": "get_weather", "parameters": {"location": "NYC"}}
static void test_standard_json_tools_cohere(testing & t) {
json tools = build_test_tools();
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto tool_call = p.standard_json_tools(
"<|START_ACTION|>[", "]<|END_ACTION|>", tools,
/* parallel */ true,
/* force */ false,
/* name_key */ "tool_name",
/* args_key */ "parameters",
/* array_wrapped */ false, // Brackets are part of section markers
/* function_is_key */ false,
/* call_id_key */ "",
/* gen_call_id_key */ "tool_call_id",
/* parameters_order */ {"tool_call_id", "tool_name", "parameters"}
);
return p.content(p.until("<|START_ACTION|>")) + p.optional(tool_call) + p.end();
});
std::string input =
"Let me search for that."
"<|START_ACTION|>["
R"({"tool_call_id": 0, "tool_name": "get_current_weather", "parameters": {"location": "NYC", "unit": "celsius"}})"
"]<|END_ACTION|>";
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
return;
}
common_chat_msg msg;
auto mapper = common_chat_peg_mapper(msg);
mapper.from_ast(ctx.ast, result);
t.assert_equal("tool calls count", 1u, msg.tool_calls.size());
if (!msg.tool_calls.empty()) {
t.assert_equal("tool name", "get_current_weather", msg.tool_calls[0].name);
t.assert_equal("tool id", "0", msg.tool_calls[0].id);
}
t.assert_true("content present", msg.content.find("Let me search") != std::string::npos);
}
// Test 3: Function-as-Key Format
// {"get_current_weather": {"id": "call-0001", "args": {"location": "NYC"}}}
static void test_standard_json_tools_function_key(testing & t) {
json tools = build_test_tools();
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto tool_call = p.standard_json_tools(
"<tool_calls>[", "]</tool_calls>", tools,
/* parallel */ true,
/* force */ false,
/* name_key */ "", // Name is the key itself
/* args_key */ "args",
/* array_wrapped */ false,
/* function_is_key */ true,
/* call_id_key */ "id",
/* gen_call_id_key */ "",
/* parameters_order */ {}
);
return p.content(p.until("<tool_calls>")) + p.optional(tool_call) + p.end();
});
std::string input =
"I'll call the weather function."
"<tool_calls>["
R"({"get_current_weather": {"id": "call-0001", "args": {"location": "NYC", "unit": "celsius"}}})"
"]</tool_calls>";
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
return;
}
common_chat_msg msg;
auto mapper = common_chat_peg_mapper(msg);
mapper.from_ast(ctx.ast, result);
t.assert_equal("tool calls count", 1u, msg.tool_calls.size());
if (!msg.tool_calls.empty()) {
t.assert_equal("tool name", "get_current_weather", msg.tool_calls[0].name);
t.assert_equal("tool id", "call-0001", msg.tool_calls[0].id);
}
t.assert_true("content present", msg.content.find("I'll call the weather") != std::string::npos);
}
// ============================================================================
// normalize_quotes_to_json Tests
// ============================================================================
// Copy of the function for isolated testing (original is static in chat-peg-parser.cpp)
static std::string normalize_quotes_to_json(const std::string & input) {
std::string result;
result.reserve(input.size() + 16);
bool in_single_quoted = false;
bool in_double_quoted = false;
for (size_t i = 0; i < input.size(); ++i) {
char c = input[i];
if (c == '\\' && i + 1 < input.size()) {
char next = input[i + 1];
if (in_single_quoted) {
if (next == '\'') {
result += '\'';
++i;
continue;
}
if (next == '"') {
result += "\\\"";
++i;
continue;
}
result += c;
result += next;
++i;
continue;
}
if (in_double_quoted) {
result += c;
result += next;
++i;
continue;
}
result += c;
continue;
}
if (c == '"') {
if (in_single_quoted) {
result += "\\\"";
} else {
in_double_quoted = !in_double_quoted;
result += c;
}
} else if (c == '\'') {
if (in_double_quoted) {
result += c;
} else if (in_single_quoted) {
in_single_quoted = false;
result += '"';
} else {
in_single_quoted = true;
result += '"';
}
} else {
result += c;
}
}
return result;
}
static void test_normalize_quotes_to_json(testing & t) {
t.test("basic single to double quotes", [](testing & t) {
std::string input = "{'key': 'value'}";
std::string expected = "{\"key\": \"value\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("basic conversion", expected, result);
});
t.test("escaped single quote inside single-quoted string", [](testing & t) {
std::string input = "{'code': 'print(\\'hello\\')'}";
std::string expected = "{\"code\": \"print('hello')\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("escaped single quote", expected, result);
});
t.test("double quote inside single-quoted string", [](testing & t) {
std::string input = "{'msg': 'He said \"hi\"'}";
std::string expected = "{\"msg\": \"He said \\\"hi\\\"\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("double quote escaping", expected, result);
});
t.test("nested backslash escapes", [](testing & t) {
std::string input = "{'path': 'C:\\\\Users\\\\test'}";
std::string expected = "{\"path\": \"C:\\\\Users\\\\test\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("backslash escaping", expected, result);
});
t.test("newline escapes", [](testing & t) {
std::string input = "{'text': 'line1\\nline2'}";
std::string expected = "{\"text\": \"line1\\nline2\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("newline escaping", expected, result);
});
t.test("mixed quotes", [](testing & t) {
std::string input = "{\"already_double\": 'single_value'}";
std::string expected = "{\"already_double\": \"single_value\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("mixed quotes", expected, result);
});
t.test("embedded quotes - the test case", test_normalize_quotes_with_embedded_quotes);
}
// Test case that mirrors the Seed-OSS failing test scenario
static void test_normalize_quotes_with_embedded_quotes(testing & t) {
// This is similar to the Seed-OSS template test case
// The input has embedded double quotes like "14" and "bar" inside string values
std::string input = "{'filename': 'foo.cpp', 'oldString': 'def foo(arg = \"14\"):\\n return arg + \"bar\"\\n', 'newString': 'def foo(arg = \"15\"):\\n pass\\n'}";
// Expected: Python single quotes -> JSON double quotes, internal double quotes escaped
std::string expected = "{\"filename\": \"foo.cpp\", \"oldString\": \"def foo(arg = \\\"14\\\"):\\n return arg + \\\"bar\\\"\\n\", \"newString\": \"def foo(arg = \\\"15\\\"):\\n pass\\n\"}";
std::string result = normalize_quotes_to_json(input);
t.assert_equal("normalize quotes with embedded double quotes", expected, result);
// Also verify the result is valid JSON
try {
json parsed = json::parse(result);
t.assert_true("result is valid JSON", true);
t.assert_equal("filename field", "foo.cpp", parsed["filename"].get<std::string>());
t.assert_true("oldString contains embedded quotes",
parsed["oldString"].get<std::string>().find("\"14\"") != std::string::npos);
t.assert_true("newString contains embedded quotes",
parsed["newString"].get<std::string>().find("\"15\"") != std::string::npos);
} catch (const std::exception & e) {
t.assert_true(std::string("JSON parse failed: ") + e.what(), false);
}
}
// ============================================================================
// TAG_WITH_TAGGED Argument Parsing Tests
// ============================================================================
// Build tools definition for edit function
static json build_edit_tool() {
json parameters_schema = json::object();
parameters_schema["type"] = "object";
parameters_schema["properties"] = json::object();
parameters_schema["properties"]["filename"] = json::object({
{"type", "string"},
{"description", "Path of file to edit"}
});
parameters_schema["properties"]["oldString"] = json::object({
{"type", "string"},
{"description", "String to replace"}
});
parameters_schema["properties"]["newString"] = json::object({
{"type", "string"},
{"description", "New (replacement) value"}
});
parameters_schema["required"] = json::array({"filename", "oldString", "newString"});
return json::array({
json{
{"type", "function"},
{"function", json{
{"name", "edit"},
{"description", "Edit a file"},
{"parameters", parameters_schema}
}}
}
});
}
// ============================================================================
// Role marker detection tests for all autoparser-handled templates
//
// Verifies that detect_user_start_marker / detect_assistant_start_marker
// return the correct boundary text between turns for every template that
// falls through to the differential autoparser (i.e. is not handled by a
// dedicated specialized template in common_chat_try_specialized_template).
//
// Markers were deduced manually from the jinja sources in models/templates/.
// ============================================================================
struct role_marker_case {
std::string template_file;
std::string expected_user_start;
std::string expected_assistant_start;
};
static void test_role_markers_all_templates(testing & t) {
// Each entry is { template filename, user_start, assistant_start } as
// produced when rendering the standard chatml-like sequences. The values
// come from reading each jinja template and tracing what text precedes
// a user/assistant message body once the autoparser strips any reasoning
// markers it detected first.
const std::vector<role_marker_case> cases = {
// ChatML family: <|im_start|>{role} ... <|im_end|>
{ "Bielik-11B-v3.0-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "HuggingFaceTB-SmolLM3-3B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "MiMo-VL.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen3.5-4B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen3-Coder.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-Qwen2.5-7B-Instruct.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" },
// DeepSeek family
{ "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "deepseek-ai-DeepSeek-V3.1.jinja", "<｜User｜>", "<｜Assistant｜>" },
{ "llama-cpp-deepseek-r1.jinja", "<｜User｜>", "<｜Assistant｜>" },
// Llama 3 header family
{ "meetkai-functionary-medium-v3.1.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meta-llama-Llama-3.1-8B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meta-llama-Llama-3.2-3B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
{ "meta-llama-Llama-3.3-70B-Instruct.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
// fireworks-ai forces a trailing assistant header even without add_generation_prompt,
// so the marker is absorbed into the common suffix and assistant_start is detected as empty.
{ "fireworks-ai-llama-3-firefunction-v2.jinja", "<|start_header_id|>user<|end_header_id|>", "<|start_header_id|>assistant<|end_header_id|>" },
// Phi/GLM/Apriel-style: <|user|> / <|assistant|>
{ "microsoft-Phi-3.5-mini-instruct.jinja", "<|user|>", "<|assistant|>" },
{ "GLM-4.6.jinja", "<|user|>", "<|assistant|>" },
{ "unsloth-Apriel-1.5.jinja", "<|user|>", "<|assistant|>" },
{ "GLM-4.7-Flash.jinja", "<|user|>", "<|assistant|>" },
// Gemma 2: <start_of_turn>{user|model}
{ "google-gemma-2-2b-it.jinja", "<start_of_turn>user", "<start_of_turn>model" },
// IBM Granite
{ "ibm-granite-granite-3.3-2B-Instruct.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>" },
{ "ibm-granite-granite-4.0.jinja", "<|start_of_role|>user<|end_of_role|>", "<|start_of_role|>assistant<|end_of_role|>" },
// Cohere R-series
{ "CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja",
"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_RESPONSE|>" },
{ "CohereForAI-c4ai-command-r-plus-tool_use.jinja",
"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" },
// Mistral: assistant content follows [/INST] immediately, no header
{ "mistralai-Mistral-Nemo-Instruct-2407.jinja", "[INST]", "" },
{ "Mistral-Small-3.2-24B-Instruct-2506.jinja", "[INST]", "" },
// Apertus uses <|user_start|> / <|assistant_start|> but the user diff
// carries the preceding <|assistant_end|> from the previous turn.
{ "Apertus-8B-Instruct.jinja", "<|user_start|>", "<|assistant_start|>" },
// Apriel 1.6 wraps the assistant body with <|begin_assistant|>, but
// <|begin_assistant|> is also the detected reasoning start, so the
// assistant_start is trimmed back to the preceding newline.
{ "Apriel-1.6-15b-Thinker-fixed.jinja", "<|begin_user|>", "<|begin_assistant|>" },
// ByteDance Seed-OSS: <seed:bos>{role}
{ "ByteDance-Seed-OSS.jinja", "<seed:bos>user", "<seed:bos>assistant" },
// GigaChat 3.1: {role}<|role_sep|>
{ "GigaChat3.1-10B-A1.8B.jinja", "user<|role_sep|>", "assistant<|role_sep|>" },
// MiniMax M2: ]~b]{user|ai}
{ "MiniMax-M2.jinja", "]~b]user", "]~b]ai" },
// HunYuan V3: <｜hy_User:opensource｜> / <｜hy_Assistant:opensource｜>
{ "tencent-Hy3.jinja", "<｜hy_User:opensource｜>", "<｜hy_Assistant:opensource｜>" },
// Nemotron Nano v2: <SPECIAL_11>{User|Assistant}; assistant marker
// is followed by a prefilled <think> block that gets included.
{ "NVIDIA-Nemotron-Nano-v2.jinja", "<SPECIAL_11>User", "<SPECIAL_11>Assistant" },
// Reka Edge: "human: " / "assistant: " — but the rendered preamble
// depends on enable_thinking, which currently confuses the user-start
// diff and trims the marker down. Lock in the observed value.
{ "Reka-Edge.jinja", "human:", "assistant:" },
// RWKV-world chat preset: "User: " / "Assistant: "
{ "llama-cpp-rwkv-world.jinja", "User:", "Assistant:" },
// Upstage Solar 100B: <|begin|>{role}... but reasoning marker absorbs
// the "<|begin|>assistant" prefix from assistant_start.
{ "upstage-Solar-Open-100B.jinja", "<|begin|>user<|content|>", "<|begin|>assistant" },
};
for (const auto & c : cases) {
t.test(c.template_file, [&](testing & t) {
common_chat_template tmpl = load_template(t, "models/templates/" + c.template_file);
struct autoparser ap;
ap.analyze_template(tmpl);
t.assert_equal("user_start", c.expected_user_start, ap.user_start);
t.assert_equal("assistant_start", c.expected_assistant_start, ap.assistant_start);
});
}
}
static void test_bailing_v3_tool_format(testing & t) {
const std::string template_source = R"JINJA(
{# Bailing V3 chat template #}
{%- if tools %}{{ tools | tojson }}{%- endif %}
{%- for message in messages %}
{%- if message.role == "user" %}
{{- '<role>HUMAN</role>' + message.content + '<|role_end|>' }}
{%- elif message.role == "assistant" %}
{{- '<role>ASSISTANT</role>' }}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- set tc = tool_call.function %}
{{- '<tool_call>' + tc.name }}
{%- for k, v in tc.arguments.items() %}
{{- '<arg_key>' + k + '</arg_key>' }}
{{- '\n<arg_value>' + v + '</arg_value>' }}
{%- endfor %}
{{- '\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|role_end|>' }}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}{{- '<role>ASSISTANT</role>' }}{%- endif %}
)JINJA";
common_chat_template tmpl(template_source, "", "");
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("arg_value_suffix", "</arg_value>", analysis.tools.arguments.value_suffix);
t.assert_true("intertag whitespace", analysis.tools.arguments.tolerate_intertag_whitespace);
generation_params inputs;
inputs.tools = json::array({
{
{ "type", "function" },
{ "function", {
{ "name", "test_function_name" },
{ "parameters", {
{ "type", "object" },
{ "properties", {
{ "param1", { { "type", "string" } } },
{ "param2", { { "type", "string" } } },
} },
} },
} },
},
});
inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE;
auto parser = analysis.build_parser(inputs, "");
const std::string output =
"<tool_call>test_function_name\n"
"<arg_key>param1</arg_key>\n"
"<arg_value>value1</arg_value>"
"<arg_key>param2</arg_key>\n"
"<arg_value>value2</arg_value>\n"
"</tool_call>";
common_peg_parse_context ctx(output, COMMON_PEG_PARSE_FLAG_LENIENT);
t.assert_true("multi-argument tool call", parser.parse(ctx).success());
}
// Test that reproduces the Seed-OSS template issue with embedded quotes
static void test_tagged_args_with_embedded_quotes(testing & t) {
json tools = build_edit_tool();
// Build a parser for TAG_WITH_TAGGED format like Seed-OSS/Nemotron
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Build tool choice for the edit function
auto tool_choice = p.choice();
for (const auto & tool_def : tools) {
if (!tool_def.contains("function")) { continue; }
const auto & function = tool_def.at("function");
std::string name = function.at("name");
const auto & params = function.at("parameters");
if (!params.contains("properties") || !params.at("properties").is_object()) { continue; }
const auto & properties = params.at("properties");
// Build argument parsers
std::vector<common_peg_parser> arg_parsers;
for (const auto & [param_name, param_schema] : properties.items()) {
auto arg = p.tool_arg(
p.tool_arg_open(p.literal("<parameter=") + p.tool_arg_name(p.literal(param_name)) + p.literal(">")) +
p.space() +
p.tool_arg_string_value(p.until("</parameter>")) +
p.space() +
p.tool_arg_close(p.literal("</parameter>"))
);
arg_parsers.push_back(p.optional(p.rule("arg-" + param_name, arg)));
}
// Build arg sequence with space() between
common_peg_parser args_seq = p.eps();
for (size_t i = 0; i < arg_parsers.size(); i++) {
if (i > 0) {
args_seq = args_seq + p.space();
}
args_seq = args_seq + arg_parsers[i];
}
auto func_parser =
p.tool_open(p.literal("<function=") + p.tool_name(p.literal(name)) + p.literal(">")) +
p.space() + args_seq + p.space() +
p.tool_close(p.literal("</function>"));
tool_choice |= p.rule("tool-" + name, p.tool(func_parser));
}
auto tool_section =
p.literal("<seed:tool_call>") + p.space() +
tool_choice +
p.space() + p.literal("</seed:tool_call>");
return p.content(p.until("<seed:tool_call>")) + p.optional(tool_section) + p.end();
});
std::string input =
"<seed:tool_call>\n"
"<function=edit>\n"
"<parameter=filename>"
"foo.cpp"
"</parameter>\n"
"<parameter=oldString>"
"def foo(arg = \"14\"):\n"
" return arg + \"bar\"\n"
"\n"
"</parameter>\n"
"<parameter=newString>"
"def foo(arg = \"15\"):\n"
" pass\n"
"\n"
"</parameter>\n"
"</function>\n"
"</seed:tool_call>";
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
return;
}
common_chat_msg msg;
auto mapper = common_chat_peg_mapper(msg);
mapper.from_ast(ctx.ast, result);
t.assert_equal("tool calls count", 1u, msg.tool_calls.size());
if (!msg.tool_calls.empty()) {
t.assert_equal("tool name", "edit", msg.tool_calls[0].name);
// Parse the arguments as JSON to verify they're valid
std::string args = msg.tool_calls[0].arguments;
try {
json parsed = json::parse(args);
t.assert_true("arguments is valid JSON", true);
// Verify each field has proper value
t.assert_equal("filename", "foo.cpp", parsed.value("filename", ""));
std::string oldString = parsed.value("oldString", "");
t.assert_true("oldString contains embedded quotes",
oldString.find("\"14\"") != std::string::npos);
t.assert_true("oldString contains bar with quotes",
oldString.find("\"bar\"") != std::string::npos);
std::string newString = parsed.value("newString", "");
t.assert_true("newString contains embedded quotes",
newString.find("\"15\"") != std::string::npos);
} catch (const std::exception & e) {
t.assert_true(std::string("arguments should be valid JSON: ") + e.what(), false);
}
}
}
Reference in New Issue View Git Blame Copy Permalink