Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.devops/cpu.Dockerfile
#	.devops/cuda.Dockerfile
#	.devops/intel.Dockerfile
#	.devops/llama-cli-cann.Dockerfile
#	.devops/musa.Dockerfile
#	.devops/openvino.Dockerfile
#	.devops/rocm.Dockerfile
#	.devops/s390x.Dockerfile
#	.devops/vulkan.Dockerfile
#	.devops/zendnn.Dockerfile
#	.github/labeler.yml
#	docs/backend/CUDA-FEDORA.md
#	docs/ops.md
#	docs/ops/SYCL.csv
#	examples/sycl/build.sh
#	examples/sycl/win-build-sycl.bat
#	ggml/src/ggml-sycl/common.cpp
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/cpy.hpp
#	ggml/src/ggml-sycl/dmmv.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/mmvq.cpp
#	ggml/src/ggml-sycl/presets.hpp
#	ggml/src/ggml-sycl/set_rows.cpp
#	ggml/src/ggml-sycl/softmax.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
#	grammars/README.md
#	tests/test-backend-ops.cpp
#	tests/test-chat-auto-parser.cpp
#	tests/test-chat.cpp
#	tests/test-jinja.cpp
#	tools/cli/cli.cpp
This commit is contained in:
Concedo
2026-06-15 20:00:57 +08:00
parent be80f5dcbc 6eab47181c
commit bdbcd049b1
40 changed files with 811 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat-auto-parser-generator.cpp
+2 -3
View File
@@ -134,7 +134,7 @@ common_peg_arena autoparser::build_parser(const generation_params & inputs, cons
auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
parser = ctx.reasoning_parser + p.space() + p.choice({
p.literal("```json") + p.space() + response_format + p.space() + p.literal("```"),
response_format
p.space() + response_format + p.space()
}) + p.end();
pure_content = false;
} else if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && jinja_caps.supports_tool_calls) {
@@ -393,8 +393,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
(schema_info.resolves_to_string(param_schema) ?
p.tool_arg_string_value(until_suffix) :
p.tool_arg_json_value(p.schema(
p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
p.space()) +
p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false))) +
p.tool_arg_close(p.literal(arguments.value_suffix)));
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
common/chat-diff-analyzer.cpp
+3 -4
View File
@@ -1229,8 +1229,8 @@ void analyze_tools::extract_argument_name_markers() {
left_result.tags["pre"] == right_result.tags["pre"] &&
left_result.tags["suffix"] == right_result.tags["suffix"]) {
// Name is inside a structure (e.g., JSON key): prefix is the shared wrapper
arguments.name_prefix = trim_whitespace(left_result.tags["pre"]);
arguments.name_suffix = trim_leading_whitespace(left_result.tags["suffix"]);
arguments.name_prefix = left_result.tags["pre"];
arguments.name_suffix = left_result.tags["suffix"];
} else if (diff.left.substr(0, ARG_FIRST.length()) == ARG_FIRST && diff.right.substr(0, ARG_SECOND.length()) == ARG_SECOND) {
// Name is directly in the diff: prefix comes from last marker in diff.prefix
auto pre_parser = build_tagged_peg_parser([&](common_peg_parser_builder & p) {
@@ -1315,8 +1315,7 @@ void analyze_tools::extract_argument_value_markers() {
value_suffix = value_suffix.substr(0, end_marker_pos);
}
}
value_suffix = trim_leading_whitespace(value_suffix);
if (!value_suffix.empty()) {
if (!trim_whitespace(value_suffix).empty()) {
arguments.value_suffix = value_suffix;
}
}
common/chat-peg-parser.cpp
+1 -1
View File
@@ -363,7 +363,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
}
if ((is_arg_value || is_arg_string_value) && current_tool) {
std::string value_content = std::string(trim_trailing_space(trim_leading_space(node.text, 1), 1));
std::string value_content = std::string(node.text);
std::string value_to_add;
if (value_content.empty() && is_arg_string_value) {
common/chat.cpp
+149
View File
@@ -1994,6 +1994,146 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
return data;
}
// Cohere2 MoE (a.k.a. "North Code") parser.
//
// The assistant turn is fully marker-wrapped:
// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
// <|START_THINKING|>{reasoning}<|END_THINKING|>
// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|>
// OR tool calls: <|START_ACTION|>[
// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...
// ]<|END_ACTION|>
// <|END_OF_TURN_TOKEN|>
//
// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is
// the template default), so the model's output continues from *inside* the thinking block. The
// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix
// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,
// regardless of whether they came from the generation prompt or the generated text.
static common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
const std::string TURN_START = "<|START_OF_TURN_TOKEN|>";
const std::string TURN_END = "<|END_OF_TURN_TOKEN|>";
const std::string CHATBOT = "<|CHATBOT_TOKEN|>";
const std::string USER = "<|USER_TOKEN|>";
const std::string SYSTEM = "<|SYSTEM_TOKEN|>";
const std::string THINK_START = "<|START_THINKING|>";
const std::string THINK_END = "<|END_THINKING|>";
const std::string TEXT_START = "<|START_TEXT|>";
const std::string TEXT_END = "<|END_TEXT|>";
const std::string ACTION_START = "<|START_ACTION|>";
const std::string ACTION_END = "<|END_ACTION|>";
const std::string RESULT_START = "<|START_TOOL_RESULT|>";
const std::string RESULT_END = "<|END_TOOL_RESULT|>";
// Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.
const std::string GEN_PREFIX = TURN_START + CHATBOT;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = THINK_START;
data.thinking_end_tag = THINK_END;
data.preserved_tokens = {
TURN_START, TURN_END, CHATBOT, USER, SYSTEM,
THINK_START, THINK_END,
TEXT_START, TEXT_END,
ACTION_START, ACTION_END,
RESULT_START, RESULT_END,
};
// Split the rendered prompt into per-role message spans. Tool results are rendered with the
// system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before
// the plain "system" one (it is a strict superset, and the role split tries delimiters in order).
data.message_spans = common_chat_split_by_role(data.prompt, {
{ "assistant", GEN_PREFIX },
{ "user", TURN_START + USER },
{ "tool", TURN_START + SYSTEM + RESULT_START },
{ "system", TURN_START + SYSTEM },
});
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
if (inputs.has_continuation()) {
const auto & msg = inputs.continue_msg;
data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
data.generation_prompt += THINK_END + TEXT_START + msg.render_content();
}
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto end = p.end();
// The thinking block is always present (the generation prompt forces <|START_THINKING|>).
// When extracting reasoning, capture its body; otherwise keep the whole block (markers
// included) inline as content, matching reasoning_format=NONE conventions.
common_peg_parser reasoning = p.eps();
if (extract_reasoning) {
reasoning = p.optional(p.literal(THINK_START) +
p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +
p.optional(p.literal(THINK_END)));
} else {
reasoning = p.optional(p.content(p.literal(THINK_START) +
p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +
p.optional(p.literal(THINK_END))));
}
auto text_content = p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;
}
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
// <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>
auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,
/* force_tool_calls = */ true,
/* name_key = */ "tool_name",
/* args_key = */ "parameters",
/* array_wrapped = */ true,
/* function_is_key = */ false,
/* call_id_key = */ "",
/* gen_call_id_key = */ "tool_call_id",
/* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });
// Content and tool calls are mutually exclusive in this format.
common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });
return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
foreach_function(inputs.tools, [&](const json & tool) {
const auto & function = tool.at("function");
auto schema = function.at("parameters");
builder.resolve_refs(schema);
});
parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }
};
}
return data;
}
namespace workaround {
static void map_developer_role_to_system(json & messages) {
@@ -2242,6 +2382,15 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_kimi_k2(tmpl, params);
}
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
// <|START_ACTION|> JSON tool calls. <|START_TEXT|> is unique to this template (the older
// Command-R templates use <|START_RESPONSE|>).
if (src.find("<|START_TEXT|>") != std::string::npos &&
src.find("<|START_ACTION|>") != std::string::npos) {
LOG_DBG("Using specialized template: Cohere2 MoE\n");
return common_chat_params_init_cohere2moe(tmpl, params);
}
if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);
common/jinja/runtime.cpp
+14 -6
View File
@@ -316,12 +316,22 @@ value filter_expression::execute_impl(context & ctx) {
JJ_DEBUG("Applying filter to %s", input->type().c_str());
auto set_filter_alias = [](auto & filter_id) {
if (filter_id == "count") {
filter_id = "length";
} else if (filter_id == "d") {
filter_id = "default";
} else if (filter_id == "e") {
filter_id = "escape";
} else if (filter_id == "trim") {
filter_id = "strip";
}
};
if (is_stmt<identifier>(filter)) {
auto filter_id = cast_stmt<identifier>(filter)->val;
if (filter_id == "trim") {
filter_id = "strip"; // alias
}
set_filter_alias(filter_id);
JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
// TODO: Refactor filters so this coercion can be done automatically
if (!input->is_undefined() && !is_val<value_string>(input) && (
@@ -345,9 +355,7 @@ value filter_expression::execute_impl(context & ctx) {
}
auto filter_id = cast_stmt<identifier>(call->callee)->val;
if (filter_id == "trim") {
filter_id = "strip"; // alias
}
set_filter_alias(filter_id);
JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
func_args args(ctx);
for (const auto & arg_expr : call->args) {
common/peg-parser.cpp
+10 -15
View File
@@ -1272,13 +1272,13 @@ common_peg_parser common_peg_parser_builder::string_content(char delimiter) {
common_peg_parser common_peg_parser_builder::double_quoted_string() {
return rule("double-quoted-string", [this]() {
return sequence({literal("\""), string_content('"'), literal("\""), space()});
return sequence({literal("\""), string_content('"'), literal("\"")});
});
}
common_peg_parser common_peg_parser_builder::single_quoted_string() {
return rule("single-quoted-string", [this]() {
return sequence({literal("'"), string_content('\''), literal("'"), space()});
return sequence({literal("'"), string_content('\''), literal("'")});
});
}
@@ -1301,25 +1301,25 @@ common_peg_parser common_peg_parser_builder::json_number() {
// At EOF in partial mode, chars returns NEED_MORE → negate propagates NEED_MORE → number not committed.
// This prevents premature commits of partial numbers (e.g. "3" when "3.14" is incoming).
auto not_number_continuation = negate(chars("[0-9.eE+-]", 1, 1));
return sequence({ optional(literal("-")), int_part, optional(frac), optional(exp), not_number_continuation, space() });
return sequence({ optional(literal("-")), int_part, optional(frac), optional(exp), not_number_continuation });
});
}
common_peg_parser common_peg_parser_builder::json_string() {
return rule("json-string", [this]() {
return sequence({literal("\""), string_content('"'), literal("\""), space()});
return sequence({literal("\""), string_content('"'), literal("\"")});
});
}
common_peg_parser common_peg_parser_builder::json_bool() {
return rule("json-bool", [this]() {
return sequence({choice({literal("true"), literal("false")}), space()});
return choice({literal("true"), literal("false")});
});
}
common_peg_parser common_peg_parser_builder::json_null() {
return rule("json-null", [this]() {
return sequence({literal("null"), space()});
return literal("null");
});
}
@@ -1334,8 +1334,7 @@ common_peg_parser common_peg_parser_builder::json_object() {
choice({
literal("}"),
sequence({members, ws, literal("}")})
}),
ws
})
});
});
}
@@ -1350,8 +1349,7 @@ common_peg_parser common_peg_parser_builder::json_array() {
choice({
literal("]"),
sequence({elements, ws, literal("]")})
}),
ws
})
});
});
}
@@ -1381,16 +1379,13 @@ common_peg_parser common_peg_parser_builder::python_number() {
common_peg_parser common_peg_parser_builder::python_bool() {
return rule("python-bool", [this]() {
return sequence({
choice({literal("True"), literal("False")}),
space()
});
return choice({literal("True"), literal("False")});
});
}
common_peg_parser common_peg_parser_builder::python_null() {
return rule("python-none", [this]() {
return sequence({literal("None"), space()});
return literal("None");
});
}