Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '11b068d06605288ce7917534b46d52b47823dc13' into concedo_experimental

Browse Source 
# Conflicts:
#	CONTRIBUTING.md
#	docs/backend/SYCL.md
#	docs/install.md
#	docs/speculative.md
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-llama-archs.cpp
#	tools/cli/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/README.md
This commit is contained in:
Concedo
2026-07-31 17:06:19 +08:00
parent 49ff71991e 11b068d066
commit 98926f27c1
81 changed files with 5314 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+26 -14
View File
@@ -1062,6 +1062,31 @@ static std::vector<ggml_backend_dev_t> parse_device_list(const std::string & val
return devices;
}
void common_print_available_devices() {
constexpr size_t MiB = 1024 * 1024;
std::vector<ggml_backend_dev_t> devices;
ggml_backend_load_all();
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
auto * dev = ggml_backend_dev_get(i);
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {
devices.push_back(dev);
}
}
printf("Available devices:\n");
if (devices.empty()) {
printf(" (none)\n");
return;
}
for (auto * dev : devices) {
size_t free, total;
ggml_backend_dev_memory(dev, &free, &total);
printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / MiB, free / MiB);
}
}
static void add_rpc_devices(const std::string & servers) {
auto rpc_servers = string_split<std::string>(servers, ',');
if (rpc_servers.empty()) {
@@ -2589,20 +2614,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
{"--list-devices"},
"print list of available devices and exit",
[](common_params &) {
ggml_backend_load_all();
std::vector<ggml_backend_dev_t> devices;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
auto * dev = ggml_backend_dev_get(i);
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {
devices.push_back(dev);
}
}
printf("Available devices:\n");
for (auto * dev : devices) {
size_t free, total;
ggml_backend_dev_memory(dev, &free, &total);
printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);
}
common_print_available_devices();
exit(0);
}
));
common/arg.h
+3
View File
@@ -123,6 +123,9 @@ struct common_params_context {
// if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message)
bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
// load all backends and print the list of available (non-CPU) devices to stdout
void common_print_available_devices();
// parse input arguments from CLI into a map
bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<common_arg, std::string> & out_map);
common/chat-peg-parser.cpp
+138
View File
@@ -1056,3 +1056,141 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, co
visit(arena, child_id);
}
}
static void minimax_m3_collect(const common_peg_ast_arena & arena,
const common_peg_ast_node & node,
const std::string & tag,
std::vector<common_peg_ast_id> & out) {
for (auto child_id : node.children) {
const auto & child = arena.get(child_id);
if (child.tag == tag) {
out.push_back(child_id);
} else {
minimax_m3_collect(arena, child, tag, out);
}
}
}
static common_peg_ast_id minimax_m3_value_of(const common_peg_ast_arena & arena, const common_peg_ast_node & node) {
for (auto child_id : node.children) {
const auto & tag = arena.get(child_id).tag;
if (tag == common_chat_peg_builder::TOOL_ARG_VALUE ||
tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE ||
tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT ||
tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) {
return child_id;
}
}
return COMMON_PEG_INVALID_AST_ID;
}
static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed);
static std::string minimax_m3_member_to_json(const common_peg_ast_arena & arena, const common_peg_ast_node & node) {
auto name_id = arena.find_by_tag(node, common_chat_peg_builder::TOOL_ARG_NAME);
if (name_id == COMMON_PEG_INVALID_AST_ID) {
return "";
}
return ordered_json(arena.get(name_id).text).dump() + ":" +
minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, node), !node.is_partial);
}
static std::string minimax_m3_container_to_json(const common_peg_ast_arena & arena,
const common_peg_ast_node & node,
bool is_object,
bool closed) {
const std::string tag = is_object ? common_chat_peg_builder::TOOL_ARG
: common_chat_peg_minimax_m3_mapper::TOOL_ARG_ITEM;
std::vector<common_peg_ast_id> entries;
minimax_m3_collect(arena, node, tag, entries);
std::string result = is_object ? "{" : "[";
bool add_comma = false;
for (auto entry_id : entries) {
const auto & entry = arena.get(entry_id);
std::string text;
if (is_object) {
text = minimax_m3_member_to_json(arena, entry);
} else {
text = minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, entry), !entry.is_partial);
}
if (text.empty()) {
continue;
}
if (add_comma) {
result += ",";
}
add_comma = true;
result += text;
}
if (closed) {
result += is_object ? "}" : "]";
}
return result;
}
static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed) {
if (id == COMMON_PEG_INVALID_AST_ID) {
return "";
}
const auto & node = arena.get(id);
if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT) {
return minimax_m3_container_to_json(arena, node, /* is_object = */ true, closed);
}
if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) {
return minimax_m3_container_to_json(arena, node, /* is_object = */ false, closed);
}
if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) {
return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : "");
}
// Numbers and booleans are written verbatim by the template
return std::string(node.text);
}
void common_chat_peg_minimax_m3_mapper::from_ast(const common_peg_ast_arena & arena,
const common_peg_parse_result & result) {
for (const auto & node : result.nodes) {
visit(arena, node);
}
}
void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {
const auto & node = arena.get(id);
if (node.tag == common_chat_peg_builder::REASONING) {
result.reasoning_content += std::string(node.text);
return;
}
if (node.tag == common_chat_peg_builder::CONTENT) {
result.content += std::string(node.text);
return;
}
if (node.tag == common_chat_peg_builder::TOOL) {
auto name_id = arena.find_by_tag(node, common_chat_peg_builder::TOOL_NAME);
if (name_id != COMMON_PEG_INVALID_AST_ID) {
common_chat_tool_call call;
call.name = std::string(arena.get(name_id).text);
call.arguments = minimax_m3_container_to_json(arena, node, /* is_object = */ true, !node.is_partial);
result.tool_calls.push_back(call);
}
return;
}
for (auto child_id : node.children) {
visit(arena, child_id);
}
}
common/chat-peg-parser.h
+12
View File
@@ -40,6 +40,18 @@ class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {
void visit(const common_peg_ast_arena & arena, common_peg_ast_id id);
};
class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper {
public:
static constexpr const char * TOOL_ARG_OBJECT = "tool-arg-object";
static constexpr const char * TOOL_ARG_ARRAY = "tool-arg-array";
static constexpr const char * TOOL_ARG_ITEM = "tool-arg-item";
common_chat_peg_minimax_m3_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}
virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);
private:
void visit(const common_peg_ast_arena & arena, common_peg_ast_id id);
};
struct content_structure;
struct tool_call_structure;
common/chat.cpp
+273
View File
@@ -830,6 +830,8 @@ const char * common_chat_format_name(common_chat_format format) {
return "peg-native";
case COMMON_CHAT_FORMAT_PEG_GEMMA4:
return "peg-gemma4";
case COMMON_CHAT_FORMAT_PEG_MINIMAX_M3:
return "peg-minimax-m3";
default:
throw std::runtime_error("Unknown chat format");
}
@@ -2284,6 +2286,264 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_t
return data;
}
static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;
data.supports_thinking = true;
data.thinking_start_tag = "<mm:think>";
data.thinking_end_tags = {"</mm:think>"};
// M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
// params use the parameter name as the tag (<file_path>...</file_path>).
const std::string NS = "]<]minimax[>[";
const std::string THINK_START = "<mm:think>";
const std::string THINK_END = "</mm:think>";
const std::string FC_START = NS + "<tool_call>";
const std::string FC_END = NS + "</tool_call>";
const std::string INVOKE_END = NS + "</invoke>";
data.preserved_tokens = {
NS,
"<tool_call>",
"</tool_call>",
THINK_START,
THINK_END,
};
data.message_delimiters = {
{ COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" },
{ COMMON_CHAT_ROLE_USER, "]~b]user" },
{ COMMON_CHAT_ROLE_TOOL, "]~b]tool" },
{ COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" },
{ COMMON_CHAT_ROLE_SYSTEM, "]~b]system" },
};
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
const std::string GEN_PROMPT = data.generation_prompt;
using mm3 = common_chat_peg_minimax_m3_mapper;
if (inputs.has_continuation()) {
const auto & msg = inputs.continue_msg;
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
data.generation_prompt += THINK_END + msg.render_content();
}
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
auto end = p.end();
auto reasoning = p.eps();
if (extract_reasoning) {
auto block = inputs.enable_thinking
? p.literal(THINK_START) + p.space() +
p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)
: p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);
// A turn without reasoning is prefixed with a bare </mm:think>, written either by the
// generation prompt (thinking_mode = "disabled") or by the model itself.
reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));
}
if (has_response_format) {
auto response_format = p.rule("response-format",
p.literal("```json") + p.space() +
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
p.space() + p.literal("```"));
return generation_prompt + reasoning + response_format + end;
}
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
return generation_prompt + reasoning + p.content(p.rest()) + end;
}
auto alternatives_of = [](const json & schema) -> std::optional<json> {
for (const auto * keyword : { "oneOf", "anyOf" }) {
if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) {
return schema.at(keyword);
}
}
return std::nullopt;
};
auto tool_choice = p.choice();
foreach_function(inputs.tools, [&](const json & tool) {
const auto & function = tool.at("function");
std::string name = function.at("name");
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
auto schema_info = common_schema_info();
schema_info.resolve_refs(params);
// The template expands argument values recursively in XML (see the to_xml() macro)
std::function<common_peg_parser(const json &, const std::string &, const std::string &)> value_of;
std::function<common_peg_parser(const json &, const std::string &)> members_of;
auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) {
const std::string close = NS + "</" + tag + ">";
return p.rule(rule_name,
p.tool_arg(
p.tool_arg_open(
p.literal(NS + "<") +
p.tool_arg_name(p.literal(tag)) +
p.literal(">")) +
value_of(schema, rule_name, close)));
};
value_of = [&](const json & schema,
const std::string & rule_name,
const std::string & close) -> common_peg_parser {
auto close_tag = p.tool_arg_close(p.literal(close));
// A string accepts anything, so a union with a string alternative is a string
if (schema_info.resolves_to_string(schema)) {
return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
}
if (auto alternatives = alternatives_of(schema)) {
std::vector<common_peg_parser> choices;
size_t index = 0;
for (const auto & alternative : *alternatives) {
const std::string alt_name = rule_name + "-" + std::to_string(index++);
// There is a risk that this breaks streaming deltas, but that's a risk we
// assume to provide tool arg streaming.
choices.push_back(value_of(alternative, alt_name, close));
}
return p.choice(choices);
}
const std::string type = schema.contains("type") && schema.at("type").is_string()
? schema.at("type").get<std::string>()
: "";
if (type == "object" && schema.contains("properties")) {
return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag;
}
if (type == "array" && schema.contains("items")) {
const std::string item_close = NS + "</item>";
auto item = p.rule(rule_name + "-item",
p.tag(mm3::TOOL_ARG_ITEM,
p.literal(NS + "<item>") +
value_of(schema.at("items"), rule_name + "-item", item_close)));
return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
}
return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag;
};
// Required properties in schema order, then any number of optional ones in any order.
members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser {
const auto & props = schema.at("properties");
std::set<std::string> required;
if (schema.contains("required")) {
schema.at("required").get_to(required);
}
std::vector<common_peg_parser> required_elements;
std::vector<common_peg_parser> optional_elements;
for (const auto & [key, key_schema] : props.items()) {
auto element = element_of(key, key_schema, rule_prefix + "-" + key);
if (required.find(key) != required.end()) {
required_elements.push_back(element);
} else {
optional_elements.push_back(element);
}
}
common_peg_parser members = p.eps();
for (size_t i = 0; i < required_elements.size(); i++) {
if (i > 0) {
members = members + p.space();
}
members = members + required_elements[i];
}
if (!optional_elements.empty()) {
common_peg_parser any_optional = p.choice();
for (const auto & element : optional_elements) {
any_optional |= element;
}
members = members + p.repeat(p.space() + any_optional, 0, -1);
}
return members;
};
common_peg_parser invoke_body =
params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps();
auto func_parser = p.tool(
p.tool_open(p.literal(NS + "<invoke name=\"") +
p.tool_name(p.literal(name)) + p.literal("\">")) +
p.space() + invoke_body + p.space() +
p.tool_close(p.literal(INVOKE_END)));
tool_choice |= p.rule("tool-" + name, func_parser);
});
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
common_peg_parser tool_calls = p.eps();
if (inputs.parallel_tool_calls) {
tool_calls = p.trigger_rule("tool-call",
p.literal(FC_START) + p.space() + tool_choice +
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
} else {
tool_calls = p.trigger_rule("tool-call",
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
}
if (!require_tools) {
tool_calls = p.optional(tool_calls);
}
auto content_before_tools = p.content(p.until(FC_START));
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
foreach_function(inputs.tools, [&](const json & tool) {
const auto & function = tool.at("function");
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
builder.resolve_refs(schema);
});
if (has_response_format) {
auto schema = inputs.json_schema;
builder.resolve_refs(schema);
}
parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
};
}
return data;
}
namespace workaround {
static void map_developer_role_to_system(json & messages) {
@@ -2721,6 +2981,15 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_gigachat_v3(tmpl, params);
}
// MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
// markup delimiters, so detect the template and use a dedicated parser.
if (src.find("]<]minimax[>[") != std::string::npos &&
src.find("<tool_call>") != std::string::npos &&
src.find("<invoke name=") != std::string::npos) {
LOG_DBG("Using specialized template: MiniMax-M3\n");
return common_chat_params_init_minimax_m3(tmpl, params);
}
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
// The template source contains the token as a variable assignment, not as a literal in markup.
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
@@ -3012,6 +3281,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
std::unique_ptr<common_chat_peg_mapper> mapper;
if (params.format == COMMON_CHAT_FORMAT_PEG_GEMMA4) {
mapper = std::make_unique<common_chat_peg_gemma4_mapper>(msg);
} else if (params.format == COMMON_CHAT_FORMAT_PEG_MINIMAX_M3) {
mapper = std::make_unique<common_chat_peg_minimax_m3_mapper>(msg);
} else {
mapper = std::make_unique<common_chat_peg_mapper>(msg);
}
@@ -3034,6 +3305,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
std::unique_ptr<common_chat_peg_mapper> mapper;
if (params.format == COMMON_CHAT_FORMAT_PEG_GEMMA4) {
mapper = std::make_unique<common_chat_peg_gemma4_mapper>(msg);
} else if (params.format == COMMON_CHAT_FORMAT_PEG_MINIMAX_M3) {
mapper = std::make_unique<common_chat_peg_minimax_m3_mapper>(msg);
} else {
mapper = std::make_unique<common_chat_peg_mapper>(msg);
}
common/chat.h
+1
View File
@@ -233,6 +233,7 @@ enum common_chat_format {
COMMON_CHAT_FORMAT_PEG_SIMPLE,
COMMON_CHAT_FORMAT_PEG_NATIVE,
COMMON_CHAT_FORMAT_PEG_GEMMA4,
COMMON_CHAT_FORMAT_PEG_MINIMAX_M3,
COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats
};
common/common.cpp
+29 -3
View File
@@ -1524,23 +1524,49 @@ done:
return res;
}
void common_context_seq_rm(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
static void common_context_seq_rm(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
auto * mem = llama_get_memory(ctx);
if (!llama_memory_seq_rm(mem, seq_id, p0, p1)) {
GGML_ABORT("%s", string_format("failed to remove sequence %d with p0=%d, p1=%d\n", seq_id, p0, p1).c_str());
}
}
void common_context_seq_cp(llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
static void common_context_seq_cp(llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
auto * mem = llama_get_memory(ctx);
llama_memory_seq_cp(mem, seq_id_src, seq_id_dst, p0, p1);
}
void common_context_seq_add(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) {
static void common_context_seq_add(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) {
auto * mem = llama_get_memory(ctx);
llama_memory_seq_add(mem, seq_id, p0, p1, delta);
}
void common_memory::init(llama_context * ctx_tgt, llama_context * ctx_dft) {
this->ctx_tgt = ctx_tgt;
this->ctx_dft = ctx_dft;
}
void common_memory::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) const {
common_context_seq_rm(ctx_tgt, seq_id, p0, p1);
if (ctx_dft) {
common_context_seq_rm(ctx_dft, seq_id, p0, p1);
}
}
void common_memory::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) const {
common_context_seq_cp(ctx_tgt, seq_id_src, seq_id_dst, p0, p1);
if (ctx_dft) {
common_context_seq_cp(ctx_dft, seq_id_src, seq_id_dst, p0, p1);
}
}
void common_memory::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) const {
common_context_seq_add(ctx_tgt, seq_id, p0, p1, delta);
if (ctx_dft) {
common_context_seq_add(ctx_dft, seq_id, p0, p1, delta);
}
}
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora) {
std::vector<llama_adapter_lora *> loras;
std::vector<float> scales;
common/common.h
+13 -5
View File
@@ -174,6 +174,7 @@ enum common_speculative_type {
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
@@ -389,7 +390,7 @@ struct common_params_speculative {
uint32_t need_n_rs_seq() const {
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
});
return needs_rs_seq ? draft.n_max : 0u;
@@ -949,10 +950,17 @@ enum common_context_seq_rm_type {
// note: clears the memory of the context
common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx);
// aborts execution on failure
void common_context_seq_rm (llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1);
void common_context_seq_add(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta);
void common_context_seq_cp (llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1);
struct common_memory {
llama_context * ctx_tgt = nullptr;
llama_context * ctx_dft = nullptr;
void init(llama_context * ctx_tgt, llama_context * ctx_dft = nullptr);
// aborts execution on failure
void seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) const;
void seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) const;
void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) const;
};
//
// Batch utils
common/speculative.cpp
+89 -33
View File
@@ -34,6 +34,7 @@ const std::map<std::string, common_speculative_type> common_speculative_type_fro
{"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3},
{"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP},
{"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH},
{"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK},
{"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE},
{"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K},
{"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V},
@@ -437,6 +438,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
int32_t n_embd_dec = 0; // draft hidden size
int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size
int32_t n_embd_tgt = 0; // target model hidden size
int32_t n_layer_tgt = 0; // target model layer count
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
@@ -478,6 +480,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
n_embd_tgt = llama_model_n_embd(model_tgt);
n_embd_dec = llama_model_n_embd(model_dft);
n_embd_enc = (int32_t) target_layer_ids_n * n_embd_tgt;
n_layer_tgt = llama_model_n_layer(model_tgt);
const int32_t n_b = (int32_t) llama_n_batch(ctx_dft);
batch = llama_batch_init(/*n_tokens=*/ n_b, /*embd=*/ n_embd_dec, /*n_seq_max=*/ 1);
@@ -510,9 +513,15 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
}
}
// turn on extraction of the target layers' input embeddings
// turn on extraction of the target layers' hidden states
for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
if (target_layer_ids[k] < n_layer_tgt) {
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
} else if (target_layer_ids[k] == n_layer_tgt) {
llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ false);
} else {
GGML_ABORT("EAGLE3: target layer id %d exceeds target n_layer %d", target_layer_ids[k], n_layer_tgt);
}
}
// turn on extraction of the draft model's pre-norm hidden state
@@ -600,7 +609,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
features_buf.resize((size_t) n_tokens * n_embd_enc, 0.0f);
for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]);
const float * layer = target_layer_ids[k] < n_layer_tgt
? llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k])
: llama_get_embeddings_nextn(ctx_tgt);
if (!layer) {
GGML_ABORT("EAGLE3: target layer %d input not extracted.", target_layer_ids[k]);
}
@@ -918,15 +929,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
int32_t block_size = 0;
llama_token mask_token_id = 0;
// draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
const bool is_dspark;
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
// scratch buffer for concatenated target features [n_tokens, n_embd_enc]
std::vector<float> features_buf;
common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq)
common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
: common_speculative_impl(type, n_seq)
, params(params.draft)
, is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
{
auto * ctx_tgt = this->params.ctx_tgt;
auto * ctx_dft = this->params.ctx_dft;
@@ -953,16 +969,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
}
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__);
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);
// DFlash input is [id_last, <mask> * (block_size-1)], so it can draft at most block_size-1 tokens per step
if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) {
LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n",
__func__, this->params.n_max, this->params.n_min, block_size, block_size - 1);
this->params.n_max = std::min(this->params.n_max, block_size - 1);
this->params.n_min = std::min(this->params.n_min, block_size - 1);
// DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most
// block_size-1 draft tokens, DSpark yield a full block_size draft tokens
const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;
if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {
LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",
__func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);
this->params.n_max = std::min(this->params.n_max, n_draft_max);
this->params.n_min = std::min(this->params.n_min, n_draft_max);
}
batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);
@@ -1126,12 +1144,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
const int32_t n = (int32_t) dp.n_past;
int32_t n_draft = params.n_max;
if (dp.n_max > 0) {
n_draft = std::min(n_draft, dp.n_max);
}
const int32_t n_draft = params.n_max;
const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * <mask>
const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);
i_block_beg[seq_id] = batch.n_tokens;
n_block [seq_id] = n_block_tokens;
for (int32_t i = 0; i < n_block_tokens; ++i) {
@@ -1163,27 +1178,57 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
auto & result = *dp.result;
// greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1
for (int32_t i = 1; i < n_block_tokens; ++i) {
common_sampler_sample(smpl, ctx_dft, beg + i, true);
if (is_dspark) {
// DSpark predicts the next token from position 0 and optionally truncates
// at the first position below the confidence threshold.
const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
const auto * cur_p = common_sampler_get_candidates(smpl, true);
for (int32_t i = 0; i < n_block_tokens; ++i) {
const int32_t idx = beg + i;
for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,
common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {
break;
}
common_sampler_sample(smpl, ctx_dft, idx, true);
const auto * cur_p = common_sampler_get_candidates(smpl, true);
for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p,
common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
}
const llama_token id = cur_p->data[0].id;
common_sampler_accept(smpl, id, true);
result.push_back(id);
}
} else {
// greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1
for (int32_t i = 1; i < n_block_tokens; ++i) {
common_sampler_sample(smpl, ctx_dft, beg + i, true);
const llama_token id = cur_p->data[0].id;
const auto * cur_p = common_sampler_get_candidates(smpl, true);
if (cur_p->data[0].p < params.p_min) {
break;
for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,
common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
}
const llama_token id = cur_p->data[0].id;
if (cur_p->data[0].p < params.p_min) {
break;
}
common_sampler_accept(smpl, id, true);
result.push_back(id);
}
common_sampler_accept(smpl, id, true);
result.push_back(id);
}
if (result.size() < (size_t) params.n_min) {
@@ -2145,6 +2190,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) {
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3";
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp";
case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash";
case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark";
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v";
@@ -2198,6 +2244,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH:
case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK:
n_max = std::max(n_max, std::max(0, spec->draft.n_max));
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
@@ -2342,6 +2389,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,
bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr;
bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;
bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr;
bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr;
@@ -2352,7 +2400,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,
bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD));
// when adding a new type - update here the logic above
static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10);
static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11);
// this list here defines the priority of the speculators
// the one with highest priority are listed first
@@ -2385,6 +2433,9 @@ common_speculative * common_speculative_init(common_params_speculative & params,
if (has_draft_dflash) {
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params));
}
if (has_draft_dspark) {
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params));
}
}
std::vector<std::unique_ptr<common_speculative_impl>> impls = {};
@@ -2409,6 +2460,11 @@ common_speculative * common_speculative_init(common_params_speculative & params,
impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(config.params, n_seq));
break;
}
case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: {
impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(
config.params, n_seq, COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple);