Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-cpu/kleidiai/kernels.cpp
#	ggml/src/ggml-cpu/kleidiai/kernels.h
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-cpu/ops.cpp
#	ggml/src/ggml-cuda/mmq.cuh
#	ggml/src/ggml-hexagon/htp/hmx-queue.c
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
#	ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
#	ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl
#	ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl
#	ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl
#	ggml/src/ggml-sycl/fattn-vec.hpp
#	tests/test-backend-ops.cpp
#	tests/test-chat-auto-parser.cpp
#	tests/test-export-graph-ops.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tools/tokenize/tokenize.cpp
This commit is contained in:
Concedo
2026-07-15 15:39:31 +08:00
parent 23ba6932b5 12127defda
commit 9001369da0
73 changed files with 6859 additions and 3973 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+92 -10
View File
@@ -698,7 +698,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
}
};
// parse the first time to get -hf option (used for remote preset)
// parse all CLI args now, so that -hf is available below for remote preset resolution
parse_cli_args();
postprocess_cpu_params(params.cpuparams, nullptr);
@@ -749,6 +749,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
params.kv_overrides.back().key[0] = 0;
}
if (!params.server_tools.empty() && !params.cors_origins_explicit) {
LOG_WRN("server tools are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
params.cors_origins = "localhost";
}
// pad tensor_buft_overrides for llama_params_fit:
const size_t ntbo = llama_max_tensor_buft_overrides();
while (params.tensor_buft_overrides.size() < ntbo) {
@@ -1078,6 +1083,7 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e
if (ctx_arg.print_usage) {
ctx_arg.print_usage(argc, argv);
}
common_log_flush(common_log_main());
exit(0);
}
if (ctx_arg.params.completion) {
@@ -1179,6 +1185,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.temp = 0.2; // lower temp by default for better quality
} else if (ex == LLAMA_EXAMPLE_SERVER) {
params.n_parallel = -1; // auto by default
} else if (ex == LLAMA_EXAMPLE_TOKENIZE) {
params.parse_special = true; // parse special tokens by default, like the old tokenize tool
}
params.use_color = tty_can_use_colors();
@@ -2746,14 +2754,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.path = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
add_opt(common_arg(
{"-mu", "--model-url"}, "MODEL_URL",
"model download url (default: unused)",
[](common_params & params, const std::string & value) {
params.model.url = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL_URL"));
add_opt(common_arg(
{ "-dr", "--docker-repo" }, "[<repo>/]<model>[:quant]",
"Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n"
@@ -2762,7 +2770,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.docker_repo = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_DOCKER_REPO"));
add_opt(common_arg(
{"-hf", "-hfr", "--hf-repo"}, "<user>/<model>[:quant]",
"Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n"
@@ -2772,14 +2780,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.hf_repo = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_REPO"));
add_opt(common_arg(
{"-hff", "--hf-file"}, "FILE",
"Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)",
[](common_params & params, const std::string & value) {
params.model.hf_file = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE"));
add_opt(common_arg(
{"-hfv", "-hfrv", "--hf-repo-v"}, "<user>/<model>[:quant]",
"Hugging Face model repository for the vocoder model (default: unused)",
@@ -2800,7 +2808,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.hf_token = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("HF_TOKEN"));
add_opt(common_arg(
{"--mtp"},
"also download the multi-token prediction (MTP) head, if available (default: unused)",
@@ -2916,6 +2924,41 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.parse_special = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--ids"},
string_format("only print the token IDs, in a Python-parseable list form like [1, 2, 3] (default: %s)", params.tokenize_ids ? "true" : "false"),
[](common_params & params) {
params.tokenize_ids = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--stdin"},
string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
[](common_params & params) {
params.tokenize_stdin = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--no-bos"},
string_format("do not add a BOS token to the prompt, even if the model normally uses one (default: %s)", params.tokenize_no_bos ? "true" : "false"),
[](common_params & params) {
params.tokenize_no_bos = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--no-parse-special"},
string_format("do not parse special tokens (chat, tool, etc) (default: %s)", !params.parse_special ? "true" : "false"),
[](common_params & params) {
params.parse_special = false;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--show-count"},
string_format("print the total number of tokens (default: %s)", params.tokenize_show_count ? "true" : "false"),
[](common_params & params) {
params.tokenize_show_count = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"-pps"},
string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),
@@ -3010,6 +3053,42 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.public_path = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_STATIC_PATH"));
add_opt(common_arg(
{"--cors-origins"}, "ORIGINS",
string_format(
"comma-separated list of allowed origins for CORS (default: %s)\n"
"if set to special value 'localhost', reflect the Origin header only if it is localhost",
params.cors_origins.c_str()),
[](common_params & params, const std::string & value) {
params.cors_origins = value;
params.cors_origins_explicit = true;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_ORIGINS"));
add_opt(common_arg(
{"--cors-methods"}, "METHODS",
string_format("comma-separated list of allowed methods for CORS (default: %s)", params.cors_methods.c_str()),
[](common_params & params, const std::string & value) {
params.cors_methods = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_METHODS"));
add_opt(common_arg(
{"--cors-headers"}, "HEADERS",
string_format("comma-separated list of allowed headers for CORS (default: %s)", params.cors_headers.c_str()),
[](common_params & params, const std::string & value) {
params.cors_headers = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_HEADERS"));
add_opt(common_arg(
{"--cors-credentials"},
{"--no-cors-credentials"},
string_format(
"whether to allow credentials for CORS (default: %s)\n"
"note: if this is enabled and --cors-origins is set to * (default), the Origin header will be echoed back, and credentials will always be allowed",
params.cors_credentials ? "enabled" : "disabled"),
[](common_params & params, bool value) {
params.cors_credentials = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_CREDENTIALS"));
add_opt(common_arg(
{"--api-prefix"}, "PREFIX",
string_format("prefix path the server serves from, without the trailing slash (default: %s)", params.api_prefix.c_str()),
@@ -3043,7 +3122,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
{"--tools"}, "TOOL1,TOOL2,...",
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
"specify \"all\" to enable all tools\n"
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime",
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"
"note: for security reasons, this will limit --cors-origins to localhost by default",
[](common_params & params, const std::string & value) {
params.server_tools = parse_csv_row(value);
}
@@ -3051,7 +3131,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
add_opt(common_arg(
{"-ag", "--agent"},
{"-no-ag", "--no-agent"},
"whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)",
"whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)\n"
"note: for security reasons, this will limit --cors-origins to localhost by default",
[](common_params & params, bool value) {
if (value) {
params.server_tools = {"all"};
@@ -3060,6 +3141,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.server_tools.clear();
params.ui_mcp_proxy = false;
}
// note: do not modify cors_origins here, as the options are not evaluated in order (user may explicitly set --cors-origins before --agent)
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT"));
add_opt(common_arg(
@@ -3506,7 +3588,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params) {
params.offline = true;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_OFFLINE"));
add_opt(common_arg(
{"-lv", "--verbosity", "--log-verbosity"}, "N",
string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n"