Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	AGENTS.md
#	CODEOWNERS
#	CONTRIBUTING.md
#	docs/backend/OPENCL.md
#	docs/development/HOWTO-add-model.md
#	examples/training/finetune.cpp
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp-drv.cpp
#	ggml/src/ggml-hexagon/htp/act-ops.c
#	ggml/src/ggml-hexagon/htp/dma-queue.c
#	ggml/src/ggml-hexagon/htp/dma-queue.h
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.c
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.h
#	ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
#	ggml/src/ggml-hexagon/htp/htp-ctx.h
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/htp-tensor.c
#	ggml/src/ggml-hexagon/htp/htp-tensor.h
#	ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h
#	ggml/src/ggml-hexagon/htp/hvx-reduce.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.h
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-hexagon/htp/unary-ops.h
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	scripts/compare-llama-bench.py
#	scripts/snapdragon/ggml-hexagon-profile.py
#	scripts/snapdragon/ggml-hexagon-trace.py
#	scripts/sync_vendor.py
#	tests/test-arg-parser.cpp
#	tests/test-chat.cpp
#	tests/test-model-load-cancel.cpp
#	tests/test-quantize-stats.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/server/README.md
#	tools/ui/src/lib/constants/settings-registry.ts
This commit is contained in:
Concedo
2026-07-25 12:20:51 +08:00
parent eea06b0a89 ed7adbfefd
commit 49dbdaaab5
81 changed files with 40233 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+62 -7
View File
@@ -5,6 +5,7 @@
#include "common.h"
#include "download.h"
#include "json-schema-to-grammar.h"
#include "llama.h"
#include "log.h"
#include "sampling.h"
#include "speculative.h"
@@ -352,6 +353,10 @@ static std::string get_default_local_path(const std::string & url) {
return fs_get_cache_file(string_split<std::string>(f, '/').back());
}
static bool spec_types_is_default(const common_params & params) {
return params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE};
}
common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {
common_download_hf_plan plan;
common_download_hf_plan plan_spec;
@@ -392,7 +397,14 @@ common_models_handler common_models_handler_init(const common_params & params, l
}
if (!params.speculative.draft.mparams.hf_repo.empty()) {
plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts);
// without a requested type, discover every sidecar the draft repo ships to infer the type later
auto opts_spec = opts;
if (spec_types_is_default(params)) {
opts_spec.download_mtp = true;
opts_spec.download_dflash = true;
opts_spec.download_eagle3 = true;
}
plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);
}
if (!params.vocoder.model.hf_repo.empty()) {
@@ -528,6 +540,20 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
};
// infer the speculative type from the sidecar shipped by the draft repo when none is requested
if (spec_types_is_default(params)) {
if (!plan_spec.mtp.local_path.empty()) {
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
plan_spec.dflash = {};
plan_spec.eagle3 = {};
} else if (!plan_spec.dflash.local_path.empty()) {
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH };
plan_spec.eagle3 = {};
} else if (!plan_spec.eagle3.local_path.empty()) {
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 };
}
}
// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
!plan_spec.dflash.local_path.empty() ||
@@ -761,6 +787,17 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
arg.c_str(), e.what(), opt.to_string().c_str()));
}
}
// TODO: remove this check after deprecating --mmap|mlock|dio
auto has_arg = [&](std::initializer_list<const char *> names) {
return std::any_of(names.begin(), names.end(), [&](const char * name) {
return seen_args.count(name);
});
};
if (has_arg({"-lm", "--load-mode"}) &&
has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {
LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");
}
};
// parse all CLI args now, so that -hf is available below for remote preset resolution
@@ -2471,27 +2508,45 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
add_opt(common_arg(
{"--mlock"},
"force system to keep model in RAM rather than swapping or compressing",
"DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing",
[](common_params & params) {
params.use_mlock = true;
LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
params.load_mode = LLAMA_LOAD_MODE_MLOCK;
}
).set_env("LLAMA_ARG_MLOCK"));
add_opt(common_arg(
{"--mmap"},
{"--no-mmap"},
string_format("whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: %s)", params.use_mmap ? "enabled" : "disabled"),
"DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",
[](common_params & params, bool value) {
params.use_mmap = value;
LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");
params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;
}
).set_env("LLAMA_ARG_MMAP"));
add_opt(common_arg(
{"-dio", "--direct-io"},
{"-ndio", "--no-direct-io"},
string_format("use DirectIO if available. (default: %s)", params.use_direct_io ? "enabled" : "disabled"),
"DEPRECATED in favor of `--load-mode`: use DirectIO if available",
[](common_params & params, bool value) {
params.use_direct_io = value;
LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");
params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;
}
).set_env("LLAMA_ARG_DIO"));
add_opt(common_arg(
{"-lm", "--load-mode"}, "MODE",
"model loading mode (default: mmap)\n"
"- none: no special loading mode\n"
"- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"
"- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"
"- dio: use DirectIO if available\n",
[](common_params & params, const std::string & value) {
/**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }
else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }
else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"