Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-13 10:18:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.devops/openvino.Dockerfile
#	.github/workflows/build-cache.yml
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/release.yml
#	ci/run.sh
#	docs/backend/OPENVINO.md
#	docs/speculative.md
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/hvx-arith.h
#	ggml/src/ggml-hexagon/htp/hvx-log.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-hexagon/htp/unary-ops.h
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-openvino/CMakeLists.txt
#	ggml/src/ggml-openvino/ggml-decoder.cpp
#	ggml/src/ggml-openvino/ggml-decoder.h
#	ggml/src/ggml-openvino/ggml-openvino-extra.cpp
#	ggml/src/ggml-openvino/ggml-openvino.cpp
#	ggml/src/ggml-openvino/openvino/op/cpy.cpp
#	ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
#	ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
#	ggml/src/ggml-openvino/openvino/op/view.cpp
#	ggml/src/ggml-openvino/openvino/op_table.cpp
#	ggml/src/ggml-openvino/openvino/op_table.h
#	ggml/src/ggml-openvino/openvino/translate_session.cpp
#	ggml/src/ggml-openvino/openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.h
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/fattn.cpp
#	scripts/pr2wt.sh
#	src/CMakeLists.txt
#	src/llama-mmap.cpp
#	src/llama-quant.cpp
#	tests/CMakeLists.txt
#	tests/test-arg-parser.cpp
#	tests/test-backend-ops.cpp
#	tests/test-llama-archs.cpp
#	tests/test-save-load-state.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/server/README.md
This commit is contained in:
Concedo
2026-08-28 22:29:03 +08:00
parent ae60e1d6e0 511f9c1379
commit 8d223ab855
104 changed files with 6236 additions and 1464 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+53
View File
@@ -1644,6 +1644,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
}
).set_env("LLAMA_ARG_CTX_SIZE"));
add_opt(common_arg(
{ "--kv-unified-per-slot" }, "N",
"context limit per parallel slot (default: unset, behavior unchanged).\n"
"when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N",
[](common_params & params, int value) {
params.kv_unified_per_slot = value;
}
).set_env("LLAMA_ARG_KV_UNIFIED_PER_SLOT").set_examples({ LLAMA_EXAMPLE_SERVER }));
add_opt(common_arg(
{"-n", "--predict", "--n-predict"}, "N",
string_format(
@@ -2721,6 +2729,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--tensor-read-lazy"}, "MODE",
"on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
"- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
"- auto: on, but only for tensors larger than 4 GiB\n"
"- off: always keep them resident",
[](common_params & params, const std::string & value) {
/**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }
else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"
@@ -4133,6 +4154,38 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.n_min = value;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
add_opt(common_arg(
{"--spec-synth-len"}, "L",
"target mean synthetic acceptance length, including the target token (benchmarking only)",
[](common_params & params, const std::string & value) {
const std::string text = string_strip(value);
size_t pos = 0;
const double length = std::stod(text, &pos);
if (pos != text.size() || length == -1.0) {
throw std::invalid_argument("invalid value");
}
params.speculative.synth_len = length;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));
add_opt(common_arg(
{"--spec-synth-rates"}, "P0,P1,...",
"comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",
[](common_params & params, const std::string & value) {
const auto values = string_split<std::string>(value, ',');
std::vector<double> rates;
rates.reserve(values.size());
for (const auto & raw : values) {
const std::string text = string_strip(raw);
size_t pos = 0;
const double rate = std::stod(text, &pos);
if (pos != text.size()) {
throw std::invalid_argument("invalid value");
}
rates.push_back(rate);
}
params.speculative.synth_rates = std::move(rates);
}
).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));
add_opt(common_arg(
{"--spec-draft-p-split", "--draft-p-split"}, "P",