Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-31 01:20:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.devops/openvino.Dockerfile
#	.github/workflows/build-cache.yml
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/release.yml
#	ci/run.sh
#	docs/backend/OPENVINO.md
#	docs/speculative.md
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/hvx-arith.h
#	ggml/src/ggml-hexagon/htp/hvx-log.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-hexagon/htp/unary-ops.h
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-openvino/CMakeLists.txt
#	ggml/src/ggml-openvino/ggml-decoder.cpp
#	ggml/src/ggml-openvino/ggml-decoder.h
#	ggml/src/ggml-openvino/ggml-openvino-extra.cpp
#	ggml/src/ggml-openvino/ggml-openvino.cpp
#	ggml/src/ggml-openvino/openvino/op/cpy.cpp
#	ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
#	ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
#	ggml/src/ggml-openvino/openvino/op/view.cpp
#	ggml/src/ggml-openvino/openvino/op_table.cpp
#	ggml/src/ggml-openvino/openvino/op_table.h
#	ggml/src/ggml-openvino/openvino/translate_session.cpp
#	ggml/src/ggml-openvino/openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.h
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/fattn.cpp
#	scripts/pr2wt.sh
#	src/CMakeLists.txt
#	src/llama-mmap.cpp
#	src/llama-quant.cpp
#	tests/CMakeLists.txt
#	tests/test-arg-parser.cpp
#	tests/test-backend-ops.cpp
#	tests/test-llama-archs.cpp
#	tests/test-save-load-state.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/server/README.md
This commit is contained in:
Concedo
2026-08-28 22:29:03 +08:00
parent ae60e1d6e0 511f9c1379
commit 8d223ab855
104 changed files with 6236 additions and 1464 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/quantize/quantize.cpp
+15 -2
View File
@@ -121,7 +121,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
static void usage(const char * executable) {
printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);
printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run] [--max-buffer-size]\n");
printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");
printf(" --allow-requantize\n");
printf(" allow requantizing tensors that have already been quantized\n");
@@ -160,7 +160,10 @@ static void usage(const char * executable) {
printf(" WARNING: this is an advanced option, use with care.\n");
printf(" --dry-run\n");
printf(" calculate and show the final quantization size without performing quantization\n");
printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n");
printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n");
printf(" --max-buffer-size MiB\n");
printf(" max amount of tensor rows kept in memory while quantizing one tensor (default: 8192)\n");
printf(" lower it to quantize models with very large tensors on a machine with little RAM\n\n");
printf("note: --include-weights and --exclude-weights cannot be used together\n\n");
printf("-----------------------------------------------------------------------------\n");
printf(" allowed quantization types\n");
@@ -466,6 +469,16 @@ int llama_quantize(int argc, char ** argv) {
}
} else if (strcmp(argv[arg_idx], "--keep-split") == 0) {
params.keep_split = true;
} else if (strcmp(argv[arg_idx], "--max-buffer-size") == 0) {
if (arg_idx == argc-1) {
usage(argv[0]);
}
const int mib = atoi(argv[++arg_idx]);
if (mib <= 0) {
fprintf(stderr, "%s: invalid --max-buffer-size '%s'\n", __func__, argv[arg_idx]);
return 1;
}
params.max_buf_size = (size_t) mib * 1024 * 1024;
} else {
usage(argv[0]);
}