Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-30 17:10:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

quantize: cap working memory size to avoid loading big tensors onto RAM (#27795)

Browse Source
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 18:31:13 +02:00
committed by GitHub
parent cb300598d5
commit 732707dff2
5 changed files with 112 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/quantize/quantize.cpp
+15 -2
View File
@@ -122,7 +122,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
static void usage(const char * executable) {
printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);
printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run] [--max-buffer-size]\n");
printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");
printf(" --allow-requantize\n");
printf(" allow requantizing tensors that have already been quantized\n");
@@ -161,7 +161,10 @@ static void usage(const char * executable) {
printf(" WARNING: this is an advanced option, use with care.\n");
printf(" --dry-run\n");
printf(" calculate and show the final quantization size without performing quantization\n");
printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n");
printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n");
printf(" --max-buffer-size MiB\n");
printf(" max amount of tensor rows kept in memory while quantizing one tensor (default: 8192)\n");
printf(" lower it to quantize models with very large tensors on a machine with little RAM\n\n");
printf("note: --include-weights and --exclude-weights cannot be used together\n\n");
printf("-----------------------------------------------------------------------------\n");
printf(" allowed quantization types\n");
@@ -467,6 +470,16 @@ int llama_quantize(int argc, char ** argv) {
}
} else if (strcmp(argv[arg_idx], "--keep-split") == 0) {
params.keep_split = true;
} else if (strcmp(argv[arg_idx], "--max-buffer-size") == 0) {
if (arg_idx == argc-1) {
usage(argv[0]);
}
const int mib = atoi(argv[++arg_idx]);
if (mib <= 0) {
fprintf(stderr, "%s: invalid --max-buffer-size '%s'\n", __func__, argv[arg_idx]);
return 1;
}
params.max_buf_size = (size_t) mib * 1024 * 1024;
} else {
usage(argv[0]);
}