From 3f7c7313a1ec51bb5d8d2f86578f8f94ea553c74 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 27 Apr 2026 10:00:42 +0300 Subject: [PATCH] cont : add spec params category --- common/arg.cpp | 88 +++++++++++++++++++++++++++++--------------------- common/arg.h | 2 ++ 2 files changed, 53 insertions(+), 37 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 4b0c747028..654faacb1e 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -102,6 +102,11 @@ common_arg & common_arg::set_sampling() { return *this; } +common_arg & common_arg::set_spec() { + is_spec = true; + return *this; +} + common_arg & common_arg::set_preset_only() { is_preset_only = true; return *this; @@ -652,11 +657,14 @@ static void common_params_print_usage(common_params_context & ctx_arg) { std::vector common_options; std::vector sampling_options; + std::vector spec_options; std::vector specific_options; for (auto & opt : ctx_arg.options) { // in case multiple LLAMA_EXAMPLE_* are set, we prioritize the LLAMA_EXAMPLE_* matching current example if (opt.is_sampling) { sampling_options.push_back(&opt); + } else if (opt.is_spec) { + spec_options.push_back(&opt); } else if (opt.in_example(ctx_arg.ex)) { specific_options.push_back(&opt); } else { @@ -667,6 +675,8 @@ static void common_params_print_usage(common_params_context & ctx_arg) { print_options(common_options); printf("\n\n----- sampling params -----\n\n"); print_options(sampling_options); + printf("\n\n----- speculative params -----\n\n"); + print_options(spec_options); // TODO: maybe convert enum llama_example to string printf("\n\n----- example-specific params -----\n\n"); print_options(specific_options); @@ -675,11 +685,14 @@ static void common_params_print_usage(common_params_context & ctx_arg) { static void common_params_print_completion(common_params_context & ctx_arg) { std::vector common_options; std::vector sampling_options; + std::vector spec_options; std::vector specific_options; for (auto & opt : ctx_arg.options) { if (opt.is_sampling) { sampling_options.push_back(&opt); + } else if (opt.is_spec) { + spec_options.push_back(&opt); } else if (opt.in_example(ctx_arg.ex)) { specific_options.push_back(&opt); } else { @@ -704,6 +717,7 @@ static void common_params_print_completion(common_params_context & ctx_arg) { print_options(common_options); print_options(sampling_options); + print_options(spec_options); print_options(specific_options); printf("\"\n\n"); @@ -3306,7 +3320,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.speculative.draft.mparams.hf_repo = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_HF_REPO")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_HF_REPO")); add_opt(common_arg( {"--spec-draft-threads", "-td", "--threads-draft"}, "N", "number of threads to use during generation (default: same as --threads)", @@ -3316,7 +3330,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.speculative.draft.cpuparams.n_threads = std::thread::hardware_concurrency(); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-threads-batch", "-tbd", "--threads-batch-draft"}, "N", "number of threads to use during batch and prompt processing (default: same as --threads-draft)", @@ -3336,7 +3350,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex throw std::invalid_argument("invalid cpumask"); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-cpu-range", "-Crd", "--cpu-range-draft"}, "lo-hi", "Ranges of CPUs for affinity. Complements --cpu-mask-draft", @@ -3346,14 +3360,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex throw std::invalid_argument("invalid range"); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-cpu-strict", "--cpu-strict-draft"}, "<0|1>", "Use strict CPU placement for draft model (default: same as --cpu-strict)", [](common_params & params, int value) { params.speculative.draft.cpuparams.strict_cpu = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-prio", "--prio-draft"}, "N", string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.speculative.draft.cpuparams.priority), @@ -3363,14 +3377,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.draft.cpuparams.priority = (enum ggml_sched_priority) prio; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-poll", "--poll-draft"}, "<0|1>", "Use polling to wait for draft model work (default: same as --poll])", [](common_params & params, int value) { params.speculative.draft.cpuparams.poll = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-cpu-mask-batch", "-Cbd", "--cpu-mask-batch-draft"}, "M", "Draft model CPU affinity mask. Complements cpu-range-draft (default: same as --cpu-mask)", @@ -3380,7 +3394,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex throw std::invalid_argument("invalid cpumask"); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-cpu-range-batch", "-Crbd", "--cpu-range-batch-draft"}, "lo-hi", "Ranges of CPUs for affinity. Complements --cpu-mask-draft-batch)", @@ -3390,14 +3404,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex throw std::invalid_argument("invalid cpumask"); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( {"--spec-draft-cpu-strict-batch", "--cpu-strict-batch-draft"}, "<0|1>", "Use strict CPU placement for draft model (default: --cpu-strict-draft)", [](common_params & params, int value) { params.speculative.draft.cpuparams_batch.strict_cpu = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-prio-batch", "--prio-batch-draft"}, "N", string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.speculative.draft.cpuparams_batch.priority), @@ -3407,14 +3421,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.draft.cpuparams_batch.priority = (enum ggml_sched_priority) prio; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-poll-batch", "--poll-batch-draft"}, "<0|1>", "Use polling to wait for draft model work (default: --poll-draft)", [](common_params & params, int value) { params.speculative.draft.cpuparams_batch.poll = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-type-k", "-ctkd", "--cache-type-k-draft"}, "TYPE", string_format( @@ -3446,14 +3460,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex "override tensor buffer type for draft model", [](common_params & params, const std::string & value) { parse_tensor_buffer_overrides(value, params.speculative.draft.tensor_buft_overrides); } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-draft-cpu-moe", "-cmoed", "--cpu-moe-draft"}, "keep all Mixture of Experts (MoE) weights in the CPU for the draft model", [](common_params & params) { params.speculative.draft.tensor_buft_overrides.push_back(llm_ffn_exps_cpu_override()); } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_CPU_MOE")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_CPU_MOE")); add_opt(common_arg( {"--spec-draft-n-cpu-moe", "--spec-draft-ncmoe", "-ncmoed", "--n-cpu-moe-draft"}, "N", "keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model", @@ -3467,7 +3481,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.speculative.draft.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()}); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE")); add_opt(common_arg( {"--spec-draft-n-max"}, "N", @@ -3475,14 +3489,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, int value) { params.speculative.draft.n_max = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MAX")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MAX")); add_opt(common_arg( {"--spec-draft-n-min"}, "N", string_format("minimum number of draft tokens to use for speculative decoding (default: %d)", params.speculative.draft.n_min), [](common_params & params, int value) { params.speculative.draft.n_min = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN")); add_opt(common_arg( {"--spec--draft-p-split", "--draft-p-split"}, "P", @@ -3490,21 +3504,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.speculative.draft.p_split = std::stof(value); } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_P_SPLIT")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_P_SPLIT")); add_opt(common_arg( {"--spec-draft-p-min", "--draft-p-min"}, "P", string_format("minimum speculative decoding probability (greedy) (default: %.2f)", (double)params.speculative.draft.p_min), [](common_params & params, const std::string & value) { params.speculative.draft.p_min = std::stof(value); } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_P_MIN")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_P_MIN")); add_opt(common_arg( {"--spec-draft-ctx-size", "-cd", "--ctx-size-draft"}, "N", string_format("size of the prompt context for the draft model (default: %d, 0 = loaded from model)", params.speculative.draft.n_ctx), [](common_params & params, int value) { params.speculative.draft.n_ctx = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_CTX_SIZE")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_CTX_SIZE")); add_opt(common_arg( {"--spec-draft-device", "-devd", "--device-draft"}, "", "comma-separated list of devices to use for offloading the draft model (none = don't offload)\n" @@ -3512,7 +3526,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.speculative.draft.devices = parse_device_list(value); } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); GGML_ASSERT(params.speculative.draft.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0 add_opt(common_arg( {"--spec-draft-ngl", "-ngld", "--gpu-layers-draft", "--n-gpu-layers-draft"}, "N", @@ -3532,21 +3546,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex fprintf(stderr, "warning: consult docs/build.md for compilation instructions\n"); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_N_GPU_LAYERS_DRAFT")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_N_GPU_LAYERS_DRAFT")); add_opt(common_arg( {"--spec-draft-model", "-md", "--model-draft"}, "FNAME", "draft model for speculative decoding (default: unused)", [](common_params & params, const std::string & value) { params.speculative.draft.mparams.path = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); add_opt(common_arg( {"--spec-draft-replace", "--spec-replace"}, "TARGET", "DRAFT", "translate the string in TARGET into DRAFT if the draft model and main model are not compatible", [](common_params & params, const std::string & tgt, const std::string & dft) { params.speculative.draft.replacements.push_back({ tgt, dft }); } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-type"}, "[none|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]", string_format("type of speculative decoding to use when no draft model is provided (default: %s)\n", @@ -3568,7 +3582,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex throw std::invalid_argument("unknown speculative decoding type without draft model"); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_TYPE")); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_TYPE")); add_opt(common_arg( {"--spec-ngram-mod-n-min"}, "N", string_format("minimum number of ngram tokens to use for ngram-based speculative decoding (default: %d)", params.speculative.ngram_mod.n_min), @@ -3578,7 +3592,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_mod.n_min = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-mod-n-max"}, "N", string_format("maximum number of ngram tokens to use for ngram-based speculative decoding (default: %d)", params.speculative.ngram_mod.n_max), @@ -3588,7 +3602,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_mod.n_max = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-mod-n-match"}, "N", string_format("ngram-mod lookup length (default: %d)", params.speculative.ngram_mod.n_match), @@ -3598,7 +3612,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_mod.n_match = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-simple-size-n"}, "N", @@ -3609,7 +3623,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_simple.size_n = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-simple-size-m"}, "N", string_format("ngram size M for ngram-simple speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_simple.size_m), @@ -3619,7 +3633,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_simple.size_m = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-simple-min-hits"}, "N", string_format("minimum hits for ngram-simple speculative decoding (default: %d)", params.speculative.ngram_simple.min_hits), @@ -3629,7 +3643,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_simple.min_hits = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-map-k-size-n"}, "N", @@ -3640,7 +3654,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_map_k.size_n = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-map-k-size-m"}, "N", string_format("ngram size M for ngram-map-k speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_map_k.size_m), @@ -3650,7 +3664,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_map_k.size_m = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-map-k-min-hits"}, "N", string_format("minimum hits for ngram-map-k speculative decoding (default: %d)", params.speculative.ngram_map_k.min_hits), @@ -3660,7 +3674,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_map_k.min_hits = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-map-k4v-size-n"}, "N", @@ -3671,7 +3685,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_map_k4v.size_n = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-map-k4v-size-m"}, "N", string_format("ngram size M for ngram-map-k4v speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_map_k4v.size_m), @@ -3681,7 +3695,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_map_k4v.size_m = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"--spec-ngram-map-k4v-min-hits"}, "N", string_format("minimum hits for ngram-map-k4v speculative decoding (default: %d)", params.speculative.ngram_map_k4v.min_hits), @@ -3691,7 +3705,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.speculative.ngram_map_k4v.min_hits = value; } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); // // TTS params diff --git a/common/arg.h b/common/arg.h index 7bd4199b73..2c2a4e38a2 100644 --- a/common/arg.h +++ b/common/arg.h @@ -26,6 +26,7 @@ struct common_arg { const char * env = nullptr; std::string help; bool is_sampling = false; // is current arg a sampling param? + bool is_spec = false; // is current arg a speculative decoding param? bool is_preset_only = false; // is current arg preset-only (not treated as CLI arg) void (*handler_void) (common_params & params) = nullptr; void (*handler_string) (common_params & params, const std::string &) = nullptr; @@ -75,6 +76,7 @@ struct common_arg { common_arg & set_excludes(std::initializer_list excludes); common_arg & set_env(const char * env); common_arg & set_sampling(); + common_arg & set_spec(); common_arg & set_preset_only(); bool in_example(enum llama_example ex); bool is_exclude(enum llama_example ex);