diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 532765be1..c185f48e5 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -893,7 +893,7 @@ static void mtp_decoding_setup(llama_model * main_model, llama_context * main_ct } draft_is_mtp = true; - speculative_state_setup(main_ctx, mtp_ctx_params, -2, COMMON_SPECULATIVE_TYPE_DRAFT_MTP); + speculative_state_setup(main_ctx, mtp_ctx_params, -1, COMMON_SPECULATIVE_TYPE_DRAFT_MTP); } //loads a model for speculative decoding. diff --git a/koboldcpp.py b/koboldcpp.py index c4c8cbfbf..5c0f16123 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -12177,7 +12177,7 @@ if __name__ == '__main__': advparser.add_argument("--defaultgenamt", help="How many tokens to generate by default, if not specified. Must be smaller than context size. Usually, your frontend GUI will override this.", type=check_range(int,64,16384), default=default_genlen) advparser.add_argument("--device", "-dev", metavar=(''), help="Set llama.cpp compatible device selection override. Comma separated. Overrides normal device choices.", default="") advparser.add_argument("--downloaddir", metavar=('[directory]'), help="Specify a directory that models will be downloaded to or searched from, if unset uses the working directory.", default="") - advparser.add_argument("--draftamount","--draft-max","--draft-n", metavar=('[tokens]'), help="How many tokens to draft per chunk before verifying results", type=int, default=default_draft_amount) + advparser.add_argument("--draftamount","--draft-max","--draft-n","--spec-draft-n-max", metavar=('[tokens]'), help="How many tokens to draft per chunk before verifying results", type=int, default=default_draft_amount) advparser.add_argument("--draftgpulayers","--gpu-layers-draft","--n-gpu-layers-draft","-ngld", metavar=('[layers]'), help="How many layers to offload to GPU for the draft model (default=full offload)", type=int, default=999) advparser.add_argument("--draftgpusplit", help="GPU layer distribution ratio for draft model (default=same as main). Only works if multi-GPUs selected for MAIN model and tensor_split is set!", metavar=('[Ratios]'), type=float, nargs='+') advparser.add_argument("--draftmodel","--model-draft","-md", metavar=('[filename]'), help="Load a small draft model for speculative decoding. It will be fully offloaded. Vocab must match the main model.", default="")