From 71898cf72844b62df1b5db74d0598ac5cf48dcbd Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:10:43 +0800 Subject: [PATCH] unlock custom contextsize --- koboldcpp.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/koboldcpp.py b/koboldcpp.py index 742805425..9d631bf8b 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2656,6 +2656,18 @@ def start_in_seperate_process(launch_args): return (output_queue, input_queue, p) if __name__ == '__main__': + + def check_range(value_type, min_value, max_value): + def range_checker(arg: str): + try: + f = value_type(arg) + except ValueError: + raise argparse.ArgumentTypeError(f'must be a valid {value_type}') + if f < min_value or f > max_value: + raise argparse.ArgumentTypeError(f'must be within [{min_value}, {max_value}]') + return f + return range_checker + print("***\nWelcome to KoboldCpp - Version " + KcppVersion) # just update version manually # print("Python version: " + sys.version) parser = argparse.ArgumentParser(description='KoboldCpp Server') @@ -2676,7 +2688,7 @@ if __name__ == '__main__': parser.add_argument("--threads", help="Use a custom number of threads if specified. Otherwise, uses an amount based on CPU cores", type=int, default=default_threads) parser.add_argument("--blasthreads", help="Use a different number of threads during BLAS if specified. Otherwise, has the same value as --threads",metavar=('[threads]'), type=int, default=0) parser.add_argument("--highpriority", help="Experimental flag. If set, increases the process CPU priority, potentially speeding up generation. Use caution.", action='store_true') - parser.add_argument("--contextsize", help="Controls the memory allocated for maximum context size, only change if you need more RAM for big contexts. (default 2048)", type=int,choices=[256, 512,1024,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152,65536], default=2048) + parser.add_argument("--contextsize", help="Controls the memory allocated for maximum context size, only change if you need more RAM for big contexts. (default 2048). Supported values are [256,512,1024,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152,65536]. IF YOU USE ANYTHING ELSE YOU ARE ON YOUR OWN.",metavar=('[256,512,1024,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152,65536]'), type=check_range(int,256,262144), default=2048) parser.add_argument("--blasbatchsize", help="Sets the batch size used in BLAS processing (default 512). Setting it to -1 disables BLAS mode, but keeps other benefits like GPU offload.", type=int,choices=[-1,32,64,128,256,512,1024,2048], default=512) parser.add_argument("--ropeconfig", help="If set, uses customized RoPE scaling from configured frequency scale and frequency base (e.g. --ropeconfig 0.25 10000). Otherwise, uses NTK-Aware scaling set automatically based on context size. For linear rope, simply set the freq-scale and ignore the freq-base",metavar=('[rope-freq-scale]', '[rope-freq-base]'), default=[0.0, 10000.0], type=float, nargs='+') parser.add_argument("--smartcontext", help="Reserving a portion of context to try processing less frequently.", action='store_true')