From 5cd9b1d23a0ac99cd78bcaf1abe0d682cf3cdd31 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Tue, 6 Feb 2024 21:48:07 +0800 Subject: [PATCH] placeholder for benchmark --- koboldcpp.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/koboldcpp.py b/koboldcpp.py index 56e243a82..c37c6d8d0 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2514,6 +2514,15 @@ def main(launch_args,start_server=True): timer_thread = threading.Timer(1, onready_subprocess) #1 second delay timer_thread.start() + if args.benchmark is not None: + start_server = False + if args.benchmark=="stdout": + print("Running benchmark (Not saving to file)...") + + else: + print("Running benchmark (Saving to file)...") + + if start_server: if args.remotetunnel: setuptunnel() @@ -2586,6 +2595,7 @@ if __name__ == '__main__': parser.add_argument("--gpulayers", help="Set number of layers to offload to GPU when using GPU. Requires GPU.",metavar=('[GPU layers]'), nargs='?', const=1, type=int, default=0) parser.add_argument("--tensor_split", help="For CUDA with ALL GPU set only, ratio to split tensors across multiple GPUs, space-separated list of proportions, e.g. 7 3", metavar=('[Ratios]'), type=float, nargs='+') parser.add_argument("--onready", help="An optional shell command to execute after the model has been loaded.", metavar=('[shell command]'), type=str, default="",nargs=1) + parser.add_argument("--benchmark", help="Do not start server, instead run benchmarks. If filename is provided, appends results to provided file.", metavar=('[filename]'), nargs='?', const="stdout", type=str, default=None) parser.add_argument("--multiuser", help="Runs in multiuser mode, which queues incoming requests instead of blocking them.", metavar=('limit'), nargs='?', const=1, type=int, default=0) parser.add_argument("--remotetunnel", help="Uses Cloudflare to create a remote tunnel, allowing you to access koboldcpp remotely over the internet even behind a firewall.", action='store_true') parser.add_argument("--foreground", help="Windows only. Sends the terminal to the foreground every time a new prompt is generated. This helps avoid some idle slowdown issues.", action='store_true')