From 24b4cd77a35788ea2e23a12b87e810b9db94658e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 7 Feb 2024 12:04:59 -0500 Subject: [PATCH] update benchmarks and api endpoints --- CHANGELOG.md | 14 ++++++-- cli/run-benchmark.py | 82 ++++++++++++++++++++++++++++--------------- modules/api/api.py | 2 ++ modules/api/server.py | 11 ++++-- modules/loader.py | 10 +++++- modules/shared.py | 2 +- wiki | 2 +- 7 files changed, 87 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ea4c4b1..0f844637a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,16 +35,21 @@ Another big release, highlights being: - Massive work integrating latest advances with [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IPEX](https://github.com/vladmandic/automatic/wiki/Intel-ARC) and [ONNX Olive](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime-&-Olive) - Full control over brightness, sharpness and color shifts and color grading during generate process directly in latent space -Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment +Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment as well as **API** improvements And it also includes fixes for all reported issues so far As of this release, default backend is set to **diffusers** as its more feature rich than **original** and supports many additional models (original backend does remain as fully supported) +Also, previous versions of **SD.Next** were tuned for balance between performance and resource usage. +With this release, focus is more on performance. +See [Benchmark](https://github.com/vladmandic/automatic/wiki/Benchmark) notes for details, but as a highlight, we are now hitting **~110-150 it/s** on a standard nVidia RTX4090 in optimal scenarios! + +Further details: - For basic instructions, see [README](https://github.com/vladmandic/automatic/blob/master/README.md) - For more details on all new features see full [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) -- For documentation, see [WIKI](https://github.com/vladmandic/automatic/wiki) +- For documentation, see [WiKi](https://github.com/vladmandic/automatic/wiki) -## Update for 2024-02-06 +## Update for 2024-02-07 - Heavily updated [Wiki](https://github.com/vladmandic/automatic/wiki) - **Control**: @@ -297,6 +302,9 @@ As of this release, default backend is set to **diffusers** as its more feature - img2img: support variable aspect ratio without explicit resize - cli: add `simple-upscale.py` script - cli: fix cmd args parsing + - cli: add `run-benchmark.py` script + - api: add `/sdapi/v1/version` endpoint + - api: add `/sdapi/v1/platform` endpoint - api: return current image in progress api if requested - api: sanitize response object - api: cleanup error logging diff --git a/cli/run-benchmark.py b/cli/run-benchmark.py index ed02026d8..9bc0721f0 100755 --- a/cli/run-benchmark.py +++ b/cli/run-benchmark.py @@ -2,34 +2,21 @@ """ sd api txt2img benchmark """ +import os import asyncio import base64 import io import json import time +import argparse from PIL import Image import sdapi from util import Map, log -options = Map({ - 'restore_faces': False, - 'prompt': 'photo of two dice on a table', - 'negative_prompt': 'foggy, blurry', - 'steps': 50, - 'batch_size': 1, - 'n_iter': 1, - 'seed': -1, - 'sampler_name': 'Euler a', - 'cfg_scale': 0, - 'width': 512, - 'height': 512 -}) - - -# batch = [1, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128] -batch = [1, 1, 2, 4, 8, 12, 16] oom = 0 +args = None +options = None async def txt2img(): @@ -46,9 +33,15 @@ async def txt2img(): else: return 0 log.debug({ 'info': info }) + if options['batch_size'] != len(data['images']): + log.error({ 'requested': options['batch_size'], 'received': len(data['images']) }) for i in range(len(data['images'])): data['images'][i] = Image.open(io.BytesIO(base64.b64decode(data['images'][i].split(',',1)[0]))) - log.debug({ 'image': data['images'][i].size }) + if args.save: + fn = os.path.join(args.save, f'benchmark-{i}-{len(data["images"])}.png') + data["images"][i].save(fn) + log.debug({ 'save': fn }) + log.debug({ "images": data["images"] }) t1 = time.perf_counter() return t1 - t0 @@ -75,28 +68,30 @@ def gb(val: float): async def main(): - log.info({ 'benchmark': { 'batch-sizes': batch } }) sdapi.quiet = True await sdapi.session() await sdapi.interrupt() + ver = await sdapi.get("/sdapi/v1/version") + log.info({ 'version': ver}) + platform = await sdapi.get("/sdapi/v1/platform") + log.info({ 'platform': platform }) opts = await sdapi.get('/sdapi/v1/options') opts = Map(opts) - log.info({ 'options': { - 'resolution': [options.width, options.height], - 'model': opts.sd_model_checkpoint, - 'vae': opts.sd_vae, - 'hypernetwork': opts.sd_hypernetwork, - 'sampler': options.sampler_name, - 'preview': opts.show_progress_every_n_steps - } }) + log.info({ 'model': opts.sd_model_checkpoint }) cpu, gpu = memstats() log.info({ 'system': { 'cpu': cpu, 'gpu': gpu }}) + batch = [1, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256] + batch = [b for b in batch if b <= args.maxbatch] + log.info({"batch-sizes": batch}) for i in range(len(batch)): if oom > 0: continue options['batch_size'] = batch[i] + warmup = await txt2img() ts = await txt2img() - if ts > 0: + if i == 0: + ts += warmup + if ts > 0.01: # cannot be faster than 10ms per run await asyncio.sleep(0) cpu, gpu = memstats() if i == 0: @@ -115,6 +110,37 @@ async def main(): if __name__ == '__main__': + log.info({ 'run-benchmark' }) + parser = argparse.ArgumentParser(description = 'run-benchmark') + parser.add_argument("--steps", type=int, default=50, required=False, help="steps") + parser.add_argument("--sampler", type=str, default='Euler a', required=False, help="max batch size") + parser.add_argument("--prompt", type=str, default='photo of two dice on a table', required=False, help="prompt") + parser.add_argument("--negative", type=str, default='foggy, blurry', required=False, help="prompt") + parser.add_argument("--maxbatch", type=int, default=16, required=False, help="max batch size") + parser.add_argument("--width", type=int, default=512, required=False, help="width") + parser.add_argument("--height", type=int, default=512, required=False, help="height") + parser.add_argument('--debug', default = False, action='store_true', help = 'debug logging') + parser.add_argument('--taesd', default = False, action='store_true', help = 'use taesd as vae') + parser.add_argument("--save", type=str, default='', required=False, help="save images to folder") + args = parser.parse_args() + if args.debug: + log.setLevel('DEBUG') + options = Map( + { + "prompt": args.prompt, + "negative_prompt": args.negative, + "steps": args.steps, + "sampler_name": args.sampler, + "width": args.width, + "height": args.height, + "full_quality": not args.taesd, + "cfg_scale": 0, + "batch_size": 1, + "n_iter": 1, + "seed": -1, + } + ) + log.info({"options": options}) try: asyncio.run(main()) except KeyboardInterrupt: diff --git a/modules/api/api.py b/modules/api/api.py index 6810e6aab..3f1aedf95 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -33,6 +33,8 @@ class Api: self.add_api_route("/sdapi/v1/motd", server.get_motd, methods=["GET"], response_model=str) self.add_api_route("/sdapi/v1/log", server.get_log_buffer, methods=["GET"], response_model=List[str]) self.add_api_route("/sdapi/v1/start", self.get_session_start, methods=["GET"]) + self.add_api_route("/sdapi/v1/version", server.get_version, methods=["GET"]) + self.add_api_route("/sdapi/v1/platform", server.get_platform, methods=["GET"]) self.add_api_route("/sdapi/v1/progress", server.get_progress, methods=["GET"], response_model=models.ResProgress) self.add_api_route("/sdapi/v1/interrupt", server.post_interrupt, methods=["POST"]) self.add_api_route("/sdapi/v1/skip", server.post_skip, methods=["POST"]) diff --git a/modules/api/server.py b/modules/api/server.py index 6c7bfed71..a4116f46b 100644 --- a/modules/api/server.py +++ b/modules/api/server.py @@ -11,9 +11,8 @@ def post_shutdown(): def get_motd(): import requests - from installer import get_version motd = '' - ver = get_version() + ver = shared.get_version() if ver.get('updated', None) is not None: motd = f"version {ver['hash']} {ver['updated']} {ver['url'].split('/')[-1]}
" if shared.opts.motd: @@ -24,6 +23,14 @@ def get_motd(): motd += res.text return motd +def get_version(): + return shared.get_version() + +def get_platform(): + from installer import get_platform as installer_get_platform + from modules.loader import get_packages as loader_get_packages + return { **installer_get_platform(), **loader_get_packages() } + def get_log_buffer(req: models.ReqLog = Depends()): lines = shared.log.buffer[:req.lines] if req.lines > 0 else shared.log.buffer.copy() if req.clear: diff --git a/modules/loader.py b/modules/loader.py index 7c8d44a43..2b502c9af 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -51,7 +51,15 @@ timer.startup.record("pydantic") import diffusers # pylint: disable=W0611,C0411 timer.startup.record("diffusers") -errors.log.info(f'Load packages: torch={getattr(torch, "__long_version__", torch.__version__)} diffusers={diffusers.__version__} gradio={gradio.__version__}') + +def get_packages(): + return { + "torch": getattr(torch, "__long_version__", torch.__version__), + "diffusers": diffusers.__version__, + "gradio": gradio.__version__, + } + +errors.log.info(f'Load packages: {get_packages()}') try: import os diff --git a/modules/shared.py b/modules/shared.py index 6bc0130ab..bd30f75c1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -430,7 +430,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_model_cpu_offload": OptionInfo(False, "Model CPU offload (--medvram)"), "diffusers_seq_cpu_offload": OptionInfo(False, "Sequential CPU offload (--lowvram)"), "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), - "diffusers_vae_slicing": OptionInfo(False, "VAE slicing"), + "diffusers_vae_slicing": OptionInfo(True, "VAE slicing"), "diffusers_vae_tiling": OptionInfo(False, "VAE tiling"), "diffusers_attention_slicing": OptionInfo(False, "Attention slicing"), "diffusers_model_load_variant": OptionInfo("default", "Preferred Model variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), diff --git a/wiki b/wiki index d094b863b..c44eeed91 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit d094b863ba531b9b4c8011cf1b878755c0af26a2 +Subproject commit c44eeed913f772b8a2c9fb9ed56b42606588c425