update benchmarks and api endpoints

CHANGELOG.md

@@ -35,16 +35,21 @@ Another big release, highlights being:
 - Massive work integrating latest advances with [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IPEX](https://github.com/vladmandic/automatic/wiki/Intel-ARC) and [ONNX Olive](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime-&-Olive)
 - Full control over brightness, sharpness and color shifts and color grading during generate process directly in latent space  
 
-Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment  
+Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment as well as **API** improvements  
 And it also includes fixes for all reported issues so far  
 
 As of this release, default backend is set to **diffusers** as its more feature rich than **original** and supports many additional models (original backend does remain as fully supported)  
 
+Also, previous versions of **SD.Next** were tuned for balance between performance and resource usage.  
+With this release, focus is more on performance.  
+See [Benchmark](https://github.com/vladmandic/automatic/wiki/Benchmark) notes for details, but as a highlight, we are now hitting **~110-150 it/s** on a standard nVidia RTX4090 in optimal scenarios!  
+
+Further details:  
 - For basic instructions, see [README](https://github.com/vladmandic/automatic/blob/master/README.md)  
 - For more details on all new features see full [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md)  
-- For documentation, see [WIKI](https://github.com/vladmandic/automatic/wiki)
+- For documentation, see [WiKi](https://github.com/vladmandic/automatic/wiki)
 
-## Update for 2024-02-06
+## Update for 2024-02-07
 
 - Heavily updated [Wiki](https://github.com/vladmandic/automatic/wiki)  
 - **Control**:  
@@ -297,6 +302,9 @@ As of this release, default backend is set to **diffusers** as its more feature
   - img2img: support variable aspect ratio without explicit resize  
   - cli: add `simple-upscale.py` script  
   - cli: fix cmd args parsing  
+  - cli: add `run-benchmark.py` script  
+  - api: add `/sdapi/v1/version` endpoint
+  - api: add `/sdapi/v1/platform` endpoint
   - api: return current image in progress api if requested  
   - api: sanitize response object  
   - api: cleanup error logging  

cli/run-benchmark.py

@@ -2,34 +2,21 @@
 """
 sd api txt2img benchmark
 """
+import os
 import asyncio
 import base64
 import io
 import json
 import time
+import argparse
 from PIL import Image
 import sdapi
 from util import Map, log
 
 
-options = Map({
-    'restore_faces': False,
-    'prompt': 'photo of two dice on a table',
-    'negative_prompt': 'foggy, blurry',
-    'steps': 50,
-    'batch_size': 1,
-    'n_iter': 1,
-    'seed': -1,
-    'sampler_name': 'Euler a',
-    'cfg_scale': 0,
-    'width': 512,
-    'height': 512
-})
-
-
-# batch = [1, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128]
-batch = [1, 1, 2, 4, 8, 12, 16]
 oom = 0
+args = None
+options = None
 
 
 async def txt2img():
@@ -46,9 +33,15 @@ async def txt2img():
     else:
         return 0
     log.debug({ 'info': info })
+    if options['batch_size'] != len(data['images']):
+        log.error({ 'requested': options['batch_size'], 'received': len(data['images']) })
     for i in range(len(data['images'])):
         data['images'][i] = Image.open(io.BytesIO(base64.b64decode(data['images'][i].split(',',1)[0])))
-        log.debug({ 'image': data['images'][i].size })
+        if args.save:
+            fn = os.path.join(args.save, f'benchmark-{i}-{len(data["images"])}.png')
+            data["images"][i].save(fn)
+            log.debug({ 'save': fn })
+    log.debug({ "images": data["images"] })
     t1 = time.perf_counter()
     return t1 - t0
 
@@ -75,28 +68,30 @@ def gb(val: float):
 
 
 async def main():
-    log.info({ 'benchmark': { 'batch-sizes': batch } })
     sdapi.quiet = True
     await sdapi.session()
     await sdapi.interrupt()
+    ver = await sdapi.get("/sdapi/v1/version")
+    log.info({ 'version': ver})
+    platform = await sdapi.get("/sdapi/v1/platform")
+    log.info({ 'platform': platform })
     opts = await sdapi.get('/sdapi/v1/options')
     opts = Map(opts)
-    log.info({ 'options': {
-        'resolution': [options.width, options.height],
-        'model': opts.sd_model_checkpoint,
-        'vae': opts.sd_vae,
-        'hypernetwork': opts.sd_hypernetwork,
-        'sampler': options.sampler_name,
-        'preview': opts.show_progress_every_n_steps
-    } })
+    log.info({ 'model': opts.sd_model_checkpoint })
     cpu, gpu = memstats()
     log.info({ 'system': { 'cpu': cpu, 'gpu': gpu }})
+    batch = [1, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256]
+    batch = [b for b in batch if b <= args.maxbatch]
+    log.info({"batch-sizes": batch})
     for i in range(len(batch)):
         if oom > 0:
             continue
         options['batch_size'] = batch[i]
+        warmup = await txt2img()
         ts = await txt2img()
-        if ts > 0:
+        if i == 0:
+            ts += warmup
+        if ts > 0.01: # cannot be faster than 10ms per run
             await asyncio.sleep(0)
             cpu, gpu = memstats()
             if i == 0:
@@ -115,6 +110,37 @@ async def main():
 
 
 if __name__ == '__main__':
+    log.info({ 'run-benchmark' })
+    parser = argparse.ArgumentParser(description = 'run-benchmark')
+    parser.add_argument("--steps", type=int, default=50, required=False, help="steps")
+    parser.add_argument("--sampler", type=str, default='Euler a', required=False, help="max batch size")
+    parser.add_argument("--prompt", type=str, default='photo of two dice on a table', required=False, help="prompt")
+    parser.add_argument("--negative", type=str, default='foggy, blurry', required=False, help="prompt")
+    parser.add_argument("--maxbatch", type=int, default=16, required=False, help="max batch size")
+    parser.add_argument("--width", type=int, default=512, required=False, help="width")
+    parser.add_argument("--height", type=int, default=512, required=False, help="height")
+    parser.add_argument('--debug', default = False, action='store_true', help = 'debug logging')
+    parser.add_argument('--taesd', default = False, action='store_true', help = 'use taesd as vae')
+    parser.add_argument("--save", type=str, default='', required=False, help="save images to folder")
+    args = parser.parse_args()
+    if args.debug:
+        log.setLevel('DEBUG')
+    options = Map(
+        {
+            "prompt": args.prompt,
+            "negative_prompt": args.negative,
+            "steps": args.steps,
+            "sampler_name": args.sampler,
+            "width": args.width,
+            "height": args.height,
+            "full_quality": not args.taesd,
+            "cfg_scale": 0,
+            "batch_size": 1,
+            "n_iter": 1,
+            "seed": -1,
+        }
+    )
+    log.info({"options": options})
     try:
         asyncio.run(main())
     except KeyboardInterrupt:

modules/api/api.py

@@ -33,6 +33,8 @@ class Api:
         self.add_api_route("/sdapi/v1/motd", server.get_motd, methods=["GET"], response_model=str)
         self.add_api_route("/sdapi/v1/log", server.get_log_buffer, methods=["GET"], response_model=List[str])
         self.add_api_route("/sdapi/v1/start", self.get_session_start, methods=["GET"])
+        self.add_api_route("/sdapi/v1/version", server.get_version, methods=["GET"])
+        self.add_api_route("/sdapi/v1/platform", server.get_platform, methods=["GET"])
         self.add_api_route("/sdapi/v1/progress", server.get_progress, methods=["GET"], response_model=models.ResProgress)
         self.add_api_route("/sdapi/v1/interrupt", server.post_interrupt, methods=["POST"])
         self.add_api_route("/sdapi/v1/skip", server.post_skip, methods=["POST"])

modules/api/server.py

@@ -11,9 +11,8 @@ def post_shutdown():
 
 def get_motd():
     import requests
-    from installer import get_version
     motd = ''
-    ver = get_version()
+    ver = shared.get_version()
     if ver.get('updated', None) is not None:
         motd = f"version <b>{ver['hash']} {ver['updated']}</b> <span style='color: var(--primary-500)'>{ver['url'].split('/')[-1]}</span><br>"
     if shared.opts.motd:
@@ -24,6 +23,14 @@ def get_motd():
             motd += res.text
     return motd
 
+def get_version():
+    return shared.get_version()
+
+def get_platform():
+    from installer import get_platform as installer_get_platform
+    from modules.loader import get_packages as loader_get_packages
+    return { **installer_get_platform(), **loader_get_packages() }
+
 def get_log_buffer(req: models.ReqLog = Depends()):
     lines = shared.log.buffer[:req.lines] if req.lines > 0 else shared.log.buffer.copy()
     if req.clear:

modules/loader.py

@@ -51,7 +51,15 @@ timer.startup.record("pydantic")
 
 import diffusers # pylint: disable=W0611,C0411
 timer.startup.record("diffusers")
-errors.log.info(f'Load packages: torch={getattr(torch, "__long_version__", torch.__version__)} diffusers={diffusers.__version__} gradio={gradio.__version__}')
+
+def get_packages():
+    return {
+        "torch": getattr(torch, "__long_version__", torch.__version__),
+        "diffusers": diffusers.__version__,
+        "gradio": gradio.__version__,
+    }
+
+errors.log.info(f'Load packages: {get_packages()}')
 
 try:
     import os

modules/shared.py

@@ -430,7 +430,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
     "diffusers_model_cpu_offload": OptionInfo(False, "Model CPU offload (--medvram)"),
     "diffusers_seq_cpu_offload": OptionInfo(False, "Sequential CPU offload (--lowvram)"),
     "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}),
-    "diffusers_vae_slicing": OptionInfo(False, "VAE slicing"),
+    "diffusers_vae_slicing": OptionInfo(True, "VAE slicing"),
     "diffusers_vae_tiling": OptionInfo(False, "VAE tiling"),
     "diffusers_attention_slicing": OptionInfo(False, "Attention slicing"),
     "diffusers_model_load_variant": OptionInfo("default", "Preferred Model variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}),