set default cuda dtype to auto

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -34,6 +34,11 @@
   - controlnet support for img2img and inpaint (in addition to previous txt2img controlnet)  
   - allow separate vae load  
   - add additional controlnets: [JasperAI](https://huggingface.co/collections/jasperai/flux1-dev-controlnets-66f27f9459d760dcafa32e08) **Depth**, **Upscaler**, **Surface**, thanks @EnragedAntelope
+- **dtype**
+  - previously `cuda_dtype` in settings defaulted to `fp16` if available  
+  - now `cuda_type` defaults to **Auto** which executes `bf16` and `fp16` tests on startup and selects best available dtype  
+    if you have specific requirements, you can still set to fp32/fp16/bf16 as desired  
+    if you have gpu that incorrectly identifies bf16 or fp16 availablity, let us know so we can improve the auto-detection  
 - **xyz grid** full refactor  
   - multi-mode: *selectable-script* and *alwayson-script*  
   - allow usage combined with other scripts  

modules/devices.py

@@ -192,39 +192,44 @@ def set_cuda_sync_mode(mode):
 
 
 def test_fp16():
-    if shared.cmd_opts.experimental:
-        if debug:
-            log.debug('Torch FP16 test skip')
-        return True
     try:
         x = torch.tensor([[1.5,.0,.0,.0]]).to(device=device, dtype=torch.float16)
         layerNorm = torch.nn.LayerNorm(4, eps=0.00001, elementwise_affine=True, dtype=torch.float16, device=device)
-        _y = layerNorm(x)
+        out = layerNorm(x)
+        if out.dtype != torch.float16:
+            raise RuntimeError('Torch FP16 test: dtype mismatch')
+        if torch.all(torch.isnan(out)).item():
+            raise RuntimeError('Torch FP16 test: NaN')
         if debug:
-            log.debug('Torch FP16 test pass')
+            log.debug('Torch FP16 test: pass')
         return True
     except Exception as ex:
-        log.warning(f'Torch FP16 test failed: Forcing FP32 operations: {ex}')
-        shared.opts.cuda_dtype = 'FP32'
-        shared.opts.no_half = True
-        shared.opts.no_half_vae = True
+        log.warning(f'Torch FP16 test fail: {ex}')
+        if shared.cmd_opts.experimental:
+            log.debug('Torch FP16 test fail: override experimental')
+            return True
         return False
 
 
 def test_bf16():
-    if shared.cmd_opts.experimental:
-        if debug:
-            log.debug('Torch BF16 test skip')
-        return True
     try:
         import torch.nn.functional as F
         image = torch.randn(1, 4, 32, 32).to(device=device, dtype=torch.bfloat16)
-        _out = F.interpolate(image, size=(64, 64), mode="nearest")
+        out = F.interpolate(image, size=(64, 64), mode="nearest")
+        if out.dtype != torch.bfloat16:
+            raise RuntimeError('Torch BF16 test: dtype mismatch')
+        if torch.all(torch.isnan(out)).item():
+            raise RuntimeError('Torch BF16 test: NaN')
         if debug:
-            log.debug('Torch BF16 test pass')
+            log.debug('Torch BF16 test: pass')
+        # if torch.cuda.is_available() and not torch.cuda.is_bf16_supported():
+        #    log.warning('Torch BF16 test: partial pass')
         return True
-    except Exception:
-        log.warning('Torch BF16 test failed: Fallback to FP16 operations')
+    except Exception as ex:
+        log.warning(f'Torch BF16 test fail: {ex}')
+        if shared.cmd_opts.experimental:
+            log.debug('Torch FP16 test fail: override experimental')
+            return True
         return False
 
 
@@ -240,7 +245,6 @@ def set_cudnn_params():
             try:
                 torch.backends.cudnn.deterministic = shared.opts.cudnn_deterministic
                 torch.use_deterministic_algorithms(shared.opts.cudnn_deterministic)
-                log.debug(f'Torch mode: deterministic={shared.opts.cudnn_deterministic}')
                 if shared.opts.cudnn_deterministic:
                     os.environ.setdefault('CUBLAS_WORKSPACE_CONFIG', ':4096:8')
                 torch.backends.cudnn.benchmark = True
@@ -291,30 +295,51 @@ def set_sdpa_params():
 
 def set_dtype():
     global dtype, dtype_vae, dtype_unet, unet_needs_upcast, inference_context, fp16_ok, bf16_ok # pylint: disable=global-statement
-    if shared.opts.cuda_dtype == 'FP32':
+    if shared.opts.cuda_dtype == 'Auto': # detect
+        if sys.platform == "darwin" or shared.cmd_opts.use_openvino: # override
+            fp16_ok = False
+            bf16_ok = False
+        else:
+            fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
+            bf16_ok = test_bf16() if bf16_ok is None else bf16_ok
+
+        if bf16_ok:
+            dtype = torch.bfloat16
+            dtype_vae = torch.bfloat16
+            dtype_unet = torch.bfloat16
+        elif fp16_ok:
+            dtype = torch.float16
+            dtype_vae = torch.float16
+            dtype_unet = torch.float16
+        else:
+            dtype = torch.float32
+            dtype_vae = torch.float32
+            dtype_unet = torch.float32
+    elif shared.opts.cuda_dtype == 'FP32':
         dtype = torch.float32
         dtype_vae = torch.float32
         dtype_unet = torch.float32
         fp16_ok = None
         bf16_ok = None
-    elif shared.opts.cuda_dtype == 'BF16' or dtype == torch.bfloat16:
+    elif shared.opts.cuda_dtype == 'BF16':
         fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
         bf16_ok = test_bf16() if bf16_ok is None else bf16_ok
         dtype = torch.bfloat16 if bf16_ok else torch.float16
         dtype_vae = torch.bfloat16 if bf16_ok else torch.float16
         dtype_unet = torch.bfloat16 if bf16_ok else torch.float16
-    elif shared.opts.cuda_dtype == 'FP16' or dtype == torch.float16:
+    elif shared.opts.cuda_dtype == 'FP16':
         fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
         bf16_ok = None
         dtype = torch.float16 if fp16_ok else torch.float32
         dtype_vae = torch.float16 if fp16_ok else torch.float32
         dtype_unet = torch.float16 if fp16_ok else torch.float32
+
     if shared.opts.no_half:
         log.info('Torch override dtype: no-half set')
         dtype = torch.float32
         dtype_vae = torch.float32
         dtype_unet = torch.float32
-    if shared.opts.no_half_vae: # set dtype again as no-half-vae options take priority
+    if shared.opts.no_half_vae:
         log.info('Torch override VAE dtype: no-half set')
         dtype_vae = torch.float32
     unet_needs_upcast = shared.opts.upcast_sampling
@@ -336,8 +361,7 @@ def set_cuda_params():
     if shared.cmd_opts.profile:
         shared.log.debug(f'Torch info: {torch.__config__.show()}')
     device_name = get_raw_openvino_device() if shared.cmd_opts.use_openvino else torch.device(get_optimal_device_name()) # pylint: disable=used-before-assignment
-    log.debug(f'Desired Torch parameters: dtype={shared.opts.cuda_dtype} no-half={shared.opts.no_half} no-half-vae={shared.opts.no_half_vae} upscast={shared.opts.upcast_sampling}')
-    log.info(f'Setting Torch parameters: device={device_name} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} fp16={fp16_ok} bf16={bf16_ok} optimization={shared.opts.cross_attention_optimization}')
+    log.info(f'Torch parameters: device={device_name} config={shared.opts.cuda_dtype} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} nohalf={shared.opts.no_half} nohalfvae={shared.opts.no_half_vae} upscast={shared.opts.upcast_sampling} deterministic={shared.opts.cudnn_deterministic} test-fp16={fp16_ok} test-bf16={bf16_ok} optimization="{shared.opts.cross_attention_optimization}"')
 
 
 args = cmd_args.parser.parse_args()
@@ -377,9 +401,9 @@ inference_context = torch.no_grad
 cuda_ok = torch.cuda.is_available()
 cpu = torch.device("cpu")
 device = device_interrogate = device_gfpgan = device_esrgan = device_codeformer = None
-dtype = torch.float16
-dtype_vae = torch.float16
-dtype_unet = torch.float16
+dtype = None
+dtype_vae = None
+dtype_unet = None
 fp16_ok = None
 bf16_ok = None
 unet_needs_upcast = False

modules/paths.py

@@ -28,7 +28,6 @@ models_config = cli.models_dir or config.get('models_dir') or 'models'
 models_path = models_config if os.path.isabs(models_config) else os.path.join(data_path, models_config)
 extensions_dir = os.path.join(data_path, "extensions")
 extensions_builtin_dir = "extensions-builtin"
-onnx_dir = os.path.join(models_path, "ONNX")
 sd_configs_path = os.path.join(script_path, "configs")
 sd_default_config = os.path.join(sd_configs_path, "v1-inference.yaml")
 sd_model_file = cli.ckpt or os.path.join(script_path, 'model.ckpt') # not used
@@ -101,7 +100,6 @@ def create_paths(opts):
     create_path(sd_configs_path)
     create_path(extensions_dir)
     create_path(extensions_builtin_dir)
-    create_path(onnx_dir)
     create_path(fix_path('temp_dir'))
     create_path(fix_path('ckpt_dir'))
     create_path(fix_path('diffusers_dir'))
@@ -111,6 +109,7 @@ def create_paths(opts):
     create_path(fix_path('lora_dir'))
     create_path(fix_path('embeddings_dir'))
     create_path(fix_path('hypernetwork_dir'))
+    create_path(fix_path('onnx_temp_dir'))
     create_path(fix_path('outdir_samples'))
     create_path(fix_path('outdir_txt2img_samples'))
     create_path(fix_path('outdir_img2img_samples'))

modules/shared.py

@@ -432,7 +432,7 @@ options_templates.update(options_section(('sd', "Execution & Models"), {
 options_templates.update(options_section(('cuda', "Compute Settings"), {
     "math_sep": OptionInfo("<h2>Execution precision</h2>", "", gr.HTML),
     "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"]}),
-    "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, {"choices": ["FP32", "FP16", "BF16"]}),
+    "cuda_dtype": OptionInfo("Auto", "Device precision type", gr.Radio, {"choices": ["Auto", "FP32", "FP16", "BF16"]}),
     "cudnn_deterministic": OptionInfo(False, "Use deterministic mode"),
 
     "model_sep": OptionInfo("<h2>Model options</h2>", "", gr.HTML),