diff --git a/CHANGELOG.md b/CHANGELOG.md
index a4e247be9..89acb5e75 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,11 @@
- controlnet support for img2img and inpaint (in addition to previous txt2img controlnet)
- allow separate vae load
- add additional controlnets: [JasperAI](https://huggingface.co/collections/jasperai/flux1-dev-controlnets-66f27f9459d760dcafa32e08) **Depth**, **Upscaler**, **Surface**, thanks @EnragedAntelope
+- **dtype**
+ - previously `cuda_dtype` in settings defaulted to `fp16` if available
+ - now `cuda_type` defaults to **Auto** which executes `bf16` and `fp16` tests on startup and selects best available dtype
+ if you have specific requirements, you can still set to fp32/fp16/bf16 as desired
+ if you have gpu that incorrectly identifies bf16 or fp16 availablity, let us know so we can improve the auto-detection
- **xyz grid** full refactor
- multi-mode: *selectable-script* and *alwayson-script*
- allow usage combined with other scripts
diff --git a/modules/devices.py b/modules/devices.py
index 509a12e1a..3791e55b1 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -192,39 +192,44 @@ def set_cuda_sync_mode(mode):
def test_fp16():
- if shared.cmd_opts.experimental:
- if debug:
- log.debug('Torch FP16 test skip')
- return True
try:
x = torch.tensor([[1.5,.0,.0,.0]]).to(device=device, dtype=torch.float16)
layerNorm = torch.nn.LayerNorm(4, eps=0.00001, elementwise_affine=True, dtype=torch.float16, device=device)
- _y = layerNorm(x)
+ out = layerNorm(x)
+ if out.dtype != torch.float16:
+ raise RuntimeError('Torch FP16 test: dtype mismatch')
+ if torch.all(torch.isnan(out)).item():
+ raise RuntimeError('Torch FP16 test: NaN')
if debug:
- log.debug('Torch FP16 test pass')
+ log.debug('Torch FP16 test: pass')
return True
except Exception as ex:
- log.warning(f'Torch FP16 test failed: Forcing FP32 operations: {ex}')
- shared.opts.cuda_dtype = 'FP32'
- shared.opts.no_half = True
- shared.opts.no_half_vae = True
+ log.warning(f'Torch FP16 test fail: {ex}')
+ if shared.cmd_opts.experimental:
+ log.debug('Torch FP16 test fail: override experimental')
+ return True
return False
def test_bf16():
- if shared.cmd_opts.experimental:
- if debug:
- log.debug('Torch BF16 test skip')
- return True
try:
import torch.nn.functional as F
image = torch.randn(1, 4, 32, 32).to(device=device, dtype=torch.bfloat16)
- _out = F.interpolate(image, size=(64, 64), mode="nearest")
+ out = F.interpolate(image, size=(64, 64), mode="nearest")
+ if out.dtype != torch.bfloat16:
+ raise RuntimeError('Torch BF16 test: dtype mismatch')
+ if torch.all(torch.isnan(out)).item():
+ raise RuntimeError('Torch BF16 test: NaN')
if debug:
- log.debug('Torch BF16 test pass')
+ log.debug('Torch BF16 test: pass')
+ # if torch.cuda.is_available() and not torch.cuda.is_bf16_supported():
+ # log.warning('Torch BF16 test: partial pass')
return True
- except Exception:
- log.warning('Torch BF16 test failed: Fallback to FP16 operations')
+ except Exception as ex:
+ log.warning(f'Torch BF16 test fail: {ex}')
+ if shared.cmd_opts.experimental:
+ log.debug('Torch FP16 test fail: override experimental')
+ return True
return False
@@ -240,7 +245,6 @@ def set_cudnn_params():
try:
torch.backends.cudnn.deterministic = shared.opts.cudnn_deterministic
torch.use_deterministic_algorithms(shared.opts.cudnn_deterministic)
- log.debug(f'Torch mode: deterministic={shared.opts.cudnn_deterministic}')
if shared.opts.cudnn_deterministic:
os.environ.setdefault('CUBLAS_WORKSPACE_CONFIG', ':4096:8')
torch.backends.cudnn.benchmark = True
@@ -291,30 +295,51 @@ def set_sdpa_params():
def set_dtype():
global dtype, dtype_vae, dtype_unet, unet_needs_upcast, inference_context, fp16_ok, bf16_ok # pylint: disable=global-statement
- if shared.opts.cuda_dtype == 'FP32':
+ if shared.opts.cuda_dtype == 'Auto': # detect
+ if sys.platform == "darwin" or shared.cmd_opts.use_openvino: # override
+ fp16_ok = False
+ bf16_ok = False
+ else:
+ fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
+ bf16_ok = test_bf16() if bf16_ok is None else bf16_ok
+
+ if bf16_ok:
+ dtype = torch.bfloat16
+ dtype_vae = torch.bfloat16
+ dtype_unet = torch.bfloat16
+ elif fp16_ok:
+ dtype = torch.float16
+ dtype_vae = torch.float16
+ dtype_unet = torch.float16
+ else:
+ dtype = torch.float32
+ dtype_vae = torch.float32
+ dtype_unet = torch.float32
+ elif shared.opts.cuda_dtype == 'FP32':
dtype = torch.float32
dtype_vae = torch.float32
dtype_unet = torch.float32
fp16_ok = None
bf16_ok = None
- elif shared.opts.cuda_dtype == 'BF16' or dtype == torch.bfloat16:
+ elif shared.opts.cuda_dtype == 'BF16':
fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
bf16_ok = test_bf16() if bf16_ok is None else bf16_ok
dtype = torch.bfloat16 if bf16_ok else torch.float16
dtype_vae = torch.bfloat16 if bf16_ok else torch.float16
dtype_unet = torch.bfloat16 if bf16_ok else torch.float16
- elif shared.opts.cuda_dtype == 'FP16' or dtype == torch.float16:
+ elif shared.opts.cuda_dtype == 'FP16':
fp16_ok = test_fp16() if fp16_ok is None else fp16_ok
bf16_ok = None
dtype = torch.float16 if fp16_ok else torch.float32
dtype_vae = torch.float16 if fp16_ok else torch.float32
dtype_unet = torch.float16 if fp16_ok else torch.float32
+
if shared.opts.no_half:
log.info('Torch override dtype: no-half set')
dtype = torch.float32
dtype_vae = torch.float32
dtype_unet = torch.float32
- if shared.opts.no_half_vae: # set dtype again as no-half-vae options take priority
+ if shared.opts.no_half_vae:
log.info('Torch override VAE dtype: no-half set')
dtype_vae = torch.float32
unet_needs_upcast = shared.opts.upcast_sampling
@@ -336,8 +361,7 @@ def set_cuda_params():
if shared.cmd_opts.profile:
shared.log.debug(f'Torch info: {torch.__config__.show()}')
device_name = get_raw_openvino_device() if shared.cmd_opts.use_openvino else torch.device(get_optimal_device_name()) # pylint: disable=used-before-assignment
- log.debug(f'Desired Torch parameters: dtype={shared.opts.cuda_dtype} no-half={shared.opts.no_half} no-half-vae={shared.opts.no_half_vae} upscast={shared.opts.upcast_sampling}')
- log.info(f'Setting Torch parameters: device={device_name} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} fp16={fp16_ok} bf16={bf16_ok} optimization={shared.opts.cross_attention_optimization}')
+ log.info(f'Torch parameters: device={device_name} config={shared.opts.cuda_dtype} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} nohalf={shared.opts.no_half} nohalfvae={shared.opts.no_half_vae} upscast={shared.opts.upcast_sampling} deterministic={shared.opts.cudnn_deterministic} test-fp16={fp16_ok} test-bf16={bf16_ok} optimization="{shared.opts.cross_attention_optimization}"')
args = cmd_args.parser.parse_args()
@@ -377,9 +401,9 @@ inference_context = torch.no_grad
cuda_ok = torch.cuda.is_available()
cpu = torch.device("cpu")
device = device_interrogate = device_gfpgan = device_esrgan = device_codeformer = None
-dtype = torch.float16
-dtype_vae = torch.float16
-dtype_unet = torch.float16
+dtype = None
+dtype_vae = None
+dtype_unet = None
fp16_ok = None
bf16_ok = None
unet_needs_upcast = False
diff --git a/modules/paths.py b/modules/paths.py
index 28c413507..52bec2d0f 100644
--- a/modules/paths.py
+++ b/modules/paths.py
@@ -28,7 +28,6 @@ models_config = cli.models_dir or config.get('models_dir') or 'models'
models_path = models_config if os.path.isabs(models_config) else os.path.join(data_path, models_config)
extensions_dir = os.path.join(data_path, "extensions")
extensions_builtin_dir = "extensions-builtin"
-onnx_dir = os.path.join(models_path, "ONNX")
sd_configs_path = os.path.join(script_path, "configs")
sd_default_config = os.path.join(sd_configs_path, "v1-inference.yaml")
sd_model_file = cli.ckpt or os.path.join(script_path, 'model.ckpt') # not used
@@ -101,7 +100,6 @@ def create_paths(opts):
create_path(sd_configs_path)
create_path(extensions_dir)
create_path(extensions_builtin_dir)
- create_path(onnx_dir)
create_path(fix_path('temp_dir'))
create_path(fix_path('ckpt_dir'))
create_path(fix_path('diffusers_dir'))
@@ -111,6 +109,7 @@ def create_paths(opts):
create_path(fix_path('lora_dir'))
create_path(fix_path('embeddings_dir'))
create_path(fix_path('hypernetwork_dir'))
+ create_path(fix_path('onnx_temp_dir'))
create_path(fix_path('outdir_samples'))
create_path(fix_path('outdir_txt2img_samples'))
create_path(fix_path('outdir_img2img_samples'))
diff --git a/modules/shared.py b/modules/shared.py
index 6b0bc0d5f..376022f8e 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -432,7 +432,7 @@ options_templates.update(options_section(('sd', "Execution & Models"), {
options_templates.update(options_section(('cuda', "Compute Settings"), {
"math_sep": OptionInfo("Execution precision
", "", gr.HTML),
"precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"]}),
- "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, {"choices": ["FP32", "FP16", "BF16"]}),
+ "cuda_dtype": OptionInfo("Auto", "Device precision type", gr.Radio, {"choices": ["Auto", "FP32", "FP16", "BF16"]}),
"cudnn_deterministic": OptionInfo(False, "Use deterministic mode"),
"model_sep": OptionInfo("Model options
", "", gr.HTML),