diff --git a/CHANGELOG.md b/CHANGELOG.md index ab2fafc0b..21e941fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ - Fix API generate save metadata - Fix Face/InstantID - Fix CivitAI update model info for all models + - Fix FP16/BF16 test on model load - Enumerate diffusers model with multiple variants ## Update for 2024-03-19 diff --git a/modules/devices.py b/modules/devices.py index 7fe12e8ea..dd9bddca5 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -184,11 +184,15 @@ def set_cuda_sync_mode(mode): def test_fp16(): if shared.cmd_opts.experimental: + if debug: + log.debug('Torch FP16 test skip') return True try: x = torch.tensor([[1.5,.0,.0,.0]]).to(device=device, dtype=torch.float16) layerNorm = torch.nn.LayerNorm(4, eps=0.00001, elementwise_affine=True, dtype=torch.float16, device=device) _y = layerNorm(x) + if debug: + log.debug('Torch FP16 test pass') return True except Exception as ex: log.warning(f'Torch FP16 test failed: Forcing FP32 operations: {ex}') @@ -197,13 +201,18 @@ def test_fp16(): shared.opts.no_half_vae = True return False + def test_bf16(): if shared.cmd_opts.experimental: + if debug: + log.debug('Torch BF16 test skip') return True try: import torch.nn.functional as F image = torch.randn(1, 4, 32, 32).to(device=device, dtype=torch.bfloat16) _out = F.interpolate(image, size=(64, 64), mode="nearest") + if debug: + log.debug('Torch BF16 test pass') return True except Exception: log.warning('Torch BF16 test failed: Fallback to FP16 operations') @@ -211,7 +220,8 @@ def test_bf16(): def set_cuda_params(): - # log.debug('Verifying Torch settings') + if debug: + log.debug(f'Verifying Torch settings: cuda={cuda_ok}') if cuda_ok: try: torch.backends.cuda.matmul.allow_tf32 = True @@ -257,7 +267,7 @@ def set_cuda_params(): pass if shared.cmd_opts.profile: shared.log.debug(f'Torch info: {torch.__config__.show()}') - global dtype, dtype_vae, dtype_unet, unet_needs_upcast, inference_context # pylint: disable=global-statement + global dtype, dtype_vae, dtype_unet, unet_needs_upcast, inference_context, fp16_ok, bf16_ok # pylint: disable=global-statement if shared.opts.cuda_dtype == 'FP32': dtype = torch.float32 dtype_vae = torch.float32 @@ -265,13 +275,13 @@ def set_cuda_params(): fp16_ok = None bf16_ok = None elif shared.opts.cuda_dtype == 'BF16' or dtype == torch.bfloat16: - fp16_ok = test_fp16() - bf16_ok = test_bf16() + fp16_ok = test_fp16() if fp16_ok is None else fp16_ok + bf16_ok = test_bf16() if bf16_ok is None else bf16_ok dtype = torch.bfloat16 if bf16_ok else torch.float16 dtype_vae = torch.bfloat16 if bf16_ok else torch.float16 dtype_unet = torch.bfloat16 if bf16_ok else torch.float16 elif shared.opts.cuda_dtype == 'FP16' or dtype == torch.float16: - fp16_ok = test_fp16() + fp16_ok = test_fp16() if fp16_ok is None else fp16_ok bf16_ok = None dtype = torch.float16 if fp16_ok else torch.float32 dtype_vae = torch.float16 if fp16_ok else torch.float32 @@ -328,6 +338,7 @@ elif sys.platform == 'darwin': else: backend = 'cpu' + inference_context = torch.no_grad cuda_ok = torch.cuda.is_available() cpu = torch.device("cpu") @@ -335,6 +346,8 @@ device = device_interrogate = device_gfpgan = device_esrgan = device_codeformer dtype = torch.float16 dtype_vae = torch.float16 dtype_unet = torch.float16 +fp16_ok = None +bf16_ok = None unet_needs_upcast = False onnx = None if args.profile: diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index 16318e1ae..ecc603d7b 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -210,7 +210,7 @@ def parse_generation_parameters(infotext, no_prompt=False): else: try: first_param, first_param_idx = next((s, i) for i, s in enumerate(params) if any(x in s.lower() for x in basic_params)) - except Exception as e: + except Exception: first_param, first_param_idx = next(iter(params)), 0 if first_param_idx > 0: for _i in range(first_param_idx):