Merge pull request #2128 from vladmandic/master

refresh dev
2026-09-19 09:14:35 +02:00 · 2023-09-03 15:59:45 -04:00
parent 13c2638bcd 1d04014df1
commit 8258313555
5 changed files with 107 additions and 78 deletions
@@ -29,7 +29,7 @@ All Individual features are not listed here, instead check [ChangeLog](CHANGELOG
 - Support for multiple diffusion models!  
  Stable Diffusion, SD-XL, Kandinsky, DeepFloyd IF, etc.  
 - Fully multiplatform with platform specific autodetection and tuning performed on install  
-  Windows / Linux / MacOS with CPU / nVidia / AMD / IntelArc / DirectML
+  Windows / Linux / MacOS with CPU / nVidia / AMD / Intel / DirectML
 - Improved prompt parser  
 - Enhanced *Lora*/*Locon*/*Lyco* code supporting latest trends in training  
 - Built-in queue management  
@@ -8,86 +8,97 @@ from torch._dynamo.backends.registry import register_backend
 from torch.fx.experimental.proxy_tensor import make_fx
 from torch._inductor.compile_fx import compile_fx
 from hashlib import sha256
+from modules import shared

@register_backend
@fake_tensor_unsupported
 def openvino_fx(subgraph, example_inputs):
-    try:
-        executor_parameters = None
-        core = Core()
-        if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0":
-            os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1")
-            model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest()
-            executor_parameters = {"model_hash_str": model_hash_str}
+    executor_parameters = None
+    core = Core()
+    if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0":
+        os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1")
+        model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest()
+        executor_parameters = {"model_hash_str": model_hash_str}

-        example_inputs.reverse()
-        cache_root = "./cache/"
-        if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None:
-            cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR")
+    example_inputs.reverse()
+    cache_root = "./cache/"
+    if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None:
+        cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR")

+    if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None:
+        device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE")
+    elif any(openvino_cpu in cpu_module.lower() for cpu_module in shared.cmd_opts.use_cpu for openvino_cpu in ["openvino", "all"]):
+        device = "CPU"
+    elif shared.cmd_opts.device_id is not None:
+        device = f"GPU.{shared.cmd_opts.device_id}"
+    elif "GPU" in core.available_devices:
        device = "GPU"
+    elif "GPU.1" in core.available_devices:
+        device = "GPU.1"
+    elif "GPU.0" in core.available_devices:
+        device = "GPU.0"
+    else:
+        device = "CPU"
+        shared.log.warning("OpenVINO: No compatible GPU detected!")
+    os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device)
+    shared.log.debug(f"OpenVINO Device: {device}")
+    if shared.opts.cuda_compile_errors and device not in core.available_devices:
+        shared.log.warning(f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices")
+    assert device in core.available_devices, f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices"

-        if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None:
-            device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE")
-            assert device in core.available_devices, "Specified device " + device + " is not in the list of OpenVINO Available Devices"
-        else:
-            os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device)
+    #Cache saving keeps increasing the partition id
+    #This loop check if non 0 partition id caches exist
+    #Takes 0.002 seconds when nothing is found
+    use_cached_file = False
+    for i in range(100):
+        file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root)
+        if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"):
+            use_cached_file = True
+            break

-        #Cache saving keeps increasing the partition id
-        #This loop check if non 0 partition id caches exist
-        #Takes 0.002 seconds when nothing is found
-        use_cached_file = False
-        for i in range(100):
-            file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root)
-            if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"):
-                use_cached_file = True
-                break
+    if use_cached_file:
+        om = core.read_model(file_name + ".xml")

-        if use_cached_file:
-            om = core.read_model(file_name + ".xml")
+        dtype_mapping = {
+            torch.float32: Type.f32,
+            torch.float64: Type.f64,
+            torch.float16: Type.f16,
+            torch.int64: Type.i64,
+            torch.int32: Type.i32,
+            torch.uint8: Type.u8,
+            torch.int8: Type.i8,
+            torch.bool: Type.boolean
+        }

-            dtype_mapping = {
-                torch.float32: Type.f32,
-                torch.float64: Type.f64,
-                torch.float16: Type.f16,
-                torch.int64: Type.i64,
-                torch.int32: Type.i32,
-                torch.uint8: Type.u8,
-                torch.int8: Type.i8,
-                torch.bool: Type.boolean
-            }
+        for idx, input_data in enumerate(example_inputs):
+            om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
+            om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
+        om.validate_nodes_and_infer_types()

-            for idx, input_data in enumerate(example_inputs):
-                om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
-                om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
-            om.validate_nodes_and_infer_types()
+        if model_hash_str is not None:
+            core.set_property({'CACHE_DIR': cache_root + '/blob'})

-            if model_hash_str is not None:
-                core.set_property({'CACHE_DIR': cache_root + '/blob'})
+        compiled_model = core.compile_model(om, device)
+        def _call(*args):
+            ov_inputs = [a.detach().cpu().numpy() for a in args]
+            ov_inputs.reverse()
+            res = compiled_model(ov_inputs)
+            result = [torch.from_numpy(res[out]) for out in compiled_model.outputs]
+            return result
+        return _call
+    else:
+        example_inputs.reverse()
+        model = make_fx(subgraph)(*example_inputs)
+        with torch.no_grad():
+            model.eval()
+        partitioner = Partitioner()
+        compiled_model = partitioner.make_partitions(model)

-            compiled_model = core.compile_model(om, device)
-            def _call(*args):
-                ov_inputs = [a.detach().cpu().numpy() for a in args]
-                ov_inputs.reverse()
-                res = compiled_model(ov_inputs)
-                result = [torch.from_numpy(res[out]) for out in compiled_model.outputs]
-                return result
-            return _call
-        else:
-            example_inputs.reverse()
-            model = make_fx(subgraph)(*example_inputs)
-            with torch.no_grad():
-                model.eval()
-            partitioner = Partitioner()
-            compiled_model = partitioner.make_partitions(model)
-
-            def _call(*args):
-                res = execute(compiled_model, *args, executor="openvino",
-                              executor_parameters=executor_parameters)
-                return res
-            return _call
-    except Exception:
-        return compile_fx(subgraph, example_inputs)
+        def _call(*args):
+            res = execute(compiled_model, *args, executor="openvino",
+                            executor_parameters=executor_parameters)
+            return res
+        return _call


 def get_cached_file_name(*args, model_hash_str, device, cache_root):
@@ -268,9 +268,19 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
    if shared.opts.diffusers_move_base and not shared.sd_model.has_accelerate:
        shared.sd_model.to(devices.device)

-    is_img2img = (sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING)
-    use_refiner_start = (is_refiner_enabled and not p.is_hr_pass and not is_img2img and p.refiner_start > 0 and p.refiner_start < 1)
-    use_denoise_start = (is_img2img and p.refiner_start > 0 and p.refiner_start < 1)
+    is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING)
+    use_refiner_start = bool(is_refiner_enabled and not p.is_hr_pass and not is_img2img and p.refiner_start > 0 and p.refiner_start < 1)
+    use_denoise_start = bool(is_img2img and p.refiner_start > 0 and p.refiner_start < 1)
+
+    def calculate_base_steps():
+        if use_refiner_start:
+            return int(p.steps // p.refiner_start + 1) if shared.sd_model_type == 'sdxl' else p.steps
+        elif use_denoise_start and shared.sd_model_type == 'sdxl':
+            return int(p.steps // (1 - p.refiner_start))
+        elif is_img2img:
+            return int(p.steps // p.denoising_strength + 1)
+        else:
+            return p.steps

    base_args = set_pipeline_args(
        model=shared.sd_model,
@@ -278,7 +288,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
        negative_prompts=negative_prompts,
        prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts,
        negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts,
-        num_inference_steps=int(p.steps // p.refiner_start + 1) if use_refiner_start else int(p.steps // (1 - p.refiner_start)) if use_denoise_start else int(p.steps // p.denoising_strength + 1) if is_img2img else p.steps,
+        num_inference_steps=calculate_base_steps(),
        eta=shared.opts.eta_ddim,
        guidance_rescale=p.diffusers_guidance_rescale,
        denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
@@ -355,13 +365,14 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro

        if shared.opts.diffusers_move_refiner and not shared.sd_refiner.has_accelerate:
            shared.sd_refiner.to(devices.device)
+        refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__)
        p.ops.append('refine')
        for i in range(len(output.images)):
            refiner_args = set_pipeline_args(
                model=shared.sd_refiner,
                prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i],
                negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i],
-                num_inference_steps=int(p.refiner_steps // (1 - p.refiner_start)) if p.refiner_start > 0 and p.refiner_start < 1 else int(p.refiner_steps // p.denoising_strength + 1),
+                num_inference_steps=int(p.refiner_steps // (1 - p.refiner_start)) if p.refiner_start > 0 and p.refiner_start < 1 and refiner_is_sdxl else int(p.refiner_steps // p.denoising_strength + 1) if refiner_is_sdxl else p.refiner_steps,
                eta=shared.opts.eta_ddim,
                strength=p.denoising_strength,
                guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale,
@@ -189,10 +189,10 @@ class StableDiffusionModelHijack:
                shared.log.info(f"Compiling pipeline={m.model.__class__.__name__} mode={opts.cuda_compile_backend}")
                import torch._dynamo # pylint: disable=unused-import,redefined-outer-name
                if shared.opts.cuda_compile_backend == "openvino_fx":
-                    torch._dynamo.reset()
-                    from modules.intel.openvino import openvino_fx, openvino_clear_caches, model_state # pylint: disable=unused-import
+                    torch._dynamo.reset() # pylint: disable=protected-access
+                    from modules.intel.openvino import openvino_fx, openvino_clear_caches # pylint: disable=unused-import
                    openvino_clear_caches()
-                    model_state.partition_id = 0
+                    torch._dynamo.eval_frame.check_if_dynamo_supported = lambda: True # pylint: disable=protected-access
                log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access
                if hasattr(torch, '_logging'):
                    torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access
@@ -822,9 +822,13 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
            try:
                if shared.opts.ipex_optimize:
                    sd_model.unet.training = False
-                    sd_model.vae.training = False
                    sd_model.unet = torch.xpu.optimize(sd_model.unet, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
-                    sd_model.vae = torch.xpu.optimize(sd_model.vae, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
+                    if hasattr(sd_model, 'vae'):
+                        sd_model.vae.training = False
+                        sd_model.vae = torch.xpu.optimize(sd_model.vae, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
+                    if hasattr(sd_model, 'movq'):
+                        sd_model.movq.training = False
+                        sd_model.movq = torch.xpu.optimize(sd_model.movq, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
                    shared.log.info("Applied IPEX Optimize.")
            except Exception as err:
                shared.log.warning(f"IPEX Optimize not supported: {err}")
@@ -845,7 +849,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                    torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access
                    torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access
                    sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init
-                    sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init
+                    if hasattr(sd_model, 'vae'):
+                        sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init
+                    if hasattr(sd_model, 'movq'):
+                        sd_model.movq.decode = torch.compile(sd_model.movq.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init
                    if shared.opts.cuda_compile_precompile:
                        sd_model("dummy prompt")
                    shared.log.info("Complilation done.")