diff --git a/README.md b/README.md index 3b4ac2972..6ce222bac 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ All Individual features are not listed here, instead check [ChangeLog](CHANGELOG - Support for multiple diffusion models! Stable Diffusion, SD-XL, Kandinsky, DeepFloyd IF, etc. - Fully multiplatform with platform specific autodetection and tuning performed on install - Windows / Linux / MacOS with CPU / nVidia / AMD / IntelArc / DirectML + Windows / Linux / MacOS with CPU / nVidia / AMD / Intel / DirectML - Improved prompt parser - Enhanced *Lora*/*Locon*/*Lyco* code supporting latest trends in training - Built-in queue management diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 0dea0706d..b39eb3cdb 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -8,86 +8,97 @@ from torch._dynamo.backends.registry import register_backend from torch.fx.experimental.proxy_tensor import make_fx from torch._inductor.compile_fx import compile_fx from hashlib import sha256 +from modules import shared @register_backend @fake_tensor_unsupported def openvino_fx(subgraph, example_inputs): - try: - executor_parameters = None - core = Core() - if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0": - os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1") - model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest() - executor_parameters = {"model_hash_str": model_hash_str} + executor_parameters = None + core = Core() + if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0": + os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1") + model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest() + executor_parameters = {"model_hash_str": model_hash_str} - example_inputs.reverse() - cache_root = "./cache/" - if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None: - cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR") + example_inputs.reverse() + cache_root = "./cache/" + if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None: + cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR") + if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None: + device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") + elif any(openvino_cpu in cpu_module.lower() for cpu_module in shared.cmd_opts.use_cpu for openvino_cpu in ["openvino", "all"]): + device = "CPU" + elif shared.cmd_opts.device_id is not None: + device = f"GPU.{shared.cmd_opts.device_id}" + elif "GPU" in core.available_devices: device = "GPU" + elif "GPU.1" in core.available_devices: + device = "GPU.1" + elif "GPU.0" in core.available_devices: + device = "GPU.0" + else: + device = "CPU" + shared.log.warning("OpenVINO: No compatible GPU detected!") + os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device) + shared.log.debug(f"OpenVINO Device: {device}") + if shared.opts.cuda_compile_errors and device not in core.available_devices: + shared.log.warning(f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices") + assert device in core.available_devices, f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices" - if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None: - device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") - assert device in core.available_devices, "Specified device " + device + " is not in the list of OpenVINO Available Devices" - else: - os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device) + #Cache saving keeps increasing the partition id + #This loop check if non 0 partition id caches exist + #Takes 0.002 seconds when nothing is found + use_cached_file = False + for i in range(100): + file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root) + if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"): + use_cached_file = True + break - #Cache saving keeps increasing the partition id - #This loop check if non 0 partition id caches exist - #Takes 0.002 seconds when nothing is found - use_cached_file = False - for i in range(100): - file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root) - if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"): - use_cached_file = True - break + if use_cached_file: + om = core.read_model(file_name + ".xml") - if use_cached_file: - om = core.read_model(file_name + ".xml") + dtype_mapping = { + torch.float32: Type.f32, + torch.float64: Type.f64, + torch.float16: Type.f16, + torch.int64: Type.i64, + torch.int32: Type.i32, + torch.uint8: Type.u8, + torch.int8: Type.i8, + torch.bool: Type.boolean + } - dtype_mapping = { - torch.float32: Type.f32, - torch.float64: Type.f64, - torch.float16: Type.f16, - torch.int64: Type.i64, - torch.int32: Type.i32, - torch.uint8: Type.u8, - torch.int8: Type.i8, - torch.bool: Type.boolean - } + for idx, input_data in enumerate(example_inputs): + om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) + om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) + om.validate_nodes_and_infer_types() - for idx, input_data in enumerate(example_inputs): - om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) - om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) - om.validate_nodes_and_infer_types() + if model_hash_str is not None: + core.set_property({'CACHE_DIR': cache_root + '/blob'}) - if model_hash_str is not None: - core.set_property({'CACHE_DIR': cache_root + '/blob'}) + compiled_model = core.compile_model(om, device) + def _call(*args): + ov_inputs = [a.detach().cpu().numpy() for a in args] + ov_inputs.reverse() + res = compiled_model(ov_inputs) + result = [torch.from_numpy(res[out]) for out in compiled_model.outputs] + return result + return _call + else: + example_inputs.reverse() + model = make_fx(subgraph)(*example_inputs) + with torch.no_grad(): + model.eval() + partitioner = Partitioner() + compiled_model = partitioner.make_partitions(model) - compiled_model = core.compile_model(om, device) - def _call(*args): - ov_inputs = [a.detach().cpu().numpy() for a in args] - ov_inputs.reverse() - res = compiled_model(ov_inputs) - result = [torch.from_numpy(res[out]) for out in compiled_model.outputs] - return result - return _call - else: - example_inputs.reverse() - model = make_fx(subgraph)(*example_inputs) - with torch.no_grad(): - model.eval() - partitioner = Partitioner() - compiled_model = partitioner.make_partitions(model) - - def _call(*args): - res = execute(compiled_model, *args, executor="openvino", - executor_parameters=executor_parameters) - return res - return _call - except Exception: - return compile_fx(subgraph, example_inputs) + def _call(*args): + res = execute(compiled_model, *args, executor="openvino", + executor_parameters=executor_parameters) + return res + return _call def get_cached_file_name(*args, model_hash_str, device, cache_root): diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index a2f3e00df..36f8bcc9c 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -268,9 +268,19 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.opts.diffusers_move_base and not shared.sd_model.has_accelerate: shared.sd_model.to(devices.device) - is_img2img = (sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING) - use_refiner_start = (is_refiner_enabled and not p.is_hr_pass and not is_img2img and p.refiner_start > 0 and p.refiner_start < 1) - use_denoise_start = (is_img2img and p.refiner_start > 0 and p.refiner_start < 1) + is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING) + use_refiner_start = bool(is_refiner_enabled and not p.is_hr_pass and not is_img2img and p.refiner_start > 0 and p.refiner_start < 1) + use_denoise_start = bool(is_img2img and p.refiner_start > 0 and p.refiner_start < 1) + + def calculate_base_steps(): + if use_refiner_start: + return int(p.steps // p.refiner_start + 1) if shared.sd_model_type == 'sdxl' else p.steps + elif use_denoise_start and shared.sd_model_type == 'sdxl': + return int(p.steps // (1 - p.refiner_start)) + elif is_img2img: + return int(p.steps // p.denoising_strength + 1) + else: + return p.steps base_args = set_pipeline_args( model=shared.sd_model, @@ -278,7 +288,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_prompts=negative_prompts, prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, - num_inference_steps=int(p.steps // p.refiner_start + 1) if use_refiner_start else int(p.steps // (1 - p.refiner_start)) if use_denoise_start else int(p.steps // p.denoising_strength + 1) if is_img2img else p.steps, + num_inference_steps=calculate_base_steps(), eta=shared.opts.eta_ddim, guidance_rescale=p.diffusers_guidance_rescale, denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None, @@ -355,13 +365,14 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.opts.diffusers_move_refiner and not shared.sd_refiner.has_accelerate: shared.sd_refiner.to(devices.device) + refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__) p.ops.append('refine') for i in range(len(output.images)): refiner_args = set_pipeline_args( model=shared.sd_refiner, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i], negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i], - num_inference_steps=int(p.refiner_steps // (1 - p.refiner_start)) if p.refiner_start > 0 and p.refiner_start < 1 else int(p.refiner_steps // p.denoising_strength + 1), + num_inference_steps=int(p.refiner_steps // (1 - p.refiner_start)) if p.refiner_start > 0 and p.refiner_start < 1 and refiner_is_sdxl else int(p.refiner_steps // p.denoising_strength + 1) if refiner_is_sdxl else p.refiner_steps, eta=shared.opts.eta_ddim, strength=p.denoising_strength, guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 93420b397..5a52e30df 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -189,10 +189,10 @@ class StableDiffusionModelHijack: shared.log.info(f"Compiling pipeline={m.model.__class__.__name__} mode={opts.cuda_compile_backend}") import torch._dynamo # pylint: disable=unused-import,redefined-outer-name if shared.opts.cuda_compile_backend == "openvino_fx": - torch._dynamo.reset() - from modules.intel.openvino import openvino_fx, openvino_clear_caches, model_state # pylint: disable=unused-import + torch._dynamo.reset() # pylint: disable=protected-access + from modules.intel.openvino import openvino_fx, openvino_clear_caches # pylint: disable=unused-import openvino_clear_caches() - model_state.partition_id = 0 + torch._dynamo.eval_frame.check_if_dynamo_supported = lambda: True # pylint: disable=protected-access log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access diff --git a/modules/sd_models.py b/modules/sd_models.py index 743876837..7c0d79413 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -822,9 +822,13 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No try: if shared.opts.ipex_optimize: sd_model.unet.training = False - sd_model.vae.training = False sd_model.unet = torch.xpu.optimize(sd_model.unet, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init - sd_model.vae = torch.xpu.optimize(sd_model.vae, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'vae'): + sd_model.vae.training = False + sd_model.vae = torch.xpu.optimize(sd_model.vae, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'movq'): + sd_model.movq.training = False + sd_model.movq = torch.xpu.optimize(sd_model.movq, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init shared.log.info("Applied IPEX Optimize.") except Exception as err: shared.log.warning(f"IPEX Optimize not supported: {err}") @@ -845,7 +849,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init - sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'vae'): + sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'movq'): + sd_model.movq.decode = torch.compile(sd_model.movq.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init if shared.opts.cuda_compile_precompile: sd_model("dummy prompt") shared.log.info("Complilation done.")