From 863fa38c2413d2d33b59232b9ac018f43bb61a1c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 23 Aug 2023 00:31:43 +0300 Subject: [PATCH] OpenVINO fix model reloading --- modules/intel/openvino/__init__.py | 11 +++++++++-- modules/sd_hijack.py | 5 ++++- modules/sd_models.py | 5 ++++- modules/shared.py | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 8113fe02c..1b9933c1b 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -1,6 +1,6 @@ import os import torch -from openvino.frontend.pytorch.torchdynamo.execute import execute +from openvino.frontend.pytorch.torchdynamo.execute import execute, partitioned_modules, compiled_cache from openvino.frontend.pytorch.torchdynamo.partition import Partitioner from openvino.runtime import Core, Type, PartialShape from torch._dynamo.backends.common import fake_tensor_unsupported @@ -12,7 +12,7 @@ from hashlib import sha256 class ModelState: def __init__(self): self.recompile = 1 - self.device = "CPU" + self.device = "GPU" self.height = 512 self.width = 512 self.batch_size = 1 @@ -112,3 +112,10 @@ def get_cached_file_name(*args, model_hash_str, device, cache_root): file_name = None model_hash_str = None return file_name + +def openvino_clear_caches(): + global partitioned_modules + global compiled_cache + + compiled_cache.clear() + partitioned_modules.clear() diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 04af28a32..93420b397 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -189,7 +189,10 @@ class StableDiffusionModelHijack: shared.log.info(f"Compiling pipeline={m.model.__class__.__name__} mode={opts.cuda_compile_backend}") import torch._dynamo # pylint: disable=unused-import,redefined-outer-name if shared.opts.cuda_compile_backend == "openvino_fx": - from modules.intel.openvino import openvino_fx + torch._dynamo.reset() + from modules.intel.openvino import openvino_fx, openvino_clear_caches, model_state # pylint: disable=unused-import + openvino_clear_caches() + model_state.partition_id = 0 log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access diff --git a/modules/sd_models.py b/modules/sd_models.py index f60cbf897..e7adea310 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -803,7 +803,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.info(f"Compiling pipeline={sd_model.__class__.__name__} shape={8 * sd_model.unet.config.sample_size} mode={shared.opts.cuda_compile_backend}") import torch._dynamo # pylint: disable=unused-import,redefined-outer-name if shared.opts.cuda_compile_backend == "openvino_fx": - from modules.intel.openvino import openvino_fx # pylint: disable=unused-import + torch._dynamo.reset() + from modules.intel.openvino import openvino_fx, openvino_clear_caches, model_state # pylint: disable=unused-import + openvino_clear_caches() + model_state.partition_id = 0 log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index 0906f19d7..b4320e7af 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -418,7 +418,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), - "diffusers_lora_loader": OptionInfo("sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['sequential apply', 'merge and apply', 'diffusers default']}), + "diffusers_lora_loader": OptionInfo("diffusers default" if cmd_opts.use_openvino else "sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['sequential apply', 'merge and apply', 'diffusers default']}), # "diffusers_force_zeros": OptionInfo(False, "Force zeros for prompts when empty"), # "diffusers_aesthetics_score": OptionInfo(6.0, "Require aesthetic score", gr.Slider, {"minimum": 0, "maximum": 10, "step": 0.1}), }))