mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
Unify settings
This commit is contained in:
Disty0
@@ -70,19 +70,22 @@ class OpenVINOGraphModule(torch.nn.Module):
|
||||
result = openvino_execute(self.gm, *args, executor_parameters=self.executor_parameters, partition_id=self.partition_id, file_name=self.file_name)
|
||||
return result
|
||||
|
||||
def get_device():
|
||||
def get_device_list():
|
||||
core = Core()
|
||||
if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None:
|
||||
device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE")
|
||||
elif shared.opts.openvino_hetero_gpu:
|
||||
return core.available_devices
|
||||
|
||||
def get_device():
|
||||
if hasattr(shared, "opts") and len(shared.opts.openvino_devices) == 1:
|
||||
return shared.opts.openvino_devices[0]
|
||||
|
||||
core = Core()
|
||||
if hasattr(shared, "opts") and len(shared.opts.openvino_devices) > 1:
|
||||
device = ""
|
||||
available_devices = core.available_devices
|
||||
available_devices = shared.opts.openvino_devices.copy()
|
||||
available_devices.remove("CPU")
|
||||
if shared.opts.openvino_remove_igpu_from_hetero and "GPU.0" in available_devices:
|
||||
available_devices.remove("GPU.0")
|
||||
for gpu in available_devices:
|
||||
device = f"{device},{gpu}"
|
||||
if not shared.opts.openvino_remove_cpu_from_hetero:
|
||||
for hetero_device in available_devices:
|
||||
device = f"{device},{hetero_device}"
|
||||
if "CPU" in shared.opts.openvino_devices:
|
||||
device = f"{device},CPU"
|
||||
device = f"HETERO:{device[1:]}"
|
||||
elif any(openvino_cpu in cpu_module.lower() for cpu_module in shared.cmd_opts.use_cpu for openvino_cpu in ["openvino", "all"]):
|
||||
@@ -98,7 +101,6 @@ def get_device():
|
||||
else:
|
||||
device = core.available_devices[-1]
|
||||
shared.log.warning(f"OpenVINO: No compatible GPU detected! Using {device}")
|
||||
os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device)
|
||||
return device
|
||||
|
||||
def get_openvino_device():
|
||||
@@ -267,7 +269,7 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs):
|
||||
om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
|
||||
om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
|
||||
om.validate_nodes_and_infer_types()
|
||||
if shared.opts.nncf_compress_weights and not (dont_use_4bit_nncf and not shared.opts.nncf_compress_vae_weights):
|
||||
if shared.opts.nncf_compress_weights and not dont_use_nncf:
|
||||
if dont_use_4bit_nncf or shared.opts.nncf_compress_weights_mode == "INT8":
|
||||
om = nncf.compress_weights(om)
|
||||
else:
|
||||
@@ -383,7 +385,7 @@ def openvino_fx(subgraph, example_inputs):
|
||||
subgraph_type[3] is torch.nn.modules.activation.SiLU):
|
||||
|
||||
dont_use_4bit_nncf = True
|
||||
dont_use_nncf = not shared.opts.nncf_compress_vae_weights
|
||||
dont_use_nncf = bool("VAE" not in shared.opts.nncf_compress_weights)
|
||||
|
||||
# SD 1.5 / SDXL Text Encoder
|
||||
elif (subgraph_type[0] is torch.nn.modules.sparse.Embedding and
|
||||
@@ -392,7 +394,7 @@ def openvino_fx(subgraph, example_inputs):
|
||||
subgraph_type[3] is torch.nn.modules.linear.Linear):
|
||||
|
||||
dont_use_faketensors = True
|
||||
dont_use_nncf = not shared.opts.nncf_compress_text_encoder_weights
|
||||
dont_use_nncf = bool("Text Encoder" not in shared.opts.nncf_compress_weights)
|
||||
|
||||
if not shared.opts.openvino_disable_model_caching:
|
||||
os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1")
|
||||
|
||||
@@ -175,7 +175,7 @@ class StableDiffusionModelHijack:
|
||||
if m.cond_stage_key == "edit":
|
||||
sd_hijack_unet.hijack_ddpm_edit()
|
||||
|
||||
if shared.opts.ipex_optimize and shared.backend == shared.Backend.ORIGINAL:
|
||||
if "Model" in shared.opts.ipex_optimize and shared.backend == shared.Backend.ORIGINAL:
|
||||
try:
|
||||
import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
|
||||
m.model.training = False
|
||||
@@ -184,7 +184,7 @@ class StableDiffusionModelHijack:
|
||||
except Exception as err:
|
||||
shared.log.warning(f"IPEX Optimize not supported: {err}")
|
||||
|
||||
if (shared.opts.cuda_compile or shared.opts.cuda_compile_vae or shared.opts.cuda_compile_upscaler) and shared.opts.cuda_compile_backend != 'none' and shared.backend == shared.Backend.ORIGINAL:
|
||||
if "Model" in shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none' and shared.backend == shared.Backend.ORIGINAL:
|
||||
try:
|
||||
import logging
|
||||
shared.log.info(f"Compiling pipeline={m.model.__class__.__name__} mode={shared.opts.cuda_compile_backend}")
|
||||
|
||||
@@ -27,12 +27,16 @@ def ipex_optimize(sd_model):
|
||||
try:
|
||||
t0 = time.time()
|
||||
import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
|
||||
if hasattr(sd_model, 'unet'):
|
||||
sd_model.unet.training = False
|
||||
sd_model.unet = ipex.optimize(sd_model.unet, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
|
||||
else:
|
||||
shared.log.warning('IPEX Optimize enabled but model has no Unet')
|
||||
if shared.opts.ipex_optimize_vae:
|
||||
if "Model" in shared.opts.ipex_optimize:
|
||||
if hasattr(sd_model, 'unet'):
|
||||
sd_model.unet.training = False
|
||||
sd_model.unet = ipex.optimize(sd_model.unet, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
|
||||
elif hasattr(sd_model, 'transformer'):
|
||||
sd_model.transformer.training = False
|
||||
sd_model.transformer = ipex.optimize(sd_model.transformer, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
|
||||
else:
|
||||
shared.log.warning('IPEX Optimize enabled but model has no Unet or Transformer')
|
||||
if "VAE" in shared.opts.ipex_optimize:
|
||||
if hasattr(sd_model, 'vae'):
|
||||
sd_model.vae.training = False
|
||||
sd_model.vae = ipex.optimize(sd_model.vae, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
|
||||
@@ -41,7 +45,7 @@ def ipex_optimize(sd_model):
|
||||
sd_model.movq = ipex.optimize(sd_model.movq, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
|
||||
else:
|
||||
shared.log.warning('Compress VAE Weights enabled but model has no VAE')
|
||||
if shared.opts.ipex_optimize_text_encoder:
|
||||
if "Text Encoder" in shared.opts.ipex_optimize:
|
||||
if hasattr(sd_model, 'text_encoder'):
|
||||
sd_model.text_encoder.training = False
|
||||
sd_model.text_encoder = ipex.optimize(sd_model.text_encoder, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init
|
||||
@@ -63,18 +67,21 @@ def nncf_compress_weights(sd_model):
|
||||
shared.compiled_model_state = CompiledModelState()
|
||||
shared.compiled_model_state.is_compiled = True
|
||||
|
||||
if hasattr(sd_model, 'unet'):
|
||||
sd_model.unet = nncf.compress_weights(sd_model.unet)
|
||||
else:
|
||||
shared.log.warning('Compress Weights enabled but model has no Unet')
|
||||
if shared.opts.nncf_compress_vae_weights:
|
||||
if "Model" in shared.opts.nncf_compress_weights:
|
||||
if hasattr(sd_model, 'unet'):
|
||||
sd_model.unet = nncf.compress_weights(sd_model.unet)
|
||||
elif hasattr(sd_model, 'transformer'):
|
||||
sd_model.transformer = nncf.compress_weights(sd_model.transformer)
|
||||
else:
|
||||
shared.log.warning('Compress Weights enabled but model has no Unet or Transformer')
|
||||
if "VAE" in shared.opts.nncf_compress_weights:
|
||||
if hasattr(sd_model, 'vae'):
|
||||
sd_model.vae = nncf.compress_weights(sd_model.vae)
|
||||
elif hasattr(sd_model, 'movq'):
|
||||
sd_model.movq = nncf.compress_weights(sd_model.movq)
|
||||
else:
|
||||
shared.log.warning('Compress VAE Weights enabled but model has no VAE')
|
||||
if shared.opts.nncf_compress_text_encoder_weights:
|
||||
if "Text Encoder" in shared.opts.nncf_compress_weights:
|
||||
if hasattr(sd_model, 'text_encoder'):
|
||||
sd_model.text_encoder = nncf.compress_weights(sd_model.text_encoder)
|
||||
if hasattr(sd_model, 'text_encoder_2'):
|
||||
@@ -166,19 +173,19 @@ def compile_torch(sd_model):
|
||||
shared.log.error(f"Torch inductor config error: {e}")
|
||||
|
||||
t0 = time.time()
|
||||
if shared.opts.cuda_compile:
|
||||
if "Model" in shared.opts.cuda_compile:
|
||||
if hasattr(sd_model, 'unet') and hasattr(sd_model.unet, 'config'):
|
||||
sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
|
||||
else:
|
||||
shared.log.warning('Model compile enabled but model has no Unet')
|
||||
if shared.opts.cuda_compile_vae:
|
||||
if "VAE" in shared.opts.cuda_compile:
|
||||
if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'decode'):
|
||||
sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
|
||||
elif hasattr(sd_model, 'movq') and hasattr(sd_model.movq, 'decode'):
|
||||
sd_model.movq.decode = torch.compile(sd_model.movq.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
|
||||
else:
|
||||
shared.log.warning('Model compile enabled but model has no VAE')
|
||||
if shared.opts.cuda_compile_text_encoder:
|
||||
if "Text Encoder" in shared.opts.cuda_compile:
|
||||
if hasattr(sd_model, 'text_encoder'):
|
||||
sd_model.text_encoder = torch.compile(sd_model.text_encoder, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph)
|
||||
if hasattr(sd_model, 'text_encoder_2'):
|
||||
@@ -200,12 +207,12 @@ def compile_diffusers(sd_model):
|
||||
sd_model = ipex_optimize(sd_model)
|
||||
if shared.opts.nncf_compress_weights and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"):
|
||||
sd_model = nncf_compress_weights(sd_model)
|
||||
if not (shared.opts.cuda_compile or shared.opts.cuda_compile_vae or shared.opts.cuda_compile_upscaler):
|
||||
if not shared.opts.cuda_compile:
|
||||
return sd_model
|
||||
if shared.opts.cuda_compile_backend == 'none':
|
||||
shared.log.warning('Model compile enabled but no backend specified')
|
||||
return sd_model
|
||||
shared.log.info(f"Model compile: pipeline={sd_model.__class__.__name__} mode={shared.opts.cuda_compile_mode} backend={shared.opts.cuda_compile_backend} fullgraph={shared.opts.cuda_compile_fullgraph} unet={shared.opts.cuda_compile} vae={shared.opts.cuda_compile_vae} upscaler={shared.opts.cuda_compile_upscaler}")
|
||||
shared.log.info(f"Model compile: pipeline={sd_model.__class__.__name__} mode={shared.opts.cuda_compile_mode} backend={shared.opts.cuda_compile_backend} fullgraph={shared.opts.cuda_compile_fullgraph} compile={shared.opts.cuda_compile}")
|
||||
if shared.opts.cuda_compile_backend == 'stable-fast':
|
||||
sd_model = compile_stablefast(sd_model)
|
||||
else:
|
||||
|
||||
+13
-24
@@ -173,6 +173,7 @@ if cmd_opts.backend is not None: # override with args
|
||||
backend = Backend.DIFFUSERS if cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL
|
||||
if cmd_opts.use_openvino: # override for openvino
|
||||
backend = Backend.DIFFUSERS
|
||||
from modules.intel.openvino import get_device_list as get_openvino_device_list
|
||||
|
||||
|
||||
class OptionInfo:
|
||||
@@ -334,10 +335,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
|
||||
"torch_gc_threshold": OptionInfo(80, "Memory usage threshold for GC", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}),
|
||||
|
||||
"cuda_compile_sep": OptionInfo("<h2>Model Compile</h2>", "", gr.HTML),
|
||||
"cuda_compile": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile UNet"),
|
||||
"cuda_compile_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile VAE"),
|
||||
"cuda_compile_text_encoder": OptionInfo(False, "Compile Text Encoder"),
|
||||
"cuda_compile_upscaler": OptionInfo(False if not cmd_opts.use_openvino else True, "Compile Upscaler"),
|
||||
"cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE", "Upscaler"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"]}),
|
||||
"cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx', 'stable-fast']}),
|
||||
"cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}),
|
||||
"cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"),
|
||||
@@ -345,29 +343,20 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
|
||||
"cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"),
|
||||
"cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"),
|
||||
"diffusers_quantization": OptionInfo(False, "Dynamic quantization with TorchAO"),
|
||||
"nncf_compress_weights": OptionInfo([], "Compress Model weights with NNCF", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": backend == Backend.DIFFUSERS}),
|
||||
|
||||
"nncf_sep": OptionInfo("<h2>NNCF</h2>", "", gr.HTML),
|
||||
"nncf_compress_weights": OptionInfo(False, "Compress Model weights with NNCF"),
|
||||
"nncf_compress_vae_weights": OptionInfo(False, "Compress VAE weights with NNCF"),
|
||||
"nncf_compress_text_encoder_weights": OptionInfo(False, "Compress TextEncoder weights with NNCF"),
|
||||
"ipex_sep": OptionInfo("<h2>IPEX</h2>", "", gr.HTML, {"visible": devices.backend == "ipex"}),
|
||||
"ipex_optimize": OptionInfo(["Model", "VAE", "Text Encoder", "Upscaler"] if devices.backend == "ipex" else [], "IPEX Optimize for Intel GPUs", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}),
|
||||
|
||||
"directml_sep": OptionInfo("<h2>DirectML</h2>", "", gr.HTML),
|
||||
"directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, {"choices": memory_providers}),
|
||||
"directml_catch_nan": OptionInfo(False, "DirectML retry ops for NaN"),
|
||||
"directml_sep": OptionInfo("<h2>IPEX and DirectML</h2>", "", gr.HTML, {"visible": devices.backend == "directml"}),
|
||||
"directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, {"choices": memory_providers, "visible": devices.backend == "directml"}),
|
||||
"directml_catch_nan": OptionInfo(False, "DirectML retry ops for NaN", gr.Checkbox, {"visible": devices.backend == "directml"}),
|
||||
|
||||
"ipex_sep": OptionInfo("<h2>IPEX</h2>", "", gr.HTML),
|
||||
"ipex_optimize": OptionInfo(False if not devices.backend == "ipex" else True, "IPEX Optimize for Intel GPUs"),
|
||||
"ipex_optimize_vae": OptionInfo(False if not devices.backend == "ipex" else True, "IPEX Optimize for Intel GPUs with VAE"),
|
||||
"ipex_optimize_text_encoder": OptionInfo(False if not devices.backend == "ipex" else True, "IPEX Optimize for Intel GPUs with Text Encoder"),
|
||||
"ipex_optimize_upscaler": OptionInfo(False if not devices.backend == "ipex" else True, "IPEX Optimize for Intel GPUs with Upscalers"),
|
||||
|
||||
"openvino_sep": OptionInfo("<h2>OpenVINO</h2>", "", gr.HTML),
|
||||
"openvino_disable_model_caching": OptionInfo(False, "OpenVINO disable model caching"),
|
||||
"openvino_hetero_gpu": OptionInfo(False, "OpenVINO use Hetero Device"),
|
||||
"openvino_remove_cpu_from_hetero": OptionInfo(False, "OpenVINO remove CPU from Hetero Device"),
|
||||
"openvino_remove_igpu_from_hetero": OptionInfo(False, "OpenVINO remove iGPU from Hetero Device"),
|
||||
"nncf_compress_weights_mode": OptionInfo("INT8", "OpenVINO compress mode for NNCF", gr.Radio, {"choices": ['INT8', 'INT4_SYM', 'INT4_ASYM', 'NF4']}),
|
||||
"nncf_compress_weights_raito": OptionInfo(1.0, "OpenVINO compress ratio for NNCF", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}),
|
||||
"openvino_sep": OptionInfo("<h2>OpenVINO</h2>", "", gr.HTML, {"visible": cmd_opts.use_openvino}),
|
||||
"openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}),
|
||||
"nncf_compress_weights_mode": OptionInfo("INT8", "OpenVINO compress mode for NNCF", gr.Radio, {"choices": ['INT8', 'INT4_SYM', 'INT4_ASYM', 'NF4'], "visible": cmd_opts.use_openvino}),
|
||||
"nncf_compress_weights_raito": OptionInfo(1.0, "OpenVINO compress ratio for NNCF", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}),
|
||||
"openvino_disable_model_caching": OptionInfo(False, "OpenVINO disable model caching", gr.Checkbox, {"visible": cmd_opts.use_openvino}),
|
||||
}))
|
||||
|
||||
options_templates.update(options_section(('advanced', "Inference Settings"), {
|
||||
|
||||
+2
-2
@@ -198,7 +198,7 @@ class UpscalerNearest(Upscaler):
|
||||
|
||||
def compile_upscaler(model, name=""):
|
||||
try:
|
||||
if modules.shared.opts.ipex_optimize_upscaler:
|
||||
if modules.shared.opts.ipex_optimize and "Upscaler" in modules.shared.opts.ipex_optimize:
|
||||
import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
|
||||
from modules.devices import dtype as devices_dtype
|
||||
model.training = False
|
||||
@@ -207,7 +207,7 @@ def compile_upscaler(model, name=""):
|
||||
except Exception as err:
|
||||
modules.shared.log.warning(f"Upscaler IPEX Optimize not supported: {err}")
|
||||
try:
|
||||
if modules.shared.opts.cuda_compile_upscaler and modules.shared.opts.cuda_compile_backend != 'none':
|
||||
if "Upscaler" in modules.shared.opts.cuda_compile and modules.shared.opts.cuda_compile_backend != 'none':
|
||||
modules.shared.log.info(f"Upscaler Compiling: {name} mode={modules.shared.opts.cuda_compile_backend}")
|
||||
import logging
|
||||
import torch._dynamo # pylint: disable=unused-import,redefined-outer-name
|
||||
|
||||
Reference in New Issue
Block a user