diff --git a/launch.py b/launch.py index 0806566c8..c03ca6e22 100755 --- a/launch.py +++ b/launch.py @@ -41,14 +41,6 @@ def init_args(): def init_paths(): global script_path, extensions_dir # pylint: disable=global-statement - try: - import olive.workflows # pylint: disable=unused-import - except ModuleNotFoundError: - pass - import modules.cmd_args - parser = modules.cmd_args.parser - installer.add_args(parser) - args, _ = parser.parse_known_args() import modules.paths modules.paths.register_paths() script_path = modules.paths.script_path diff --git a/modules/onnx_impl/__init__.py b/modules/onnx_impl/__init__.py index 40813c7a8..a12220a21 100644 --- a/modules/onnx_impl/__init__.py +++ b/modules/onnx_impl/__init__.py @@ -100,9 +100,9 @@ def preprocess_pipeline(p, refiner_enabled: bool): if "ONNX" not in shared.opts.diffusers_pipeline: shared.log.warning(f"Unsupported pipeline for 'olive-ai' compile backend: {shared.opts.diffusers_pipeline}. You should select one of the ONNX pipelines.") - return + return shared.sd_model - if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "olive-ai": + if shared.opts.cuda_compile_backend == "olive-ai" and len(shared.opts.cuda_compile) != 1: compile_height = p.height compile_width = p.width if (shared.compiled_model_state is None or @@ -131,6 +131,8 @@ def preprocess_pipeline(p, refiner_enabled: bool): sd_models.reload_model_weights(op='model') shared.sd_model = shared.sd_model.preprocess(p) + return shared.sd_model + def initialize(): global initialized # pylint: disable=global-statement diff --git a/modules/onnx_impl/pipelines.py b/modules/onnx_impl/pipelines.py index bfebce4ac..9f389be29 100644 --- a/modules/onnx_impl/pipelines.py +++ b/modules/onnx_impl/pipelines.py @@ -370,9 +370,9 @@ class OnnxRawPipeline(PipelineBase): } out_dir = converted_dir - submodels_for_olive = [] - if shared.opts.cuda_compile_backend == "olive-ai": + submodels_for_olive = [] + if "Text Encoder" in shared.opts.cuda_compile: if not self.is_refiner: submodels_for_olive.append("text_encoder") @@ -384,34 +384,34 @@ class OnnxRawPipeline(PipelineBase): submodels_for_olive.append("vae_encoder") submodels_for_olive.append("vae_decoder") - if len(submodels_for_olive) == 0: - log.warning("Olive: Skipping olive run.") - else: - log.warning("Olive implementation is experimental. It contains potentially an issue and is subject to change at any time.") + if len(submodels_for_olive) == 0: + log.warning("Olive: Skipping olive run.") + else: + log.warning("Olive implementation is experimental. It contains potentially an issue and is subject to change at any time.") - in_dir = converted_dir + in_dir = converted_dir - if p.width != p.height: - log.warning("Olive: Different width and height are detected. The quality of the result is not guaranteed.") + if p.width != p.height: + log.warning("Olive: Different width and height are detected. The quality of the result is not guaranteed.") - if shared.opts.olive_static_dims: - sess_options = DynamicSessionOptions() - sess_options.enable_static_dims({ - "is_sdxl": self._is_sdxl, - "is_refiner": self.is_refiner, + if shared.opts.olive_static_dims: + sess_options = DynamicSessionOptions() + sess_options.enable_static_dims({ + "is_sdxl": self._is_sdxl, + "is_refiner": self.is_refiner, - "hidden_batch_size": p.batch_size if disable_classifier_free_guidance else p.batch_size * 2, - "height": p.height, - "width": p.width, - }) - kwargs["sess_options"] = sess_options + "hidden_batch_size": p.batch_size if disable_classifier_free_guidance else p.batch_size * 2, + "height": p.height, + "width": p.width, + }) + kwargs["sess_options"] = sess_options - try: - out_dir = self.run_olive(submodels_for_olive, in_dir) - except Exception: - log.error(f"Olive: Failed to run olive passes: model='{self.original_filename}'.") - shutil.rmtree(shared.opts.onnx_temp_dir, ignore_errors=True) - shutil.rmtree(os.path.join(shared.opts.onnx_cached_models_path, self.original_filename), ignore_errors=True) + try: + out_dir = self.run_olive(submodels_for_olive, in_dir) + except Exception: + log.error(f"Olive: Failed to run olive passes: model='{self.original_filename}'.") + shutil.rmtree(shared.opts.onnx_temp_dir, ignore_errors=True) + shutil.rmtree(os.path.join(shared.opts.onnx_cached_models_path, self.original_filename), ignore_errors=True) pipeline = self.derive_properties(load_pipeline(self.constructor, out_dir, **kwargs)) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index dcf6d15a7..10c914716 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -27,18 +27,25 @@ def process_diffusers(p: processing.StableDiffusionProcessing): def is_refiner_enabled(): return p.enable_hr and p.refiner_steps > 0 and p.refiner_start > 0 and p.refiner_start < 1 and shared.sd_refiner is not None - if getattr(p, 'init_images', None) is not None and len(p.init_images) > 0: - tgt_width, tgt_height = 8 * math.ceil(p.init_images[0].width / 8), 8 * math.ceil(p.init_images[0].height / 8) - if p.init_images[0].width != tgt_width or p.init_images[0].height != tgt_height: - shared.log.debug(f'Resizing init images: original={p.init_images[0].width}x{p.init_images[0].height} target={tgt_width}x{tgt_height}') - p.init_images = [images.resize_image(1, image, tgt_width, tgt_height, upscaler_name=None) for image in p.init_images] - p.height = tgt_height - p.width = tgt_width - hypertile_set(p) - if getattr(p, 'mask', None) is not None and p.mask.size != (tgt_width, tgt_height): - p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) - if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): - p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) + def resize_images(): + if getattr(p, 'image', None) is not None and getattr(p, 'init_images', None) is None: + p.init_images = [p.image] + if getattr(p, 'init_images', None) is not None and len(p.init_images) > 0: + tgt_width, tgt_height = 8 * math.ceil(p.init_images[0].width / 8), 8 * math.ceil(p.init_images[0].height / 8) + if p.init_images[0].size != (tgt_width, tgt_height): + shared.log.debug(f'Resizing init images: original={p.init_images[0].width}x{p.init_images[0].height} target={tgt_width}x{tgt_height}') + p.init_images = [images.resize_image(1, image, tgt_width, tgt_height, upscaler_name=None) for image in p.init_images] + p.height = tgt_height + p.width = tgt_width + sd_hijack_hypertile.hypertile_set(p) + if getattr(p, 'mask', None) is not None and p.mask.size != (tgt_width, tgt_height): + p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) + if getattr(p, 'init_mask', None) is not None and p.init_mask.size != (tgt_width, tgt_height): + p.init_mask = images.resize_image(1, p.init_mask, tgt_width, tgt_height, upscaler_name=None) + if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): + p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) + return tgt_width, tgt_height + return p.width, p.height def hires_resize(latents): # input=latents output=pil if not torch.is_tensor(latents): @@ -399,7 +406,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): p.task_args['sag_scale'] = p.sag_scale else: shared.log.warning(f'SAG incompatible scheduler: current={sd_model.scheduler.__class__.__name__} supported={supported}') - if shared.opts.cuda_compile_backend == "olive-ai": + if sd_model.__class__.__name__ == "OnnxRawPipeline": sd_model = preprocess_onnx_pipeline(p, is_refiner_enabled()) return sd_model @@ -540,7 +547,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if (latent_scale_mode is not None or p.hr_force) and p.denoising_strength > 0: p.ops.append('hires') shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - preprocess_onnx_pipeline(p, is_refiner_enabled()) + if shared.sd_model.__class__.__name__ == "OnnxRawPipeline": + shared.sd_model = preprocess_onnx_pipeline(p, is_refiner_enabled()) recompile_model(hires=True) update_sampler(shared.sd_model, second_pass=True) hires_args = set_pipeline_args( diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index 615cc24f5..8728d3326 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -152,22 +152,16 @@ def compile_stablefast(sd_model): def compile_torch(sd_model): - if shared.opts.cuda_compile_backend == "olive-ai": - if shared.compiled_model_state is None: - shared.compiled_model_state = CompiledModelState() - return sd_model try: import torch._dynamo # pylint: disable=unused-import,redefined-outer-name torch._dynamo.reset() # pylint: disable=protected-access shared.log.debug(f"Model compile available backends: {torch._dynamo.list_backends()}") # pylint: disable=protected-access if shared.opts.cuda_compile_backend == "openvino_fx": optimize_openvino() - """ elif shared.opts.cuda_compile_backend == "olive-ai": if shared.compiled_model_state is None: shared.compiled_model_state = CompiledModelState() return sd_model - """ log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index 5af58f7e8..59896d356 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -362,7 +362,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE", "Upscaler"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"]}), - "cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx', 'stable-fast']}), + "cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx', 'stable-fast', 'olive-ai']}), "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}), "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"), "cuda_compile_precompile": OptionInfo(False, "Model compile precompile"), @@ -382,10 +382,11 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "nncf_compress_weights_raito": OptionInfo(1.0, "OpenVINO compress ratio for NNCF", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), "openvino_disable_model_caching": OptionInfo(False, "OpenVINO disable model caching", gr.Checkbox, {"visible": cmd_opts.use_openvino}), - "directml_sep": OptionInfo("

IPEX and DirectML

", "", gr.HTML, {"visible": devices.backend == "directml"}), + "directml_sep": OptionInfo("

DirectML

", "", gr.HTML, {"visible": devices.backend == "directml"}), "directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, {"choices": memory_providers, "visible": devices.backend == "directml"}), "directml_catch_nan": OptionInfo(False, "DirectML retry ops for NaN", gr.Checkbox, {"visible": devices.backend == "directml"}), - "directml_olive_sep": OptionInfo("

DirectML and Olive

", "", gr.HTML), + + "olive_sep": OptionInfo("

Olive

", "", gr.HTML), "olive_float16": OptionInfo(True, 'Olive use FP16 on optimization (will use FP32 if unchecked)'), "olive_vae_encoder_float32": OptionInfo(False, 'Olive force FP32 for VAE Encoder (if Img2Img generates NaN, enable this option and remove previously optimized model)'), "olive_static_dims": OptionInfo(True, 'Olive use static dimensions (make inference faster with OrtTransformersOptimization)'), @@ -438,8 +439,8 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "disable_accelerate": OptionInfo(False, "Disable accelerate"), "diffusers_force_zeros": OptionInfo(False, "Force zeros for prompts when empty", gr.Checkbox, {"visible": False}), "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"), - "diffusers_force_inpaint": OptionInfo(False, 'Diffusers force inpaint pipeline'), - "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds (experimental)", gr.Radio, {"choices": ['default', 'weighted']}), + "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds", gr.Radio, {"choices": ['default', 'weighted']}), + "huggingface_token": OptionInfo('', 'HuggingFace token'), "onnx_sep": OptionInfo("

ONNX Runtime

", "", gr.HTML), @@ -463,7 +464,6 @@ options_templates.update(options_section(('system-paths', "System Paths"), { "embeddings_dir": OptionInfo(os.path.join(paths.models_path, 'embeddings'), "Folder with textual inversion embeddings", folder=True), "hypernetwork_dir": OptionInfo(os.path.join(paths.models_path, 'hypernetworks'), "Folder with Hypernetwork models", folder=True), "control_dir": OptionInfo(os.path.join(paths.models_path, 'control'), "Folder with Control models", folder=True), - "olive_temp_dir": OptionInfo(os.path.join(paths.models_path, 'Olive', 'temp'), "Directory for olive optimization process", folder=True), "codeformer_models_path": OptionInfo(os.path.join(paths.models_path, 'Codeformer'), "Folder with codeformer models", folder=True), "gfpgan_models_path": OptionInfo(os.path.join(paths.models_path, 'GFPGAN'), "Folder with GFPGAN models", folder=True), "esrgan_models_path": OptionInfo(os.path.join(paths.models_path, 'ESRGAN'), "Folder with ESRGAN models", folder=True), @@ -479,6 +479,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), { "openvino_cache_path": OptionInfo('cache', "Directory for OpenVINO cache", folder=True), "temp_dir": OptionInfo("", "Directory for temporary images; leave empty for default", folder=True), "clean_temp_dir_at_start": OptionInfo(True, "Cleanup non-default temporary directory when starting webui"), + "onnx_temp_dir": OptionInfo(os.path.join(paths.models_path, 'ONNX', 'temp'), "Directory for ONNX conversion and Olive optimization process", folder=True), })) options_templates.update(options_section(('saving-images', "Image Options"), { diff --git a/modules/ui.py b/modules/ui.py index c30860955..a56a05ce2 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -417,6 +417,7 @@ def create_ui(startup_timer = None): loadsave.add_block(interface, ifid) loadsave.add_component(f"webui/Tabs@{tabs.elem_id}", tabs) loadsave.setup_ui() + if opts.notification_audio_enable and os.path.exists(os.path.join(script_path, opts.notification_audio_path)): gr.Audio(interactive=False, value=os.path.join(script_path, opts.notification_audio_path), elem_id="audio_notification", visible=False) diff --git a/requirements.txt b/requirements.txt index 2b699627c..3d2de0d0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,6 @@ lmdb lpips omegaconf open-clip-torch -opencv-contrib-python-headless onnx optimum piexif @@ -35,6 +34,7 @@ rich safetensors scipy tb_nightly +tensordict toml torchdiffeq voluptuous @@ -43,27 +43,34 @@ scikit-image basicsr fasteners dctorch +pymatting +matplotlib +peft +orjson +httpx==0.24.1 compel==2.0.2 torchsde==0.2.6 +clip-interrogator==0.6.0 antlr4-python3-runtime==4.9.3 requests==2.31.0 tqdm==4.66.1 -accelerate==0.20.3 -opencv-python-headless==4.7.0.72 -diffusers==0.21.4 +accelerate==0.26.1 +opencv-contrib-python-headless==4.8.1.78 +diffusers==0.25.1 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.17.1 +huggingface_hub==0.20.3 numexpr==2.8.4 -numpy==1.24.4 -numba==0.57.1 +numpy==1.26.2 +numba==0.58.1 pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 -transformers==4.32.1 +tokenizers==0.15.1 +transformers==4.37.1 tomesd==0.1.3 -urllib3==1.26.15 -Pillow==9.5.0 -timm==0.9.7 +urllib3==1.26.18 +Pillow==10.2.0 +timm==0.9.12 pydantic==1.10.13 -typing-extensions==4.8.0 +typing-extensions==4.9.0 \ No newline at end of file