From e2fe0ef36fc31c44e9064b6fd1d8c6dfc139be1d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 14 Sep 2023 15:41:58 -0400 Subject: [PATCH 01/37] fix en filter by folder on windows --- javascript/extraNetworks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index b59129103..0a0cfa9b7 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -160,7 +160,7 @@ function setupExtraNetworksForTab(tabname) { const searchTerm = search.value.toLowerCase(); gradioApp().querySelectorAll(`#${tabname}_extra_tabs div.card`).forEach((elem) => { let text = `${elem.querySelector('.name').textContent.toLowerCase()} ${elem.querySelector('.search_term').textContent.toLowerCase()}`; - text = text.replace('models--', 'Diffusers'); + text = text.replace('models--', 'Diffusers').replace('\\', '/'); elem.style.display = text.indexOf(searchTerm) === -1 ? 'none' : ''; }); searchTimer = null; From 083ab521fa6e8fa77a7494619f2d0739fd499ade Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 14 Sep 2023 20:01:30 -0400 Subject: [PATCH 02/37] cleanup --- CHANGELOG.md | 4 ++-- modules/ui.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8af1d23a5..1137f59c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,11 +22,11 @@ Major changes how **hires** works as well as support for a very interesting new - diffusers: - allow loading of sd/sdxl models from safetensors without online connectivity - support for new model: [wuerstchen](https://huggingface.co/warp-ai/wuerstchen) - its a high-resolution model (1024px+) that nearly doubls performance of sd-xl with much lower resource requirements + its a high-resolution model (1024px+) thats ~40% faster than sd-xl with a bit lower resource requirements go to *models -> huggingface -> search "warp-ai/wuerstchen" -> download* its nearly 12gb in size, so be patient :) - minor re-layout of the main ui -- update **ui hints** +- updated **ui hints** - updated **models -> civitai** - search and download loras - find previews for already downloaded models or loras diff --git a/modules/ui.py b/modules/ui.py index c1154d9d9..d6f7bb62c 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -937,7 +937,7 @@ def create_ui(startup_timer = None): elif info.folder is not None: with FormRow(): res = comp(label=info.label, value=fun(), elem_id=elem_id, elem_classes="folder-selector", **args) - ui_common.create_browse_button(res, f"folder_{key}") + # ui_common.create_browse_button(res, f"folder_{key}") else: try: res = comp(label=info.label, value=fun(), elem_id=elem_id, **args) From 496bdf7c555f6ef4e6a8c63d591493aa2d23f1de Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 08:39:25 -0400 Subject: [PATCH 03/37] fix backend switch --- javascript/extraNetworks.js | 1 + modules/dml/__init__.py | 16 +++++++--------- modules/processing_diffusers.py | 6 +++--- modules/scripts.py | 2 +- modules/sd_models.py | 31 ++++++++++++++++--------------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 0a0cfa9b7..ce2e8cf03 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -162,6 +162,7 @@ function setupExtraNetworksForTab(tabname) { let text = `${elem.querySelector('.name').textContent.toLowerCase()} ${elem.querySelector('.search_term').textContent.toLowerCase()}`; text = text.replace('models--', 'Diffusers').replace('\\', '/'); elem.style.display = text.indexOf(searchTerm) === -1 ? 'none' : ''; + console.log({ search: searchTerm, text, display: elem.style.display }); }); searchTimer = null; }, 100); diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py index f14f479a2..3661559ec 100644 --- a/modules/dml/__init__.py +++ b/modules/dml/__init__.py @@ -9,7 +9,7 @@ default_memory_provider = "None" if platform.system() == "Windows": memory_providers.append("Performance Counter") default_memory_provider = "Performance Counter" -do_nothing = lambda: None +do_nothing = lambda: None # pylint: disable=unnecessary-lambda-assignment def _set_memory_provider(): from modules.shared import opts, cmd_opts, log @@ -63,7 +63,7 @@ def directml_init(): return True, None def directml_do_hijack(): - import modules.dml.hijack + import modules.dml.hijack # pylint: disable=unused-import from modules.devices import device if not torch.dml.has_float64_support(device): @@ -79,9 +79,9 @@ class OverrideItem(NamedTuple): message: Optional[str] opts_override_table = { - "diffusers_generator_device": OverrideItem("cpu", None, "DirectML does not support torch Generator API."), - "diffusers_model_cpu_offload": OverrideItem(False, None, "Diffusers' model CPU offloading does not support DirectML devices."), - "diffusers_seq_cpu_offload": OverrideItem(False, lambda opts: opts.diffusers_pipeline != "Stable Diffusion XL", "Diffusers' sequential CPU offloading is available only on StableDiffusionXLPipeline with DirectML devices."), + "diffusers_generator_device": OverrideItem("cpu", None, "DirectML does not support torch Generator API"), + "diffusers_model_cpu_offload": OverrideItem(False, None, "Diffusers model CPU offloading does not support DirectML devices"), + "diffusers_seq_cpu_offload": OverrideItem(False, lambda opts: opts.diffusers_pipeline != "Stable Diffusion XL", "Diffusers sequential CPU offloading is available only on StableDiffusionXLPipeline with DirectML devices"), } def directml_override_opts(): @@ -96,11 +96,9 @@ def directml_override_opts(): if getattr(shared.opts, key) != item.value and (item.condition is None or item.condition(shared.opts)): count += 1 setattr(shared.opts, key, item.value) - if item.message is not None: - shared.log.warning(item.message) - shared.log.warning(f'{key} is automatically overriden to {item.value}.') + shared.log.warning(f'Overriding: {key}={item.value} {item.message if item.message is not None else ""}') if count > 0: - shared.log.info(f'{count} options are automatically overriden. If you want to keep them from overriding, run with --experimental argument.') + shared.log.info(f'Options override: count={count}. If you want to keep them from overriding, run with --experimental argument.') _set_memory_provider() diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d22c73a13..8ef2fe638 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -78,8 +78,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro model.vae.to(devices.device) latents.to(model.vae.device) - needs_upcasting = model.vae.dtype == torch.float16 and model.vae.config.force_upcast - if needs_upcasting: # this is done by diffusers automatically if output_type != 'latent' + upcast = (model.vae.dtype == torch.float16) and model.vae.config.force_upcast and hasattr(model, 'upcast_vae') + if upcast: # this is done by diffusers automatically if output_type != 'latent' model.upcast_vae() latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) @@ -87,7 +87,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.opts.diffusers_move_unet and not model.has_accelerate: model.unet.to(unet_device) t1 = time.time() - shared.log.debug(f'VAE decode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={model.vae.config.get("force_upcast", None)} images={latents.shape[0]} latents={latents.shape} time={round(t1-t0, 3)}s') + shared.log.debug(f'VAE decode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={upcast} images={latents.shape[0]} latents={latents.shape} time={round(t1-t0, 3)}s') return decoded def full_vae_encode(image, model): diff --git a/modules/scripts.py b/modules/scripts.py index b81ce2aa1..c5eb6afe8 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -398,7 +398,7 @@ class ScriptRunner: dropdown.init_field = init_field dropdown.change(fn=select_script, inputs=[dropdown], outputs=[script.group for script in self.selectable_scripts]) - + def onload_script_visibility(params): title = params.get('Script', None) if title: diff --git a/modules/sd_models.py b/modules/sd_models.py index 318d2915e..aa457408f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -586,7 +586,7 @@ model_data = ModelData() def change_backend(): - shared.log.info(f'Pipeline changed: {shared.backend}') + shared.log.info(f'Backend changed: {shared.backend}') unload_model_weights() checkpoints_loaded.clear() from modules.sd_samplers import list_samplers @@ -762,7 +762,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No diffusers_load_config.pop('safety_checker', None) diffusers_load_config.pop('requires_safety_checker', None) diffusers_load_config.pop('load_safety_checker', None) - shared.log.debug(f'Model {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access + shared.log.debug(f'Setting {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access except Exception as e: shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') return @@ -773,8 +773,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.scheduler.name = 'DDIM' if (shared.opts.diffusers_model_cpu_offload or shared.cmd_opts.medvram) and (shared.opts.diffusers_seq_cpu_offload or shared.cmd_opts.lowvram): - shared.log.warning(f'Model {op}: Model CPU offload (--medvram) and Sequential CPU offload (--lowvram) are not compatible') - shared.log.debug(f'Model {op}: disabling model CPU offload and --medvram') + shared.log.warning(f'Setting {op}: Model CPU offload and Sequential CPU offload are not compatible') + shared.log.debug(f'Setting {op}: disabling model CPU offload') shared.opts.diffusers_model_cpu_offload=False shared.cmd_opts.medvram=False @@ -783,7 +783,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.has_accelerate = False if hasattr(sd_model, "enable_model_cpu_offload"): if (shared.cmd_opts.medvram and devices.backend != "directml") or shared.opts.diffusers_model_cpu_offload: - shared.log.debug(f'Model {op}: enable model CPU offload') + shared.log.debug(f'Setting {op}: enable model CPU offload') if shared.opts.diffusers_move_base or shared.opts.diffusers_move_unet or shared.opts.diffusers_move_refiner: shared.opts.diffusers_move_base = False shared.opts.diffusers_move_unet = False @@ -793,7 +793,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.has_accelerate = True if hasattr(sd_model, "enable_sequential_cpu_offload"): if shared.cmd_opts.lowvram or shared.opts.diffusers_seq_cpu_offload: - shared.log.debug(f'Model {op}: enable sequential CPU offload') + shared.log.debug(f'Setting {op}: enable sequential CPU offload') if shared.opts.diffusers_move_base or shared.opts.diffusers_move_unet or shared.opts.diffusers_move_refiner: shared.opts.diffusers_move_base = False shared.opts.diffusers_move_unet = False @@ -803,19 +803,19 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.has_accelerate = True if hasattr(sd_model, "enable_vae_slicing"): if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_slicing: - shared.log.debug(f'Model {op}: enable VAE slicing') + shared.log.debug(f'Setting {op}: enable VAE slicing') sd_model.enable_vae_slicing() else: sd_model.disable_vae_slicing() if hasattr(sd_model, "enable_vae_tiling"): if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_tiling: - shared.log.debug(f'Model {op}: enable VAE tiling') + shared.log.debug(f'Setting {op}: enable VAE tiling') sd_model.enable_vae_tiling() else: sd_model.disable_vae_tiling() if hasattr(sd_model, "enable_attention_slicing"): if shared.cmd_opts.lowvram or shared.opts.diffusers_attention_slicing: - shared.log.debug(f'Model {op}: enable attention slicing') + shared.log.debug(f'Setting {op}: enable attention slicing') sd_model.enable_attention_slicing() else: sd_model.disable_attention_slicing() @@ -832,11 +832,11 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if shared.opts.no_half_vae: devices.dtype_vae = torch.float32 sd_model.vae.to(devices.dtype_vae) - shared.log.debug(f'Model {op} VAE: name={sd_vae.loaded_vae_file} upcast={sd_model.vae.config.get("force_upcast", None)}') + shared.log.debug(f'Setting {op} VAE: name={sd_vae.loaded_vae_file} upcast={sd_model.vae.config.get("force_upcast", None)}') if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'): sd_model.enable_xformers_memory_efficient_attention() if shared.opts.opt_channelslast: - shared.log.debug(f'Model {op}: enable channels last') + shared.log.debug(f'Setting {op}: enable channels last') sd_model.unet.to(memory_format=torch.channels_last) base_sent_to_cpu=False @@ -1163,20 +1163,21 @@ def disable_offload(sd_model): def unload_model_weights(op='model'): - from modules import sd_hijack if op == 'model' or op == 'dict': if model_data.sd_model: - if shared.backend == shared.Backend.ORIGINAL: + if shared.backend != shared.Backend.ORIGINAL: # moving from diffusers=>original + from modules import sd_hijack model_data.sd_model.to(devices.cpu) sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - else: + else: # moving from original=>diffusers disable_offload(model_data.sd_model) model_data.sd_model.to('meta') model_data.sd_model = None shared.log.debug(f'Unload weights {op}: {memory_stats()}') else: if model_data.sd_refiner: - if shared.backend == shared.Backend.ORIGINAL: + if shared.backend != shared.Backend.ORIGINAL: + from modules import sd_hijack model_data.sd_model.to(devices.cpu) sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) else: From 0d0240314bf04212e200282c613d9176290b430d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 09:14:47 -0400 Subject: [PATCH 04/37] add process/postprocess logging --- modules/postprocessing.py | 3 +++ modules/scripts_postprocessing.py | 31 ++++--------------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/modules/postprocessing.py b/modules/postprocessing.py index c82aac540..d090e00dd 100644 --- a/modules/postprocessing.py +++ b/modules/postprocessing.py @@ -17,6 +17,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp outputs = [] params = {} if extras_mode == 1: + shared.log.debug(f'process: mode=batch folder={image_folder}') for img in image_folder: if isinstance(img, Image.Image): image = img @@ -29,6 +30,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp image_names.append(fn) image_ext.append(ext) elif extras_mode == 2: + shared.log.debug(f'process: mode=folder folder={input_dir}') assert not shared.cmd_opts.hide_ui_dir_config, '--hide-ui-dir-config option must be disabled' assert input_dir, 'input directory not selected' image_list = shared.listfiles(input_dir) @@ -50,6 +52,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp else: outpath = opts.outdir_samples or opts.outdir_extras_samples for image, name, ext in zip(image_data, image_names, image_ext): + shared.log.debug(f'process: image={image} {args}') infotext = '' if shared.state.interrupted: shared.log.debug('Postprocess interrupted') diff --git a/modules/scripts_postprocessing.py b/modules/scripts_postprocessing.py index 7baa4c738..3c8720fc5 100644 --- a/modules/scripts_postprocessing.py +++ b/modules/scripts_postprocessing.py @@ -1,6 +1,5 @@ import os import gradio as gr - from modules import errors, shared @@ -15,15 +14,9 @@ class ScriptPostprocessing: controls = None args_from = None args_to = None - - order = 1000 - """scripts will be ordred by this value in postprocessing UI""" - - name = None - """this function should return the title of the script.""" - - group = None - """A gr.Group component that has all script's UI inside it""" + order = 1000 # scripts will be ordred by this value in postprocessing UI + name = None # this function should return the title of the script + group = None # A gr.Group component that has all script's UI inside it def ui(self): """ @@ -61,25 +54,19 @@ class ScriptPostprocessingRunner: def initialize_scripts(self, scripts_data): self.scripts = [] - for script_class, path, _basedir, _script_module in scripts_data: script: ScriptPostprocessing = script_class() script.filename = path - if script.name == "Simple Upscale": continue - self.scripts.append(script) def create_script_ui(self, script, inputs): script.args_from = len(inputs) script.args_to = len(inputs) - script.controls = wrap_call(script.ui, script.filename, "ui") - for control in script.controls.values(): control.custom_script_source = os.path.basename(script.filename) - inputs += list(script.controls.values()) script.args_to = len(inputs) @@ -87,56 +74,46 @@ class ScriptPostprocessingRunner: if self.scripts is None: import modules.scripts self.initialize_scripts(modules.scripts.postprocessing_scripts_data) - scripts_order = shared.opts.postprocessing_operation_order def script_score(name): for i, possible_match in enumerate(scripts_order): if possible_match == name: return i - return len(self.scripts) script_scores = {script.name: (script_score(script.name), script.order, script.name, original_index) for original_index, script in enumerate(self.scripts)} - return sorted(self.scripts, key=lambda x: script_scores[x.name]) def setup_ui(self): inputs = [] - for script in self.scripts_in_preferred_order(): with gr.Row() as group: self.create_script_ui(script, inputs) - script.group = group - self.ui_created = True return inputs def run(self, pp: PostprocessedImage, args): for script in self.scripts_in_preferred_order(): shared.state.job = script.name - script_args = args[script.args_from:script.args_to] - process_args = {} for (name, _component), value in zip(script.controls.items(), script_args): process_args[name] = value - + shared.log.debug(f'postprocess: script={script.name} args={process_args}') script.process(pp, **process_args) def create_args_for_run(self, scripts_args): if not self.ui_created: with gr.Blocks(analytics_enabled=False): self.setup_ui() - scripts = self.scripts_in_preferred_order() args = [None] * max([x.args_to for x in scripts]) for script in scripts: script_args_dict = scripts_args.get(script.name, None) if script_args_dict is not None: - for i, name in enumerate(script.controls): args[script.args_from + i] = script_args_dict.get(name, None) From c7f3f57093e1d9c59dce9cd4d0eb68f647e20684 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 09:50:45 -0400 Subject: [PATCH 05/37] fix hires preview --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 01f39e13c..a6d686b9a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -174,7 +174,7 @@ class State: """sets self.current_image from self.current_latent if enough sampling steps have been made after the last call to this""" if not parallel_processing_allowed: return - if self.sampling_step - self.current_image_sampling_step >= opts.show_progress_every_n_steps and opts.live_previews_enable and opts.show_progress_every_n_steps != -1: + if abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps and opts.live_previews_enable and opts.show_progress_every_n_steps > 0: self.do_set_current_image() def do_set_current_image(self): From d3177fc5603b4a93876cb559192eaa0ba660237c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 10:09:06 -0400 Subject: [PATCH 06/37] fix api typing for scripts --- modules/api/api.py | 8 +++++--- modules/api/models.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/api/api.py b/modules/api/api.py index cf721d445..12c814be8 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -2,7 +2,7 @@ import io import time import base64 from io import BytesIO -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional from threading import Lock from secrets import compare_digest from fastapi import FastAPI, APIRouter, Depends @@ -180,10 +180,12 @@ class Api: i2ilist = [script.name for script in scripts.scripts_img2img.scripts if script.name is not None] return models.ScriptsList(txt2img = t2ilist, img2img = i2ilist) - def get_script_info(self): + def get_script_info(self, script_name: Optional[str] = None): res = [] for script_list in [scripts.scripts_txt2img.scripts, scripts.scripts_img2img.scripts]: - res += [script.api_info for script in script_list if script.api_info is not None] + for script in script_list: + if script.api_info is not None and (script_name is None or script_name == script.api_info.name): + res.append(script.api_info) return res def get_script(self, script_name, script_runner): diff --git a/modules/api/models.py b/modules/api/models.py index da4158dcd..6bd4eccfe 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -303,7 +303,7 @@ class ScriptArg(BaseModel): minimum: Optional[Any] = Field(default=None, title="Minimum", description="Minimum allowed value for the argumentin UI") maximum: Optional[Any] = Field(default=None, title="Minimum", description="Maximum allowed value for the argumentin UI") step: Optional[Any] = Field(default=None, title="Minimum", description="Step for changing value of the argumentin UI") - choices: Optional[List[str]] = Field(default=None, title="Choices", description="Possible values for the argument") + choices: Optional[Any] = Field(default=None, title="Choices", description="Possible values for the argument") class ScriptInfo(BaseModel): From 396135d56773d9d403c4f96eb845d15e86b38412 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 10:18:50 -0400 Subject: [PATCH 07/37] fix tomes --- modules/sd_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/sd_models.py b/modules/sd_models.py index aa457408f..81df6963a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1211,3 +1211,5 @@ def apply_token_merging(sd_model, token_merging_ratio=0): sd_model.applied_token_merged_ratio = token_merging_ratio except: shared.log.warning(f'Token merging not supported: pipeline={sd_model.__class__.__name__}') + else: + sd_model.applied_token_merged_ratio = 0 From 2309398be8e554c93b45c7a0e5444e41a27a2395 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 11:38:38 -0400 Subject: [PATCH 08/37] temp fix sdxl lora --- extensions-builtin/Lora/lora.py | 10 ++---- html/locale_en.json | 2 +- modules/lora_diffusers.py | 57 +++++++++++++++++++++------------ modules/shared.py | 35 +++++--------------- wiki | 2 +- 5 files changed, 49 insertions(+), 57 deletions(-) diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py index fbac7e8fc..02bb2092c 100644 --- a/extensions-builtin/Lora/lora.py +++ b/extensions-builtin/Lora/lora.py @@ -145,11 +145,11 @@ def assign_lora_names_to_compvis_modules(sd_model): sd_model.lora_layer_mapping = lora_layer_mapping -def load_diffuser_lora(name, lora_on_disk, multiplier): +def load_diffuser_lora(name, lora_on_disk, multiplier, num_loras): lora = LoraModule(name, lora_on_disk) lora.mtime = os.path.getmtime(lora_on_disk.filename) from modules.lora_diffusers import load_diffusers_lora - load_diffusers_lora(name, lora_on_disk, multiplier) + load_diffusers_lora(name, lora_on_disk, multiplier, num_loras) return lora @@ -241,22 +241,18 @@ def load_loras(names, multipliers=None): for i, name in enumerate(names): lora = already_loaded.get(name, None) if shared.backend == shared.Backend.ORIGINAL else None - lora_on_disk = loras_on_disk[i] - if lora_on_disk is not None: if lora is None or os.path.getmtime(lora_on_disk.filename) > lora.mtime: try: if shared.backend == shared.Backend.DIFFUSERS: - lora = load_diffuser_lora(name, lora_on_disk, multipliers[i] if multipliers else 1.0) + lora = load_diffuser_lora(name, lora_on_disk, multipliers[i] if multipliers else 1.0, len(names)) else: lora = load_lora(name, lora_on_disk) except Exception as e: errors.display(e, f"loading Lora {lora_on_disk.filename}") continue - lora.mentioned_name = name - lora_on_disk.read_hash() if lora is None: diff --git a/html/locale_en.json b/html/locale_en.json index 2c6e77e49..0238f9a67 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -585,7 +585,7 @@ {"id":"","label":"Enable attention slicing","localized":"","hint":"Performs attention computation in steps instead of all at once. Slower inference times, but greatly reduced memory usage"}, {"id":"","label":"Diffusers model loading variant","localized":"","hint":""}, {"id":"","label":"Diffusers VAE loading variant","localized":"","hint":""}, - {"id":"","label":"Diffusers LoRA loading variant","localized":"","hint":"'sequential apply' loads and applies each LoRA in order of appearance, 'merge and apply' loads all LoRAs and merges them in-memory before applying to model, 'diffusers default' uses single LoRA loading method"}, + {"id":"","label":"Diffusers LoRA loading variant","localized":"","hint":"'sequential apply' loads and applies each LoRA in order of appearance, 'merge and apply' loads all LoRAs and merges them in-memory before applying to model, 'diffusers' uses diffusers default LoRA loading method"}, {"id":"","label":"Torch inference mode","localized":"","hint":"Use torch inference mode"}, {"id":"","label":"inference-mode","localized":"","hint":"Use torch.inference_mode"}, {"id":"","label":"no-grad","localized":"","hint":"Use torch.no_grad"}, diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py index 59bf57b24..82d7a10b0 100644 --- a/modules/lora_diffusers.py +++ b/modules/lora_diffusers.py @@ -1,3 +1,4 @@ +import time import diffusers import diffusers.models.lora as diffusers_lora # from modules import shared @@ -7,13 +8,15 @@ import modules.shared as shared lora_state = { # TODO Lora state for Diffusers 'multiplier': [], 'active': False, - 'loaded': 0, - 'all_loras': [] + 'loaded': [], + 'all_loras': [], } def unload_diffusers_lora(): try: pipe = shared.sd_model - if shared.opts.diffusers_lora_loader == "diffusers default": + if shared.opts.diffusers_lora_loader == "diffusers": + if len(lora_state['loaded']) > 1: + pipe.unfuse_lora() pipe.unload_lora_weights() pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ @@ -29,22 +32,30 @@ def unload_diffusers_lora(): if shared.opts.diffusers_lora_loader == "sequential apply": lora_network.unapply_to() lora_state['active'] = False - lora_state['loaded'] = 0 + lora_state['loaded'].clear() lora_state['all_loras'] = [] lora_state['multiplier'] = [] - except Exception as e: - shared.log.error(f"Diffusers LoRA unloading failed: {e}") + shared.log.error(f"LoRA unload failed: {e}") -def load_diffusers_lora(name, lora, strength = 1.0): +def load_diffusers_lora(name, lora, strength = 1.0, num_loras = 1): + if f'{lora.filename}:{strength}' in lora_state['loaded']: + shared.log.info(f'LoRA cached: {name} strength={strength}') + return try: + t0 = time.time() pipe = shared.sd_model lora_state['active'] = True - lora_state['loaded'] += 1 lora_state['multiplier'].append(strength) - if shared.opts.diffusers_lora_loader == "diffusers default": - pipe.load_lora_weights(lora.filename, cache_dir=shared.opts.diffusers_dir, local_files_only=True, lora_scale=strength) + fuse = 0 + if shared.opts.diffusers_lora_loader.startswith("diffusers"): + pipe.load_lora_weights(lora.filename, cache_dir=shared.opts.diffusers_dir, local_files_only=True, lora_scale=strength, low_cpu_mem_usage=True) + if num_loras > 1: + t2 = time.time() + pipe.fuse_lora(lora_scale=strength) + fuse = time.time() - t2 + lora_state['loaded'].append(f'{lora.filename}:{strength}') else: from safetensors.torch import load_file lora_sd = load_file(lora.filename) @@ -60,20 +71,24 @@ def load_diffusers_lora(name, lora, strength = 1.0): lora_network.to(shared.device, dtype=pipe.unet.dtype) lora_network.apply_to(multiplier=strength) lora_state['all_loras'].append(lora_network) - shared.log.info(f"LoRA loaded: {name} strength={strength} loader={shared.opts.diffusers_lora_loader}") + lora_state['loaded'].append(f'{lora.filename}:{strength}') + t1 = time.time() + fuse = f'fuse={fuse:.2f}s' if fuse > 0 else '' + shared.log.info(f'LoRA loaded: {name} strength={strength} loader="{shared.opts.diffusers_lora_loader}" lora={t1-t0:.2f}s {fuse}') except Exception as e: - shared.log.error(f"LoRA loading failed: {name} {e}") + lines = str(e).splitlines() + shared.log.error(f'LoRA loading failed: {name} loader="{shared.opts.diffusers_lora_loader}" {lines[0]}') # Diffusersで動くLoRA。このファイル単独で完結する。 # LoRA module for Diffusers. This file works independently. -import bisect -import math -from typing import Any, Dict, List, Mapping, Optional, Union -from diffusers import UNet2DConditionModel -from tqdm import tqdm -from transformers import CLIPTextModel -import torch +import bisect # pylint: disable=wrong-import-order +import math # pylint: disable=wrong-import-order +from typing import Any, Dict, List, Mapping, Optional, Union # pylint: disable=wrong-import-order +from diffusers import UNet2DConditionModel # pylint: disable=wrong-import-order +from tqdm import tqdm # pylint: disable=wrong-import-order +from transformers import CLIPTextModel # pylint: disable=wrong-import-order +import torch # pylint: disable=wrong-import-order def make_unet_conversion_map() -> Dict[str, str]: @@ -496,7 +511,7 @@ class LoRANetwork(torch.nn.Module): # pylint: disable=abstract-method for lora in tqdm(self.text_encoder_loras + self.unet_loras): lora.restore_from(multiplier) - def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True): + def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True): # pylint: disable=arguments-differ # convert SDXL Stability AI's state dict to Diffusers' based state dict map_keys = list(UNET_CONVERSION_MAP.keys()) # prefix of U-Net modules map_keys.sort() @@ -514,7 +529,7 @@ class LoRANetwork(torch.nn.Module): # pylint: disable=abstract-method # because V2 LoRA is based on U-Net created by use_linear_projection=False my_state_dict = self.state_dict() for key in state_dict.keys(): - if state_dict[key].size() != my_state_dict[key].size(): + if state_dict[key].size() != my_state_dict[key].size(): # pylint: disable=unsubscriptable-object # print(f"convert {key} from {state_dict[key].size()} to {my_state_dict[key].size()}") state_dict[key] = state_dict[key].view(my_state_dict[key].size()) diff --git a/modules/shared.py b/modules/shared.py index a6d686b9a..252baa251 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -438,7 +438,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), - "diffusers_lora_loader": OptionInfo("diffusers default" if cmd_opts.use_openvino else "sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['sequential apply', 'merge and apply', 'diffusers default']}), + "diffusers_lora_loader": OptionInfo("diffusers", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}), "diffusers_force_zeros": OptionInfo(True, "Force zeros for prompts when empty"), "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"), })) @@ -832,6 +832,7 @@ else: opts.data['sd_backend'] = 'diffusers' if backend == Backend.DIFFUSERS else 'original' opts.data['uni_pc_lower_order_final'] = opts.schedulers_use_loworder opts.data['uni_pc_order'] = opts.schedulers_solver_order +opts.data['diffusers_lora_loader'] = 'diffusers' # TODO broken in diffusers=0.21 log.info(f'Engine: backend={backend} compute={devices.backend} mode={devices.inference_context.__name__} device={devices.get_optimal_device_name()}') log.info(f'Device: {print_dict(devices.get_gpu_info())}') @@ -885,37 +886,17 @@ def reload_gradio_theme(theme_name=None): log.info(f'Loading UI theme: name={theme_name} style={opts.theme_style}') -class TotalTQDM: +class TotalTQDM: # compatibility with previous global-tqdm def __init__(self): - self._tqdm = None - + pass def reset(self): - self._tqdm = tqdm.tqdm( - desc="Total", - total=state.job_count * state.sampling_steps, - position=1, - ) - + pass def update(self): - if not opts.multiple_tqdm or cmd_opts.disable_console_progressbars: - return - if self._tqdm is None: - self.reset() - self._tqdm.update() - + pass def updateTotal(self, new_total): - if not opts.multiple_tqdm or cmd_opts.disable_console_progressbars: - return - if self._tqdm is None: - self.reset() - self._tqdm.total = new_total - + pass def clear(self): - if self._tqdm is not None: - self._tqdm.refresh() - self._tqdm.close() - self._tqdm = None - + pass total_tqdm = TotalTQDM() diff --git a/wiki b/wiki index fea51bf38..d43376f66 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit fea51bf38c010520dbf30fb8cb58043f94fb2e8e +Subproject commit d43376f66fe454d2911a3b284077910df2b16b23 From b67e986ec30a32cb4c192bdba4af3124d0a3911b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 11:53:14 -0400 Subject: [PATCH 09/37] fix loading hypernetwork --- javascript/extraNetworks.js | 5 ++++- modules/hypernetworks/hypernetwork.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index ce2e8cf03..e48919ef6 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -187,7 +187,10 @@ function setupExtraNetworksForTab(tabname) { const intersectionObserver = new IntersectionObserver((entries) => { if (!en) return; - for (const el of Array.from(gradioApp().querySelectorAll('.extra-networks-page'))) el.style.height = `${window.opts.extra_networks_height}vh`; + for (const el of Array.from(gradioApp().querySelectorAll('.extra-networks-page'))) { + el.style.height = `${window.opts.extra_networks_height}vh`; + el.parentElement.style.width = '-webkit-fill-available'; + } if (entries[0].intersectionRatio > 0) { if (window.opts.extra_networks_card_cover === 'cover') { en.style.transition = ''; diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index d9cc95e86..7b319b1b7 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -221,10 +221,10 @@ class Hypernetwork: torch.save(optimizer_saved_dict, f"{filename}.optim") def load(self, filename): - self.filename = filename + self.filename = filename if os.path.exists(filename) else os.path.join(shared.opts.hypernetwork_dir, filename) if self.name is None: - self.name = os.path.splitext(os.path.basename(filename))[0] - with progress.open(filename, 'rb', description=f'Loading hypernetwork: [cyan]{filename}', auto_refresh=True, console=shared.console) as f: + self.name = os.path.splitext(os.path.basename(self.filename))[0] + with progress.open(self.filename, 'rb', description=f'Loading hypernetwork: [cyan]{self.filename}', auto_refresh=True, console=shared.console) as f: state_dict = torch.load(f, map_location='cpu') self.layer_structure = state_dict.get('layer_structure', [1, 2, 1]) self.optional_info = state_dict.get('optional_info', None) From d833b853ffc3111e9fa6bb3408dcffad39aebeda Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 15 Sep 2023 13:41:56 -0400 Subject: [PATCH 10/37] downgrade diffusers --- CHANGELOG.md | 5 +++++ modules/processing_diffusers.py | 12 ++++++++++-- modules/prompt_parser_diffusers.py | 3 ++- modules/sd_models.py | 2 ++ requirements.txt | 2 +- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1137f59c9..dd92b2c7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log for SD.Next +## Update for 2023-09-15 + +Downgrade of `diffusers` to 0.20.2 due to critical issue with model offloading +This means that new model **Wuerstchen** is not supported until diffusers issue is resolved + ## Update for 2023-09-13 Started as a mostly a service release with quite a few fixes, but then... diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 8ef2fe638..94a3a88be 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -187,8 +187,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_embed = None negative_pooled = None prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) - if shared.opts.prompt_attention in {'Compel parser', 'Full parser'} and 'StableDiffusion' in model.__class__.__name__: - prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, prompts_2, negative_prompts_2, is_refiner, kwargs.pop("clip_skip", None)) + parser = 'Fixed attention' + if shared.opts.prompt_attention != 'Fixed attention' and 'StableDiffusion' in model.__class__.__name__: + try: + prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, prompts_2, negative_prompts_2, is_refiner, kwargs.pop("clip_skip", None)) + parser = shared.opts.prompt_attention + except Exception as e: + shared.log.error(f'Prompt parser: {e}') if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None: if type(pooled) == list: @@ -244,7 +249,10 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if 'negative_pooled_prompt_embeds' in clean: clean['negative_pooled_prompt_embeds'] = clean['negative_pooled_prompt_embeds'].shape if torch.is_tensor(clean['negative_pooled_prompt_embeds']) else type(clean['negative_pooled_prompt_embeds']) clean['generator'] = generator_device + clean['parser'] = parser shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} task={sd_models.get_diffusers_task(model)} set={clean}') + # components = [{ k: getattr(v, 'device', None) } for k, v in model.components.items()] + # shared.log.debug(f'Diffuser pipeline components: {components}') return args def recompile_model(hires=False): diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 57cbf7f17..4e1c8da05 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -179,6 +179,7 @@ def compel_encode_prompt( return prompt_embed, positive_pooled, negative_embed, negative_pooled # neither base+sdxl nor refiner+sdxl - positive, negative = compel_te1(prompt), compel_te1(negative_prompt) + positive = compel_te1(prompt) + negative = compel_te1(negative_prompt) [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length([positive, negative]) return prompt_embed, None, negative_embed, None diff --git a/modules/sd_models.py b/modules/sd_models.py index 81df6963a..951c1217a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -762,6 +762,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No diffusers_load_config.pop('safety_checker', None) diffusers_load_config.pop('requires_safety_checker', None) diffusers_load_config.pop('load_safety_checker', None) + diffusers_load_config.pop('config_files', None) + diffusers_load_config.pop('local_files_only', None) shared.log.debug(f'Setting {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access except Exception as e: shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') diff --git a/requirements.txt b/requirements.txt index 67aad81ef..d926ae426 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ requests==2.31.0 tqdm==4.66.1 accelerate==0.20.3 opencv-python-headless==4.7.0.72 -diffusers==0.21.1 +diffusers==0.20.2 einops==0.4.1 gradio==3.43.2 huggingface_hub==0.17.1 From 71631b7a368083507ed7cba3bb438df887dc422d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 16 Sep 2023 13:48:22 +0300 Subject: [PATCH 11/37] Diffusers add DPM SDE sampler --- modules/paths.py | 2 +- modules/sd_models.py | 2 +- modules/sd_samplers_diffusers.py | 3 +++ modules/shared.py | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/paths.py b/modules/paths.py index a502be0cf..ee9e9d194 100644 --- a/modules/paths.py +++ b/modules/paths.py @@ -71,7 +71,7 @@ def create_paths(opts, log=None): try: relpath = os.path.relpath(fullpath, script_path) opts.data[folder] = relpath - except: + except Exception: opts.data[folder] = fullpath return opts.data[folder] diff --git a/modules/sd_models.py b/modules/sd_models.py index 951c1217a..b9acd87a4 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1211,7 +1211,7 @@ def apply_token_merging(sd_model, token_merging_ratio=0): ) shared.log.debug(f'Applying token merging: ratio={token_merging_ratio}') sd_model.applied_token_merged_ratio = token_merging_ratio - except: + except Exception: shared.log.warning(f'Token merging not supported: pipeline={sd_model.__class__.__name__}') else: sd_model.applied_token_merged_ratio = 0 diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 7e34b30ed..ec9388588 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -8,6 +8,7 @@ try: DEISMultistepScheduler, DPMSolverMultistepScheduler, DPMSolverSinglestepScheduler, + DPMSolverSDEScheduler, EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, HeunDiscreteScheduler, @@ -30,6 +31,7 @@ config = { 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True }, 'DPM 1S': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, 'DPM 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, + 'DPM SDE': { 'use_karras_sigmas': False }, 'Euler a': { }, 'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False }, 'Heun': { 'use_karras_sigmas': False }, @@ -52,6 +54,7 @@ samplers_data_diffusers = [ sd_samplers_common.SamplerData('KDPM2 a', lambda model: DiffusionSampler('KDPM2 a', KDPM2AncestralDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('DPM 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverSinglestepScheduler, model), [], {}), sd_samplers_common.SamplerData('DPM 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + sd_samplers_common.SamplerData('DPM SDE', lambda model: DiffusionSampler('DPM SDE', DPMSolverSDEScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), diff --git a/modules/shared.py b/modules/shared.py index 252baa251..bc8f896b5 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -558,7 +558,7 @@ options_templates.update(options_section(('live-preview', "Live Previews"), { })) options_templates.update(options_section(('sampler-params', "Sampler Settings"), { - "show_samplers": OptionInfo(["Default", "Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), + "show_samplers": OptionInfo(["Default", "Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM SDE", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}), @@ -570,7 +570,7 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), "schedulers_use_karras": OptionInfo(True, "Samplers use Karras sigmas where applicable"), "schedulers_use_loworder": OptionInfo(True, "Samplers use simplified solvers in final steps where applicable"), "schedulers_use_thresholding": OptionInfo(False, "Samplers use dynamic thresholding where applicable"), - "schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "Samplers DPM solver algorithm", gr.Radio, lambda: {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver++']}), + "schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "Samplers DPM solver algorithm", gr.Radio, lambda: {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver', 'sde-dpmsolver++']}), "schedulers_beta_schedule": OptionInfo("default", "Samplers override beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), 'schedulers_beta_start': OptionInfo(0, "Samplers override beta start", gr.Number, {}), 'schedulers_beta_end': OptionInfo(0, "Samplers override beta end", gr.Number, {}), From b13494a1421ad7f73143f929222c556c115a8867 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 16 Sep 2023 14:38:55 +0300 Subject: [PATCH 12/37] Diffusers fix hires sampler --- modules/processing_diffusers.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 94a3a88be..d913c4291 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -281,9 +281,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro pass #Do nothing if compile is disabled is_karras_compatible = shared.sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers - use_sampler = p.sampler_name if not p.is_hr_pass else p.latent_sampler - if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != use_sampler) and (use_sampler != 'Default') and is_karras_compatible: - sampler = sd_samplers.all_samplers_map.get(use_sampler, None) + if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name) and (p.sampler_name != 'Default') and is_karras_compatible: + sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op @@ -380,6 +379,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro output.images = hires_resize(latents=output.images) if latent_scale_mode is not None or p.hr_force: p.ops.append('hires') + if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.latent_sampler) and (p.latent_sampler != 'Default') and is_karras_compatible: + sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) + if sampler is None: + sampler = sd_samplers.all_samplers_map.get("UniPC") + sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op recompile_model(hires=True) sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) hires_args = set_pipeline_args( @@ -412,7 +416,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.sd_model.to(devices.cpu) devices.torch_gc() - if (not hasattr(shared.sd_refiner.scheduler, 'name')) or (shared.sd_refiner.scheduler.name != p.latent_sampler) and (p.sampler_name != 'Default'): + if (not hasattr(shared.sd_refiner.scheduler, 'name')) or (shared.sd_refiner.scheduler.name != p.latent_sampler) and (p.latent_sampler != 'Default'): sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") From 793b5f492166b0f4a4a7094d3b843e9fe1bde24f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 16 Sep 2023 16:10:02 +0300 Subject: [PATCH 13/37] OpenVINO Lora support --- installer.py | 4 +- modules/intel/openvino/__init__.py | 248 +++++++++++++++++++---------- modules/lora_diffusers.py | 4 + modules/processing_diffusers.py | 20 +-- modules/sd_models.py | 9 +- modules/shared.py | 1 + 6 files changed, 188 insertions(+), 98 deletions(-) diff --git a/installer.py b/installer.py index 1f853cc24..ae227723d 100644 --- a/installer.py +++ b/installer.py @@ -429,8 +429,8 @@ def check_torch(): torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0a0 intel_extension_for_pytorch==2.0.110+gitba7f6c1 -f https://developer.intel.com/ipex-whl-stable-xpu') elif allow_openvino and args.use_openvino: #Remove this after 2.1.0 releases - log.info('Using OpenVINO with Torch Nightly CPU') - torch_command = os.environ.get('TORCH_COMMAND', '--pre torch==2.1.0.dev20230713+cpu torchvision==0.16.0.dev20230713+cpu -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html') + log.info('Using OpenVINO') + torch_command = os.environ.get('TORCH_COMMAND', '--pre torch==2.1.0.dev20230726+cpu torchvision==0.16.0.dev20230726+cpu -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html') else: machine = platform.machine() if sys.platform == 'darwin': diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index b6bcec4ab..6162b674b 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -7,24 +7,20 @@ from torch._dynamo.backends.common import fake_tensor_unsupported from torch._dynamo.backends.registry import register_backend from torch.fx.experimental.proxy_tensor import make_fx from torch._inductor.compile_fx import compile_fx +from torch.utils._pytree import tree_flatten from hashlib import sha256 +import functools from modules import shared, devices -@register_backend -@fake_tensor_unsupported -def openvino_fx(subgraph, example_inputs): - executor_parameters = None +def openvino_clear_caches(): + global partitioned_modules + global compiled_cache + + compiled_cache.clear() + partitioned_modules.clear() + +def get_device(): core = Core() - if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0": - os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1") - model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest() - executor_parameters = {"model_hash_str": model_hash_str} - - example_inputs.reverse() - cache_root = "./cache/" - if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None: - cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR") - if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None: device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") elif any(openvino_cpu in cpu_module.lower() for cpu_module in shared.cmd_opts.use_cpu for openvino_cpu in ["openvino", "all"]): @@ -43,83 +39,169 @@ def openvino_fx(subgraph, example_inputs): os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device) shared.log.debug(f"OpenVINO Device: {device}") if shared.opts.cuda_compile_errors and device not in core.available_devices: - shared.log.warning(f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices") + shared.log.error(f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices") assert device in core.available_devices, f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices" - #Cache saving keeps increasing the partition id - #This loop check if non 0 partition id caches exist - #Takes 0.002 seconds when nothing is found - use_cached_file = False - for i in range(100): - file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root) - if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"): - use_cached_file = True - break + return device - if use_cached_file: - om = core.read_model(file_name + ".xml") +def cache_root_path(): + cache_root = "./cache/" + if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None: + cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR") + return cache_root - dtype_mapping = { - torch.float32: Type.f32, - torch.float64: Type.f64, - torch.float16: Type.f16, - torch.int64: Type.i64, - torch.int32: Type.i32, - torch.uint8: Type.u8, - torch.int8: Type.i8, - torch.bool: Type.boolean - } +def cached_model_name(model_hash_str, device, args, cache_root, reversed = False): + if model_hash_str is None: + return None - for idx, input_data in enumerate(example_inputs): - om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) - om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) - om.validate_nodes_and_infer_types() + model_cache_dir = cache_root + "/model/" - if model_hash_str is not None: - core.set_property({'CACHE_DIR': cache_root + '/blob'}) + try: + os.makedirs(model_cache_dir, exist_ok=True) + file_name = model_cache_dir + model_hash_str + "_" + device + except OSError as error: + shared.log.error(f"Cache directory {cache_root} cannot be created. Model caching is disabled. Error: {error}") + return None - compiled_model = core.compile_model(om, device) - def _call(*args): - ov_inputs = [a.detach().cpu().numpy() for a in args] - ov_inputs.reverse() - res = compiled_model(ov_inputs) - result = [torch.from_numpy(res[out]) for out in compiled_model.outputs] - return result - return _call - else: - example_inputs.reverse() - model = make_fx(subgraph)(*example_inputs) - with devices.inference_context(): - model.eval() - partitioner = Partitioner() - compiled_model = partitioner.make_partitions(model) + inputs_str = "" + for input_data in args: + if reversed: + inputs_str = "_" + str(input_data.type()) + str(input_data.size())[11:-1].replace(" ", "") + inputs_str + else: + inputs_str += "_" + str(input_data.type()) + str(input_data.size())[11:-1].replace(" ", "") + inputs_str = sha256(inputs_str.encode('utf-8')).hexdigest() + file_name += inputs_str - def _call(*args): - res = execute(compiled_model, *args, executor="openvino", - executor_parameters=executor_parameters) - return res - return _call - - -def get_cached_file_name(*args, model_hash_str, device, cache_root): - file_name = None - if model_hash_str is not None: - model_cache_dir = cache_root + "/model/" - try: - os.makedirs(model_cache_dir, exist_ok=True) - file_name = model_cache_dir + model_hash_str + "_" + device - for input_data in args: - if file_name is not None: - file_name += "_" + str(input_data.type()) + str(input_data.size())[11:-1].replace(" ", "") - except OSError as error: - print("Cache directory ", cache_root, " cannot be created. Model caching is disabled. Error: ", error) - file_name = None - model_hash_str = None return file_name -def openvino_clear_caches(): - global partitioned_modules - global compiled_cache +def openvino_compile_cached_model(cached_model_path, *example_inputs): + core = Core() + om = core.read_model(cached_model_path + ".xml") - compiled_cache.clear() - partitioned_modules.clear() + dtype_mapping = { + torch.float32: Type.f32, + torch.float64: Type.f64, + torch.float16: Type.f16, + torch.int64: Type.i64, + torch.int32: Type.i32, + torch.uint8: Type.u8, + torch.int8: Type.i8, + torch.bool: Type.boolean + } + + for idx, input_data in enumerate(example_inputs): + om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) + om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) + om.validate_nodes_and_infer_types() + + core.set_property({'CACHE_DIR': cache_root_path() + '/blob'}) + + compiled_model = core.compile_model(om, get_device()) + + return compiled_model + +def execute_cached(compiled_model, *args): + model_state = shared.compiled_model_state + flat_args, _ = tree_flatten(args) + ov_inputs = [a.detach().cpu().numpy() for a in flat_args] + + if (model_state.cn_model == "None"): + ov_inputs.reverse() + + res = compiled_model(ov_inputs) + result = [torch.from_numpy(res[out]) for out in compiled_model.outputs] + return result + +def check_fully_supported(self, graph_module): + num_fused = 0 + for node in graph_module.graph.nodes: + if node.op == "call_module" and "fused_" in node.name: + num_fused += 1 + elif node.op != "placeholder" and node.op != "output": + return False + if num_fused == 1: + return True + return False + +Partitioner.check_fully_supported = functools.partial(check_fully_supported, Partitioner) + +@register_backend +@fake_tensor_unsupported +def openvino_fx(subgraph, example_inputs): + model_state = shared.compiled_model_state + executor_parameters = None + inputs_reversed = False + if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0": + os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1") + # Create a hash to be used for caching + model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest() + if (model_state.cn_model != "None" and model_state.partition_id == 0): + model_hash_str = model_hash_str + model_state.cn_model + + if (model_state.lora_model != "None"): + model_hash_str = model_hash_str + model_state.lora_model + + executor_parameters = {"model_hash_str": model_hash_str} + # Check if the model was fully supported and already cached + example_inputs.reverse() + inputs_reversed = True + maybe_fs_cached_name = cached_model_name(model_hash_str + "_fs", get_device(), example_inputs, cache_root_path()) + + if os.path.isfile(maybe_fs_cached_name + ".xml") and os.path.isfile(maybe_fs_cached_name + ".bin"): + if (model_state.cn_model != "None" and model_state.cn_model in maybe_fs_cached_name): + example_inputs_reordered = [] + if (os.path.isfile(maybe_fs_cached_name + ".txt")): + f = open(maybe_fs_cached_name + ".txt", "r") + for input_data in example_inputs: + shape = f.readline() + if (str(input_data.size()) != shape): + for idx1, input_data1 in enumerate(example_inputs): + if (str(input_data1.size()).strip() == str(shape).strip()): + example_inputs_reordered.append(example_inputs[idx1]) + example_inputs = example_inputs_reordered + + # Model is fully supported and already cached. Run the cached OV model directly. + compiled_model = openvino_compile_cached_model(maybe_fs_cached_name, *example_inputs) + + def _call(*args): + if (model_state.cn_model != "None" and model_state.cn_model in maybe_fs_cached_name): + args_reordered = [] + if (os.path.isfile(maybe_fs_cached_name + ".txt")): + f = open(maybe_fs_cached_name + ".txt", "r") + for input_data in args: + shape = f.readline() + if (str(input_data.size()) != shape): + for idx1, input_data1 in enumerate(args): + if (str(input_data1.size()).strip() == str(shape).strip()): + args_reordered.append(args[idx1]) + args = args_reordered + + res = execute_cached(compiled_model, *args) + model_state.partition_id = model_state.partition_id + 1 + return res + return _call + else: + maybe_fs_cached_name = None + + if inputs_reversed: + example_inputs.reverse() + model = make_fx(subgraph)(*example_inputs) + for node in model.graph.nodes: + if node.target == torch.ops.aten.mul_.Tensor: + node.target = torch.ops.aten.mul.Tensor + with torch.no_grad(): + model.eval() + partitioner = Partitioner() + compiled_model = partitioner.make_partitions(model) + + if executor_parameters is not None and 'model_hash_str' in executor_parameters: + # Check if the model is fully supported. + fully_supported = partitioner.check_fully_supported(compiled_model) + if fully_supported: + executor_parameters["model_hash_str"] += "_fs" + + def _call(*args): + res = execute(compiled_model, *args, executor="openvino", + executor_parameters=executor_parameters) #, file_name=maybe_fs_cached_name) + return res + return _call diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py index 82d7a10b0..72af05722 100644 --- a/modules/lora_diffusers.py +++ b/modules/lora_diffusers.py @@ -35,6 +35,8 @@ def unload_diffusers_lora(): lora_state['loaded'].clear() lora_state['all_loras'] = [] lora_state['multiplier'] = [] + if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": + shared.compiled_model_state.lora_model = "None" except Exception as e: shared.log.error(f"LoRA unload failed: {e}") @@ -74,6 +76,8 @@ def load_diffusers_lora(name, lora, strength = 1.0, num_loras = 1): lora_state['loaded'].append(f'{lora.filename}:{strength}') t1 = time.time() fuse = f'fuse={fuse:.2f}s' if fuse > 0 else '' + if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": + shared.compiled_model_state.lora_model = str(lora_state['loaded']) shared.log.info(f'LoRA loaded: {name} strength={strength} loader="{shared.opts.diffusers_lora_loader}" lora={t1-t0:.2f}s {fuse}') except Exception as e: lines = str(e).splitlines() diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d913c4291..6b03640f3 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -260,9 +260,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.opts.cuda_compile_backend == "openvino_fx": compile_height = p.height if not hires else p.hr_upscale_to_y compile_width = p.width if not hires else p.hr_upscale_to_x - if (not hasattr(shared.sd_model, "compiled_model_state") or (not shared.sd_model.compiled_model_state.first_pass - and (shared.sd_model.compiled_model_state.height != compile_height or shared.sd_model.compiled_model_state.width != compile_width - or shared.sd_model.compiled_model_state.batch_size != p.batch_size))): + if (shared.compiled_model_state is None or (not shared.compiled_model_state.first_pass + and (shared.compiled_model_state.height != compile_height or shared.compiled_model_state.width != compile_width + or shared.compiled_model_state.batch_size != p.batch_size))): shared.log.info("OpenVINO: Resolution change detected") shared.log.info("OpenVINO: Recompiling base model") sd_models.unload_model_weights(op='model') @@ -271,15 +271,17 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.log.info("OpenVINO: Recompiling refiner") sd_models.unload_model_weights(op='refiner') sd_models.reload_model_weights(op='refiner') - shared.sd_model.compiled_model_state.height = compile_height - shared.sd_model.compiled_model_state.width = compile_width - shared.sd_model.compiled_model_state.batch_size = p.batch_size - shared.sd_model.compiled_model_state.first_pass = False + shared.compiled_model_state.height = compile_height + shared.compiled_model_state.width = compile_width + shared.compiled_model_state.batch_size = p.batch_size + shared.compiled_model_state.first_pass = False else: pass #Can be implemented for TensorRT or Olive else: pass #Do nothing if compile is disabled + recompile_model() + is_karras_compatible = shared.sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name) and (p.sampler_name != 'Default') and is_karras_compatible: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) @@ -316,8 +318,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro unload_diffusers_lora() return results - recompile_model() - if shared.opts.diffusers_move_base and not shared.sd_model.has_accelerate: shared.sd_model.to(devices.device) @@ -379,12 +379,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro output.images = hires_resize(latents=output.images) if latent_scale_mode is not None or p.hr_force: p.ops.append('hires') + recompile_model(hires=True) if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.latent_sampler) and (p.latent_sampler != 'Default') and is_karras_compatible: sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op - recompile_model(hires=True) sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) hires_args = set_pipeline_args( model=shared.sd_model, diff --git a/modules/sd_models.py b/modules/sd_models.py index b9acd87a4..f27068914 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -118,10 +118,13 @@ class CheckpointInfo: #Used by OpenVINO, can be used with TensorRT or Olive class CompiledModelState: def __init__(self): + self.first_pass = True self.height = 512 self.width = 512 self.batch_size = 1 - self.first_pass = True + self.partition_id = 0 + self.cn_model = "None" + self.lora_model = "None" class NoWatermark: @@ -888,8 +891,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No from modules.intel.openvino import openvino_fx, openvino_clear_caches # pylint: disable=unused-import openvino_clear_caches() torch._dynamo.eval_frame.check_if_dynamo_supported = lambda: True # pylint: disable=protected-access - sd_model.compiled_model_state = CompiledModelState() - sd_model.compiled_model_state.first_pass = True if not shared.opts.cuda_compile_precompile else False + shared.compiled_model_state = CompiledModelState() + shared.compiled_model_state.first_pass = True if not shared.opts.cuda_compile_precompile else False log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index bc8f896b5..616ea76d9 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -1048,4 +1048,5 @@ sd_model = None sd_refiner = None sd_model_type = '' sd_refiner_type = '' +compiled_model_state = None sys.modules[__name__].__class__ = Shared From 214d14ef53c6f732391aa53a19d4df3cdae3b4ea Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 10:40:56 -0400 Subject: [PATCH 14/37] fix double before-hires save --- modules/processing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/processing.py b/modules/processing.py index b5e437097..2acca730f 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -1017,7 +1017,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) x_sample = validate_sample(x_sample) image = Image.fromarray(x_sample) - save_intermediate(image, i) image = images.resize_image(1, image, target_width, target_height, upscaler_name=self.hr_upscaler) image = np.array(image).astype(np.float32) / 255.0 image = np.moveaxis(image, 2, 0) From 4e209fe87fd11c5c59fcdfb80b43b43567c51be4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 16 Sep 2023 20:05:53 +0300 Subject: [PATCH 15/37] OpenVINO fix caching and recompile when using Lora --- extensions-builtin/Lora/lora.py | 20 +++ modules/intel/openvino/__init__.py | 244 ++++++++++++++++++++++++----- modules/lora_diffusers.py | 8 +- modules/processing_diffusers.py | 8 +- modules/sd_models.py | 89 ++++++----- 5 files changed, 286 insertions(+), 83 deletions(-) diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py index 02bb2092c..f23bc6b76 100644 --- a/extensions-builtin/Lora/lora.py +++ b/extensions-builtin/Lora/lora.py @@ -239,6 +239,22 @@ def load_loras(names, multipliers=None): failed_to_load_loras = [] + recompile_model = False + if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": + if len(names) == len(shared.compiled_model_state.lora_model): + for i, name in enumerate(names): + if shared.compiled_model_state.lora_model[i] != f"{name}:{multipliers[i]}": + recompile_model = True + break + else: + recompile_model = True + shared.compiled_model_state.lora_model = [] + if recompile_model: + sd_models.unload_model_weights(op='model') + shared.opts.cuda_compile = False + sd_models.reload_model_weights(op='model') + shared.opts.cuda_compile = True + for i, name in enumerate(names): lora = already_loaded.get(name, None) if shared.backend == shared.Backend.ORIGINAL else None lora_on_disk = loras_on_disk[i] @@ -265,6 +281,10 @@ def load_loras(names, multipliers=None): if len(failed_to_load_loras) > 0: sd_hijack.model_hijack.comments.append("Failed to find Loras: " + ", ".join(failed_to_load_loras)) + + if recompile_model: + shared.log.info("Lora: Recompiling model") + shared.sd_model = sd_models.compile_diffusers(shared.sd_model) def lora_calc_updown(lora, module, target): diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 6162b674b..88b43610d 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -1,23 +1,50 @@ import os import torch -from openvino.frontend.pytorch.torchdynamo.execute import execute, partitioned_modules, compiled_cache +from openvino.frontend import FrontEndManager +from openvino.frontend.pytorch.fx_decoder import TorchFXPythonDecoder from openvino.frontend.pytorch.torchdynamo.partition import Partitioner -from openvino.runtime import Core, Type, PartialShape +from openvino.runtime import Core, Type, PartialShape, serialize from torch._dynamo.backends.common import fake_tensor_unsupported from torch._dynamo.backends.registry import register_backend from torch.fx.experimental.proxy_tensor import make_fx -from torch._inductor.compile_fx import compile_fx from torch.utils._pytree import tree_flatten +from types import MappingProxyType from hashlib import sha256 import functools from modules import shared, devices -def openvino_clear_caches(): - global partitioned_modules - global compiled_cache +compiled_cache = {} +max_openvino_partitions = 0 +partitioned_modules = {} - compiled_cache.clear() - partitioned_modules.clear() +DEFAULT_OPENVINO_PYTHON_CONFIG = MappingProxyType( + { + "use_python_fusion_cache": True, + "allow_single_op_fusion": True, + }, +) + +class OpenVINOGraphModule(torch.nn.Module): + def __init__(self, gm, partition_id, use_python_fusion_cache, model_hash_str: str = None, file_name=""): + super().__init__() + self.gm = gm + self.partition_id = partition_id + self.executor_parameters = {"use_python_fusion_cache": use_python_fusion_cache, + "model_hash_str": model_hash_str} + self.file_name = file_name + self.perm_fallback = False + + def __call__(self, *args): + #if self.perm_fallback: + # return self.gm(*args) + + #try: + result = openvino_execute(self.gm, *args, executor_parameters=self.executor_parameters, partition_id=self.partition_id, file_name=self.file_name) + #except Exception: + # self.perm_fallback = True + # return self.gm(*args) + + return result def get_device(): core = Core() @@ -74,6 +101,107 @@ def cached_model_name(model_hash_str, device, args, cache_root, reversed = False return file_name +def check_fully_supported(self, graph_module): + num_fused = 0 + for node in graph_module.graph.nodes: + if node.op == "call_module" and "fused_" in node.name: + num_fused += 1 + elif node.op != "placeholder" and node.op != "output": + return False + if num_fused == 1: + return True + return False + +Partitioner.check_fully_supported = functools.partial(check_fully_supported, Partitioner) + +def execute( + gm, + *args, + executor = "openvino", + executor_parameters = None, + file_name = "" +): + if executor == "openvino": + return openvino_execute_partitioned(gm, *args, executor_parameters=executor_parameters, file_name=file_name) + elif executor == "strictly_openvino": + return openvino_execute(gm, *args, executor_parameters=executor_parameters, file_name=file_name) + + msg = "Received unexpected value for 'executor': {0}. Allowed values are: openvino, strictly_openvino.".format(executor) + raise ValueError(msg) + +def execute_cached(compiled_model, *args): + flat_args, _ = tree_flatten(args) + ov_inputs = [a.detach().cpu().numpy() for a in flat_args] + + if (shared.compiled_model_state.cn_model == []): + ov_inputs.reverse() + + res = compiled_model(ov_inputs) + result = [torch.from_numpy(res[out]) for out in compiled_model.outputs] + return result + +def openvino_clear_caches(): + global partitioned_modules + global compiled_cache + + compiled_cache.clear() + partitioned_modules.clear() + +def openvino_compile(gm, *args, model_hash_str: str = None, file_name=""): + core = Core() + + device = get_device() + cache_root = cache_root_path() + + if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"): + om = core.read_model(file_name + ".xml") + else: + fe_manager = FrontEndManager() + fe = fe_manager.load_by_framework("pytorch") + + input_shapes = [] + input_types = [] + for input_data in args: + input_types.append(input_data.type()) + input_shapes.append(input_data.size()) + + decoder = TorchFXPythonDecoder(gm, gm, input_shapes=input_shapes, input_types=input_types) + + im = fe.load(decoder) + + om = fe.convert(im) + + if (file_name is not None): + serialize(om, file_name + ".xml", file_name + ".bin") + if (shared.compiled_model_state.cn_model != []): + f = open(file_name + ".txt", "w") + for input_data in args: + f.write(str(input_data.size())) + f.write("\n") + f.close() + + dtype_mapping = { + torch.float32: Type.f32, + torch.float64: Type.f64, + torch.float16: Type.f16, + torch.int64: Type.i64, + torch.int32: Type.i32, + torch.uint8: Type.u8, + torch.int8: Type.i8, + torch.bool: Type.boolean + } + + for idx, input_data in enumerate(args): + om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) + om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) + om.validate_nodes_and_infer_types() + + if model_hash_str is not None: + core.set_property({'CACHE_DIR': cache_root + '/blob'}) + + compiled = core.compile_model(om, device) + return compiled + def openvino_compile_cached_model(cached_model_path, *example_inputs): core = Core() om = core.read_model(cached_model_path + ".xml") @@ -100,46 +228,92 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs): return compiled_model -def execute_cached(compiled_model, *args): - model_state = shared.compiled_model_state +def openvino_execute(gm, *args, executor_parameters=None, partition_id, file_name=""): + executor_parameters = executor_parameters or DEFAULT_OPENVINO_PYTHON_CONFIG + + use_cache = executor_parameters.get( + "use_python_fusion_cache", + DEFAULT_OPENVINO_PYTHON_CONFIG["use_python_fusion_cache"], + ) + global compiled_cache + + model_hash_str = executor_parameters.get("model_hash_str", None) + if model_hash_str is not None: + model_hash_str = model_hash_str + str(partition_id) + + if use_cache and (partition_id in compiled_cache): + compiled = compiled_cache[partition_id] + else: + if (shared.compiled_model_state.cn_model != [] and file_name is not None + and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin")): + compiled = openvino_compile_cached_model(file_name, *args) + else: + compiled = openvino_compile(gm, *args, model_hash_str=model_hash_str, file_name=file_name) + compiled_cache[partition_id] = compiled + flat_args, _ = tree_flatten(args) ov_inputs = [a.detach().cpu().numpy() for a in flat_args] - if (model_state.cn_model == "None"): - ov_inputs.reverse() + res = compiled(ov_inputs) - res = compiled_model(ov_inputs) - result = [torch.from_numpy(res[out]) for out in compiled_model.outputs] - return result + results1 = [torch.from_numpy(res[out]) for out in compiled.outputs] + if len(results1) == 1: + return results1[0] + return results1 -def check_fully_supported(self, graph_module): - num_fused = 0 - for node in graph_module.graph.nodes: +def openvino_execute_partitioned(gm, *args, executor_parameters=None, file_name=""): + executor_parameters = executor_parameters or DEFAULT_OPENVINO_PYTHON_CONFIG + + global partitioned_modules + + use_python_fusion_cache = executor_parameters.get( + "use_python_fusion_cache", + DEFAULT_OPENVINO_PYTHON_CONFIG["use_python_fusion_cache"], + ) + model_hash_str = executor_parameters.get("model_hash_str", None) + + signature = str(id(gm)) + for idx, input_data in enumerate(args): + if isinstance(input_data, torch.Tensor): + signature = signature + "_" + str(idx) + ":" + str(input_data.type())[6:] + ":" + str(input_data.size())[11:-1].replace(" ", "") + else: + signature = signature + "_" + str(idx) + ":" + type(input_data).__name__ + ":val(" + str(input_data) + ")" + + if signature not in partitioned_modules: + partitioned_modules[signature] = partition_graph(gm, use_python_fusion_cache=use_python_fusion_cache, + model_hash_str=model_hash_str, file_name=file_name) + + return partitioned_modules[signature](*args) + +def partition_graph(gm, use_python_fusion_cache: bool, model_hash_str: str = None, file_name=""): + global max_openvino_partitions + for node in gm.graph.nodes: if node.op == "call_module" and "fused_" in node.name: - num_fused += 1 - elif node.op != "placeholder" and node.op != "output": - return False - if num_fused == 1: - return True - return False + openvino_submodule = getattr(gm, node.name) + gm.delete_submodule(node.target) + gm.add_submodule( + node.target, + OpenVINOGraphModule(openvino_submodule, shared.compiled_model_state.partition_id, use_python_fusion_cache, + model_hash_str=model_hash_str, file_name=file_name), + ) + shared.compiled_model_state.partition_id = shared.compiled_model_state.partition_id + 1 -Partitioner.check_fully_supported = functools.partial(check_fully_supported, Partitioner) + return gm @register_backend @fake_tensor_unsupported def openvino_fx(subgraph, example_inputs): - model_state = shared.compiled_model_state executor_parameters = None inputs_reversed = False if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0": os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1") # Create a hash to be used for caching model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest() - if (model_state.cn_model != "None" and model_state.partition_id == 0): - model_hash_str = model_hash_str + model_state.cn_model + if (shared.compiled_model_state.cn_model != [] and shared.compiled_model_state.partition_id == 0): + model_hash_str = model_hash_str + str(shared.compiled_model_state.cn_model) - if (model_state.lora_model != "None"): - model_hash_str = model_hash_str + model_state.lora_model + if (shared.compiled_model_state.lora_model != []): + model_hash_str = model_hash_str + str(shared.compiled_model_state.lora_model) executor_parameters = {"model_hash_str": model_hash_str} # Check if the model was fully supported and already cached @@ -148,7 +322,7 @@ def openvino_fx(subgraph, example_inputs): maybe_fs_cached_name = cached_model_name(model_hash_str + "_fs", get_device(), example_inputs, cache_root_path()) if os.path.isfile(maybe_fs_cached_name + ".xml") and os.path.isfile(maybe_fs_cached_name + ".bin"): - if (model_state.cn_model != "None" and model_state.cn_model in maybe_fs_cached_name): + if (shared.compiled_model_state.cn_model != [] and str(shared.compiled_model_state.cn_model) in maybe_fs_cached_name): example_inputs_reordered = [] if (os.path.isfile(maybe_fs_cached_name + ".txt")): f = open(maybe_fs_cached_name + ".txt", "r") @@ -164,7 +338,7 @@ def openvino_fx(subgraph, example_inputs): compiled_model = openvino_compile_cached_model(maybe_fs_cached_name, *example_inputs) def _call(*args): - if (model_state.cn_model != "None" and model_state.cn_model in maybe_fs_cached_name): + if (shared.compiled_model_state.cn_model != [] and str(shared.compiled_model_state.cn_model) in maybe_fs_cached_name): args_reordered = [] if (os.path.isfile(maybe_fs_cached_name + ".txt")): f = open(maybe_fs_cached_name + ".txt", "r") @@ -177,11 +351,11 @@ def openvino_fx(subgraph, example_inputs): args = args_reordered res = execute_cached(compiled_model, *args) - model_state.partition_id = model_state.partition_id + 1 + shared.compiled_model_state.partition_id = shared.compiled_model_state.partition_id + 1 return res return _call else: - maybe_fs_cached_name = None + maybe_fs_cached_name = "" if inputs_reversed: example_inputs.reverse() @@ -202,6 +376,6 @@ def openvino_fx(subgraph, example_inputs): def _call(*args): res = execute(compiled_model, *args, executor="openvino", - executor_parameters=executor_parameters) #, file_name=maybe_fs_cached_name) + executor_parameters=executor_parameters, file_name=maybe_fs_cached_name) return res return _call diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py index 72af05722..c6ce96d1e 100644 --- a/modules/lora_diffusers.py +++ b/modules/lora_diffusers.py @@ -35,8 +35,6 @@ def unload_diffusers_lora(): lora_state['loaded'].clear() lora_state['all_loras'] = [] lora_state['multiplier'] = [] - if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": - shared.compiled_model_state.lora_model = "None" except Exception as e: shared.log.error(f"LoRA unload failed: {e}") @@ -58,6 +56,8 @@ def load_diffusers_lora(name, lora, strength = 1.0, num_loras = 1): pipe.fuse_lora(lora_scale=strength) fuse = time.time() - t2 lora_state['loaded'].append(f'{lora.filename}:{strength}') + if shared.compiled_model_state is not None: #filename breaks caching + shared.compiled_model_state.lora_model.append(f'{name}:{strength}') else: from safetensors.torch import load_file lora_sd = load_file(lora.filename) @@ -74,10 +74,10 @@ def load_diffusers_lora(name, lora, strength = 1.0, num_loras = 1): lora_network.apply_to(multiplier=strength) lora_state['all_loras'].append(lora_network) lora_state['loaded'].append(f'{lora.filename}:{strength}') + if shared.compiled_model_state is not None: #filename breaks caching + shared.compiled_model_state.lora_model.append(f'{name}:{strength}') t1 = time.time() fuse = f'fuse={fuse:.2f}s' if fuse > 0 else '' - if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": - shared.compiled_model_state.lora_model = str(lora_state['loaded']) shared.log.info(f'LoRA loaded: {name} strength={strength} loader="{shared.opts.diffusers_lora_loader}" lora={t1-t0:.2f}s {fuse}') except Exception as e: lines = str(e).splitlines() diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 6b03640f3..c52193e5b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -260,10 +260,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.opts.cuda_compile_backend == "openvino_fx": compile_height = p.height if not hires else p.hr_upscale_to_y compile_width = p.width if not hires else p.hr_upscale_to_x - if (shared.compiled_model_state is None or (not shared.compiled_model_state.first_pass - and (shared.compiled_model_state.height != compile_height or shared.compiled_model_state.width != compile_width + if (shared.compiled_model_state is None or + (not shared.compiled_model_state.first_pass + and (shared.compiled_model_state.height != compile_height + or shared.compiled_model_state.width != compile_width or shared.compiled_model_state.batch_size != p.batch_size))): - shared.log.info("OpenVINO: Resolution change detected") + shared.log.info("OpenVINO: Parameter change detected") shared.log.info("OpenVINO: Recompiling base model") sd_models.unload_model_weights(op='model') sd_models.reload_model_weights(op='model') diff --git a/modules/sd_models.py b/modules/sd_models.py index f27068914..36f00d0af 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -123,8 +123,8 @@ class CompiledModelState: self.width = 512 self.batch_size = 1 self.partition_id = 0 - self.cn_model = "None" - self.lora_model = "None" + self.cn_model = [] + self.lora_model = [] class NoWatermark: @@ -657,6 +657,50 @@ def detect_pipeline(f: str, op: str = 'model'): pipeline = None, None return pipeline, guess +def compile_diffusers(sd_model): + try: + if shared.opts.ipex_optimize: + import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import + sd_model.unet.training = False + sd_model.unet = ipex.optimize(sd_model.unet, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'vae'): + sd_model.vae.training = False + sd_model.vae = ipex.optimize(sd_model.vae, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'movq'): + sd_model.movq.training = False + sd_model.movq = ipex.optimize(sd_model.movq, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init + shared.log.info("Applied IPEX Optimize.") + except Exception as err: + shared.log.warning(f"IPEX Optimize not supported: {err}") + + try: + if shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none': + shared.log.info(f"Compiling pipeline={sd_model.__class__.__name__} shape={8 * sd_model.unet.config.sample_size} mode={shared.opts.cuda_compile_backend}") + import torch._dynamo # pylint: disable=unused-import,redefined-outer-name + if shared.opts.cuda_compile_backend == "openvino_fx": + torch._dynamo.reset() # pylint: disable=protected-access + from modules.intel.openvino import openvino_fx, openvino_clear_caches # pylint: disable=unused-import + openvino_clear_caches() + torch._dynamo.eval_frame.check_if_dynamo_supported = lambda: True # pylint: disable=protected-access + if shared.compiled_model_state is None: + shared.compiled_model_state = CompiledModelState() + shared.compiled_model_state.first_pass = True if not shared.opts.cuda_compile_precompile else False + log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access + if hasattr(torch, '_logging'): + torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access + torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access + torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access + sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'vae'): + sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + if hasattr(sd_model, 'movq'): + sd_model.movq.decode = torch.compile(sd_model.movq.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init + if shared.opts.cuda_compile_precompile: + sd_model("dummy prompt") + shared.log.info("Complilation done.") + return sd_model + except Exception as err: + shared.log.warning(f"Model compile not supported: {err}") def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument import torch # pylint: disable=reimported,redefined-outer-name @@ -869,45 +913,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No base_sent_to_cpu=True elif not sd_model.has_accelerate: sd_model.to(devices.device) - try: - if shared.opts.ipex_optimize: - sd_model.unet.training = False - sd_model.unet = torch.xpu.optimize(sd_model.unet, dtype=devices.dtype_unet, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init - if hasattr(sd_model, 'vae'): - sd_model.vae.training = False - sd_model.vae = torch.xpu.optimize(sd_model.vae, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init - if hasattr(sd_model, 'movq'): - sd_model.movq.training = False - sd_model.movq = torch.xpu.optimize(sd_model.movq, dtype=devices.dtype_vae, inplace=True, weights_prepack=False) # pylint: disable=attribute-defined-outside-init - shared.log.info("Applied IPEX Optimize.") - except Exception as err: - shared.log.warning(f"IPEX Optimize not supported: {err}") - try: - if shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none': - shared.log.info(f"Compiling pipeline={sd_model.__class__.__name__} shape={8 * sd_model.unet.config.sample_size} mode={shared.opts.cuda_compile_backend}") - import torch._dynamo # pylint: disable=unused-import,redefined-outer-name - if shared.opts.cuda_compile_backend == "openvino_fx": - torch._dynamo.reset() # pylint: disable=protected-access - from modules.intel.openvino import openvino_fx, openvino_clear_caches # pylint: disable=unused-import - openvino_clear_caches() - torch._dynamo.eval_frame.check_if_dynamo_supported = lambda: True # pylint: disable=protected-access - shared.compiled_model_state = CompiledModelState() - shared.compiled_model_state.first_pass = True if not shared.opts.cuda_compile_precompile else False - log_level = logging.WARNING if shared.opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access - if hasattr(torch, '_logging'): - torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access - torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access - torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access - sd_model.unet = torch.compile(sd_model.unet, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init - if hasattr(sd_model, 'vae'): - sd_model.vae.decode = torch.compile(sd_model.vae.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init - if hasattr(sd_model, 'movq'): - sd_model.movq.decode = torch.compile(sd_model.movq.decode, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph=shared.opts.cuda_compile_fullgraph) # pylint: disable=attribute-defined-outside-init - if shared.opts.cuda_compile_precompile: - sd_model("dummy prompt") - shared.log.info("Complilation done.") - except Exception as err: - shared.log.warning(f"Model compile not supported: {err}") + + sd_model = compile_diffusers(sd_model) if sd_model is None: shared.log.error('Diffuser model not loaded') From 3c3b3b92912ba6600ba26735439724d0f68a7d15 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 13:46:22 -0400 Subject: [PATCH 16/37] simplify ti loading --- modules/processing.py | 6 ++++-- modules/processing_diffusers.py | 9 +++------ modules/sd_models.py | 6 +++--- modules/sd_samplers_compvis.py | 4 ---- modules/shared.py | 2 +- modules/textual_inversion/textual_inversion.py | 5 ----- 6 files changed, 11 insertions(+), 21 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 2acca730f..d4d635497 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -686,8 +686,8 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.all_subseeds = subseed else: p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))] - if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings: - modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings() + if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and shared.backend == shared.Backend.ORIGINAL: + modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=False) if p.scripts is not None: p.scripts.process(p) infotexts = [] @@ -990,6 +990,8 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE) latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None") + if latent_scale_mode is not None: + self.hr_force = False # no need to force anything if self.enable_hr and (latent_scale_mode is None or self.hr_force): if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0: shared.log.warning(f"Cannot find upscaler for hires: {self.hr_upscaler}") diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index c52193e5b..4c64c79a1 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -3,6 +3,7 @@ import inspect import typing import torch import torchvision.transforms.functional as TF +import diffusers import modules.devices as devices import modules.shared as shared import modules.sd_samplers as sd_samplers @@ -15,12 +16,6 @@ from modules.processing import StableDiffusionProcessing import modules.prompt_parser_diffusers as prompt_parser_diffusers -try: - import diffusers -except Exception as ex: - shared.log.error(f'Failed to import diffusers: {ex}') - - def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_prompts): results = [] if p.enable_hr and p.hr_upscaler != 'None' and p.denoising_strength > 0 and len(getattr(p, 'init_images', [])) == 0: @@ -171,6 +166,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return prompts, negative_prompts, prompts_2, negative_prompts_2 def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): + if hasattr(model, 'embedding_db'): + del model.embedding_db try: is_refiner = model.text_encoder.__class__.__name__ != 'CLIPTextModel' except Exception: diff --git a/modules/sd_models.py b/modules/sd_models.py index 36f00d0af..95203739b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -780,7 +780,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.error(f'Failed loading {op}: {checkpoint_info.path} {e}') return else: - diffusers_load_config["local_files_only "] = True + diffusers_load_config["local_files_only"] = True diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema pipeline, model_type = detect_pipeline(checkpoint_info.path, op) if pipeline is None: @@ -873,10 +873,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.vae = vae if shared.opts.diffusers_vae_upcast != 'default': if shared.opts.diffusers_vae_upcast == 'true': - sd_model.vae.config["force_upcast"] = True + # sd_model.vae.config["force_upcast"] = True sd_model.vae.config.force_upcast = True else: - sd_model.vae.config["force_upcast"] = False + # sd_model.vae.config["force_upcast"] = False sd_model.vae.config.force_upcast = False if shared.opts.no_half_vae: devices.dtype_vae = torch.float32 diff --git a/modules/sd_samplers_compvis.py b/modules/sd_samplers_compvis.py index 19fe750a1..bd3e95c70 100644 --- a/modules/sd_samplers_compvis.py +++ b/modules/sd_samplers_compvis.py @@ -45,7 +45,6 @@ class VanillaStableDiffusionSampler: def launch_sampling(self, steps, func): state.sampling_steps = steps state.sampling_step = 0 - try: return func() except sd_samplers_common.InterruptedException: @@ -53,11 +52,8 @@ class VanillaStableDiffusionSampler: def p_sample_ddim_hook(self, x_dec, cond, ts, unconditional_conditioning, *args, **kwargs): x_dec, ts, cond, unconditional_conditioning = self.before_sample(x_dec, ts, cond, unconditional_conditioning) - res = self.orig_p_sample_ddim(x_dec, cond, ts, *args, unconditional_conditioning=unconditional_conditioning, **kwargs) - x_dec, ts, cond, unconditional_conditioning, res = self.after_sample(x_dec, ts, cond, unconditional_conditioning, res) - return res def before_sample(self, x, ts, cond, unconditional_conditioning): diff --git a/modules/shared.py b/modules/shared.py index 616ea76d9..7ee44b668 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -9,7 +9,6 @@ import urllib.request from urllib.parse import urlparse from enum import Enum import gradio as gr -import tqdm import fasteners from rich.console import Console from modules import errors, ui_components, shared_items, cmd_args @@ -887,6 +886,7 @@ def reload_gradio_theme(theme_name=None): class TotalTQDM: # compatibility with previous global-tqdm + # import tqdm def __init__(self): pass def reset(self): diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index a9e200822..b959034ef 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -159,11 +159,6 @@ class EmbeddingDatabase: self.register_embedding(embedding, shared.sd_model) except Exception: self.skipped_embeddings[name] = embedding - try: - text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() - text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] - except Exception: - text_inv_tokens = [] def load_from_file(self, path, filename): name, ext = os.path.splitext(filename) From 3fa354813821617ec88eed9aef56fd1ca0eddf36 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 16 Sep 2023 22:15:07 +0300 Subject: [PATCH 17/37] Diffusers fix compile --- modules/sd_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 95203739b..4ae268eea 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -698,9 +698,9 @@ def compile_diffusers(sd_model): if shared.opts.cuda_compile_precompile: sd_model("dummy prompt") shared.log.info("Complilation done.") - return sd_model except Exception as err: shared.log.warning(f"Model compile not supported: {err}") + return sd_model def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument import torch # pylint: disable=reimported,redefined-outer-name From cb43af03a2a3c48718b6b992b4b80d952b95c9c9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 16:44:48 -0400 Subject: [PATCH 18/37] get api extra-networks --- modules/api/api.py | 29 ++++++++++++++++++++++++ modules/api/models.py | 14 ++++++++++++ modules/lora_diffusers.py | 6 ++--- modules/processing.py | 2 +- modules/ui_extra_networks_checkpoints.py | 1 + 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/modules/api/api.py b/modules/api/api.py index 12c814be8..36a22e423 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -148,6 +148,7 @@ class Api: self.add_api_route("/sdapi/v1/scripts", self.get_scripts_list, methods=["GET"], response_model=models.ScriptsList) self.add_api_route("/sdapi/v1/script-info", self.get_script_info, methods=["GET"], response_model=List[models.ScriptInfo]) self.add_api_route("/sdapi/v1/log", self.get_log_buffer, methods=["GET"], response_model=List) # bypass auth + self.add_api_route("/sdapi/v1/extra-networks", self.get_extra_networks, methods=["GET"], response_model=List[models.ExtraNetworkItem]) self.default_script_arg_txt2img = [] self.default_script_arg_img2img = [] @@ -502,6 +503,34 @@ class Api: "skipped": convert_embeddings(db.skipped_embeddings), } + def get_extra_networks(self, page: Optional[str] = None, name: Optional[str] = None, filename: Optional[str] = None, title: Optional[str] = None, fullname: Optional[str] = None, hash: Optional[str] = None): # pylint: disable=redefined-builtin + import modules.ui_extra_networks + res = [] + for pg in modules.ui_extra_networks.extra_pages: + if page is not None and pg.name != page.lower(): + continue + for item in pg.items: + if name is not None and item.get('name', '') != name: + continue + if title is not None and item.get('title', '') != title: + continue + if filename is not None and item.get('filename', '') != filename: + continue + if fullname is not None and item.get('fullname', '') != fullname: + continue + if hash is not None and (item.get('shorthash', None) or item.get('hash')) != hash: + continue + res.append({ + 'name': item.get('name', ''), + 'type': pg.name, + 'title': item.get('title', None), + 'fullname': item.get('fullname', None), + 'filename': item.get('filename', None), + 'hash': item.get('shorthash', None) or item.get('hash'), + "preview": item.get('preview', None), + }) + return res + def refresh_checkpoints(self): return shared.refresh_checkpoints() diff --git a/modules/api/models.py b/modules/api/models.py index 6bd4eccfe..9241592eb 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -272,6 +272,20 @@ class StyleItem(BaseModel): filename: Optional[str] = Field(title="Filename") preview: Optional[str] = Field(title="Preview") +class ExtraNetworkItem(BaseModel): + name: str = Field(title="Name") + type: str = Field(title="Type") + title: Optional[str] = Field(title="Title") + fullname: Optional[str] = Field(title="Fullname") + filename: Optional[str] = Field(title="Filename") + hash: Optional[str] = Field(title="Hash") + preview: Optional[str] = Field(title="Preview image URL") + # description: Optional[str] = Field(title="Description") + # info: Optional[str] = Field(title="Information") + # metadata: Optional[Any] = Field(title="Metadata") + # local: Optional[str] = Field(title="Local") + + class ArtistItem(BaseModel): name: str = Field(title="Name") score: float = Field(title="Score") diff --git a/modules/lora_diffusers.py b/modules/lora_diffusers.py index c6ce96d1e..1e9890f02 100644 --- a/modules/lora_diffusers.py +++ b/modules/lora_diffusers.py @@ -15,7 +15,7 @@ def unload_diffusers_lora(): try: pipe = shared.sd_model if shared.opts.diffusers_lora_loader == "diffusers": - if len(lora_state['loaded']) > 1: + if len(lora_state['loaded']) > 1 and hasattr(pipe, "unfuse_lora"): pipe.unfuse_lora() pipe.unload_lora_weights() pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 @@ -51,7 +51,7 @@ def load_diffusers_lora(name, lora, strength = 1.0, num_loras = 1): fuse = 0 if shared.opts.diffusers_lora_loader.startswith("diffusers"): pipe.load_lora_weights(lora.filename, cache_dir=shared.opts.diffusers_dir, local_files_only=True, lora_scale=strength, low_cpu_mem_usage=True) - if num_loras > 1: + if num_loras > 1 and hasattr(pipe, "fuse_lora"): t2 = time.time() pipe.fuse_lora(lora_scale=strength) fuse = time.time() - t2 @@ -535,6 +535,6 @@ class LoRANetwork(torch.nn.Module): # pylint: disable=abstract-method for key in state_dict.keys(): if state_dict[key].size() != my_state_dict[key].size(): # pylint: disable=unsubscriptable-object # print(f"convert {key} from {state_dict[key].size()} to {my_state_dict[key].size()}") - state_dict[key] = state_dict[key].view(my_state_dict[key].size()) + state_dict[key] = state_dict[key].view(my_state_dict[key].size()) # pylint: disable=unsubscriptable-object return super().load_state_dict(state_dict, strict) diff --git a/modules/processing.py b/modules/processing.py index d4d635497..fe0e78e6a 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -1015,7 +1015,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)) lowres_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0) batch_images = [] - for i, x_sample in enumerate(lowres_samples): + for _i, x_sample in enumerate(lowres_samples): x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) x_sample = validate_sample(x_sample) image = Image.fromarray(x_sample) diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 7159445b5..4a4a18e1d 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -18,6 +18,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): path, _ext = os.path.splitext(checkpoint.filename) yield { "name": checkpoint.name_for_extra, + "title": checkpoint.title, "filename": path, "fullname": checkpoint.filename, "hash": checkpoint.shorthash, From 2878f66c574866497174401470a919059880627f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 16:53:10 -0400 Subject: [PATCH 19/37] fix paths --- modules/paths.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/paths.py b/modules/paths.py index ee9e9d194..d8791776a 100644 --- a/modules/paths.py +++ b/modules/paths.py @@ -68,6 +68,8 @@ def create_paths(opts, log=None): fullpath = os.path.join(data_path, tgt) if len(data_path) > 0 and os.path.isabs(data_path): return fullpath + if os.path.isabs(fullpath) and os.path.exists(fullpath): + return fullpath try: relpath = os.path.relpath(fullpath, script_path) opts.data[folder] = relpath From 9ca17487e0cf5339c91d0db04b535f022a65d423 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 17:09:51 -0400 Subject: [PATCH 20/37] diffusers round width/height --- modules/processing_diffusers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 4c64c79a1..9c47ae957 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -1,4 +1,5 @@ import time +import math import inspect import typing import torch @@ -305,13 +306,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro task_specific_kwargs={} if sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: p.ops.append('txt2img') - task_specific_kwargs = {"height": p.height, "width": p.width} + task_specific_kwargs = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: p.ops.append('img2img') task_specific_kwargs = {"image": p.init_images, "strength": p.denoising_strength} elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: p.ops.append('inpaint') - task_specific_kwargs = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": p.height, "width": p.width} + task_specific_kwargs = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} if shared.state.interrupted or shared.state.skipped: unload_diffusers_lora() From 12d6173b5779a4de8e90e8ec3e886182ca752f30 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 17:19:19 -0400 Subject: [PATCH 21/37] fix bad styles names --- modules/styles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/styles.py b/modules/styles.py index abeeb5e08..c9b8ca85f 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -1,7 +1,8 @@ # We need this so Python doesn't complain about the unknown StableDiffusionProcessing-typehint at runtime from __future__ import annotations -import csv +import re import os +import csv import json from installer import log from modules import paths @@ -9,7 +10,7 @@ from modules import paths class Style(): def __init__(self, name: str, prompt: str = "", negative_prompt: str = "", extra: str = "", filename: str = "", preview: str = ""): - self.name = name + self.name = re.sub(r'[\t\r\n]', '', name).strip() self.prompt = prompt self.negative_prompt = negative_prompt self.extra = extra From 2daf61ae7a60eeca06dfd4cbac6892e0ab244304 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 16 Sep 2023 17:36:03 -0400 Subject: [PATCH 22/37] presort en --- modules/sd_models.py | 3 +++ modules/styles.py | 1 + modules/ui_extra_networks_textual_inversion.py | 1 + 3 files changed, 5 insertions(+) diff --git a/modules/sd_models.py b/modules/sd_models.py index 4ae268eea..a7e8fec55 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -149,6 +149,7 @@ def checkpoint_tiles(use_short=False): # pylint: disable=unused-argument def list_models(): t0 = time.time() + global checkpoints_list # pylint: disable=global-statement checkpoints_list.clear() checkpoint_aliases.clear() if shared.opts.sd_disable_ckpt or shared.backend == shared.Backend.DIFFUSERS: @@ -175,6 +176,7 @@ def list_models(): shared.log.warning(f"Checkpoint not found: {shared.cmd_opts.ckpt}") shared.log.info(f'Available models: {shared.opts.ckpt_dir} items={len(checkpoints_list)} time={time.time()-t0:.2f}s') + checkpoints_list = dict(sorted(checkpoints_list.items(), key=lambda cp: cp[1].filename)) if len(checkpoints_list) == 0: if not shared.cmd_opts.no_download: key = input('Download the default model? (y/N) ') @@ -193,6 +195,7 @@ def list_models(): if checkpoint_info.name is not None: checkpoint_info.register() + def update_model_hashes(): txt = [] lst = [ckpt for ckpt in checkpoints_list.values() if ckpt.hash is None] diff --git a/modules/styles.py b/modules/styles.py index c9b8ca85f..8491a43a6 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -74,6 +74,7 @@ class StyleDatabase: list_folder(fn) list_folder(self.path) + self.styles = dict(sorted(self.styles.items(), key=lambda style: style[1].filename)) log.debug(f'Loaded styles: folder={self.path} items={len(self.styles.keys())}') def get_style_prompts(self, styles): diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py index 520c630d1..82b7e45c9 100644 --- a/modules/ui_extra_networks_textual_inversion.py +++ b/modules/ui_extra_networks_textual_inversion.py @@ -39,6 +39,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values()) else: embeddings = [] + embeddings = list(sorted(embeddings, key=lambda emb: emb.filename)) for embedding in embeddings: path, _ext = os.path.splitext(embedding.filename) tags = {} From 36e08235388e44a58a7d082d83ac833ede9d4c61 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 17 Sep 2023 01:28:25 +0300 Subject: [PATCH 23/37] Cleanup --- extensions-builtin/Lora/lora.py | 2 +- modules/intel/openvino/__init__.py | 2 +- modules/sd_models.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py index f23bc6b76..481ccd629 100644 --- a/extensions-builtin/Lora/lora.py +++ b/extensions-builtin/Lora/lora.py @@ -284,7 +284,7 @@ def load_loras(names, multipliers=None): if recompile_model: shared.log.info("Lora: Recompiling model") - shared.sd_model = sd_models.compile_diffusers(shared.sd_model) + sd_models.compile_diffusers(shared.sd_model) def lora_calc_updown(lora, module, target): diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 88b43610d..460062a7b 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -363,7 +363,7 @@ def openvino_fx(subgraph, example_inputs): for node in model.graph.nodes: if node.target == torch.ops.aten.mul_.Tensor: node.target = torch.ops.aten.mul.Tensor - with torch.no_grad(): + with devices.inference_context(): model.eval() partitioner = Partitioner() compiled_model = partitioner.make_partitions(model) diff --git a/modules/sd_models.py b/modules/sd_models.py index a7e8fec55..113fa2e63 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -703,7 +703,6 @@ def compile_diffusers(sd_model): shared.log.info("Complilation done.") except Exception as err: shared.log.warning(f"Model compile not supported: {err}") - return sd_model def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument import torch # pylint: disable=reimported,redefined-outer-name @@ -917,7 +916,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No elif not sd_model.has_accelerate: sd_model.to(devices.device) - sd_model = compile_diffusers(sd_model) + compile_diffusers(sd_model) if sd_model is None: shared.log.error('Diffuser model not loaded') From 3389edf6f8e8887a0634c3dde5d3fbd014b4a858 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 17 Sep 2023 03:46:16 +0300 Subject: [PATCH 24/37] Fix model unloading --- modules/sd_models.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 113fa2e63..5278015e6 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -593,7 +593,11 @@ model_data = ModelData() def change_backend(): shared.log.info(f'Backend changed: {shared.backend}') - unload_model_weights() + if shared.backend == shared.Backend.ORIGINAL: + change_from = shared.Backend.DIFFUSERS + else: + change_from = shared.Backend.ORIGINAL + unload_model_weights(change_from=change_from) checkpoints_loaded.clear() from modules.sd_samplers import list_samplers list_samplers(shared.backend) @@ -1176,21 +1180,21 @@ def disable_offload(sd_model): remove_hook_from_module(model, recurse=True) -def unload_model_weights(op='model'): +def unload_model_weights(op='model', change_from='none'): if op == 'model' or op == 'dict': if model_data.sd_model: - if shared.backend != shared.Backend.ORIGINAL: # moving from diffusers=>original + if (shared.backend == shared.Backend.ORIGINAL and change_from != shared.Backend.DIFFUSERS) or change_from == shared.Backend.ORIGINAL: from modules import sd_hijack model_data.sd_model.to(devices.cpu) sd_hijack.model_hijack.undo_hijack(model_data.sd_model) - else: # moving from original=>diffusers + else: disable_offload(model_data.sd_model) model_data.sd_model.to('meta') model_data.sd_model = None shared.log.debug(f'Unload weights {op}: {memory_stats()}') else: if model_data.sd_refiner: - if shared.backend != shared.Backend.ORIGINAL: + if (shared.backend == shared.Backend.ORIGINAL and change_from != shared.Backend.DIFFUSERS) or change_from == shared.Backend.ORIGINAL: from modules import sd_hijack model_data.sd_model.to(devices.cpu) sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) From e0c8d37d5e6a94e420919178dee842f776758ce7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 17 Sep 2023 16:14:40 +0300 Subject: [PATCH 25/37] Fix DPM SDE and update IPEX defaults --- modules/intel/ipex/attention.py | 2 +- modules/processing_diffusers.py | 6 +++--- modules/shared.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/intel/ipex/attention.py b/modules/intel/ipex/attention.py index d7335bfaf..fc4ab6e26 100644 --- a/modules/intel/ipex/attention.py +++ b/modules/intel/ipex/attention.py @@ -65,7 +65,7 @@ original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_a def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False): #ARC GPUs can't allocate more than 4GB to a single block, Slice it: shape_one, batch_size_attention, query_tokens, shape_four = query.shape - block_multiply = 2.4 if query.dtype == torch.float32 else 1.2 + block_multiply = 3.6 if query.dtype == torch.float32 else 1.8 block_size = (shape_one * batch_size_attention * query_tokens * shape_four) / 1024 * block_multiply #MB split_slice_size = batch_size_attention if block_size >= 4000: diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 9c47ae957..9cd74f162 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -283,7 +283,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro recompile_model() is_karras_compatible = shared.sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers - if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name) and (p.sampler_name != 'Default') and is_karras_compatible: + if ((not hasattr(shared.sd_model.scheduler, 'name')) or (p.sampler_name == 'DPM SDE') or (shared.sd_model.scheduler.name != p.sampler_name)) and (p.sampler_name != 'Default') and is_karras_compatible: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") @@ -380,7 +380,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if latent_scale_mode is not None or p.hr_force: p.ops.append('hires') recompile_model(hires=True) - if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.latent_sampler) and (p.latent_sampler != 'Default') and is_karras_compatible: + if ((not hasattr(shared.sd_model.scheduler, 'name')) or (p.latent_sampler == 'DPM SDE') or (shared.sd_model.scheduler.name != p.latent_sampler)) and (p.latent_sampler != 'Default') and is_karras_compatible: sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") @@ -416,7 +416,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.sd_model.to(devices.cpu) devices.torch_gc() - if (not hasattr(shared.sd_refiner.scheduler, 'name')) or (shared.sd_refiner.scheduler.name != p.latent_sampler) and (p.latent_sampler != 'Default'): + if ((not hasattr(shared.sd_refiner.scheduler, 'name')) or (p.latent_sampler == 'DPM SDE') or (shared.sd_refiner.scheduler.name != p.latent_sampler)) and (p.latent_sampler != 'Default'): sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") diff --git a/modules/shared.py b/modules/shared.py index 7ee44b668..bed228abf 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -434,7 +434,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, lambda: {"choices": ['default', 'true', 'false']}), "diffusers_vae_slicing": OptionInfo(True, "Enable VAE slicing"), "diffusers_vae_tiling": OptionInfo(False if cmd_opts.use_openvino else True, "Enable VAE tiling"), - "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), + "diffusers_attention_slicing": OptionInfo(True if devices.backend == "ipex" else False, "Enable attention slicing"), "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), "diffusers_lora_loader": OptionInfo("diffusers", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}), From 4887b0a6317dd316d9ae20f92d8cb859e1564e0d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 17 Sep 2023 10:16:33 -0400 Subject: [PATCH 26/37] add before process callback --- modules/processing.py | 2 ++ modules/scripts.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/modules/processing.py b/modules/processing.py index fe0e78e6a..7f3a46b43 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -572,6 +572,8 @@ def print_profile(profile, msg: str): def process_images(p: StableDiffusionProcessing) -> Processed: if not hasattr(p.sd_model, 'sd_checkpoint_info'): return None + if p.scripts is not None: + p.scripts.before_process(p) stored_opts = {} for k, v in p.override_settings.copy().items(): orig = shared.opts.data.get(k, None) or shared.opts.data_labels[k].default diff --git a/modules/scripts.py b/modules/scripts.py index c5eb6afe8..697798975 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -67,6 +67,20 @@ class Script: """ pass # pylint: disable=unnecessary-pass + def setup(self, p, *args): + """For AlwaysVisible scripts, this function is called when the processing object is set up, before any processing starts. + args contains all values returned by components from ui(). + """ + pass + + def before_process(self, p, *args): + """ + This function is called very early during processing begins for AlwaysVisible scripts. + You can modify the processing object (p) here, inject hooks, etc. + args contains all values returned by components from ui() + """ + pass + def process(self, p, *args): """ This function is called before processing begins for AlwaysVisible scripts. @@ -437,6 +451,17 @@ class ScriptRunner: s.report() return processed + def before_process(self, p, **kwargs): + s = ScriptSummary('before-process') + for script in self.alwayson_scripts: + try: + script_args = p.script_args[script.args_from:script.args_to] + script.before_process(p, *script_args, **kwargs) + except Exception as e: + errors.display(e, f"Error running before process: {script.filename}") + s.record(script.title()) + s.report() + def process(self, p, **kwargs): s = ScriptSummary('process') for script in self.alwayson_scripts: From f4492f4c8688446ce644c727cf1ccc18ee927bcd Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 17 Sep 2023 10:47:25 -0400 Subject: [PATCH 27/37] optimize en search --- javascript/extraNetworks.js | 36 ++++++++++++++++++++++++++++-------- modules/ui_extra_networks.py | 6 +++--- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index e48919ef6..43ef498a8 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -136,6 +136,31 @@ function saveCardDescription(event) { event.preventDefault(); } +async function filterExtraNetworksForTab(tabname, searchTerm) { + let found = 0; + let items = 0; + const t0 = performance.now(); + const cards = Array.from(gradioApp().querySelectorAll(`#${tabname}_extra_tabs div.card`)); + cards.forEach((elem) => { + items += 1; + if (searchTerm === '') { + elem.style.display = ''; + } else { + let text = `${elem.querySelector('.name').textContent.toLowerCase()} ${elem.querySelector('.search_term').textContent}`; + text = text.toLowerCase().replace('models--', 'Diffusers').replace('\\', '/'); + if (text.indexOf(searchTerm) === -1) { + elem.style.display = 'none'; + } else { + elem.style.display = ''; + found += 1; + } + } + }); + const t1 = performance.now(); + if (found > 0) log(`filterExtraNetworks: text=${searchTerm} items=${items} match=${found} time=${Math.round(1000 * (t1 - t0)) / 1000000}`); + else log(`filterExtraNetworks: text=all items=${items} time=${Math.round(1000 * (t1 - t0)) / 1000000}`); +} + function setupExtraNetworksForTab(tabname) { gradioApp().querySelector(`#${tabname}_extra_tabs`).classList.add('extra-networks'); const tabs = gradioApp().querySelector(`#${tabname}_extra_tabs > div`); @@ -157,15 +182,9 @@ function setupExtraNetworksForTab(tabname) { search.addEventListener('input', (evt) => { if (searchTimer) clearTimeout(searchTimer); searchTimer = setTimeout(() => { - const searchTerm = search.value.toLowerCase(); - gradioApp().querySelectorAll(`#${tabname}_extra_tabs div.card`).forEach((elem) => { - let text = `${elem.querySelector('.name').textContent.toLowerCase()} ${elem.querySelector('.search_term').textContent.toLowerCase()}`; - text = text.replace('models--', 'Diffusers').replace('\\', '/'); - elem.style.display = text.indexOf(searchTerm) === -1 ? 'none' : ''; - console.log({ search: searchTerm, text, display: elem.style.display }); - }); + filterExtraNetworksForTab(tabname, search.value.toLowerCase()); searchTimer = null; - }, 100); + }, 150); }); let hoverTimer = null; @@ -273,6 +292,7 @@ function tryToRemoveExtraNetworkFromPrompt(textarea, text) { } function refreshExtraNetworks(tabname) { + console.log('refreshExtraNetworks', tabname, gradioApp().querySelector(`#${tabname}_extra_networks textarea`)?.value); gradioApp().querySelector(`#${tabname}_extra_networks textarea`)?.dispatchEvent(new Event('input')); } diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index bc33ba7a4..62662b9fb 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -387,9 +387,9 @@ def create_ui(container, button, tabname, skip_indexing = False): for page in extra_pages: page.create_page(ui.tabname, skip_indexing) with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab"): - page_elem = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page") - page_elem.change(fn=lambda: None, _js=f'() => refreshExtraNetworks("{tabname}")', inputs=[], outputs=[]) - ui.pages.append(page_elem) + hmtl = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page") + # hmtl.change(fn=lambda: None, _js=f'() => refreshExtraNetworks("{tabname}")', inputs=[], outputs=[]) + ui.pages.append(hmtl) def toggle_visibility(is_visible): is_visible = not is_visible From d1302c09e38ca6445630351e20c7ea8b7de70daf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 17 Sep 2023 15:06:21 -0400 Subject: [PATCH 28/37] update pre-commit and fix ops --- extensions-builtin/Lora/lora.py | 2 +- html/locale_en.json | 2 +- javascript/midnight-barbie.css | 2 +- javascript/style.css | 4 ++-- modules/processing.py | 5 +++-- modules/scripts.py | 4 ++-- modules/shared.py | 4 +--- modules/ui_extra_networks_textual_inversion.py | 2 +- webui.bat | 2 +- 9 files changed, 13 insertions(+), 14 deletions(-) diff --git a/extensions-builtin/Lora/lora.py b/extensions-builtin/Lora/lora.py index 481ccd629..f45595cf6 100644 --- a/extensions-builtin/Lora/lora.py +++ b/extensions-builtin/Lora/lora.py @@ -281,7 +281,7 @@ def load_loras(names, multipliers=None): if len(failed_to_load_loras) > 0: sd_hijack.model_hijack.comments.append("Failed to find Loras: " + ", ".join(failed_to_load_loras)) - + if recompile_model: shared.log.info("Lora: Recompiling model") sd_models.compile_diffusers(shared.sd_model) diff --git a/html/locale_en.json b/html/locale_en.json index 0238f9a67..e26871b17 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -63,7 +63,7 @@ {"id":"","label":"UI position","localized":"","hint":"Location of extra networks"}, {"id":"","label":"cover","localized":"","hint":"cover full area"}, {"id":"","label":"inline","localized":"","hint":"inline with all additional elelemtns (scrollable)"}, - {"id":"","label":"sidebar","localized":"","hint":"sidebar on the right side of the screen"}, + {"id":"","label":"sidebar","localized":"","hint":"sidebar on the right side of the screen"}, {"id":"","label":"UI height (%)","localized":"","hint":""}, {"id":"","label":"UI sidebar width (%)","localized":"","hint":""}, {"id":"","label":"UI card preview lazy loading","localized":"","hint":""}, diff --git a/javascript/midnight-barbie.css b/javascript/midnight-barbie.css index a52980c27..1ee4d7f10 100644 --- a/javascript/midnight-barbie.css +++ b/javascript/midnight-barbie.css @@ -314,4 +314,4 @@ svg.feather.feather-image, .feather .feather-image { display: none } --size-9: 64px; --size-14: 64px; } -/*Midnight-Barbie, By Nyxxia*/ \ No newline at end of file +/*Midnight-Barbie, By Nyxxia*/ diff --git a/javascript/style.css b/javascript/style.css index e06d61642..52554cca3 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -249,8 +249,8 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .extra-network-cards .card:hover .overlay .tags { display: block; } .extra-network-cards .card:hover .overlay .description { display: block; } .extra-network-cards .card:hover .preview { box-shadow: none; filter: grayscale(100%); } -#txt2img_description, #img2img_description { max-height: 63px; overflow-y: auto !important; } -#txt2img_description > label > textarea, #img2img_description > label > textarea { font-size: 0.9em } +#txt2img_description, #img2img_description { max-height: 63px; overflow-y: auto !important; } +#txt2img_description > label > textarea, #img2img_description > label > textarea { font-size: 0.9em } /* controlnet */ .controlnet_control_type .controlnet_control_type_filter_group .wrap:last-of-type { display: grid; grid-auto-flow: row; grid-template-columns: repeat(4, minmax(0, 1fr)); } diff --git a/modules/processing.py b/modules/processing.py index 7f3a46b43..69f77d919 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -462,7 +462,8 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su if all_negative_prompts is None: all_negative_prompts = p.all_negative_prompts comment = ', '.join(comments) if comments is not None and type(comments) is list else None - + ops = list(set(p.ops)) + ops.reverse() args = { # basic "Steps": p.steps, @@ -488,7 +489,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su "Backend": 'Diffusers' if shared.backend == shared.Backend.DIFFUSERS else 'Original', "Version": git_commit, "Comment": comment, - "Operations": '; '.join(p.ops).replace('"', '') if len(p.ops) > 0 else 'none', + "Operations": '; '.join(ops).replace('"', '') if len(p.ops) > 0 else 'none', } if 'txt2img' in p.ops: pass diff --git a/modules/scripts.py b/modules/scripts.py index 697798975..5e15f1c19 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -71,7 +71,7 @@ class Script: """For AlwaysVisible scripts, this function is called when the processing object is set up, before any processing starts. args contains all values returned by components from ui(). """ - pass + pass # pylint: disable=unnecessary-pass def before_process(self, p, *args): """ @@ -79,7 +79,7 @@ class Script: You can modify the processing object (p) here, inject hooks, etc. args contains all values returned by components from ui() """ - pass + pass # pylint: disable=unnecessary-pass def process(self, p, *args): """ diff --git a/modules/shared.py b/modules/shared.py index bed228abf..a3bde1288 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -486,14 +486,13 @@ options_templates.update(options_section(('saving-images', "Image Options"), { "n_rows": OptionInfo(-1, "Grid row count", gr.Slider, {"minimum": -1, "maximum": 16, "step": 1}), "save_sep_options": OptionInfo("

Intermediate Image Saving

", "", gr.HTML), - "save_init_img": OptionInfo(True, "Save copy of img2img init images"), + "save_init_img": OptionInfo(False, "Save copy of img2img init images"), "save_images_before_highres_fix": OptionInfo(False, "Save copy of image before applying highres fix"), "save_images_before_refiner": OptionInfo(False, "Save copy of image before running refiner"), "save_images_before_face_restoration": OptionInfo(False, "Save copy of image before doing face restoration"), "save_images_before_color_correction": OptionInfo(False, "Save copy of image before applying color correction"), "save_mask": OptionInfo(False, "Save copy of the inpainting greyscale mask"), "save_mask_composite": OptionInfo(False, "Save copy of inpainting masked composite"), - })) options_templates.update(options_section(('saving-paths', "Image Naming & Paths"), { @@ -521,7 +520,6 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths" "outdir_grids": OptionInfo("", "Output directory for grids", component_args=hide_dirs, folder=True), "outdir_txt2img_grids": OptionInfo("outputs/grids", 'Output directory for txt2img grids', component_args=hide_dirs, folder=True), "outdir_img2img_grids": OptionInfo("outputs/grids", 'Output directory for img2img grids', component_args=hide_dirs, folder=True), - })) options_templates.update(options_section(('ui', "User Interface"), { diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py index 82b7e45c9..438ce86f1 100644 --- a/modules/ui_extra_networks_textual_inversion.py +++ b/modules/ui_extra_networks_textual_inversion.py @@ -39,7 +39,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values()) else: embeddings = [] - embeddings = list(sorted(embeddings, key=lambda emb: emb.filename)) + embeddings = sorted(embeddings, key=lambda emb: emb.filename) for embedding in embeddings: path, _ext = os.path.splitext(embedding.filename) tags = {} diff --git a/webui.bat b/webui.bat index 022be1367..2d12762c6 100755 --- a/webui.bat +++ b/webui.bat @@ -28,7 +28,7 @@ if %ERRORLEVEL% == 0 goto :activate_venv for /f "delims=" %%i in ('CALL %PYTHON% -c "import sys; print(sys.executable)"') do set PYTHON_FULLNAME="%%i" echo Using python: %PYTHON_FULLNAME% -echo Creating VENV: %VENV_DIR% +echo Creating VENV: %VENV_DIR% %PYTHON_FULLNAME% -m venv "%VENV_DIR%" >tmp/stdout.txt 2>tmp/stderr.txt if %ERRORLEVEL% == 0 goto :activate_venv echo Failed creating VENV: "%VENV_DIR%" From 792893e68e357088046b4e652919b491496e1ab0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 17 Sep 2023 15:36:05 -0400 Subject: [PATCH 29/37] fix filename gen --- html/locale_en.json | 2 +- modules/images.py | 17 +++++++++-------- modules/processing.py | 2 +- modules/shared.py | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/html/locale_en.json b/html/locale_en.json index e26871b17..692a82929 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -431,7 +431,7 @@ {"id":"","label":"Create text file next to every image with generation parameters","localized":"","hint":""}, {"id":"","label":"Create JSON log file for each saved image","localized":"","hint":"Save image information to a JSON file"}, {"id":"","label":"Save copy of image before doing face restoration","localized":"","hint":""}, - {"id":"","label":"Save copy of image before applying highres fix","localized":"","hint":""}, + {"id":"","label":"Save copy of image before applying hires","localized":"","hint":""}, {"id":"","label":"Save copy of image before applying color correction","localized":"","hint":""}, {"id":"","label":"Save copy of the inpainting greyscale mask","localized":"","hint":""}, {"id":"","label":"Save copy of inpainting masked composite","localized":"","hint":""}, diff --git a/modules/images.py b/modules/images.py index 49f83669f..39d9e7246 100644 --- a/modules/images.py +++ b/modules/images.py @@ -273,7 +273,7 @@ re_nonletters = re.compile(r'[\s' + string.punctuation + ']+') re_pattern = re.compile(r"(.*?)(?:\[([^\[\]]+)\]|$)") re_pattern_arg = re.compile(r"(.*)<([^>]*)>$") max_filename_part_length = 128 -NOTHING_AND_SKIP_PREVIOUS_TEXT = object() +NOTHING = object() def sanitize_filename_part(text, replace_spaces=True): @@ -291,13 +291,13 @@ def sanitize_filename_part(text, replace_spaces=True): class FilenameGenerator: replacements = { - 'batch_number': lambda self: NOTHING_AND_SKIP_PREVIOUS_TEXT if self.p is None or self.p.batch_size == 1 else self.p.batch_index + 1, + 'batch_number': lambda self: NOTHING if self.index <= 1 else self.index, 'cfg': lambda self: self.p and self.p.cfg_scale, 'clip_skip': lambda self: self.p and self.p.clip_skip, 'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'), 'datetime': lambda self, *args: self.datetime(*args), # accepts formats: [datetime], [datetime], [datetime