From 88cc6f63e19eb95a31a557405cb1d3a72d4112f3 Mon Sep 17 00:00:00 2001 From: ljleb Date: Wed, 2 Aug 2023 15:26:04 -0400 Subject: [PATCH 01/12] add callback --- modules/processing.py | 41 +++++++++++++++++++++++++++-------------- modules/scripts.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 7b83f2fc3..c65f16a5d 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -444,8 +444,13 @@ def fix_seed(p): p.subseed = get_fixed_seed(p.subseed) -def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_subseeds, comments=None, iteration=0, position_in_batch=0): # pylint: disable=unused-argument - index = position_in_batch + iteration * p.batch_size +def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_subseeds, comments=None, iteration=0, position_in_batch=0, index=None, all_negative_prompts=None): # pylint: disable=unused-argument + if index is None: + index = position_in_batch + iteration * p.batch_size + + if all_negative_prompts is None: + all_negative_prompts = p.all_negative_prompts + generation_params = { "Steps": p.steps, "Seed": all_seeds[index], @@ -487,7 +492,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su generation_params['Token merging ratio hr'] = token_merging_ratio_hr if token_merging_ratio_hr != 0 else None generation_params.update(p.extra_generation_params) generation_params_text = ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in generation_params.items() if v is not None]) - negative_prompt_text = f"\nNegative prompt: {p.all_negative_prompts[index]}" if p.all_negative_prompts[index] else "" + negative_prompt_text = f"\nNegative prompt: {all_negative_prompts[index]}" if all_negative_prompts[index] else "" return f"{all_prompts[index]}{negative_prompt_text}\n{generation_params_text}".strip() @@ -599,9 +604,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: else: p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))] - def infotext(iteration=0, position_in_batch=0): - return create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds, comments, iteration, position_in_batch) - if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings: model_hijack.embedding_db.load_textual_inversion_embeddings() if p.scripts is not None: @@ -709,6 +711,17 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: devices.torch_gc() if p.scripts is not None: p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) + p.seeds = seeds + p.subseeds = subseeds + if p.scripts is not None: + p.prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size] + p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size] + batch_params = scripts.PostprocessBatchListArgs(list(x_samples_ddim)) + p.scripts.postprocess_batch_list(p, batch_params, batch_number=n) + x_samples_ddim = batch_params.images + + def infotext(index=0): + return create_infotext(p, p.prompts, p.seeds, p.subseeds, index=index, all_negative_prompts=p.negative_prompts) for i, x_sample in enumerate(x_samples_ddim): p.batch_index = i @@ -721,9 +734,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_face_restoration: orig = p.restore_faces p.restore_faces = False - info=infotext(n, i) + info = infotext(i) p.restore_faces = orig - images.save_image(Image.fromarray(x_sample), path=p.outpath_samples, basename="", seed=seeds[i], prompt=prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-face-restoration") + images.save_image(Image.fromarray(x_sample), path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-face-restoration") p.ops.append('face') x_sample = modules.face_restoration.restore_faces(x_sample) image = Image.fromarray(x_sample) @@ -735,16 +748,16 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_color_correction: orig = p.color_corrections p.color_corrections = None - info=infotext(n, i) + info = infotext(i) p.color_corrections = orig image_without_cc = apply_overlay(image, p.paste_to, i, p.overlay_images) - images.save_image(image_without_cc, path=p.outpath_samples, basename="", seed=seeds[i], prompt=prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-color-correction") + images.save_image(image_without_cc, path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-color-correction") p.ops.append('color') image = apply_color_correction(p.color_corrections[i], image) image = apply_overlay(image, p.paste_to, i, p.overlay_images) if shared.opts.samples_save and not p.do_not_save_samples: - images.save_image(image, p.outpath_samples, "", seeds[i], prompts[i], shared.opts.samples_format, info=infotext(n, i), p=p) - text = infotext(n, i) + images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=infotext(i), p=p) + text = infotext(i) infotexts.append(text) image.info["parameters"] = text output_images.append(image) @@ -752,9 +765,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: image_mask = p.mask_for_overlay.convert('RGB') image_mask_composite = Image.composite(image.convert('RGBA').convert('RGBa'), Image.new('RGBa', image.size), images.resize_image(3, p.mask_for_overlay, image.width, image.height).convert('L')).convert('RGBA') if shared.opts.save_mask: - images.save_image(image_mask, p.outpath_samples, "", seeds[i], prompts[i], shared.opts.samples_format, info=infotext(n, i), p=p, suffix="-mask") + images.save_image(image_mask, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=infotext(i), p=p, suffix="-mask") if shared.opts.save_mask_composite: - images.save_image(image_mask_composite, p.outpath_samples, "", seeds[i], prompts[i], shared.opts.samples_format, info=infotext(n, i), p=p, suffix="-mask-composite") + images.save_image(image_mask_composite, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=infotext(i), p=p, suffix="-mask-composite") if shared.opts.return_mask: output_images.append(image_mask) if shared.opts.return_mask_composite: diff --git a/modules/scripts.py b/modules/scripts.py index 55baa562b..8efc44018 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -18,6 +18,11 @@ class PostprocessImageArgs: self.image = image +class PostprocessBatchListArgs: + def __init__(self, images): + self.images = images + + class Script: name = None filename = None @@ -108,6 +113,23 @@ class Script: """ pass # pylint: disable=unnecessary-pass + def postprocess_batch_list(self, p, pp: PostprocessBatchListArgs, *args, **kwargs): + """ + Same as postprocess_batch(), but receives batch images as a list of 3D tensors instead of a 4D tensor. + This is useful when you want to update the entire batch instead of individual images. + You can modify the postprocessing object (pp) to update the images in the batch, remove images, add images, etc. + If the number of images is different from the batch size when returning, + then the script has the responsibility to also update the following attributes in the processing object (p): + - p.prompts + - p.negative_prompts + - p.seeds + - p.subseeds + + **kwargs will have same items as process_batch, and also: + - batch_number - index of current batch, from 0 to number of batches-1 + """ + pass # pylint: disable=unnecessary-pass + def postprocess(self, p, processed, *args): """ This function is called after processing ends for AlwaysVisible scripts. @@ -457,6 +479,18 @@ class ScriptRunner: errors.display(e, f'Running script before postprocess batch: {script.filename}') log.debug(f'Script postprocess-batch: {s}') + def postprocess_batch_list(self, p, pp: PostprocessBatchListArgs, **kwargs): + s = [] + for script in self.alwayson_scripts: + try: + t0 = time.time() + args = p.per_script_args.get(script.title(), p.script_args[script.args_from:script.args_to]) + script.postprocess_batch_list(p, pp, *args, **kwargs) + s.append(f'{script.title()}:{round(time.time()-t0, 2)}s') + except Exception as e: + errors.display(e, f'Running script before postprocess batch list: {script.filename}') + log.debug(f'Script postprocess-batch-list: {s}') + def postprocess_image(self, p, pp: PostprocessImageArgs): s = [] for script in self.alwayson_scripts: From 65aee8cf7bd7262c058182a37d0def62f767d57c Mon Sep 17 00:00:00 2001 From: ljleb Date: Wed, 2 Aug 2023 15:54:41 -0400 Subject: [PATCH 02/12] refact --- modules/processing.py | 26 ++++++++++++-------------- modules/scripts.py | 1 - 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index c65f16a5d..a313c89d7 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -648,20 +648,20 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.state.interrupted: shared.log.debug(f'Process interrupted: {n}/{p.n_iter}') break - prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size] - negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size] - seeds = p.all_seeds[n * p.batch_size:(n + 1) * p.batch_size] - subseeds = p.all_subseeds[n * p.batch_size:(n + 1) * p.batch_size] + p.prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size] + p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size] + p.seeds = p.all_seeds[n * p.batch_size:(n + 1) * p.batch_size] + p.subseeds = p.all_subseeds[n * p.batch_size:(n + 1) * p.batch_size] if p.scripts is not None: - p.scripts.before_process_batch(p, batch_number=n, prompts=prompts, seeds=seeds, subseeds=subseeds) - if len(prompts) == 0: + p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) + if len(p.prompts) == 0: break - prompts, extra_network_data = extra_networks.parse_prompts(prompts) + p.prompts, extra_network_data = extra_networks.parse_prompts(p.prompts) if not p.disable_extra_networks: with devices.autocast(): extra_networks.activate(p, extra_network_data) if p.scripts is not None: - p.scripts.process_batch(p, batch_number=n, prompts=prompts, seeds=seeds, subseeds=subseeds) + p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) if n == 0: with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file: processed = Processed(p, [], p.seed, "") @@ -673,13 +673,13 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: shared.state.job = f"Batch {n+1} out of {p.n_iter}" if shared.backend == shared.Backend.ORIGINAL: - uc = get_conds_with_caching(prompt_parser.get_learned_conditioning, negative_prompts, p.steps * step_multiplier, cached_uc) - c = get_conds_with_caching(prompt_parser.get_multicond_learned_conditioning, prompts, p.steps * step_multiplier, cached_c) + uc = get_conds_with_caching(prompt_parser.get_learned_conditioning, p.negative_prompts, p.steps * step_multiplier, cached_uc) + c = get_conds_with_caching(prompt_parser.get_multicond_learned_conditioning, p.prompts, p.steps * step_multiplier, cached_c) if len(model_hijack.comments) > 0: for comment in model_hijack.comments: comments[comment] = 1 with devices.without_autocast() if devices.unet_needs_upcast else devices.autocast(): - samples_ddim = p.sample(conditioning=c, unconditional_conditioning=uc, seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, prompts=prompts) + samples_ddim = p.sample(conditioning=c, unconditional_conditioning=uc, seeds=p.seeds, subseeds=p.subseeds, subseed_strength=p.subseed_strength, prompts=p.prompts) x_samples_ddim = [decode_first_stage(p.sd_model, samples_ddim[i:i+1].to(dtype=devices.dtype_vae))[0].cpu() for i in range(samples_ddim.size(0))] try: for x in x_samples_ddim: @@ -701,7 +701,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: elif shared.backend == shared.Backend.DIFFUSERS: from modules.processing_diffusers import process_diffusers - x_samples_ddim = process_diffusers(p, seeds, prompts, negative_prompts) + x_samples_ddim = process_diffusers(p, p.seeds, p.prompts, p.negative_prompts) else: raise ValueError(f"Unknown backend {shared.backend}") @@ -711,8 +711,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: devices.torch_gc() if p.scripts is not None: p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) - p.seeds = seeds - p.subseeds = subseeds if p.scripts is not None: p.prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size] p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size] diff --git a/modules/scripts.py b/modules/scripts.py index 8efc44018..bfd04c30e 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -124,7 +124,6 @@ class Script: - p.negative_prompts - p.seeds - p.subseeds - **kwargs will have same items as process_batch, and also: - batch_number - index of current batch, from 0 to number of batches-1 """ From 7c4fdbff1b7b21c6dd7411f9f1e14e17483c5c5a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Aug 2023 08:56:38 +0200 Subject: [PATCH 03/12] update taesd --- extensions-builtin/sd-extension-system-info | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- modules/processing.py | 3 +- modules/sd_models.py | 10 +-- modules/sd_samplers.py | 4 +- modules/sd_samplers_common.py | 21 ++--- modules/sd_vae_taesd.py | 88 ------------------- modules/shared.py | 28 +++++-- modules/taesd/sd_vae_taesd.py | 57 +++++++++++++ modules/taesd/taesd.py | 93 +++++++++++++++++++++ modules/txt2img.py | 5 +- modules/ui.py | 2 + webui.py | 3 + 13 files changed, 200 insertions(+), 118 deletions(-) delete mode 100644 modules/sd_vae_taesd.py create mode 100644 modules/taesd/sd_vae_taesd.py create mode 100644 modules/taesd/taesd.py diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 9d3c0ca0f..19d190e71 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 9d3c0ca0f2dc8f8973b3d08f5ec1fa8bbd726155 +Subproject commit 19d190e71b2f1399623519db741d2a6bf8d2c86c diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 5ae9b4a1a..4b815cc35 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 5ae9b4a1a0c7d9a2938e75aaf052ab078623066f +Subproject commit 4b815cc351ca5eeec489f573a9a6c2dbd47374d7 diff --git a/modules/processing.py b/modules/processing.py index 7b83f2fc3..7a24d156f 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -86,7 +86,7 @@ class StableDiffusionProcessing: """ The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing """ - def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, ddim_discretize: str = None, s_min_uncond: float = 0.0, s_churn: float = 0.0, s_tmax: float = None, s_tmin: float = 0.0, s_noise: float = 1.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument + def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, ddim_discretize: str = None, s_min_uncond: float = 0.0, s_churn: float = 0.0, s_tmax: float = None, s_tmin: float = 0.0, s_noise: float = 1.0, quality: bool = True, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument self.outpath_samples: str = outpath_samples self.outpath_grids: str = outpath_grids @@ -151,6 +151,7 @@ class StableDiffusionProcessing: self.is_hr_pass = False self.enable_hr = None self.refiner_start = 0 + self.quality = quality self.ops = [] shared.opts.data['clip_skip'] = clip_skip diff --git a/modules/sd_models.py b/modules/sd_models.py index d010ffd6b..f5c35b95f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -534,10 +534,7 @@ def change_backend(): def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument - if op != 'model' and checkpoint_info is None and (shared.cmd_opts.ckpt is None or shared.cmd_opts.ckpt.lower() == 'none'): - return import torch # pylint: disable=reimported,redefined-outer-name - devices.set_cuda_params() if timer is None: timer = Timer() import logging @@ -570,7 +567,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return - shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}') sd_model = None try: @@ -580,7 +576,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if model_name is not None: shared.log.info(f'Loading diffuser {op}: {model_name}') model_file = modelloader.download_diffusers_model(hub_id=model_name) + devices.set_cuda_params() try: + shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}') sd_model = diffusers.DiffusionPipeline.from_pretrained(model_file, **diffusers_load_config) except Exception as e: shared.log.error(f'Diffusers failed loading model: {model_file} {e}') @@ -592,8 +590,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if checkpoint_info is None: unload_model_weights(op=op) return - shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') + devices.set_cuda_params() vae = None if op == 'model' or op == 'refiner': vae_file, vae_source = sd_vae.resolve_vae(checkpoint_info.filename) @@ -601,8 +599,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if vae is not None: diffusers_load_config["vae"] = vae + shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') if not os.path.isfile(checkpoint_info.path): try: + shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}') sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) except Exception as e: shared.log.error(f'Diffusers {op} failed loading model: {checkpoint_info.path} {e}') diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 21777c800..4f298fbce 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -23,9 +23,7 @@ def list_samplers(backend_name = shared.backend): samplers = all_samplers samplers_for_img2img = all_samplers samplers_map = {} - shared.log.debug(f'Samplers enumerated: {[x.name for x in all_samplers]}') - -list_samplers() + shared.log.debug(f'Available samplers: {[x.name for x in all_samplers]}') def find_sampler_config(name): diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 40b16d50b..a18667d47 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -2,17 +2,16 @@ from collections import namedtuple import numpy as np import torch from PIL import Image -from modules import devices, processing, images, sd_vae_approx, sd_samplers, sd_vae_taesd -from modules.shared import opts, state +from modules import devices, processing, images, sd_vae_approx, sd_samplers import modules.shared as shared - +import modules.taesd.sd_vae_taesd as sd_vae_taesd SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = {"Full VAE": 0, "Approximate NN": 1, "Approximate simple": 2, "TAESD": 3} def setup_img2img_steps(p, steps=None): - if opts.img2img_fix_steps or steps is not None: + if shared.opts.img2img_fix_steps or steps is not None: requested_steps = (steps or p.steps) steps = int(requested_steps / min(p.denoising_strength, 0.999)) if p.denoising_strength > 0 else 0 t_enc = requested_steps - 1 @@ -25,7 +24,7 @@ def setup_img2img_steps(p, steps=None): def single_sample_to_image(sample, approximation=None): if approximation is None: - approximation = approximation_indexes.get(opts.show_progress_type, 0) + approximation = approximation_indexes.get(shared.opts.show_progress_type, 0) if approximation == 0: x_sample = processing.decode_first_stage(shared.sd_model, sample.unsqueeze(0))[0] * 0.5 + 0.5 elif approximation == 1: @@ -33,8 +32,9 @@ def single_sample_to_image(sample, approximation=None): elif approximation == 2: x_sample = sd_vae_approx.cheap_approximation(sample) * 0.5 + 0.5 elif approximation == 3: - x_sample = sample * 1.5 - x_sample = sd_vae_taesd.model()(x_sample.to(devices.device, devices.dtype).unsqueeze(0))[0].detach() + # x_sample = sample * 1.5 + # x_sample = sd_vae_taesd.model()(x_sample.to(devices.device, devices.dtype).unsqueeze(0))[0].detach() + x_sample = sd_vae_taesd.decode(sample, 'sd') else: shared.log.warning(f"Unknown image decode type: {approximation}") return Image.new(mode="RGB", size=(512, 512)) @@ -52,10 +52,11 @@ def samples_to_image_grid(samples, approximation=None): def store_latent(decoded): - state.current_latent = decoded - if opts.live_previews_enable and opts.show_progress_every_n_steps > 0 and shared.state.sampling_step % opts.show_progress_every_n_steps == 0: + shared.state.current_latent = decoded + if shared.opts.live_previews_enable and shared.opts.show_progress_every_n_steps > 0 and shared.state.sampling_step % shared.opts.show_progress_every_n_steps == 0: if not shared.parallel_processing_allowed: - shared.state.assign_current_image(sample_to_image(decoded)) + image = sample_to_image(decoded) + shared.state.assign_current_image(image) def is_sampler_using_eta_noise_seed_delta(p): diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py deleted file mode 100644 index 74ad13926..000000000 --- a/modules/sd_vae_taesd.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Tiny AutoEncoder for Stable Diffusion -(DNN for encoding / decoding SD's latent space) - -https://github.com/madebyollin/taesd -""" -import os -import torch -import torch.nn as nn - -from modules import devices, paths_internal - -sd_vae_taesd = None - - -def conv(n_in, n_out, **kwargs): - return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs) - - -class Clamp(nn.Module): - @staticmethod - def forward(x): - return torch.tanh(x / 3) * 3 - - -class Block(nn.Module): - def __init__(self, n_in, n_out): - super().__init__() - self.conv = nn.Sequential(conv(n_in, n_out), nn.ReLU(), conv(n_out, n_out), nn.ReLU(), conv(n_out, n_out)) - self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity() - self.fuse = nn.ReLU() - - def forward(self, x): - return self.fuse(self.conv(x) + self.skip(x)) - - -def decoder(): - return nn.Sequential( - Clamp(), conv(4, 64), nn.ReLU(), - Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False), - Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False), - Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False), - Block(64, 64), conv(64, 3), - ) - - -class TAESD(nn.Module): # pylint: disable=abstract-method - latent_magnitude = 3 - latent_shift = 0.5 - - def __init__(self, decoder_path="taesd_decoder.pth"): - """Initialize pretrained TAESD on the given device from the given checkpoints.""" - super().__init__() - self.decoder = decoder() - self.decoder.load_state_dict( - torch.load(decoder_path, map_location='cpu' if devices.device.type != 'cuda' else None)) - - @staticmethod - def unscale_latents(x): - """[0, 1] -> raw latents""" - return x.sub(TAESD.latent_shift).mul(2 * TAESD.latent_magnitude) - - -def download_model(model_path): - model_url = 'https://github.com/madebyollin/taesd/raw/main/taesd_decoder.pth' - - if not os.path.exists(model_path): - os.makedirs(os.path.dirname(model_path), exist_ok=True) - - print(f'Downloading TAESD decoder to: {model_path}') - torch.hub.download_url_to_file(model_url, model_path) - - -def model(): - global sd_vae_taesd # pylint: disable=global-statement - - if sd_vae_taesd is None: - model_path = os.path.join(paths_internal.models_path, "VAE-taesd", "taesd_decoder.pth") - download_model(model_path) - - if os.path.exists(model_path): - sd_vae_taesd = TAESD(model_path) - sd_vae_taesd.eval() - sd_vae_taesd.to(devices.device, devices.dtype) - else: - raise FileNotFoundError('TAESD model not found') - - return sd_vae_taesd.decoder diff --git a/modules/shared.py b/modules/shared.py index 3a3ce09f3..8b0ff3109 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -171,13 +171,11 @@ class State: return import modules.sd_samplers # pylint: disable=W0621 try: - if opts.show_progress_grid: - self.assign_current_image(modules.sd_samplers.samples_to_image_grid(self.current_latent)) - else: - self.assign_current_image(modules.sd_samplers.sample_to_image(self.current_latent)) - except Exception: - pass - self.current_image_sampling_step = self.sampling_step + image = modules.sd_samplers.samples_to_image_grid(self.current_latent) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(self.current_latent) + self.assign_current_image(image) + self.current_image_sampling_step = self.sampling_step + except Exception as e: + log.error(f'Error setting current image: step={self.sampling_step} {e}') def assign_current_image(self, image): self.current_image = image @@ -984,6 +982,22 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s def backend(self): return Backend.ORIGINAL if opts.data['sd_backend'] == 'original' else Backend.DIFFUSERS + @property + def sd_model_type(self): + try: + if backend == Backend.ORIGINAL: + model_type = 'ldm' + elif "StableDiffusionXL" in self.sd_model.__class__.__name__: + model_type = 'sdxl' + elif "StableDiffusion" in self.sd_model.__class__.__name__: + model_type = 'sd' + elif "Kandinsky" in self.sd_model.__class__.__name__: + model_type = 'kandinsky' + else: + model_type = self.sd_model.__class__.__name__ + except Exception: + model_type = 'unknown' + return model_type sd_model = None sd_refiner = None diff --git a/modules/taesd/sd_vae_taesd.py b/modules/taesd/sd_vae_taesd.py new file mode 100644 index 000000000..17a7bceee --- /dev/null +++ b/modules/taesd/sd_vae_taesd.py @@ -0,0 +1,57 @@ +""" +Tiny AutoEncoder for Stable Diffusion +(DNN for encoding / decoding SD's latent space) + +https://github.com/madebyollin/taesd +""" +import os +from modules import devices, paths_internal +from modules.taesd.taesd import TAESD + +taesd_models = { 'sd-decoder': None, 'sd-encoder': None, 'sdxl-decoder': None, 'sdxl-encoder': None } + +def download_model(model_path): + model_name = os.path.basename(model_path) + model_url = f'https://github.com/madebyollin/taesd/raw/main/{model_name}' + if not os.path.exists(model_path): + os.makedirs(os.path.dirname(model_path), exist_ok=True) + from modules.shared import log + log.info(f'Downloading TAESD decoder: {model_path}') + import torch + torch.hub.download_url_to_file(model_url, model_path) + + +def model(model_class = 'sd', model_type = 'decoder'): + vae = taesd_models[f'{model_class}-{model_type}'] + if vae is None: + model_path = os.path.join(paths_internal.models_path, "TAESD", f"tae{model_class}_{model_type}.pth") + download_model(model_path) + if os.path.exists(model_path): + taesd_models[f'{model_class}-{model_type}'] = TAESD(decoder_path=model_path, encoder_path=None) if model_type == 'decoder' else TAESD(encoder_path=model_path, decoder_path=None) + vae = taesd_models[f'{model_class}-{model_type}'] + vae.eval() + vae.to(devices.device, devices.dtype_vae) + else: + raise FileNotFoundError('TAESD model not found') + if vae is None: + return None + else: + return vae.decoder if model_type == 'decoder' else vae.encoder + + +def decode(latents): + from modules import shared + model_class = shared.sd_model_type + if 'sd' not in model_class: + return None + vae = taesd_models[f'{model_class}-decoder'] + if vae is None: + model_path = os.path.join(paths_internal.models_path, "TAESD", f"tae{model_class}_decoder.pth") + download_model(model_path) + if os.path.exists(model_path): + taesd_models[f'{model_class}-decoder'] = TAESD(decoder_path=model_path, encoder_path=None) + vae = taesd_models[f'{model_class}-decoder'] + vae.to(devices.device, devices.dtype_vae) + enc = latents.unsqueeze(0).to(devices.device, devices.dtype_vae) + image = vae.decoder(enc).clamp(0, 1).detach() + return image[0] diff --git a/modules/taesd/taesd.py b/modules/taesd/taesd.py new file mode 100644 index 000000000..0355a81ff --- /dev/null +++ b/modules/taesd/taesd.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +""" +Tiny AutoEncoder for Stable Diffusion +(DNN for encoding / decoding SD's latent space) +""" +import torch +import torch.nn as nn + +def conv(n_in, n_out, **kwargs): + return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs) + +class Clamp(nn.Module): + def forward(self, x): + return torch.tanh(x / 3) * 3 + +class Block(nn.Module): + def __init__(self, n_in, n_out): + super().__init__() + self.conv = nn.Sequential(conv(n_in, n_out), nn.ReLU(), conv(n_out, n_out), nn.ReLU(), conv(n_out, n_out)) + self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity() + self.fuse = nn.ReLU() + def forward(self, x): + return self.fuse(self.conv(x) + self.skip(x)) + +def Encoder(): + return nn.Sequential( + conv(3, 64), Block(64, 64), + conv(64, 64, stride=2, bias=False), Block(64, 64), Block(64, 64), Block(64, 64), + conv(64, 64, stride=2, bias=False), Block(64, 64), Block(64, 64), Block(64, 64), + conv(64, 64, stride=2, bias=False), Block(64, 64), Block(64, 64), Block(64, 64), + conv(64, 4), + ) + +def Decoder(): + return nn.Sequential( + Clamp(), conv(4, 64), nn.ReLU(), + Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False), + Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False), + Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False), + Block(64, 64), conv(64, 3), + ) + +class TAESD(nn.Module): + latent_magnitude = 3 + latent_shift = 0.5 + + def __init__(self, encoder_path="taesd_encoder.pth", decoder_path="taesd_decoder.pth"): + """Initialize pretrained TAESD on the given device from the given checkpoints.""" + super().__init__() + self.encoder = Encoder() + self.decoder = Decoder() + if encoder_path is not None: + self.encoder.load_state_dict(torch.load(encoder_path, map_location="cpu")) + if decoder_path is not None: + self.decoder.load_state_dict(torch.load(decoder_path, map_location="cpu")) + + @staticmethod + def scale_latents(x): + """raw latents -> [0, 1]""" + return x.div(2 * TAESD.latent_magnitude).add(TAESD.latent_shift).clamp(0, 1) + + @staticmethod + def unscale_latents(x): + """[0, 1] -> raw latents""" + return x.sub(TAESD.latent_shift).mul(2 * TAESD.latent_magnitude) + + +@torch.no_grad() +def main(): + from PIL import Image + import sys + import torchvision.transforms.functional as TF + dev = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu") + print("Using device", dev) + taesd = TAESD().to(dev) + for im_path in sys.argv[1:]: + im = TF.to_tensor(Image.open(im_path).convert("RGB")).unsqueeze(0).to(dev) + + # encode image, quantize, and save to file + im_enc = taesd.scale_latents(taesd.encoder(im)).mul_(255).round_().byte() + enc_path = im_path + ".encoded.png" + TF.to_pil_image(im_enc[0]).save(enc_path) + print(f"Encoded {im_path} to {enc_path}") + + # load the saved file, dequantize, and decode + im_enc = taesd.unscale_latents(TF.to_tensor(Image.open(enc_path)).unsqueeze(0).to(dev)) + im_dec = taesd.decoder(im_enc).clamp(0, 1) + dec_path = im_path + ".decoded.png" + print(f"Decoded {enc_path} to {dec_path}") + TF.to_pil_image(im_dec[0]).save(dec_path) + +if __name__ == "__main__": + main() diff --git a/modules/txt2img.py b/modules/txt2img.py index 64cb978ac..b9c0add2f 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -5,9 +5,9 @@ from modules.ui import plaintext_to_html from modules.memstats import memory_stats -def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument +def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, quality: bool, override_settings_texts, *args): # pylint: disable=unused-argument - shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_start={refiner_start}||refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}args={args}') + shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_start={refiner_start}||refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|quality={quality}|override_settings_texts={override_settings_texts}|args={args}') if shared.sd_model is None: shared.log.warning('Model not loaded') @@ -55,6 +55,7 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step refiner_start=refiner_start, refiner_prompt=refiner_prompt, refiner_negative=refiner_negative, + quality=quality, override_settings=override_settings, ) p.scripts = modules.scripts.scripts_txt2img diff --git a/modules/ui.py b/modules/ui.py index 7ca1261ca..aaf1cdf68 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -379,6 +379,7 @@ def create_ui(startup_timer = None): cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG Scale', value=6.0, elem_id="txt2img_cfg_scale") clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True) with FormRow(elem_classes="checkboxes-row", variant="compact"): + quality = gr.Checkbox(label='Decode quality', value=True, elem_id="txt2img_quality") restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces") tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling") @@ -445,6 +446,7 @@ def create_ui(startup_timer = None): show_second_pass, denoising_strength, hr_scale, hr_upscaler, hr_second_pass_steps, hr_resize_x, hr_resize_y, refiner_start, refiner_prompt, refiner_negative, + quality, override_settings, ] + custom_inputs, outputs=[ diff --git a/webui.py b/webui.py index 72e878c40..7119f1238 100644 --- a/webui.py +++ b/webui.py @@ -105,6 +105,9 @@ def initialize(): shared.disable_extensions() check_rollback_vae() + modules.sd_samplers.list_samplers() + startup_timer.record("samplers") + modules.sd_vae.refresh_vae_list() startup_timer.record("vae") From 095ef45d3f0f9285c1a1315195749ee7d5312a39 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Aug 2023 09:33:55 +0200 Subject: [PATCH 04/12] linting update --- modules/api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/api/api.py b/modules/api/api.py index 0a35a8fcf..ce5bda675 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -448,7 +448,7 @@ class Api: def get_samplers(self): return [{"name": sampler[0], "aliases":sampler[2], "options":sampler[3]} for sampler in sd_samplers.all_samplers] - + def get_sd_vaes(self): return [{"model_name": x, "filename": vae_dict[x]} for x in vae_dict.keys()] From e2ee02a1e853f18e9ca6286766f8997f8eabeeaf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Aug 2023 09:38:11 +0200 Subject: [PATCH 05/12] refactor taesd --- modules/sd_samplers_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index a18667d47..5790d4689 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -34,7 +34,7 @@ def single_sample_to_image(sample, approximation=None): elif approximation == 3: # x_sample = sample * 1.5 # x_sample = sd_vae_taesd.model()(x_sample.to(devices.device, devices.dtype).unsqueeze(0))[0].detach() - x_sample = sd_vae_taesd.decode(sample, 'sd') + x_sample = sd_vae_taesd.decode(sample) else: shared.log.warning(f"Unknown image decode type: {approximation}") return Image.new(mode="RGB", size=(512, 512)) From cfe14884a038d0e3e26c9432055bf23715c2d54a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Aug 2023 12:06:56 +0000 Subject: [PATCH 06/12] add taesd for sdxl --- CHANGELOG.md | 9 +++++++++ modules/processing.py | 3 +-- modules/processing_diffusers.py | 2 -- modules/txt2img.py | 5 ++--- modules/ui.py | 2 -- pyproject.toml | 1 + wiki | 2 +- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c5aa5c5..9fc34c3a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log for SD.Next +## Update for 2023-08-05 + +- general: + - new torch 2.0 with ipex (intel arc) + - additional callbacks for extensions + - update requirements +- diffusers + - sd-xl: vaesd live preview decoder + ## Update for 2023-07-30 Smaller release, but IMO worth a post... diff --git a/modules/processing.py b/modules/processing.py index 7a24d156f..7b83f2fc3 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -86,7 +86,7 @@ class StableDiffusionProcessing: """ The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing """ - def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, ddim_discretize: str = None, s_min_uncond: float = 0.0, s_churn: float = 0.0, s_tmax: float = None, s_tmin: float = 0.0, s_noise: float = 1.0, quality: bool = True, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument + def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, ddim_discretize: str = None, s_min_uncond: float = 0.0, s_churn: float = 0.0, s_tmax: float = None, s_tmin: float = 0.0, s_noise: float = 1.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument self.outpath_samples: str = outpath_samples self.outpath_grids: str = outpath_grids @@ -151,7 +151,6 @@ class StableDiffusionProcessing: self.is_hr_pass = False self.enable_hr = None self.refiner_start = 0 - self.quality = quality self.ops = [] shared.opts.data['clip_skip'] = clip_skip diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 8ee247f9c..18c86116b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -218,6 +218,4 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if p.is_hr_pass: shared.log.warning('Diffusers not implemented: hires fix') - - return results diff --git a/modules/txt2img.py b/modules/txt2img.py index b9c0add2f..c4e01df9d 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -5,9 +5,9 @@ from modules.ui import plaintext_to_html from modules.memstats import memory_stats -def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, quality: bool, override_settings_texts, *args): # pylint: disable=unused-argument +def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument - shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_start={refiner_start}||refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|quality={quality}|override_settings_texts={override_settings_texts}|args={args}') + shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_start={refiner_start}||refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}|args={args}') if shared.sd_model is None: shared.log.warning('Model not loaded') @@ -55,7 +55,6 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step refiner_start=refiner_start, refiner_prompt=refiner_prompt, refiner_negative=refiner_negative, - quality=quality, override_settings=override_settings, ) p.scripts = modules.scripts.scripts_txt2img diff --git a/modules/ui.py b/modules/ui.py index aaf1cdf68..7ca1261ca 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -379,7 +379,6 @@ def create_ui(startup_timer = None): cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG Scale', value=6.0, elem_id="txt2img_cfg_scale") clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True) with FormRow(elem_classes="checkboxes-row", variant="compact"): - quality = gr.Checkbox(label='Decode quality', value=True, elem_id="txt2img_quality") restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces") tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling") @@ -446,7 +445,6 @@ def create_ui(startup_timer = None): show_second_pass, denoising_strength, hr_scale, hr_upscaler, hr_second_pass_steps, hr_resize_x, hr_resize_y, refiner_start, refiner_prompt, refiner_negative, - quality, override_settings, ] + custom_inputs, outputs=[ diff --git a/pyproject.toml b/pyproject.toml index c6c71dfef..34a413446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ ignore = [ "C408", # Rewrite as a literal "E402", # Module level import not at top of file "F401", # Imported but unused + "EXE001", # Shebang present "ISC003", # Implicit string concatenation "RUF005", # Consider concatenation "RUF012", # Mutable class attributes diff --git a/wiki b/wiki index f76cc3a9a..35142f02a 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f76cc3a9ac124882f58f35ba3dfe930744109456 +Subproject commit 35142f02aee984aac261d1e1f563768d008398f6 From 64273169b9a9a146a8dd354e53b463e514d4cb8f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Aug 2023 13:43:30 +0000 Subject: [PATCH 07/12] fix diffusers inpaint --- modules/img2img.py | 3 ++- modules/processing_diffusers.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/img2img.py b/modules/img2img.py index 7ec50aaab..18a7aba85 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -106,7 +106,8 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s elif mode == 2: # inpaint if init_img_with_mask is None: return - image, mask = init_img_with_mask["image"], init_img_with_mask["mask"] + image = init_img_with_mask["image"] + mask = init_img_with_mask["mask"] alpha_mask = ImageOps.invert(image.split()[-1]).convert('L').point(lambda x: 255 if x > 0 else 0, mode='1') mask = ImageChops.lighter(alpha_mask, mask.convert('L')).convert('L') image = image.convert("RGB") diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 18c86116b..bb270b887 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -125,7 +125,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro task_specific_kwargs = {"image": p.init_images, "strength": p.denoising_strength} elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: p.ops.append('inpaint') - task_specific_kwargs = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength} + task_specific_kwargs = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": p.height, "width": p.width} # TODO diffusers use transformers for prompt parsing # from modules.prompt_parser import parse_prompt_attention From 489d0382cfd95ba4420d94dce4d2432c0cb54dd8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 5 Aug 2023 17:26:18 +0300 Subject: [PATCH 08/12] IPEX Diffusers fix cannot allocate more than 4GB --- installer.py | 2 +- modules/ipex_specific/__init__.py | 3 + modules/ipex_specific/diffusers.py | 111 +++++++++++++++++++++++++++++ modules/sd_vae.py | 7 +- modules/shared.py | 2 +- webui.sh | 2 +- 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 modules/ipex_specific/diffusers.py diff --git a/installer.py b/installer.py index b1b5d1520..644b6b4b8 100644 --- a/installer.py +++ b/installer.py @@ -328,7 +328,7 @@ def check_torch(): torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.1a0 torchvision==0.15.2a0 intel_extension_for_pytorch==2.0.110+xpu -f https://developer.intel.com/ipex-whl-stable-xpu') os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0 intel-extension-for-tensorflow[gpu]') else: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0a0 torchvision intel_extension_for_pytorch==2.0.110+gitba7f6c1 -f https://developer.intel.com/ipex-whl-stable-xpu') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0a0 intel_extension_for_pytorch==2.0.110+gitba7f6c1 -f https://developer.intel.com/ipex-whl-stable-xpu') else: machine = platform.machine() if sys.platform == 'darwin': diff --git a/modules/ipex_specific/__init__.py b/modules/ipex_specific/__init__.py index b71027b35..8dbec2830 100644 --- a/modules/ipex_specific/__init__.py +++ b/modules/ipex_specific/__init__.py @@ -4,6 +4,7 @@ import torch import intel_extension_for_pytorch as ipex from modules import shared from modules.sd_hijack_utils import CondFunc +from .diffusers import ipex_diffusers #ControlNet depth_leres++ class DummyDataParallel(torch.nn.Module): @@ -149,3 +150,5 @@ def ipex_init(): weight if weight is not None else torch.ones(input.size()[1], device=shared.device), bias if bias is not None else torch.zeros(input.size()[1], device=shared.device), *args, **kwargs), lambda orig_func, input, *args, **kwargs: input.device != torch.device("cpu")) + + ipex_diffusers() diff --git a/modules/ipex_specific/diffusers.py b/modules/ipex_specific/diffusers.py new file mode 100644 index 000000000..7253e8cd5 --- /dev/null +++ b/modules/ipex_specific/diffusers.py @@ -0,0 +1,111 @@ +import torch +import intel_extension_for_pytorch as ipex +import diffusers + +#ARC GPUs can't allocate more than 4GB to a single block: +class SlicedAttnProcessor: + r""" + Processor for implementing sliced attention. + + Args: + slice_size (`int`, *optional*): + The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and + `attention_head_dim` must be a multiple of the `slice_size`. + """ + + def __init__(self, slice_size): + self.slice_size = slice_size + + def __call__(self, attn: diffusers.models.attention_processor.Attention, hidden_states, encoder_hidden_states=None, attention_mask=None): + residual = hidden_states + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + dim = query.shape[-1] + query = attn.head_to_batch_dim(query) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + batch_size_attention, query_tokens, shape_three = query.shape + hidden_states = torch.zeros( + (batch_size_attention, query_tokens, dim // attn.heads), device=query.device, dtype=query.dtype + ) + + block_size = (batch_size_attention * query_tokens * shape_three) / 1024 * 1.2 #MB + split_2_slice_size = query_tokens + if block_size >= 4000: + do_split_2 = True + #Find something divisible with the query_tokens + while ((self.slice_size * split_2_slice_size * shape_three) / 1024 * 1.2) > 4000: + split_2_slice_size = split_2_slice_size // 2 + else: + do_split_2 = False + + for i in range(batch_size_attention // self.slice_size): + start_idx = i * self.slice_size + end_idx = (i + 1) * self.slice_size + + if do_split_2: + for i2 in range(query_tokens // split_2_slice_size): + start_idx_2 = i2 * split_2_slice_size + end_idx_2 = (i2 + 1) * split_2_slice_size + + query_slice = query[start_idx:end_idx, start_idx_2:end_idx_2] + key_slice = key[start_idx:end_idx, start_idx_2:end_idx_2] + attn_mask_slice = attention_mask[start_idx:end_idx, start_idx_2:end_idx_2] if attention_mask is not None else None + + attn_slice = attn.get_attention_scores(query_slice, key_slice, attn_mask_slice) + attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx, start_idx_2:end_idx_2]) + + hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = attn_slice + else: + query_slice = query[start_idx:end_idx] + key_slice = key[start_idx:end_idx] + attn_mask_slice = attention_mask[start_idx:end_idx] if attention_mask is not None else None + + attn_slice = attn.get_attention_scores(query_slice, key_slice, attn_mask_slice) + + attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx]) + + hidden_states[start_idx:end_idx] = attn_slice + + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + +def ipex_diffusers(): + diffusers.models.attention_processor.SlicedAttnProcessor = SlicedAttnProcessor diff --git a/modules/sd_vae.py b/modules/sd_vae.py index e87af749f..1edd39961 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -196,9 +196,10 @@ def load_vae_diffusers(_model, vae_file=None, vae_source="from unknown source"): try: import diffusers if os.path.isfile(vae_file): - # load_config passed to from_single_file doesn't apply - # from_single_file by default downloads VAE1.5 config - shared.log.warning("Using SDXL VAE loaded from singular file will result in low contrast images.") + if shared.opts.diffusers_pipeline == "Stable Diffusion XL": + # load_config passed to from_single_file doesn't apply + # from_single_file by default downloads VAE1.5 config + shared.log.warning("Using SDXL VAE loaded from singular file will result in low contrast images.") vae = diffusers.AutoencoderKL.from_single_file(vae_file) vae = vae.to(devices.dtype_vae) else: diff --git a/modules/shared.py b/modules/shared.py index 8b0ff3109..eb8c685c0 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -407,7 +407,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, lambda: {"choices": ['default', 'true', 'false']}), "diffusers_vae_slicing": OptionInfo(True, "Enable VAE slicing"), "diffusers_vae_tiling": OptionInfo(False, "Enable VAE tiling"), - "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), + "diffusers_attention_slicing": OptionInfo(True if devices.backend == "ipex" else False, "Enable attention slicing"), "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}), # "diffusers_force_zeros": OptionInfo(False, "Force zeros for prompts when empty"), diff --git a/webui.sh b/webui.sh index 550008f2d..2800b2d3d 100755 --- a/webui.sh +++ b/webui.sh @@ -96,7 +96,7 @@ if [[ ! -z "${ACCELERATE}" ]] && [ ${ACCELERATE}="True" ] && [ -x "$(command -v then echo "Launching accelerate launch.py..." exec accelerate launch --num_cpu_threads_per_process=6 launch.py "$@" -elif [[ -z "${first_launch}" ]] && [[ $(uname -a) != *WSL2* ]] && [ -x "$(command -v ipexrun)" ] && [ -x "$(command -v numactl)" ] && [ -x "$(command -v sycl-ls)" ] +elif [[ "$@" == *"--use-ipex"* ]] && [[ -z "${first_launch}" ]] && [[ $(uname -a) != *WSL2* ]] && [ -x "$(command -v ipexrun)" ] && [ -x "$(command -v numactl)" ] && [ -x "$(command -v sycl-ls)" ] then echo "Launching ipexrun launch.py..." exec ipexrun launch.py "$@" From 4234555566a8a7d3b3505a784b85c9ffc2c0ac49 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Aug 2023 14:37:06 +0000 Subject: [PATCH 09/12] update --- CHANGELOG.md | 8 +++++--- TODO.md | 1 - modules/devices.py | 8 ++++---- modules/shared.py | 5 ++--- requirements.txt | 1 + 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc34c3a6..bb3dbb74a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,14 @@ ## Update for 2023-08-05 +- diffusers: + - vaesd live preview (sd and sd-xl) + - fix inpainting (sd and sd-xl) - general: - new torch 2.0 with ipex (intel arc) - additional callbacks for extensions - - update requirements -- diffusers - - sd-xl: vaesd live preview decoder + enables latest comfyui extension + - update requirements ## Update for 2023-07-30 diff --git a/TODO.md b/TODO.md index 08c7e4655..6428c9e68 100644 --- a/TODO.md +++ b/TODO.md @@ -15,7 +15,6 @@ Stuff to be added, in no particular order... - Add Hires - Add Lora/Lyco mixer - Add ControlNet - - Fix SD-XL Img2img/Inpaint - Add SD and SD-XL Pix2Pix - Fix DeepFloyd IF model - Redo Prompt parser for diffusers diff --git a/modules/devices.py b/modules/devices.py index 8a30e2ee4..5f15eeb98 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -119,9 +119,9 @@ def set_cuda_params(): shared.log.debug('Verifying Torch settings') if cuda_ok: try: - torch.backends.cuda.matmul.allow_tf32 = shared.opts.cuda_allow_tf32 - torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = shared.opts.cuda_allow_tf16_reduced - torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = shared.opts.cuda_allow_tf16_reduced + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = True except Exception: pass if torch.backends.cudnn.is_available(): @@ -130,7 +130,7 @@ def set_cuda_params(): if shared.opts.cudnn_benchmark: shared.log.debug('Torch enable cuDNN benchmark') torch.backends.cudnn.benchmark_limit = 0 - torch.backends.cudnn.allow_tf32 = shared.opts.cuda_allow_tf32 + torch.backends.cudnn.allow_tf32 = True except Exception: pass global dtype, dtype_vae, dtype_unet, unet_needs_upcast # pylint: disable=global-statement diff --git a/modules/shared.py b/modules/shared.py index eb8c685c0..f898e6406 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -380,8 +380,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "rollback_vae": OptionInfo(False, "Attempt VAE roll back when produced NaN values (experimental)"), "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "), "cudnn_benchmark": OptionInfo(False, "Enable full-depth cuDNN benchmark feature"), - "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"), - "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"), + # "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"), + # "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"), "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"), "cuda_compile_backend": OptionInfo("none", "Model compile backend (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex']}), "cuda_compile_mode": OptionInfo("default", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['default', 'reduce-overhead', 'max-autotune']}), @@ -396,7 +396,6 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_allow_safetensors": OptionInfo(True, 'Diffusers allow loading from safetensors files'), "diffusers_pipeline": OptionInfo(pipelines[0], 'Diffusers pipeline', gr.Dropdown, lambda: {"choices": pipelines}), - "diffusers_refiner_latents": OptionInfo(True, "Use latents when using refiner"), "diffusers_move_base": OptionInfo(False, "Move base model to CPU when using refiner"), "diffusers_move_refiner": OptionInfo(True, "Move refiner model to CPU when not in use"), "diffusers_move_unet": OptionInfo(False, "Move UNet to CPU while VAE decoding"), diff --git a/requirements.txt b/requirements.txt index e3ab89323..713365b0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,6 +52,7 @@ opencv-python-headless==4.7.0.72 diffusers==0.19.3 einops==0.4.1 gradio==3.32.0 +huggingface_hub==0.16.4 numexpr==2.8.4 numpy==1.23.5 numba==0.57.0 From 8aba6d82889ba572a2c71eafc5fa2d210e3dca5f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 5 Aug 2023 19:29:52 +0300 Subject: [PATCH 10/12] IPEX fix BF16 --- modules/ipex_specific/__init__.py | 11 ++++++++--- modules/ipex_specific/diffusers.py | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/ipex_specific/__init__.py b/modules/ipex_specific/__init__.py index 8dbec2830..9206b95f2 100644 --- a/modules/ipex_specific/__init__.py +++ b/modules/ipex_specific/__init__.py @@ -115,10 +115,15 @@ def ipex_init(): CondFunc('torch.nn.modules.Linear.forward', lambda orig_func, self, input: orig_func(self, input.to(self.weight.data.dtype)), lambda orig_func, self, input: input.dtype != self.weight.data.dtype) + CondFunc('torch.nn.functional.layer_norm', + lambda orig_func, input, normalized_shape=None, weight=None, *args, **kwargs: + orig_func(input.to(weight.data.dtype), normalized_shape, weight, *args, **kwargs), + lambda orig_func, input, normalized_shape=None, weight=None, *args, **kwargs: + input.dtype != weight.data.dtype and weight is not None) #Diffusers bfloat16: - CondFunc('torch.nn.modules.Conv2d._conv_forward', - lambda orig_func, self, input, weight, bias=None: orig_func(self, input.to(weight.data.dtype), weight, bias=bias), - lambda orig_func, self, input, weight, bias=None: input.dtype != weight.data.dtype) + CondFunc('torch.nn.functional.conv2d', + lambda orig_func, input, weight, *args, **kwargs: orig_func(input.to(weight.data.dtype), weight, *args, **kwargs), + lambda orig_func, input, weight, *args, **kwargs: input.dtype != weight.data.dtype) #Functions that does not work with the XPU: #UniPC: diff --git a/modules/ipex_specific/diffusers.py b/modules/ipex_specific/diffusers.py index 7253e8cd5..2312fcf8b 100644 --- a/modules/ipex_specific/diffusers.py +++ b/modules/ipex_specific/diffusers.py @@ -52,12 +52,13 @@ class SlicedAttnProcessor: (batch_size_attention, query_tokens, dim // attn.heads), device=query.device, dtype=query.dtype ) - block_size = (batch_size_attention * query_tokens * shape_three) / 1024 * 1.2 #MB + block_multiply = 2.4 if query.dtype == torch.float32 else 1.2 + block_size = (batch_size_attention * query_tokens * shape_three) / 1024 * block_multiply #MB split_2_slice_size = query_tokens if block_size >= 4000: do_split_2 = True #Find something divisible with the query_tokens - while ((self.slice_size * split_2_slice_size * shape_three) / 1024 * 1.2) > 4000: + while ((self.slice_size * split_2_slice_size * shape_three) / 1024 * block_multiply) > 4000: split_2_slice_size = split_2_slice_size // 2 else: do_split_2 = False From a22862d5c6da68c463e2b82f2a187efe858bf125 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 6 Aug 2023 02:20:45 +0300 Subject: [PATCH 11/12] IPEX fix embedding on FP32 and BF16 --- modules/ipex_specific/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/ipex_specific/__init__.py b/modules/ipex_specific/__init__.py index 9206b95f2..f7af296c1 100644 --- a/modules/ipex_specific/__init__.py +++ b/modules/ipex_specific/__init__.py @@ -112,15 +112,25 @@ def ipex_init(): CondFunc('torch.nn.modules.GroupNorm.forward', lambda orig_func, self, input: orig_func(self, input.to(self.weight.data.dtype)), lambda orig_func, self, input: input.dtype != self.weight.data.dtype) + #FP32: CondFunc('torch.nn.modules.Linear.forward', lambda orig_func, self, input: orig_func(self, input.to(self.weight.data.dtype)), lambda orig_func, self, input: input.dtype != self.weight.data.dtype) + #Embedding FP32: + CondFunc('torch.bmm', + lambda orig_func, input, mat2, *args, **kwargs: orig_func(input, mat2.to(input.dtype), *args, **kwargs), + lambda orig_func, input, mat2, *args, **kwargs: input.dtype != mat2.dtype) + #BF16: CondFunc('torch.nn.functional.layer_norm', lambda orig_func, input, normalized_shape=None, weight=None, *args, **kwargs: orig_func(input.to(weight.data.dtype), normalized_shape, weight, *args, **kwargs), lambda orig_func, input, normalized_shape=None, weight=None, *args, **kwargs: input.dtype != weight.data.dtype and weight is not None) - #Diffusers bfloat16: + #Embedding BF16 + CondFunc('torch.cat', + lambda orig_func, input, *args, **kwargs: orig_func([input[0].to(input[1].dtype), input[1], input[2].to(input[1].dtype)], *args, **kwargs), + lambda orig_func, input, *args, **kwargs: len(input) == 3 and (input[0].dtype != input[1].dtype or input[2].dtype != input[1].dtype)) + #Diffusers BF16: CondFunc('torch.nn.functional.conv2d', lambda orig_func, input, weight, *args, **kwargs: orig_func(input.to(weight.data.dtype), weight, *args, **kwargs), lambda orig_func, input, weight, *args, **kwargs: input.dtype != weight.data.dtype) From dc739b9f50a4a70ef26e614f664dada53212bc12 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 6 Aug 2023 07:06:58 +0000 Subject: [PATCH 12/12] fix taesd for original backend --- CHANGELOG.md | 2 ++ extensions-builtin/sd-dynamic-thresholding | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- modules/shared.py | 1 + modules/taesd/sd_vae_taesd.py | 6 +++++- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb3dbb74a..b8156706d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Update for 2023-08-05 +Another minor update, but it unlocks some cool new items... + - diffusers: - vaesd live preview (sd and sd-xl) - fix inpainting (sd and sd-xl) diff --git a/extensions-builtin/sd-dynamic-thresholding b/extensions-builtin/sd-dynamic-thresholding index 5349f0087..c60fe071e 160000 --- a/extensions-builtin/sd-dynamic-thresholding +++ b/extensions-builtin/sd-dynamic-thresholding @@ -1 +1 @@ -Subproject commit 5349f008721480a572ab9a917533afdd0dae7b9e +Subproject commit c60fe071e5938974a52611f87ca2fd1878aa6d15 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 4b815cc35..af34f5144 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 4b815cc351ca5eeec489f573a9a6c2dbd47374d7 +Subproject commit af34f514499933d5e7e1641a6b13d56411e45e76 diff --git a/modules/shared.py b/modules/shared.py index f898e6406..3dd194ff5 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -1000,4 +1000,5 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s sd_model = None sd_refiner = None +sd_model_type = '' sys.modules[__name__].__class__ = Shared diff --git a/modules/taesd/sd_vae_taesd.py b/modules/taesd/sd_vae_taesd.py index 17a7bceee..40151ba6b 100644 --- a/modules/taesd/sd_vae_taesd.py +++ b/modules/taesd/sd_vae_taesd.py @@ -5,6 +5,7 @@ Tiny AutoEncoder for Stable Diffusion https://github.com/madebyollin/taesd """ import os +from PIL import Image from modules import devices, paths_internal from modules.taesd.taesd import TAESD @@ -42,8 +43,11 @@ def model(model_class = 'sd', model_type = 'decoder'): def decode(latents): from modules import shared model_class = shared.sd_model_type + if model_class == 'ldm': + model_class = 'sd' if 'sd' not in model_class: - return None + shared.log.warning(f'TAESD unsupported model type: {model_class}') + return Image.new('RGB', (8, 8), color = (0, 0, 0)) vae = taesd_models[f'{model_class}-decoder'] if vae is None: model_path = os.path.join(paths_internal.models_path, "TAESD", f"tae{model_class}_decoder.pth")