From a29b740cee92798742beeed3c8ac31e4565becb4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 14 Oct 2023 17:03:49 -0400 Subject: [PATCH] reset pipeline and handle hypertile errors --- modules/processing.py | 2 +- modules/processing_diffusers.py | 25 ++++++++++++++----------- modules/sd_hijack_hypertile.py | 20 ++++++++++++-------- modules/upscaler.py | 2 +- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 6705db2a1..e1cd956eb 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -1223,7 +1223,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): image = images.resize_image(3, image, self.width, self.height) if shared.backend == shared.Backend.DIFFUSERS: unprocessed.append(image) - self.init_images = [image] # assign early for diffusers + self.init_images = [image] # TODO assign early for diffusers if image_mask is not None: if self.inpainting_fill != 1: image = modules.masking.fill(image, latent_mask) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index ea32d1165..40dfac131 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -32,9 +32,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro p.height = tgt_height p.width = tgt_width hypertile_set(p) - if p.mask is not None: + if getattr(p, 'mask', None) is not None: p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) - if p.mask_for_overlay is not None: + if getattr(p, 'mask_for_overlay', None) is not None: p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) def hires_resize(latents): # input=latents output=pil @@ -44,11 +44,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro latents = torch.nn.functional.interpolate(latents, size=(p.hr_upscale_to_y // 8, p.hr_upscale_to_x // 8), mode=latent_upscaler["mode"], antialias=latent_upscaler["antialias"]) first_pass_images = vae_decode(latents=latents, model=shared.sd_model, full_quality=p.full_quality, output_type='pil') p.init_images = [] - for first_pass_image in first_pass_images: + for img in first_pass_images: if latent_upscaler is None: - init_image = images.resize_image(1, first_pass_image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler) + init_image = images.resize_image(1, img, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler) else: - init_image = first_pass_image + init_image = img # if is_refiner_enabled: # init_image = vae_encode(init_image, model=shared.sd_model, full_quality=p.full_quality) p.init_images.append(init_image) @@ -185,13 +185,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: p.ops.append('txt2img') task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} - elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('img2img') task_args = {"image": p.init_images, "strength": p.denoising_strength} - elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INSTRUCT: + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INSTRUCT and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('instruct') task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8), "image": p.init_images, "strength": p.denoising_strength} - elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING: + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING and len(getattr(p, 'init_images' ,[])) > 0: p.ops.append('inpaint') if getattr(p, 'mask', None) is None: p.mask = TF.to_pil_image(torch.ones_like(TF.to_tensor(p.init_images[0]))).convert("L") @@ -201,6 +201,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return task_args def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): + if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} @@ -261,7 +262,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro pass # shared.log.debug(f'Diffuser not supported: pipeline={pipeline.__class__.__name__} task={sd_models.get_diffusers_task(model)} arg={arg}') # shared.log.debug(f'Diffuser pipeline: {pipeline.__class__.__name__} possible={possible}') - hypertile_set(p, hr=hasattr(p, 'init_images') and len(p.init_images) > 0) + hypertile_set(p, hr=len(getattr(p, 'init_images', []))) clean = args.copy() clean.pop('callback', None) clean.pop('callback_steps', None) @@ -351,7 +352,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro else: return p.steps - # pipeline type is set earlier in processing.py + # pipeline type is set earlier in processing, but check for sanity + if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: # reset pipeline + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) base_args = set_pipeline_args( model=shared.sd_model, prompts=prompts, @@ -420,11 +423,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro strength=p.denoising_strength, desc='Hires', ) - # p.steps += hires_args['num_inference_steps'] try: output = shared.sd_model(**hires_args) # pylint: disable=not-callable except AssertionError as e: shared.log.info(e) + p.init_images = [] # optional refiner pass or decode if is_refiner_enabled: diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index 946a7a789..b4b9cd6a2 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -48,25 +48,30 @@ def split_attention(layer: nn.Module, tile_size: int=256, min_tile_size: int=256 reset_needed = True nhs = possible_tile_sizes(height, tile_size, min_tile_size, swap_size) # possible sub-grids that fit into the image nws = possible_tile_sizes(width, tile_size, min_tile_size, swap_size) - make_ns = lambda: (nhs[random.randint(0, len(nhs) - 1)], nws[random.randint(0, len(nws) - 1)]) # pylint: disable=unnecessary-lambda-assignment def reset_nhs(): - nonlocal nws, make_ns, ar + nonlocal nhs, ar ar = height / width # Aspect ratio nhs = possible_tile_sizes(height, tile_size, min_tile_size, swap_size) - make_ns = lambda: (nhs[random.randint(0, len(nhs) - 1)], nws[random.randint(0, len(nws) - 1)]) # pylint: disable=unnecessary-lambda-assignment def reset_nws(): - nonlocal nws, make_ns, ar + nonlocal nws, ar ar = height / width # Aspect ratio nws = possible_tile_sizes(width, tile_size, min_tile_size, swap_size) - make_ns = lambda: (nhs[random.randint(0, len(nhs) - 1)], nws[random.randint(0, len(nws) - 1)]) # pylint: disable=unnecessary-lambda-assignment def self_attn_forward(forward: Callable) -> Callable: @wraps(forward) def wrapper(*args, **kwargs): - global height, width, max_h, max_w, reset_needed # pylint: disable=global-statement - nh, nw = make_ns() + global height, width, max_h, max_w, reset_needed, error_reported # pylint: disable=global-statement x = args[0] + try: + nh = nhs[random.randint(0, len(nhs) - 1)] + nw = nws[random.randint(0, len(nws) - 1)] + except Exception as e: + if not error_reported: + error_reported = True + log.error(f'Hypertile error: width={width} height={height} {e}') + out = forward(x, *args[1:], **kwargs) + return out if x.ndim == 4: # VAE # TODO: VAE breaks for diffusers when using non-standard sizes if nh * nw > 1: @@ -108,7 +113,6 @@ def split_attention(layer: nn.Module, tile_size: int=256, min_tile_size: int=256 out = rearrange(out, "(b nh nw) hw c -> b nh nw hw c", nh=nh, nw=nw) out = rearrange(out, "b nh nw (h w) c -> b (nh h nw w) c", h=h // nh, w=w // nw) except Exception as e: - global error_reported # pylint: disable=global-statement if not error_reported: error_reported = True log.error(f'Hypertile error: width={width} height={height} {e}') diff --git a/modules/upscaler.py b/modules/upscaler.py index fcf967ede..45cdef63a 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -40,7 +40,7 @@ class Upscaler: self.mod_scale = None self.model_download_path = None if self.user_path is not None and len(self.user_path) > 0 and not os.path.exists(self.user_path): - modules.shared.log(f'Upscaler create: folder={self.user_path}') + modules.shared.log.info(f'Upscaler create: folder={self.user_path}') if self.model_path is None and self.name: self.model_path = os.path.join(modules.shared.models_path, self.name) if self.model_path and create_dirs: