From fab013d31c3b39b74b9400dcb9b520ff3aeadf80 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 22 Oct 2023 11:11:28 -0400 Subject: [PATCH] fix diffusers base/hires/refiner workflows --- CHANGELOG.md | 4 +-- modules/processing.py | 1 + modules/processing_diffusers.py | 49 +++++++++++++++++++++--------- modules/prompt_parser_diffusers.py | 13 ++++---- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd0013bee..ece6a095a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Service release addressing all zero-day issues reported so far... - fix new style filename template - fix image name template using model name - fix model path using relative path -- fix `torch-rocm` and `tensorflow-rocm` version detection (thanks @xangelix) +- fix `torch-rocm` and `tensorflow-rocm` version detection, thanks @xangelix - fix chainner upscalers color clipping - force second requirements check on startup - remove lyco, multiple_tqdm @@ -26,7 +26,7 @@ Service release addressing all zero-day issues reported so far... - clearly mark external themes in ui - new option: *settings -> images -> keep incomplete* can be used to skip vae decode on aborted/skipped/interrupted image generations -- update `openvino` (thanks @disty0) +- update `openvino`, thanks @disty0 - update `typing-extensions` ### Dev branch diff --git a/modules/processing.py b/modules/processing.py index db1eeda12..10fca683d 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -158,6 +158,7 @@ class StableDiffusionProcessing: self.clip_skip = clip_skip self.iteration = 0 self.is_hr_pass = False + self.is_refiner_pass = False self.hr_force = False self.enable_hr = None self.refiner_steps = 5 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 84f425f5a..7266ab197 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -1,3 +1,4 @@ +import os import time import math import inspect @@ -20,8 +21,6 @@ from modules.sd_hijack_hypertile import hypertile_set def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_prompts): results = [] - if p.enable_hr and p.hr_upscaler != 'None' and p.denoising_strength > 0 and len(getattr(p, 'init_images', [])) == 0: - p.is_hr_pass = True is_refiner_enabled = p.enable_hr and p.refiner_steps > 0 and p.refiner_start > 0 and p.refiner_start < 1 and shared.sd_refiner is not None if hasattr(p, 'init_images') and len(p.init_images) > 0: @@ -62,11 +61,16 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro for j in range(len(decoded)): images.save_image(decoded[j], path=p.outpath_samples, basename="", seed=seeds[i], prompt=prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix=suffix) - def diffusers_callback(_step: int, _timestep: int, latents: torch.FloatTensor): - shared.state.sampling_step += 1 - shared.state.sampling_steps = p.steps + def diffusers_callback(step: int, _timestep: int, latents: torch.FloatTensor): + shared.state.sampling_step = step if p.is_hr_pass: - shared.state.sampling_steps += p.hr_second_pass_steps + shared.state.job = 'hires' + shared.state.sampling_steps = p.hr_second_pass_steps # add optional hires + elif p.is_refiner_pass: + shared.state.job = 'refine' + shared.state.sampling_steps = calculate_refiner_steps() # add optional refiner + else: + shared.state.sampling_steps = p.steps # base steps shared.state.current_latent = latents if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') @@ -221,6 +225,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro parser = shared.opts.prompt_attention except Exception as e: shared.log.error(f'Prompt parser encode: {e}') + if os.environ.get('SD_PROMPT_DEBUG', None) is not None: + errors.display(e, 'Prompt parser encode') if 'prompt' in possible: if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None: if type(pooled) == list: @@ -355,6 +361,14 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro else: return p.steps + def calculate_refiner_steps(): + if p.refiner_start > 0 and p.refiner_start < 1 and refiner_is_sdxl: + refiner_steps = int(p.refiner_steps // (1 - p.refiner_start)) + else: + refiner_steps = int(p.refiner_steps // p.denoising_strength + 1) if refiner_is_sdxl else p.refiner_steps + p.refiner_steps = min(99, refiner_steps) + return p.refiner_steps + # pipeline type is set earlier in processing, but check for sanity if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: # reset pipeline shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) @@ -373,7 +387,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clip_skip=p.clip_skip, desc='Base', ) - # p.steps = base_args['num_inference_steps'] p.extra_generation_params['CFG rescale'] = p.diffusers_guidance_rescale p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1 else None try: @@ -393,6 +406,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return results # optional hires pass + if p.enable_hr and p.hr_upscaler != 'None' and p.denoising_strength > 0 and len(getattr(p, 'init_images', [])) == 0: + p.is_hr_pass = True latent_scale_mode = shared.latent_upscale_modes.get(p.hr_upscaler, None) if (hasattr(p, "hr_upscaler") and p.hr_upscaler is not None) else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None") if p.is_hr_pass: p.init_hr() @@ -427,6 +442,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro except AssertionError as e: shared.log.info(e) p.init_images = [] + p.is_hr_pass = False # optional refiner pass or decode if is_refiner_enabled: @@ -446,26 +462,25 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.sd_refiner.to(devices.device) refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__) p.ops.append('refine') + p.is_refiner_pass = True shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) + shared.sd_refiner = sd_models.set_diffuser_pipe(shared.sd_refiner, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) for i in range(len(output.images)): image = output.images[i] - # if (image.shape[2] == 3) and (image.shape[0] % 8 != 0 or image.shape[1] % 8 != 0): - # shared.log.warning(f'Refiner requires image size to be divisible by 8: {image.shape}') - # results.append(image) - # return results noise_level = round(350 * p.denoising_strength) output_type='latent' if hasattr(shared.sd_refiner, 'vae') else 'np' if shared.sd_refiner.__class__.__name__ == 'StableDiffusionUpscalePipeline': image = vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil') p.extra_generation_params['Noise level'] = noise_level output_type = 'np' + calculate_refiner_steps() refiner_args = set_pipeline_args( model=shared.sd_refiner, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i], negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i], - num_inference_steps=int(p.refiner_steps // (1 - p.refiner_start)) if p.refiner_start > 0 and p.refiner_start < 1 and refiner_is_sdxl else int(p.refiner_steps // p.denoising_strength + 1) if refiner_is_sdxl else p.refiner_steps, + num_inference_steps=p.refiner_steps, eta=shared.opts.scheduler_eta, - strength=p.denoising_strength, + # strength=p.denoising_strength, noise_level=noise_level, # StableDiffusionUpscalePipeline only guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, guidance_rescale=p.diffusers_guidance_rescale, @@ -476,7 +491,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clip_skip=p.clip_skip, desc='Refiner', ) - # p.steps += refiner_args['num_inference_steps'] try: refiner_output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable except AssertionError as e: @@ -491,9 +505,14 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.log.debug('Moving to CPU: model=refiner') shared.sd_refiner.to(devices.cpu) devices.torch_gc() + p.is_refiner_pass = True # final decode since there is no refiner if not is_refiner_enabled: - results = vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality) + if output is not None and output.images is not None and len(output.images) > 0: + results = vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality) + else: + shared.log.warning('Processing returned no results') + results = [] return results diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index d4db74254..600c66d4a 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -6,8 +6,7 @@ from compel.embeddings_provider import BaseTextualInversionManager, EmbeddingsPr from modules import shared, prompt_parser -debug_output = os.environ.get('SD_PROMPT_DEBUG', None) -debug = shared.log.info if debug_output is not None else lambda *args, **kwargs: None +debug = shared.log.info if os.environ.get('SD_PROMPT_DEBUG', None) is not None else lambda *args, **kwargs: None CLIP_SKIP_MAPPING = { @@ -106,11 +105,11 @@ def prepare_embedding_providers(pipe, clip_skip): shared.log.warning(f"Prompt parser unsupported: clip_skip={clip_skip}") clip_skip = 2 embedding_type = CLIP_SKIP_MAPPING[clip_skip] - if hasattr(pipe, "tokenizer") and hasattr(pipe, "text_encoder"): - embedding = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type) + if getattr(pipe, "tokenizer", None) is not None and getattr(pipe, "text_encoder", None) is not None: + embedding = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type, device=pipe.device) embeddings_providers.append(embedding) - if hasattr(pipe, "tokenizer_2") and hasattr(pipe, "text_encoder_2"): - embedding = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, truncate=False, returned_embeddings_type=embedding_type) + if getattr(pipe, "tokenizer_2", None) is not None and getattr(pipe, "text_encoder_2", None) is not None: + embedding = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, truncate=False, returned_embeddings_type=embedding_type, device=pipe.device) embeddings_providers.append(embedding) return embeddings_providers @@ -140,7 +139,7 @@ def get_weighted_text_embeddings_sdxl(pipe, prompt: str = "", neg_prompt: str = ns = [get_prompts_with_weights(p) for p in [neg_prompt, neg_prompt_2]] negatives = [t for t, w in ns] negative_weights = [w for t, w in ns] - if hasattr(pipe, "tokenizer_2") and not hasattr(pipe, "tokenizer"): + if getattr(pipe, "tokenizer_2", None) is not None and getattr(pipe, "tokenizer", None) is None: positives.pop(0) positive_weights.pop(0) negatives.pop(0)