From 965e5a95f14e29a9c612d452a683d64045c19520 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Sep 2023 14:13:27 -0400 Subject: [PATCH] refactor diffusers tasks --- CHANGELOG.md | 5 ++- html/locale_en.json | 1 + modules/processing.py | 23 ++++------- modules/processing_diffusers.py | 48 +++++++++++++--------- modules/sd_models.py | 71 ++++++++++++++++----------------- modules/shared.py | 1 + 6 files changed, 78 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc3ea6a94..0f3462ed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,8 @@ Upgrades are still possible and supported, but above is recommended for best exp *note*: this will trigger model hash recaclulation on first model use - **Diffusers**: - better pipeline auto-detect when loading from safetensors + also, new setting: *settings -> diffusers -> force inpaint* + as some models behave better when in *inpaint* mode even for normal *img2img* tasks - **SDXL Inpaint** - Although any model can be used for inpainiting, there is a case to be made for dedicated inpainting models as they are tuned to inpaint and not generate @@ -102,7 +104,8 @@ Upgrades are still possible and supported, but above is recommended for best exp *Models -> Valida -> Calculate hashes* - **Compute** - **Intel Arc/IPEX**: - - more optimizations, built-in binary wheels for Windows + - tons of optimizations, built-in binary wheels for Windows + i have to say, intel arc/ipex is getting to be quite a player, especially with openvino thanks @Disty0 @Nuullll - **AMD ROCm**: - updated installer to support detect `ROCm` *5.4/5.5/5.6/5.7* diff --git a/html/locale_en.json b/html/locale_en.json index 65b762a7f..e3b1118d8 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -590,6 +590,7 @@ {"id":"","label":"Enable VAE slicing","localized":"","hint":"Decodes batch latents one image at a time with limited VRAM. Small performance boost in VAE decode on multi-image batches"}, {"id":"","label":"Enable VAE tiling","localized":"","hint":"Divide large images into overlapping tiles with limited VRAM. Results in a minor increase in processing time"}, {"id":"","label":"Enable attention slicing","localized":"","hint":"Performs attention computation in steps instead of all at once. Slower inference times, but greatly reduced memory usage"}, + {"id":"","label":"Diffusers force inpaint pipeline","localized":"","hint":"Force inpaint pipeline for all img2img tasks. Results in slightly different results that may be more precise"}, {"id":"","label":"Diffusers model loading variant","localized":"","hint":""}, {"id":"","label":"Diffusers VAE loading variant","localized":"","hint":""}, {"id":"","label":"Diffusers LoRA loading variant","localized":"","hint":"'sequential apply' loads and applies each LoRA in order of appearance, 'merge and apply' loads all LoRAs and merges them in-memory before applying to model, 'diffusers' uses diffusers default LoRA loading method"}, diff --git a/modules/processing.py b/modules/processing.py index 96b21b719..95bd10004 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -516,10 +516,10 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}" args["Init image hash"] = getattr(p, 'init_img_hash', None) args["Conditional mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None - args['Resize mode'] = p.resize_mode - args["Mask blur"] = p.mask_blur if p.mask is not None and p.mask_blur > 0 else None - args["Noise multiplier"] = p.initial_noise_multiplier if p.initial_noise_multiplier != 1.0 else None - args["Denoising strength"] = p.denoising_strength + args['Resize mode'] = getattr(p, 'resize_mode', None) + args["Mask blur"] = p.mask_blur if getattr(p, 'mask', None) is not None and getattr(p, 'mask_blur', 0) > 0 else None + args["Noise multiplier"] = p.initial_noise_multiplier if getattr(p, 'initial_noise_multiplier', 1.0) != 1.0 else None + args["Denoising strength"] = getattr(p, 'denoising_strength', None) if 'face' in p.ops: args["Face restoration"] = shared.opts.face_restoration_model if 'color' in p.ops: @@ -933,7 +933,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == shared.Backend.DIFFUSERS: - modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE) + shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE) self.width = self.width or 512 self.height = self.height or 512 @@ -990,9 +990,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.restore_faces = orig2 images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], shared.opts.samples_format, info=info, suffix="-before-hires") - if shared.backend == shared.Backend.DIFFUSERS: - modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE) - latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None") if latent_scale_mode is not None: self.hr_force = False # no need to force anything @@ -1097,10 +1094,10 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None: - modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING) + shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING) self.sd_model.dtype = self.sd_model.unet.dtype elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None: - modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE) if self.sampler_name == "PLMS": self.sampler_name = 'UniPC' @@ -1213,12 +1210,6 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.image_conditioning = self.img2img_image_conditioning(image, self.init_latent, image_mask) def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): - if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None: - modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING) - self.sd_model.dtype = self.sd_model.unet.dtype - elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None: - modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - x = create_random_tensors([4, self.height // 8, self.width // 8], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) x *= self.initial_noise_multiplier samples = self.sampler.sample_img2img(self, self.init_latent, x, conditioning, unconditional_conditioning, image_conditioning=self.image_conditioning) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index dc895be6d..7c5fb3bb1 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -176,6 +176,26 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_prompts_2.append(negative_prompts_2[-1]) return prompts, negative_prompts, prompts_2, negative_prompts_2 + def task_specific_kwargs(model): + task_args = {} + if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: + p.ops.append('txt2img') + task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: + p.ops.append('img2img') + task_args = {"image": p.init_images, "strength": p.denoising_strength} + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INSTRUCT: + p.ops.append('instruct') + task_args = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8), "image": p.init_images, "strength": p.denoising_strength} + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING: + p.ops.append('inpaint') + if getattr(p, 'mask', None) is None: + p.mask = TF.to_pil_image(torch.ones_like(TF.to_tensor(p.init_images[0]))).convert("L") + width = 8 * math.ceil(p.init_images[0].width / 8) + height = 8 * math.ceil(p.init_images[0].height / 8) + task_args = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": height, "width": width} + return task_args + def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): # if hasattr(model, 'embedding_db'): # del model.embedding_db @@ -231,10 +251,16 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if 'callback' in possible: args['callback'] = diffusers_callback for arg in kwargs: - if arg in possible: + if arg in possible: # add kwargs args[arg] = kwargs[arg] else: pass + task_kwargs = task_specific_kwargs(model) + for arg in task_kwargs: + if arg in possible and arg not in args: # task specific args should not override args + args[arg] = task_kwargs[arg] + else: + pass # shared.log.debug(f'Diffuser not supported: pipeline={pipeline.__class__.__name__} task={sd_models.get_diffusers_task(model)} arg={arg}') # shared.log.debug(f'Diffuser pipeline: {pipeline.__class__.__name__} possible={possible}') clean = args.copy() @@ -308,21 +334,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro p.init_images.append(p.init_images[-1]) if lora_state['active']: cross_attention_kwargs['scale'] = lora_state['multiplier'] - task_specific_kwargs={} - if sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: - p.ops.append('txt2img') - task_specific_kwargs = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} - elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: - p.ops.append('img2img') - task_specific_kwargs = {"image": p.init_images, "strength": p.denoising_strength} - elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INSTRUCT: - p.ops.append('instruct') - task_specific_kwargs = {"height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8), "image": p.init_images, "strength": p.denoising_strength} - elif sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING: - p.ops.append('inpaint') - if p.mask is None: - p.mask = TF.to_pil_image(torch.ones_like(TF.to_tensor(p.init_images[0]))).convert("L") - task_specific_kwargs = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)} if shared.state.interrupted or shared.state.skipped: if lora_state['active']: @@ -346,6 +357,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro else: return p.steps + # pipeline type is set earlier in processing.py base_args = set_pipeline_args( model=shared.sd_model, prompts=prompts, @@ -360,7 +372,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', clip_skip=p.clip_skip, desc='Base', - **task_specific_kwargs ) # p.steps = base_args['num_inference_steps'] p.extra_generation_params['CFG rescale'] = p.diffusers_guidance_rescale @@ -392,13 +403,13 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro output.images = hires_resize(latents=output.images) if latent_scale_mode is not None or p.hr_force: p.ops.append('hires') + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) recompile_model(hires=True) if ((not hasattr(shared.sd_model.scheduler, 'name')) or (p.latent_sampler == 'DPM SDE') or (shared.sd_model.scheduler.name != p.latent_sampler)) and (p.latent_sampler != 'Default') and is_karras_compatible: sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op - sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) hires_args = set_pipeline_args( model=shared.sd_model, prompts=prompts, @@ -449,6 +460,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.sd_refiner.to(devices.device) refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__) p.ops.append('refine') + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) for i in range(len(output.images)): image = output.images[i] # if (image.shape[2] == 3) and (image.shape[0] % 8 != 0 or image.shape[1] % 8 != 0): diff --git a/modules/sd_models.py b/modules/sd_models.py index 497a67192..8817bd7f6 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -540,16 +540,10 @@ class ModelData: self.lock = threading.Lock() def get_sd_model(self): - if self.sd_model is None and shared.opts.sd_model_checkpoint != 'None': + if self.sd_model is None and shared.opts.sd_model_checkpoint != 'None' and not self.lock.locked(): with self.lock: try: - if shared.backend == shared.Backend.ORIGINAL: - reload_model_weights(op='model') - elif shared.backend == shared.Backend.DIFFUSERS: - reload_model_weights(op='model') - # load_diffuser(op='model') - else: - shared.log.error(f"Unknown Execution backend: {shared.backend}") + self.sd_model = reload_model_weights(op='model') self.initial = False except Exception as e: shared.log.error("Failed to load stable diffusion model") @@ -561,15 +555,10 @@ class ModelData: self.sd_model = v def get_sd_refiner(self): - if self.sd_refiner is None and shared.opts.sd_model_refiner != 'None': + if self.sd_refiner is None and shared.opts.sd_model_refiner != 'None' and not self.lock.locked(): with self.lock: try: - if shared.backend == shared.Backend.ORIGINAL: - reload_model_weights(op='refiner') - elif shared.backend == shared.Backend.DIFFUSERS: - load_diffuser(op='refiner') - else: - shared.log.error(f"Unknown Execution backend: {shared.backend}") + self.sd_refiner = reload_model_weights(op='refiner') self.initial = False except Exception as e: shared.log.error("Failed to load stable diffusion model") @@ -585,6 +574,7 @@ model_data = ModelData() def change_backend(): shared.log.info(f'Backend changed: {shared.backend}') + shared.log.warning('Server restart required to apply all changes') if shared.backend == shared.Backend.ORIGINAL: change_from = shared.Backend.DIFFUSERS else: @@ -967,6 +957,17 @@ class DiffusersTaskType(Enum): INSTRUCT = 4 +def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: + if pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline": + return DiffusersTaskType.INSTRUCT + elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): + return DiffusersTaskType.IMAGE_2_IMAGE + elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING.values(): + return DiffusersTaskType.INPAINTING + else: + return DiffusersTaskType.TEXT_2_IMAGE + + def set_diffuser_pipe(pipe, new_pipe_type): sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None) @@ -974,8 +975,9 @@ def set_diffuser_pipe(pipe, new_pipe_type): has_accelerate = getattr(pipe, "has_accelerate", None) embedding_db = getattr(pipe, "embedding_db", None) - if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE and (pipe.__class__.__name__ == "StableDiffusionXLPipeline" or pipe.__class__.__name__ == 'StableDiffusionXLImg2ImgPipeline'): - new_pipe_type = DiffusersTaskType.INPAINTING # sdxl works better with init mask + if shared.opts.diffusers_force_inpaint: + if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + new_pipe_type = DiffusersTaskType.INPAINTING # sdxl may work better with init mask try: if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe) @@ -985,18 +987,18 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) except Exception: # pylint: disable=unused-variable # shared.log.error(f'Failed to change: type={new_pipe_type} pipeline={pipe.__class__.__name__} {e}') - return + return pipe if pipe.__class__ == new_pipe.__class__: - return - + return pipe new_pipe.sd_checkpoint_info = sd_checkpoint_info new_pipe.sd_model_checkpoint = sd_model_checkpoint new_pipe.sd_model_hash = sd_model_hash new_pipe.has_accelerate = has_accelerate new_pipe.embedding_db = embedding_db - model_data.sd_model = new_pipe - shared.log.debug(f"Pipeline class changed from {pipe.__class__.__name__} to {new_pipe.__class__.__name__}") + shared.log.debug(f"Pipeline class change: original={pipe.__class__.__name__} target={new_pipe.__class__.__name__}") + pipe = new_pipe + return pipe def get_native(pipe: diffusers.DiffusionPipeline): @@ -1013,17 +1015,6 @@ def get_native(pipe: diffusers.DiffusionPipeline): return size -def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: - if pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline": - return DiffusersTaskType.INSTRUCT - elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): - return DiffusersTaskType.IMAGE_2_IMAGE - elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING.values(): - return DiffusersTaskType.INPAINTING - else: - return DiffusersTaskType.TEXT_2_IMAGE - - def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): from modules import lowvram, sd_hijack checkpoint_info = checkpoint_info or select_checkpoint(op=op) @@ -1127,7 +1118,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model') next_checkpoint_info = info or select_checkpoint(op='dict' if load_dict else 'model') if load_dict else None if checkpoint_info is None: unload_model_weights(op=op) - return + return None orig_state = copy.deepcopy(shared.state) shared.state = shared.State() shared.state.begin(f'load-{op}') @@ -1143,7 +1134,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model') else: current_checkpoint_info = getattr(sd_model, 'sd_checkpoint_info', None) if current_checkpoint_info is not None and checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename: - return + return None if not getattr(sd_model, 'has_accelerate', False): if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: lowvram.send_everything_to_cpu() @@ -1172,9 +1163,16 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model') reload_model_weights(reuse_dict=True) # ok we loaded dict now lets redo and load model on top of it shared.state.end() shared.state = orig_state - return model_data.sd_model if op == 'model' or op == 'dict' else model_data.sd_refiner + # data['sd_model_checkpoint'] + if op == 'model' or op == 'dict': + shared.opts.data["sd_model_checkpoint"] = checkpoint_info.title + return model_data.sd_model + else: + shared.opts.data["sd_model_refiner"] = checkpoint_info.title + return model_data.sd_refiner # fallback + shared.log.info(f"Loading using fallback: {op} model={checkpoint_info.title}") try: load_model_weights(sd_model, checkpoint_info, state_dict, timer) except Exception: @@ -1191,6 +1189,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model') shared.state.end() shared.state = orig_state shared.log.info(f"Loaded: {op} time={timer.summary()}") + return sd_model def disable_offload(sd_model): diff --git a/modules/shared.py b/modules/shared.py index 464cacde1..c8a8b8cda 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -456,6 +456,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_lora_loader": OptionInfo("diffusers" if cmd_opts.use_openvino else "sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}), "diffusers_force_zeros": OptionInfo(True, "Force zeros for prompts when empty"), "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"), + "diffusers_force_inpaint": OptionInfo(False, 'Diffusers force inpaint pipeline'), })) options_templates.update(options_section(('system-paths', "System Paths"), {