From 75a8c1f9d097040258b7e1c3e5dbb263af22a558 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 10 Jul 2023 11:44:52 -0400 Subject: [PATCH] enable basic img2img --- CHANGELOG.md | 6 +++++- TODO.md | 1 + modules/processing.py | 13 ++++++------- modules/sd_models.py | 13 +++++++++++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cb46d1af..acbe327ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,16 @@ ## Update for 07/10/2023 +Service release with some fixes and enhancements: + - diffusers: - option to move base and/or refiner model to cpu to free up vram - model downloader options to specify model variant / revision / mirror - now you can download `fp16` variant directly for reduced memory footprint + - basic **img2img** workflow (*sketch* and *inpaint* are not supported yet) + note that **sd-xl** img2img workflows are architecturaly different so it will take longer to implement - updated hints for settings -- extra network: +- extra networks: - fix corrupt display on refesh when new extra network type found - additional ui tweaks - generate thumbnails from previews only if preview resolution is above 1k diff --git a/TODO.md b/TODO.md index 5477ee142..cdbe28d62 100644 --- a/TODO.md +++ b/TODO.md @@ -62,3 +62,4 @@ Tech that can be integrated as part of the core workflow... - upate `gradio` - extra network refresh breaks if new extra network type found - [sd-xl lora](https://civitai.com/models/104913/fcstyledxl) +- sd-xl img2img with configurable steps diff --git a/modules/processing.py b/modules/processing.py index 0b664b34d..76af2f8e7 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -225,14 +225,13 @@ class StableDiffusionProcessing: # identify itself with a field common to all models. The conditioning_key is also hybrid. if backend == Backend.DIFFUSERS: log.warning('Diffusers not implemented: img2img_image_conditioning') - return None if isinstance(self.sd_model, LatentDepth2ImageDiffusion): return self.depth2img_image_conditioning(source_image) - if self.sd_model.cond_stage_key == "edit": + if hasattr(self.sd_model, 'cond_stage_key') and self.sd_model.cond_stage_key == "edit": return self.edit_image_conditioning(source_image) - if self.sampler.conditioning_key in {'hybrid', 'concat'}: + if hasattr(self.sampler, 'conditioning_key') and self.sampler.conditioning_key in {'hybrid', 'concat'}: return self.inpainting_image_conditioning(source_image, latent_image, image_mask=image_mask) - if self.sampler.conditioning_key == "crossattn-adm": + if hasattr(self.sampler, 'conditioning_key') and self.sampler.conditioning_key == "crossattn-adm": return self.unclip_image_conditioning(source_image) # Dummy zero conditioning if we're not using inpainting or depth model. return latent_image.new_zeros(latent_image.shape[0], 5, 1, 1) @@ -752,7 +751,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: x_samples_ddim = output.images - if p.enable_hr: + if p.is_hr_pass: log.warning('Diffusers not implemented: hires fix') if lora_state['active']: @@ -1130,8 +1129,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): if backend == Backend.ORIGINAL: self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image)) else: - # we don't pre-encode the latents for diffusers to allow the UI to stay general for different model types - self.init_latent = None + # TODO Diffusers don't pre-encode the latents for diffusers to allow the UI to stay general for different model types + self.init_latent = torch.Tensor(1) if self.resize_mode == 3: self.init_latent = torch.nn.functional.interpolate(self.init_latent, size=(self.height // opt_f, self.width // opt_f), mode="bilinear") diff --git a/modules/sd_models.py b/modules/sd_models.py index ac4fceaa5..4ed74663a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -806,12 +806,21 @@ def set_diffuser_pipe(pipe, new_pipe_type): pipe_name = pipe.__class__.__name__ pipe_name = pipe_name.replace("Img2Img", "").replace("Inpaint", "") + new_pipe_cls_str = None if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: new_pipe_cls_str = pipe_name elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: - new_pipe_cls_str = pipe_name.replace("Pipeline", "Img2ImgPipeline") + tmp_pipe_name = pipe_name.replace("Pipeline", "Img2ImgPipeline") + if hasattr(diffusers, tmp_pipe_name): + new_pipe_cls_str = pipe_name.replace("Pipeline", "Img2ImgPipeline") elif new_pipe_type == DiffusersTaskType.INPAINTING: - new_pipe_cls_str = pipe_name.replace("Pipeline", "InpaintPipeline") + tmp_pipe_name = pipe_name.replace("Pipeline", "InpaintPipeline") + if hasattr(diffusers, tmp_pipe_name): + new_pipe_cls_str = pipe_name.replace("Pipeline", "InpaintPipeline") + + if new_pipe_cls_str is None: + shared.log.warning(f'Diffusers unknown pipeline: {tmp_pipe_name}') + new_pipe_cls_str = pipe_name new_pipe_cls = getattr(diffusers, new_pipe_cls_str)