improve inpainting quality

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Update for 2023-12-27
+## Update for 2023-12-28
 
 - **Control**  
   - native implementation of all image control methods:  
@@ -48,6 +48,8 @@
   - **Schedulers**  
     - add timesteps range, changing it will make scheduler to be over-complete or under-complete  
     - add rescale betas with zero SNR option (applicable to Euler, Euler a and DDIM, allows for higher dynamic range)  
+  - **Inpaint**  
+    - improved quality when using mask blur and padding  
   - **UI**  
     - 3 new native UI themes: **orchid-dreams**, **emerald-paradise** and **timeless-beige**, thanks @illu_Zn
     - more dynamic controls depending on the backend (original or diffusers)  

modules/processing.py

@@ -1227,9 +1227,9 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         self.image_mask = mask
         self.latent_mask = None
         self.mask_for_overlay = None
-        self.mask_blur = mask_blur
         self.mask_blur_x: int = 4
         self.mask_blur_y: int = 4
+        self.mask_blur = mask_blur
         self.inpainting_fill = inpainting_fill
         self.inpaint_full_res = inpaint_full_res
         self.inpaint_full_res_padding = inpaint_full_res_padding
@@ -1251,15 +1251,13 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
 
     @property
     def mask_blur(self):
-        if self.mask_blur_x == self.mask_blur_y:
-            return self.mask_blur_x
-        return None
+        mask_blur = max(self.mask_blur_x, self.mask_blur_y)
+        return mask_blur
 
     @mask_blur.setter
     def mask_blur(self, value):
-        if isinstance(value, int):
-            self.mask_blur_x = value
-            self.mask_blur_y = value
+        self.mask_blur_x = value
+        self.mask_blur_y = value
 
     def init(self, all_prompts, all_seeds, all_subseeds):
         if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None and not self.is_control:

modules/processing_diffusers.py

@@ -164,34 +164,17 @@ def process_diffusers(p: StableDiffusionProcessing):
             p.ops.append('inpaint')
             if getattr(p, 'mask', None) is None:
                 p.mask = TF.to_pil_image(torch.ones_like(TF.to_tensor(p.init_images[0]))).convert("L")
+            p.mask = shared.sd_model.mask_processor.blur(p.mask, blur_factor=p.mask_blur)
             width = 8 * math.ceil(p.init_images[0].width / 8)
             height = 8 * math.ceil(p.init_images[0].height / 8)
-            # option-1: use images as inputs
             task_args = {
                 'image': p.init_images,
                 'mask_image': p.mask,
                 'strength': p.denoising_strength,
                 'height': height,
                 'width': width,
+                # 'padding_mask_crop': p.inpaint_full_res_padding # done back in main processing method
             }
-            """ # option-2: preprocess images into latents using diffusers
-            vae_scale_factor = 2 ** (len(model.vae.config.block_out_channels) - 1)
-            image_processor = diffusers.image_processor.VaeImageProcessor(vae_scale_factor=vae_scale_factor)
-            mask_processor = diffusers.image_processor.VaeImageProcessor(vae_scale_factor=vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True)
-            init_image = image_processor.preprocess(p.init_images[0], width=width, height=height)
-            mask_image = mask_processor.preprocess(p.mask, width=width, height=height)
-            task_args = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": height, "width": width}
-            """
-            """ # option-2: manually assemble masked image latents
-            masked_image_latents = []
-            mask_image = TF.to_tensor(p.mask)
-            for init_image in p.init_images:
-                init_image = TF.to_tensor(p.init_images[0])
-                masked_image = init_image * (mask_image > 0.5)
-                masked_image_latents.append(torch.cat([masked_image, mask_image], dim=0))
-            masked_image_latents = torch.stack(masked_image_latents, dim=0).to(shared.device)
-            task_args = {"image": p.init_images, "mask_image": mask_image, "masked_image_latents": masked_image_latents, "strength": p.denoising_strength, "height": height, "width": width}
-            """
         if model.__class__.__name__ == 'LatentConsistencyModelPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0:
             p.ops.append('lcm')
             init_latents = [vae_encode(image, model=shared.sd_model, full_quality=p.full_quality).squeeze(dim=0) for image in p.init_images]

modules/ui.py

@@ -472,7 +472,7 @@ def create_hires_inputs(tab):
                 hr_force = gr.Checkbox(label='Force Hires', value=False, elem_id=f"{tab}_hr_force")
             with FormRow(elem_id=f"{tab}_hires_fix_row2", variant="compact"):
                 hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='Hires steps', elem_id=f"{tab}_steps_alt", value=20)
-                hr_scale = gr.Slider(minimum=1.0, maximum=4.0, step=0.05, label="Upscale by", value=2.0, elem_id=f"{tab}_hr_scale")
+                hr_scale = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Upscale by", value=2.0, elem_id=f"{tab}_hr_scale")
             with FormRow(elem_id=f"{tab}_hires_fix_row3", variant="compact"):
                 hr_resize_x = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize width to", value=0, elem_id=f"{tab}_hr_resize_x")
                 hr_resize_y = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize height to", value=0, elem_id=f"{tab}_hr_resize_y")