fix inpaint

CHANGELOG.md

@@ -17,6 +17,7 @@
   - Updated logic for calculating **steps** when using base/hires/refiner workflows  
   - Safe model offloading for non-standard models  
   - Fix **DPM SDE** scheduler  
+  - Better support for SD 1.5 **inpainting** models
   - Update to `diffusers==0.23.0`  
 - **Extra networks**  
   - Use multi-threading for 5x load speedup  

modules/processing_diffusers.py

@@ -223,7 +223,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         return task_args
 
     def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs):
-
         if hasattr(model, "set_progress_bar_config"):
             model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba')
         args = {}
@@ -408,8 +407,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         return max(2, int(steps))
 
     # pipeline type is set earlier in processing, but check for sanity
-    if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: # reset pipeline
-        shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+    if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0:
+        shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset pipeline
+    if hasattr(shared.sd_model, 'unet') and hasattr(shared.sd_model.unet, 'config') and hasattr(shared.sd_model.unet.config, 'in_channels') and shared.sd_model.unet.config.in_channels == 9:
+        shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # force pipeline
+        if len(getattr(p, 'init_images' ,[])) == 0:
+            p.init_images = [TF.to_pil_image(torch.rand((3, p.height, p.width)))]
     base_args = set_pipeline_args(
         model=shared.sd_model,
         prompts=prompts,

modules/sd_models.py

@@ -899,15 +899,26 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                 if model_type.startswith('Stable Diffusion'):
                     diffusers_load_config['force_zeros_for_empty_prompt '] = shared.opts.diffusers_force_zeros
                     diffusers_load_config['requires_aesthetics_score'] = shared.opts.diffusers_aesthetics_score
-                    diffusers_load_config['config_files'] = {
-                        'v1': 'configs/v1-inference.yaml',
-                        'v2': 'configs/v2-inference-768-v.yaml',
-                        'xl': 'configs/sd_xl_base.yaml',
-                        'xl_refiner': 'configs/sd_xl_refiner.yaml',
-                    }
+                    if 'inpainting' in checkpoint_info.path.lower():
+                        diffusers_load_config['config_files'] = {
+                            'v1': 'configs/v1-inpainting-inference.yaml',
+                            'v2': 'configs/v2-inference-768-v.yaml',
+                            'xl': 'configs/sd_xl_base.yaml',
+                            'xl_refiner': 'configs/sd_xl_refiner.yaml',
+                        }
+                    else:
+                        diffusers_load_config['config_files'] = {
+                            'v1': 'configs/v1-inference.yaml',
+                            'v2': 'configs/v2-inference-768-v.yaml',
+                            'xl': 'configs/sd_xl_base.yaml',
+                            'xl_refiner': 'configs/sd_xl_refiner.yaml',
+                        }
                 if hasattr(pipeline, 'from_single_file'):
                     diffusers_load_config['use_safetensors'] = True
                     sd_model = pipeline.from_single_file(checkpoint_info.path, **diffusers_load_config)
+                    if sd_model is not None and hasattr(sd_model, 'unet') and hasattr(sd_model.unet, 'config') and 'inpainting' in checkpoint_info.path.lower():
+                        shared.log.debug('Model patch: type=inpaint')
+                        sd_model.unet.config.in_channels = 9
                 elif hasattr(pipeline, 'from_ckpt'):
                     sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config)
                 else:

scripts/xyz_grid.py

@@ -254,7 +254,7 @@ axis_options = [
     AxisOption("[Second pass] hires steps", int, apply_field("hr_second_pass_steps")),
     AxisOption("[Second pass] CFG scale", float, apply_field("image_cfg_scale")),
     AxisOption("[Second pass] guidance rescale", float, apply_field("diffusers_guidance_rescale")),
-    AxisOption("[Refiner] model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: sorted(sd_models.checkpoints_list)),
+    AxisOption("[Refiner] model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)),
     AxisOption("[Refiner] refiner start", float, apply_field("refiner_start")),
     AxisOption("[Refiner] refiner steps", float, apply_field("refiner_steps")),
     AxisOption("[TOME] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')),