add flux inpaint and img2img

CHANGELOG.md

@@ -1,12 +1,13 @@
 # Change Log for SD.Next
 
-## Update for 2024-09-03
+## Update for 2024-09-04
 
-### Highlights
+### Highlights for 2024-09-04
 
 Major refactor of [FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) support:
 - Full **ControlNet** support, better **LoRA** support, full **prompt attention** support,
 - Faster execution, more flexible loading, additional quantization options, and more...
+- Added **image-to-image**, **inpaint** and **outpaint** modes  
 - Since both *Optimum-Quanto* and *BitsAndBytes* libraries are limited in their platform support matrix,  
   try enabling **NNCF** for quantization/compression on-the-fly!
 
@@ -14,7 +15,7 @@ And support for [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b) text to v
 
 Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;)
 
-### Details
+### Details for 2024-09-04
 
 **Major refactor of FLUX.1 support:**
 - allow configuration of individual FLUX.1 model components: *transformer, text-encoder, vae*  
@@ -37,6 +38,9 @@ Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;)
   not recommended due to massive duplication of components, but added due to popular demand  
   each such model is 20-32GB in size vs ~11GB for typical unet fine-tune  
 - improve logging, warn when attempting to load unet as base model  
+- **img2img**, **inpaint** and **outpaint** support  
+  *note* flux may require higher denoising strength than typical sd/sdxl models  
+  *note*: img2img is not yet supported with controlnet  
 - transformer/unet support *fp8/fp4* quantization  
   this brings supported quants to: *nf4/fp8/fp4/qint8/qint4*
 - vae support *fp16*  

installer.py

@@ -443,7 +443,7 @@ def check_diffusers():
         log.debug(f'Diffusers {"install" if minor == 0 else "upgrade"}')
         if minor > 0:
             pip('uninstall --yes diffusers', ignore=True, quiet=True, uv=False)
-        pip('install --upgrade git+https://github.com/huggingface/diffusers@8ba90aa706a733f45d83508a5b221da3c59fe4cd', ignore=False, quiet=True, uv=False)
+        pip('install --upgrade git+https://github.com/huggingface/diffusers@300055172988c73b72d5190fad5c30999c6717be', ignore=False, quiet=True, uv=False)
 
 
 # check onnx version

modules/model_flux.py

@@ -238,5 +238,13 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch
     if vae is not None:
         components['vae'] = vae
     shared.log.debug(f'Loading FLUX: preloaded={list(components)}')
+    if repo_id == 'sayakpaul/flux.1-dev-nf4':
+        repo_id = 'black-forest-labs/FLUX.1-dev' # TODO fix for since sayakpaul model is missing model_index.json
     pipe = diffusers.FluxPipeline.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **components, **diffusers_load_config)
+    try:
+        diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flux"] = diffusers.FluxPipeline
+        diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flux"] = diffusers.FluxImg2ImgPipeline
+        diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flux"] = diffusers.FluxInpaintPipeline
+    except Exception:
+        pass
     return pipe

modules/sd_models.py

@@ -1488,7 +1488,7 @@ def set_diffuser_pipe(pipe, new_pipe_type):
         return pipe
 
     # skip specific pipelines
-    if n in ['StableDiffusionReferencePipeline', 'StableDiffusionAdapterPipeline', 'AnimateDiffPipeline', 'AnimateDiffSDXLPipeline', 'FluxPipeline', 'FluxControlNetPipeline']: # TODO flux does not have inpaint/img2img yet
+    if n in ['StableDiffusionReferencePipeline', 'StableDiffusionAdapterPipeline', 'AnimateDiffPipeline', 'AnimateDiffSDXLPipeline', 'FluxControlNetPipeline']: # TODO flux does not have inpaint/img2img yet
         return pipe
     if 'Onnx' in pipe.__class__.__name__:
         return pipe

scripts/cogvideo.py

@@ -60,10 +60,11 @@ class Script(scripts.Script):
         return [model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video]
 
     def load(self, model, txt):
-        if shared.sd_model_type != 'cogvideox' and model != 'None':
+        if (shared.sd_model_type != 'cogvideox' or shared.sd_model.sd_model_checkpoint != model) and model != 'None':
             sd_models.unload_model_weights('model')
             shared.log.info(f'CogVideoX load: model="{model}"')
             try:
+                shared.sd_model = None
                 shared.sd_model = diffusers.CogVideoXPipeline.from_pretrained(model, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir)
                 shared.sd_model.sd_checkpoint_info = sd_models.CheckpointInfo(model)
                 shared.sd_model.sd_model_checkpoint = model
@@ -71,7 +72,6 @@ class Script(scripts.Script):
                 shared.log.error(f'Loading CogVideoX: {e}')
                 if debug:
                     errors.display(e, 'CogVideoX')
-            devices.torch_gc()
         if shared.sd_model_type == 'cogvideox' and model != 'None':
             shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXPipeline if txt else diffusers.CogVideoXVideoToVideoPipeline, shared.sd_model)
             shared.sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba')
@@ -80,13 +80,14 @@ class Script(scripts.Script):
             shared.log.info(f'CogVideoX unload: model={model}')
             shared.sd_model = None
             devices.torch_gc(force=True)
+        devices.torch_gc()
 
     def offload(self, offload):
         if shared.sd_model_type != 'cogvideox':
             return
         if offload == 'none':
             sd_models.move_model(shared.sd_model, devices.device)
-        shared.log.info(f'CogVideoX: offload={offload}')
+        shared.log.debug(f'CogVideoX: offload={offload}')
         if offload == 'balanced':
             sd_models.apply_balanced_offload(shared.sd_model)
         if offload == 'model':
@@ -97,6 +98,7 @@ class Script(scripts.Script):
         shared.sd_model.vae.enable_slicing()
         shared.sd_model.vae.enable_tiling()
 
+    """
     def prepare(self, p, video):
         import imageio # TODO dont use imageio
         from torchvision import transforms
@@ -108,6 +110,7 @@ class Script(scripts.Script):
         tensor = torch.stack(frames).to(devices.device).permute(1, 0, 2, 3).unsqueeze(0).to(devices.dtype)
         encoded = shared.sd_model.vae.encode(tensor)[0].sample()
         return encoded
+    """
 
     def generate(self, p: processing.StableDiffusionProcessing):
         if shared.sd_model_type != 'cogvideox':
@@ -136,10 +139,10 @@ class Script(scripts.Script):
                 callback_on_step_end_tensor_inputs=['latents'],
             )
             if getattr(p, 'image', False):
-                raise ValueError('CogVideoX: image not supported')
+                raise ValueError('CogVideoX: image not supported') # TODO image2video
                 # args['latents'] = self.prepare(p, [p.image])
             elif getattr(p, 'video', False):
-                raise ValueError('CogVideoX: video not supported')
+                raise ValueError('CogVideoX: video not supported') # TODO video2video
                 # args['video'] = self.prepare(p, p.video)
             else:
                 args['num_frames'] = p.frames # only txt2vid has num_frames
@@ -180,8 +183,8 @@ class Script(scripts.Script):
         self.load(model, txt)
         self.offload(offload)
         frames = self.generate(p)
-        info = 'whatever'
-        processed = processing.Processed(p, images_list=frames, info=info)
+        devices.torch_gc()
+        processed = processing.Processed(p, images_list=frames)
         shared.state.end()
         return processed