From 25a301105f9eec022fb109a56f35cfc7d1b6959d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 4 Sep 2024 18:20:40 -0400 Subject: [PATCH] add flux inpaint and img2img --- CHANGELOG.md | 10 +++++++--- installer.py | 2 +- modules/model_flux.py | 8 ++++++++ modules/sd_models.py | 2 +- scripts/cogvideo.py | 17 ++++++++++------- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e799e57ea..278b18718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # Change Log for SD.Next -## Update for 2024-09-03 +## Update for 2024-09-04 -### Highlights +### Highlights for 2024-09-04 Major refactor of [FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) support: - Full **ControlNet** support, better **LoRA** support, full **prompt attention** support, - Faster execution, more flexible loading, additional quantization options, and more... +- Added **image-to-image**, **inpaint** and **outpaint** modes - Since both *Optimum-Quanto* and *BitsAndBytes* libraries are limited in their platform support matrix, try enabling **NNCF** for quantization/compression on-the-fly! @@ -14,7 +15,7 @@ And support for [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b) text to v Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;) -### Details +### Details for 2024-09-04 **Major refactor of FLUX.1 support:** - allow configuration of individual FLUX.1 model components: *transformer, text-encoder, vae* @@ -37,6 +38,9 @@ Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;) not recommended due to massive duplication of components, but added due to popular demand each such model is 20-32GB in size vs ~11GB for typical unet fine-tune - improve logging, warn when attempting to load unet as base model +- **img2img**, **inpaint** and **outpaint** support + *note* flux may require higher denoising strength than typical sd/sdxl models + *note*: img2img is not yet supported with controlnet - transformer/unet support *fp8/fp4* quantization this brings supported quants to: *nf4/fp8/fp4/qint8/qint4* - vae support *fp16* diff --git a/installer.py b/installer.py index 5654c94ed..413887ff3 100644 --- a/installer.py +++ b/installer.py @@ -443,7 +443,7 @@ def check_diffusers(): log.debug(f'Diffusers {"install" if minor == 0 else "upgrade"}') if minor > 0: pip('uninstall --yes diffusers', ignore=True, quiet=True, uv=False) - pip('install --upgrade git+https://github.com/huggingface/diffusers@8ba90aa706a733f45d83508a5b221da3c59fe4cd', ignore=False, quiet=True, uv=False) + pip('install --upgrade git+https://github.com/huggingface/diffusers@300055172988c73b72d5190fad5c30999c6717be', ignore=False, quiet=True, uv=False) # check onnx version diff --git a/modules/model_flux.py b/modules/model_flux.py index e0109cb07..62c41d379 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -238,5 +238,13 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch if vae is not None: components['vae'] = vae shared.log.debug(f'Loading FLUX: preloaded={list(components)}') + if repo_id == 'sayakpaul/flux.1-dev-nf4': + repo_id = 'black-forest-labs/FLUX.1-dev' # TODO fix for since sayakpaul model is missing model_index.json pipe = diffusers.FluxPipeline.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **components, **diffusers_load_config) + try: + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flux"] = diffusers.FluxPipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flux"] = diffusers.FluxImg2ImgPipeline + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flux"] = diffusers.FluxInpaintPipeline + except Exception: + pass return pipe diff --git a/modules/sd_models.py b/modules/sd_models.py index f4e7b44a6..8ea02af68 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1488,7 +1488,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): return pipe # skip specific pipelines - if n in ['StableDiffusionReferencePipeline', 'StableDiffusionAdapterPipeline', 'AnimateDiffPipeline', 'AnimateDiffSDXLPipeline', 'FluxPipeline', 'FluxControlNetPipeline']: # TODO flux does not have inpaint/img2img yet + if n in ['StableDiffusionReferencePipeline', 'StableDiffusionAdapterPipeline', 'AnimateDiffPipeline', 'AnimateDiffSDXLPipeline', 'FluxControlNetPipeline']: # TODO flux does not have inpaint/img2img yet return pipe if 'Onnx' in pipe.__class__.__name__: return pipe diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index 71fab47b9..bcf1ecc93 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -60,10 +60,11 @@ class Script(scripts.Script): return [model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video] def load(self, model, txt): - if shared.sd_model_type != 'cogvideox' and model != 'None': + if (shared.sd_model_type != 'cogvideox' or shared.sd_model.sd_model_checkpoint != model) and model != 'None': sd_models.unload_model_weights('model') shared.log.info(f'CogVideoX load: model="{model}"') try: + shared.sd_model = None shared.sd_model = diffusers.CogVideoXPipeline.from_pretrained(model, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir) shared.sd_model.sd_checkpoint_info = sd_models.CheckpointInfo(model) shared.sd_model.sd_model_checkpoint = model @@ -71,7 +72,6 @@ class Script(scripts.Script): shared.log.error(f'Loading CogVideoX: {e}') if debug: errors.display(e, 'CogVideoX') - devices.torch_gc() if shared.sd_model_type == 'cogvideox' and model != 'None': shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXPipeline if txt else diffusers.CogVideoXVideoToVideoPipeline, shared.sd_model) shared.sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba') @@ -80,13 +80,14 @@ class Script(scripts.Script): shared.log.info(f'CogVideoX unload: model={model}') shared.sd_model = None devices.torch_gc(force=True) + devices.torch_gc() def offload(self, offload): if shared.sd_model_type != 'cogvideox': return if offload == 'none': sd_models.move_model(shared.sd_model, devices.device) - shared.log.info(f'CogVideoX: offload={offload}') + shared.log.debug(f'CogVideoX: offload={offload}') if offload == 'balanced': sd_models.apply_balanced_offload(shared.sd_model) if offload == 'model': @@ -97,6 +98,7 @@ class Script(scripts.Script): shared.sd_model.vae.enable_slicing() shared.sd_model.vae.enable_tiling() + """ def prepare(self, p, video): import imageio # TODO dont use imageio from torchvision import transforms @@ -108,6 +110,7 @@ class Script(scripts.Script): tensor = torch.stack(frames).to(devices.device).permute(1, 0, 2, 3).unsqueeze(0).to(devices.dtype) encoded = shared.sd_model.vae.encode(tensor)[0].sample() return encoded + """ def generate(self, p: processing.StableDiffusionProcessing): if shared.sd_model_type != 'cogvideox': @@ -136,10 +139,10 @@ class Script(scripts.Script): callback_on_step_end_tensor_inputs=['latents'], ) if getattr(p, 'image', False): - raise ValueError('CogVideoX: image not supported') + raise ValueError('CogVideoX: image not supported') # TODO image2video # args['latents'] = self.prepare(p, [p.image]) elif getattr(p, 'video', False): - raise ValueError('CogVideoX: video not supported') + raise ValueError('CogVideoX: video not supported') # TODO video2video # args['video'] = self.prepare(p, p.video) else: args['num_frames'] = p.frames # only txt2vid has num_frames @@ -180,8 +183,8 @@ class Script(scripts.Script): self.load(model, txt) self.offload(offload) frames = self.generate(p) - info = 'whatever' - processed = processing.Processed(p, images_list=frames, info=info) + devices.torch_gc() + processed = processing.Processed(p, images_list=frames) shared.state.end() return processed