diff --git a/CHANGELOG.md b/CHANGELOG.md index ea57477fb..90eda3736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,20 +7,23 @@ ### SD.Next Xmass edition: *What's new?* While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: -- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, brand new LoRA loader, system memory management, on-the-fly quantization, improved gguf loader, etc. +- New memory management engine + list of changes that went into this one is long: changes to GPU offloading, brand new LoRA loader, system memory management, on-the-fly quantization, improved gguf loader, etc. but main goal is enabling modern large models to run on standard consumer GPUs without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks - New [documentation website](https://vladmandic.github.io/sdnext-docs/) with full search and tons of new documentation - New settings panel with simplified and streamlined configuration -We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) +We've also added support for several new models such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) And several new SOTA video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) And a lot of **Control** and **IPAdapter** goodies - for **SDXL** there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* models -- for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model -- for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models as well as [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter +- for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models, + a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model as well as [XLabs](https://huggingface.co/XLabs-AI/flux-ip-adapter-v2) IP-adapter +- for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models + as well as [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) @@ -49,6 +52,10 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full *note*: when selecting tiles in control settings, you can also specify non-square ratios in which case it will use context-aware image resize to maintain overall composition *note*: available tiling options can be set in settings -> control +- **IP-Adapter** + - FLUX.1 [XLabs](https://huggingface.co/XLabs-AI/flux-ip-adapter-v2) v1 and v2 IP-adapter + - FLUX.1 secondary guidance, enabled using *Attention guidance* in advanced menu + - SD 3.5 [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model diff --git a/installer.py b/installer.py index d186cc5ac..6aa8bb060 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = 'b64ca6c11cbc1644c22f1dae441c8124d588bb14' # diffusers commit hash + sha = '233dffdc3f56b26abaaba8363a5dd30dab7f0e40' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/errors.py b/modules/errors.py index 5f2c54cb7..6302057d7 100644 --- a/modules/errors.py +++ b/modules/errors.py @@ -36,7 +36,7 @@ def print_error_explanation(message): log.error(line) -def display(e: Exception, task, suppress=[]): +def display(e: Exception, task: str, suppress=[]): log.error(f"{task or 'error'}: {type(e).__name__}") console.print_exception(show_locals=False, max_frames=16, extra_lines=1, suppress=suppress, theme="ansi_dark", word_wrap=False, width=console.width) @@ -48,7 +48,7 @@ def display_once(e: Exception, task): already_displayed[task] = 1 -def run(code, task): +def run(code, task: str): try: code() except Exception as e: diff --git a/modules/ipadapter.py b/modules/ipadapter.py index c1b6ed52f..aa010c33d 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -11,12 +11,13 @@ import json from PIL import Image import diffusers import transformers -from modules import processing, shared, devices, sd_models +from modules import processing, shared, devices, sd_models, errors clip_loaded = None adapters_loaded = [] CLIP_ID = "h94/IP-Adapter" +OPEN_ID = "openai/clip-vit-large-patch14" SIGLIP_ID = 'google/siglip-so400m-patch14-384' ADAPTERS_NONE = { 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, @@ -136,14 +137,22 @@ def crop_images(images, crops): return images -def unapply(pipe): # pylint: disable=arguments-differ +def unapply(pipe, unload: bool = False): # pylint: disable=arguments-differ if len(adapters_loaded) == 0: return try: if hasattr(pipe, 'set_ip_adapter_scale'): pipe.set_ip_adapter_scale(0) - pipe.unload_ip_adapter() - if hasattr(pipe, 'unet') and hasattr(pipe.unet, 'config') and pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj': + if unload: + shared.log.debug('IP adapter unload') + pipe.unload_ip_adapter() + if hasattr(pipe, 'unet'): + module = pipe.unet + elif hasattr(pipe, 'transformer'): + module = pipe.transformer + else: + module = None + if module is not None and hasattr(module, 'config') and module.config.encoder_hid_dim_type == 'ip_image_proj': pipe.unet.encoder_hid_proj = None pipe.config.encoder_hid_dim_type = None pipe.unet.set_default_attn_processor() @@ -171,8 +180,8 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st clip_repo = SIGLIP_ID clip_subfolder = None elif shared.sd_model_type == 'f1': - shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') - return False + clip_repo = OPEN_ID + clip_subfolder = None else: shared.log.error(f'IP adapter: unknown model type: {adapter_name}') return False @@ -181,13 +190,22 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}': try: if shared.sd_model_type == 'sd3': - pipe.image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) else: - pipe.image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) - shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}') + if clip_subfolder is None: + image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) + shared.log.debug(f'IP adapter load: encoder="{clip_repo}" cls={pipe.image_encoder.__class__.__name__}') + else: + image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) + shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}') + if hasattr(pipe, 'register_modules'): + pipe.register_modules(image_encoder=image_encoder) + else: + pipe.image_encoder = image_encoder clip_loaded = f'{clip_repo}/{clip_subfolder}' except Exception as e: shared.log.error(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" {e}') + errors.display(e, 'IP adapter: type=encoder') return False sd_models.move_model(pipe.image_encoder, devices.device) return True @@ -198,12 +216,17 @@ def load_feature_extractor(pipe): if pipe.feature_extractor is None: try: if shared.sd_model_type == 'sd3': - pipe.feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) else: - pipe.feature_extractor = transformers.CLIPImageProcessor() + feature_extractor = transformers.CLIPImageProcessor() + if hasattr(pipe, 'register_modules'): + pipe.register_modules(feature_extractor=feature_extractor) + else: + pipe.feature_extractor = feature_extractor shared.log.debug(f'IP adapter load: extractor={pipe.feature_extractor.__class__.__name__}') except Exception as e: shared.log.error(f'IP adapter load: extractor {e}') + errors.display(e, 'IP adapter: type=extractor') return False return True @@ -266,7 +289,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt adapters = [ADAPTERS.get(adapter_name, None) for adapter_name in adapter_names if adapter_name.lower() != 'none'] if len(adapters) == 0: - unapply(pipe) + unapply(pipe, getattr(p, 'ip_adapter_unload', False)) if hasattr(p, 'ip_adapter_images'): del p.ip_adapter_images return False @@ -286,7 +309,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt shared.log.error('IP adapter: no image provided') adapters = [] # unload adapter if previously loaded as it will cause runtime errors if len(adapters) == 0: - unapply(pipe) + unapply(pipe, getattr(p, 'ip_adapter_unload', False)) if hasattr(p, 'ip_adapter_images'): del p.ip_adapter_images return False @@ -335,4 +358,5 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt shared.log.info(f'IP adapter: {ip_str} image={adapter_images} mask={adapter_masks is not None} time={t1-t0:.2f}') except Exception as e: shared.log.error(f'IP adapter load: adapters={adapter_names} repo={repos} folders={subfolders} names={names} {e}') + errors.display(e, 'IP adapter: type=adapter') return True diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index 8fe54c198..b7a56c40d 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -17,17 +17,22 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- unapply() if p.pag_scale == 0: return - if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: - shared.log.warning(f'PAG: pipeline={c} not implemented') - return None if 'PAG' in shared.sd_model.__class__.__name__: pass elif detect.is_sd15(c): + if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: + shared.log.warning(f'PAG: pipeline={c} not implemented') + return None orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionPAGPipeline, shared.sd_model) elif detect.is_sdxl(c): + if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: + shared.log.warning(f'PAG: pipeline={c} not implemented') + return None orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionXLPAGPipeline, shared.sd_model) + elif detect.is_f1(c): + p.task_args['true_cfg_scale'] = p.pag_scale else: shared.log.warning(f'PAG: pipeline={c} required={StableDiffusionPipeline.__name__}') return None diff --git a/modules/processing.py b/modules/processing.py index f39393f95..6d9b64c17 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -451,7 +451,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.native: from modules import ipadapter - ipadapter.unapply(shared.sd_model) + ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False)) if shared.opts.include_mask: if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images): diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index 60c70b9dc..5ca4ca578 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -57,6 +57,7 @@ class Script(scripts.Script): mask_galleries = [] with gr.Row(): num_adapters = gr.Slider(label="Active IP adapters", minimum=1, maximum=MAX_ADAPTERS, step=1, value=1, scale=1) + unload_adapter = gr.Checkbox(label='Unload adapter', value=False, interactive=True) for i in range(MAX_ADAPTERS): with gr.Accordion(f'Adapter {i+1}', visible=i==0) as unit: with gr.Row(): @@ -85,7 +86,7 @@ class Script(scripts.Script): layers_label = gr.HTML('InstantStyle: advanced layer activation', visible=False) layers = gr.Text(label='Layer scales', placeholder='{\n"down": {"block_2": [0.0, 1.0]},\n"up": {"block_0": [0.0, 1.0, 0.0]}\n}', rows=1, type='text', interactive=True, lines=5, visible=False, show_label=False) layers_active.change(fn=self.display_advanced, inputs=[layers_active], outputs=[layers_label, layers]) - return [num_adapters] + adapters + scales + files + crops + starts + ends + masks + [layers_active] + [layers] + return [num_adapters] + [unload_adapter] + adapters + scales + files + crops + starts + ends + masks + [layers_active] + [layers] def process(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ if not shared.native: @@ -94,6 +95,7 @@ class Script(scripts.Script): if len(args) == 0: return units = args.pop(0) + unload = args.pop(0) if getattr(p, 'ip_adapter_names', []) == []: p.ip_adapter_names = args[:MAX_ADAPTERS][:units] if getattr(p, 'ip_adapter_scales', [0.0]) == [0.0]: @@ -110,6 +112,7 @@ class Script(scripts.Script): p.ip_adapter_masks = args[MAX_ADAPTERS*6:MAX_ADAPTERS*7][:units] p.ip_adapter_masks = [x for x in p.ip_adapter_masks if x] layers_active, layers = args[MAX_ADAPTERS*7:MAX_ADAPTERS*8] + p.ip_adapter_unload = unload if layers_active and len(layers) > 0: try: layers = json.loads(layers)