From b3eb03b7ba87f5376034970998d22a26399d290c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 19 Nov 2024 18:07:06 -0500 Subject: [PATCH] add slg Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 11 ++++-- installer.py | 2 +- modules/ipadapter.py | 39 +++++++++++++++---- scripts/skip_layer_guidance.py | 71 ++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 scripts/skip_layer_guidance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 872f4618b..587632778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ First, a massive update to docs including new UI top-level **info** tab with acc **Workflow Improvements**: - Native Docker support -- SD3x & Flux.1: more ControlNets, all-in-one-safetensors, DPM samplers, etc. +- SD3x & Flux.1: more ControlNets, all-in-one-safetensors, DPM samplers, skip-layer-guidance, etc. - XYZ grid: benchmarking, video creation, etc. - Enhanced prompt parsing - UI improvements @@ -66,15 +66,18 @@ And quite a few more improvements and fixes since the last update - for full det ``, `` detailed caption and tags with optional analyze - Model improvements: - - SD3: ControlNets: + - SD35: **ControlNets**: - *InstantX Canny, Pose, Depth, Tile* - *Alimama Inpainting, SoftEdge* - *note*: that just like with FLUX.1 or any large model, ControlNet are also large and can push your system over the limit e.g. SD3 controlnets vary from 1GB to over 4GB in size - - SD3: all-in-one safetensors + - SD35: **All-in-one** safetensors - *examples*: [large](https://civitai.com/models/882666/sd35-large-google-flan?modelVersionId=1003031), [medium](https://civitai.com/models/900327) - *note*: enable *bnb* on-the-fly quantization for even bigger gains - - FlowMatch samplers: + - SD35: **skip-layer-guidance** + - enable in *scripts -> slg* + - allows for granular strength/start/stop control of guidance for each layer of the model + - **FlowMatch samplers**: - Applicable to SD 3.x and Flux.1 models - Complete family: *DPM2, DPM2a, DPM2++, DPM2++ 2M, DPM2++ 2S, DPM2++ SDE, DPM2++ 2M SDE, DPM2++ 3M SDE* - [NoobAI XL ControlNets](https://huggingface.co/collections/Eugeoter/controlnext-673161eae023f413e0432799), thanks @lbeltrame diff --git a/installer.py b/installer.py index 6f48e5790..7b1ca2fb0 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '345907f32de71c8ca67f3d9d00e37127192da543' + sha = '99c0483b67427de467f11aa35d54678fd36a7ea2' pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 4b67b9bb4..d5bfbec8c 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -35,8 +35,15 @@ ADAPTERS_SDXL = { 'Plus Face ViT-H SDXL': { 'name': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', 'repo': 'h94/IP-Adapter', 'subfolder': 'sdxl_models' }, 'Ostris Composition ViT-H SDXL': { 'name': 'ip_plus_composition_sdxl.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' }, } -ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL } -ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL } +ADAPTERS_SD3 = { + 'InstantX Large': { 'name': 'ip-adapter.bin', 'repo': 'InstantX/SD3.5-Large-IP-Adapter' }, +} +ADAPTERS_F1 = { + 'XLabs AI v1': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter' }, + 'XLabs AI v2': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter-v2' }, +} +ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 } +ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 } def get_adapters(): @@ -45,6 +52,10 @@ def get_adapters(): ADAPTERS = ADAPTERS_SD15 elif shared.sd_model_type == 'sdxl': ADAPTERS = ADAPTERS_SDXL + elif shared.sd_model_type == 'sd3': + ADAPTERS = ADAPTERS_SD3 + elif shared.sd_model_type == 'f1': + ADAPTERS = ADAPTERS_F1 else: ADAPTERS = ADAPTERS_NONE return list(ADAPTERS) @@ -55,7 +66,7 @@ def get_images(input_images): if input_images is None or len(input_images) == 0: shared.log.error('IP adapter: no init images') return None - if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': + if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']: shared.log.error('IP adapter: base model not supported') return None if isinstance(input_images, str): @@ -147,7 +158,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt if hasattr(p, 'ip_adapter_images'): del p.ip_adapter_images return False - if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': + if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']: shared.log.error(f'IP adapter: model={shared.sd_model_type} class={pipe.__class__.__name__} not supported') return False if hasattr(p, 'ip_adapter_scales'): @@ -172,6 +183,9 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt for i in range(len(adapter_masks)): adapter_masks[i] = mask_processor.preprocess(adapter_masks[i], height=p.height, width=p.width) adapter_masks = mask_processor.preprocess(adapter_masks, height=p.height, width=p.width) + if adapter_images is None: + shared.log.error('IP adapter: no image provided') + return False if len(adapters) < len(adapter_images): adapter_images = adapter_images[:len(adapters)] if len(adapters) < len(adapter_masks): @@ -212,13 +226,24 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt clip_subfolder = 'models/image_encoder' else: clip_subfolder = 'sdxl_models/image_encoder' - elif 'ViT-H' in adapter_name: + if 'ViT-H' in adapter_name: clip_subfolder = 'models/image_encoder' # this is vit-h elif 'ViT-G' in adapter_name: clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g else: - shared.log.error(f'IP adapter: unknown model type: {adapter_name}') - return False + if shared.sd_model_type == 'sd': + clip_subfolder = 'models/image_encoder' + elif shared.sd_model_type == 'sdxl': + clip_subfolder = 'sdxl_models/image_encoder' + elif shared.sd_model_type == 'sd3': + shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') + return False + elif shared.sd_model_type == 'f1': + shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') + return False + else: + shared.log.error(f'IP adapter: unknown model type: {adapter_name}') + return False # load feature extractor used by ip adapter if pipe.feature_extractor is None: diff --git a/scripts/skip_layer_guidance.py b/scripts/skip_layer_guidance.py new file mode 100644 index 000000000..4196db037 --- /dev/null +++ b/scripts/skip_layer_guidance.py @@ -0,0 +1,71 @@ +import sys +import gradio as gr +from modules import scripts, processing, shared + + +registered = False + + +class Script(scripts.Script): + def __init__(self): + super().__init__() + self.register() + + def title(self): + return 'SLG: Skip Layer Guidance' + + def show(self, is_img2img): + return shared.native + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Row(): + layers = gr.Textbox(label='Skip guidance layers', value='7,8,9') + with gr.Row(): + scale = gr.Slider(label='Guidance strength', minimum=0.0, maximum=1.0, step=0.01, value=1.0) + with gr.Row(): + start = gr.Slider(label='Guidance start', minimum=0.0, maximum=1.0, step=0.01, value=0.01) + stop = gr.Slider(label='Guidance stop', minimum=0.0, maximum=1.0, step=0.01, value=0.2) + return [layers, scale, start, stop] + + def register(self): # register xyz grid elements + global registered # pylint: disable=global-statement + if registered: + return + registered = True + def apply_task_args(field): + def fun(p, x, xs): # pylint: disable=unused-argument + try: + val = str(x).replace('"', '') + val = [int(layer.strip()) for layer in val.split(',')] + except Exception: + return + if len(val) > 0: + shared.log.debug(f'SLG: {field}={val}') + p.task_args[field] = val + return fun + + xyz_classes = [v for k, v in sys.modules.items() if 'xyz_grid_classes' in k][0] + options = [ + xyz_classes.AxisOption("[SLG] Layers", str, apply_task_args("skip_guidance_layers")), + ] + for option in options: + if option not in xyz_classes.axis_options: + xyz_classes.axis_options.append(option) + + + def run(self, p: processing.StableDiffusionProcessing, layers: str = '', scale: float = 1.0, start: float = 1.0, stop: float = 1.0): # pylint: disable=arguments-differ, unused-argument + if shared.sd_model_type != 'sd3': + return + p.task_args['skip_layer_guidance_scale'] = float(scale) + p.task_args['skip_layer_guidance_start'] = float(start) + p.task_args['skip_layer_guidance_stop'] = float(stop) + parsed = [] + try: + parsed = [int(layer.strip()) for layer in layers.split(',')] + except Exception: + return + if len(parsed) == 0: + return + p.task_args['skip_guidance_layers'] = parsed + shared.log.info(f'SLG: layers={parsed} scale={scale} start={start} stop={stop}')