diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ac15643..e2fcfe1b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - **Attention** - *TODO*: see [Attention docs](https://vladmandic.github.io/sdnext-docs/Attention) for details and usage instructions *note*: attention now has its own settings section in *settings -> cross attention* + *note*: this is a breaking change - if you had custom attention settings in previous releases, you will need to re-apply them in the new settings section - new `sparse-attention` method that can be combined with other attention methods to reduce memory usage and improve performance on large models - **Internal** diff --git a/TODO.md b/TODO.md index a12d3234c..8ed48d892 100644 --- a/TODO.md +++ b/TODO.md @@ -9,6 +9,7 @@ - Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322, @vladmandic - Control tab verify overrides handling, @vladmandic - LTX: Create pre-quant for LTX-2.5 +- Modular guiders ## Features @@ -51,11 +52,8 @@ ### Modular -*Pending finalization of modular pipelines implementation and development of compatibility layer* - - Switch to modular pipelines - Feature: Transformers unified cache handler -- Refactor: [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915) - [MagCache](https://github.com/huggingface/diffusers/pull/12744) - [SmoothCache](https://github.com/huggingface/diffusers/issues/11135) - [STG](https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#spatiotemporal-skip-guidance) diff --git a/modules/modular.py b/modules/modular.py index c4c8d3ebf..193ac76da 100644 --- a/modules/modular.py +++ b/modules/modular.py @@ -1,27 +1,26 @@ -import time +import os import diffusers from modules import shared from modules.logger import log -modular_map= { - 'StableDiffusionXLPipeline': 'StableDiffusionXLAutoBlocks', - 'StableDiffusionXLImg2ImgPipeline': 'StableDiffusionXLAutoBlocks', - 'StableDiffusionXLInpaintPipeline': 'StableDiffusionXLAutoBlocks', - 'FluxPipeline': 'FluxAutoBlocks', - 'FluxImg2ImgPipeline': 'FluxAutoBlocks', - 'FluxInpaintPipeline': 'FluxAutoBlocks', - 'WanPipeline': 'WanAutoBlocks', - 'WanImageToVideoPipeline': 'WanAutoBlocks', - 'QwenImagePipeline': 'QwenImageAutoBlocks', - 'QwenImageEditPipeline': 'QwenImageEditAutoBlocks', -} +debug = os.environ.get('SD_MODULAR_DEBUG', None) is not None + + +def get_modular_class_name(diffusion_pipeline: diffusers.DiffusionPipeline) -> str: + name = diffusion_pipeline.__class__.__name__ + name = name.replace('Pipeline', '').replace('Img2Img', '').replace('Inpaint', '').replace('ImageToVideo', '') + name = f'{name}AutoBlocks' + modular_cls = getattr(diffusers, name, None) + if debug: + log.trace(f'Modular lookup: key={name} source={diffusion_pipeline.__class__.__name__} target={modular_cls.__name__ if modular_cls else None}') + return modular_cls def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: if not shared.opts.model_modular_enable: return False - compatible = diffusion_pipeline.__class__.__name__ in modular_map + compatible = get_modular_class_name(diffusion_pipeline) is not None if not compatible: log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline') return compatible @@ -35,28 +34,15 @@ def is_guider(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffusers.ModularPipeline: modular_pipe = None try: - t0 = time.time() - modular_cls = modular_map.get(diffusion_pipeline.__class__.__name__, None) + modular_cls = get_modular_class_name(diffusion_pipeline) if modular_cls is None: raise ValueError(f'unknown: cls={diffusion_pipeline.__class__.__name__}') - modular_cls = getattr(diffusers, modular_cls, None) - if modular_cls is None: - raise ValueError(f'invalid: cls={diffusion_pipeline.__class__.__name__}') modular_blocks = modular_cls() modular_pipe = modular_blocks.init_pipeline() components_dct = {k: v for k, v in diffusion_pipeline.components.items() if v is not None} modular_pipe.update_components(**components_dct, **diffusion_pipeline.parameters) modular_pipe.original_pipe = diffusion_pipeline - t1 = time.time() - log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__} time={t1 - t0:.2f}') - """ - for expected_input_param in modular_pipe.blocks.inputs: - name = expected_input_param.name - default = expected_input_param.default - kwargs_type = expected_input_param.kwargs_type - log.trace(f'Modular input: name={name} type={kwargs_type} default={default}') - """ - + log.debug(f'Modular convert: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__}') except Exception as e: log.error(f'Modular: {e}') raise e diff --git a/modules/modular_guiders.py b/modules/modular_guiders.py index 00e45f83b..b39e26f70 100644 --- a/modules/modular_guiders.py +++ b/modules/modular_guiders.py @@ -61,7 +61,7 @@ def set_guider(p: processing.StableDiffusionProcessing): guider_cls = guider_info['cls'] guider_args = {} for k, v in base_args.items(): - if v is not None and v >= 0.0: + if isinstance(v, float): guider_args[k] = v log.warning('Guiders: partially implemented') # TODO: guiders for k, v in guider_info['args'].items(): @@ -94,3 +94,5 @@ def set_guider(p: processing.StableDiffusionProcessing): except Exception as e: log.error(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args} {e}') return + else: + log.warning(f'Guider: name={guidance_name} cls=None args={guider_args}') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index b8ae288b0..9ff88d4d1 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -469,9 +469,11 @@ def process_decode(p: processing.StableDiffusionProcessing, output): if not hasattr(output, 'images') and hasattr(output, 'frames'): log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] - if getattr(p, 'video_still', False) and hasattr(output, 'images') and output.images is not None: + if hasattr(output, 'latents') and hasattr(output, 'images') and (output.images is None): + output.images = output.latents # modular pipelines may return latents instead of images + if getattr(p, 'video_still', False) and hasattr(output, 'images') and (output.images is not None): output.images = output.images[:1] # only the first frame derives from real latents; the rest decode from padding - if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image): + if (output.images is not None) and (len(output.images) > 0) and isinstance(output.images[0], Image.Image): sd_models.offload_ondemand(shared.sd_model) # in-pipe decode paths return materialized frames; the vae seam in processing_vae never runs return attach_audio(output.images, audio) model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner @@ -622,7 +624,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): images = shared.history.last_latent output = SimpleNamespace(images=images) if images is not None else None - if (output is None or (hasattr(output, 'images') and len(output.images) == 0)) and has_images: + if (output is None or (hasattr(output, 'images') and (output.images is None or len(output.images) == 0))) and has_images: if output is not None: log.debug('Processing: using input as base output') output.images = p.init_images diff --git a/modules/processing_prompt.py b/modules/processing_prompt.py index cab96ce23..6eac4a8bf 100644 --- a/modules/processing_prompt.py +++ b/modules/processing_prompt.py @@ -3,7 +3,7 @@ from __future__ import annotations import os from typing import TYPE_CHECKING import torch -from modules import shared, errors, timer, prompt_parser_diffusers +from modules import shared, errors, timer, prompt_parser_diffusers, processing_helpers from modules.logger import log if TYPE_CHECKING: @@ -189,6 +189,9 @@ def set_prompt(p: StableDiffusionProcessing, args = set_fallback_prompt(args, possible, prompts=prompts, negative_prompts=negative_prompts, prompts_2=None, negative_prompts_2=None) prompt_attention = 'fixed' + if processing_helpers.is_modular(): + return prompt_attention, args + if 'prompt_embeds' not in args and 'negative_prompt_embeds' not in args: # pass secondary prompts as-in args = set_fallback_prompt(args, possible, prompts=None, negative_prompts=None, prompts_2=prompts_2, negative_prompts_2=negative_prompts_2) diff --git a/modules/ui_guidance.py b/modules/ui_guidance.py index c451d2863..d9d7670d3 100644 --- a/modules/ui_guidance.py +++ b/modules/ui_guidance.py @@ -1,9 +1,7 @@ import gradio as gr from modules import shared from modules import ui_symbols, ui_components - - -guiders = ['Default', 'CFG', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG'] +from modules.modular_guiders import guiders def create_guidance_inputs(tab): @@ -11,7 +9,7 @@ def create_guidance_inputs(tab): with gr.Group(): with gr.Row(elem_id=f"{tab}_guider_row", elem_classes=['flexbox'], visible=shared.opts.model_modular_enable): - guidance_name = gr.Dropdown(choices=guiders, value='Default', label='Guider', elem_id=f"{tab}_guider") + guidance_name = gr.Dropdown(choices=guiders.keys(), value='Default', label='Guider', elem_id=f"{tab}_guider") guidance_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_guider_docs") guidance_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_name], outputs=[]) with gr.Row(visible=shared.opts.model_modular_enable): @@ -123,6 +121,6 @@ def create_guidance_inputs(tab): cfg_true = gr.Slider(minimum=-1.0, maximum=30.0, step=0.05, label='Attention guidance', value=-1.0, elem_id=f"{tab}_cfg_true") cfg_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_cfg_adaptive") - _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args + _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args # TODO modular: guidance args are not implemented standard_args = [cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end] return guidance_args + standard_args diff --git a/scripts/xyz/xyz_grid_classes.py b/scripts/xyz/xyz_grid_classes.py index 2dd9ed356..655fc6dfb 100644 --- a/scripts/xyz/xyz_grid_classes.py +++ b/scripts/xyz/xyz_grid_classes.py @@ -264,7 +264,6 @@ axis_options = [ AxisOption("[Guidance] End", float, apply_field("cfg_end")), AxisOption("[Guidance] Image scale", float, apply_field("cfg_image")), AxisOption("[Guidance] Rescale", float, apply_field("cfg_rescale")), - AxisOption("[Guidance] Modular name", str, apply_guidance, choices=lambda: ['Default', 'CFG', 'Auto', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG']), AxisOption("[Refine] Upscaler", str, apply_field("hr_upscaler"), cost=0.3, choices=lambda: [x.name for x in shared.sd_upscalers]), AxisOption("[Refine] Sampler", str, apply_hr_sampler_name, fmt=format_value_add_label, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.visible_samplers()]), AxisOption("[Refine] Denoising strength", float, apply_field("denoising_strength")),