work on convert-to-modular

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -17,6 +17,7 @@
 - **Attention**
   - *TODO*: see [Attention docs](https://vladmandic.github.io/sdnext-docs/Attention) for details and usage instructions  
     *note*: attention now has its own settings section in *settings -> cross attention*  
+    *note*: this is a breaking change - if you had custom attention settings in previous releases, you will need to re-apply them in the new settings section  
   - new `sparse-attention` method that can be combined with other attention methods  
     to reduce memory usage and improve performance on large models  
 - **Internal**

TODO.md

@@ -9,6 +9,7 @@
 - Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322, @vladmandic
 - Control tab verify overrides handling, @vladmandic
 - LTX: Create pre-quant for LTX-2.5
+- Modular guiders
 
 ## Features
 
@@ -51,11 +52,8 @@
 
 ### Modular
 
-*Pending finalization of modular pipelines implementation and development of compatibility layer*
-
 - Switch to modular pipelines
 - Feature: Transformers unified cache handler
-- Refactor: [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915)
 - [MagCache](https://github.com/huggingface/diffusers/pull/12744)
 - [SmoothCache](https://github.com/huggingface/diffusers/issues/11135)
 - [STG](https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#spatiotemporal-skip-guidance)

modules/modular.py

@@ -1,27 +1,26 @@
-import time
+import os
 import diffusers
 from modules import shared
 from modules.logger import log
 
 
-modular_map= {
-    'StableDiffusionXLPipeline': 'StableDiffusionXLAutoBlocks',
-    'StableDiffusionXLImg2ImgPipeline': 'StableDiffusionXLAutoBlocks',
-    'StableDiffusionXLInpaintPipeline': 'StableDiffusionXLAutoBlocks',
-    'FluxPipeline': 'FluxAutoBlocks',
-    'FluxImg2ImgPipeline': 'FluxAutoBlocks',
-    'FluxInpaintPipeline': 'FluxAutoBlocks',
-    'WanPipeline': 'WanAutoBlocks',
-    'WanImageToVideoPipeline': 'WanAutoBlocks',
-    'QwenImagePipeline': 'QwenImageAutoBlocks',
-    'QwenImageEditPipeline': 'QwenImageEditAutoBlocks',
-}
+debug = os.environ.get('SD_MODULAR_DEBUG', None) is not None
+
+
+def get_modular_class_name(diffusion_pipeline: diffusers.DiffusionPipeline) -> str:
+    name = diffusion_pipeline.__class__.__name__
+    name = name.replace('Pipeline', '').replace('Img2Img', '').replace('Inpaint', '').replace('ImageToVideo', '')
+    name = f'{name}AutoBlocks'
+    modular_cls = getattr(diffusers, name, None)
+    if debug:
+        log.trace(f'Modular lookup: key={name} source={diffusion_pipeline.__class__.__name__} target={modular_cls.__name__ if modular_cls else None}')
+    return modular_cls
 
 
 def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool:
     if not shared.opts.model_modular_enable:
         return False
-    compatible = diffusion_pipeline.__class__.__name__ in modular_map
+    compatible = get_modular_class_name(diffusion_pipeline) is not None
     if not compatible:
         log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline')
     return compatible
@@ -35,28 +34,15 @@ def is_guider(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool:
 def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffusers.ModularPipeline:
     modular_pipe = None
     try:
-        t0 = time.time()
-        modular_cls = modular_map.get(diffusion_pipeline.__class__.__name__, None)
+        modular_cls = get_modular_class_name(diffusion_pipeline)
         if modular_cls is None:
             raise ValueError(f'unknown: cls={diffusion_pipeline.__class__.__name__}')
-        modular_cls = getattr(diffusers, modular_cls, None)
-        if modular_cls is None:
-            raise ValueError(f'invalid: cls={diffusion_pipeline.__class__.__name__}')
         modular_blocks = modular_cls()
         modular_pipe = modular_blocks.init_pipeline()
         components_dct = {k: v for k, v in diffusion_pipeline.components.items() if v is not None}
         modular_pipe.update_components(**components_dct, **diffusion_pipeline.parameters)
         modular_pipe.original_pipe = diffusion_pipeline
-        t1 = time.time()
-        log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__} time={t1 - t0:.2f}')
-        """
-        for expected_input_param in modular_pipe.blocks.inputs:
-            name = expected_input_param.name
-            default = expected_input_param.default
-            kwargs_type = expected_input_param.kwargs_type
-            log.trace(f'Modular input: name={name} type={kwargs_type} default={default}')
-        """
-
+        log.debug(f'Modular convert: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__}')
     except Exception as e:
         log.error(f'Modular: {e}')
         raise e

modules/modular_guiders.py

@@ -61,7 +61,7 @@ def set_guider(p: processing.StableDiffusionProcessing):
     guider_cls = guider_info['cls']
     guider_args = {}
     for k, v in base_args.items():
-        if v is not None and v >= 0.0:
+        if isinstance(v, float):
             guider_args[k] = v
     log.warning('Guiders: partially implemented') # TODO: guiders
     for k, v in guider_info['args'].items():
@@ -94,3 +94,5 @@ def set_guider(p: processing.StableDiffusionProcessing):
         except Exception as e:
             log.error(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args} {e}')
             return
+    else:
+        log.warning(f'Guider: name={guidance_name} cls=None args={guider_args}')

modules/processing_diffusers.py

@@ -469,9 +469,11 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
         if not hasattr(output, 'images') and hasattr(output, 'frames'):
             log.debug(f'Generated: frames={len(output.frames[0])}')
             output.images = output.frames[0]
-        if getattr(p, 'video_still', False) and hasattr(output, 'images') and output.images is not None:
+        if hasattr(output, 'latents') and hasattr(output, 'images') and (output.images is None):
+            output.images = output.latents # modular pipelines may return latents instead of images
+        if getattr(p, 'video_still', False) and hasattr(output, 'images') and (output.images is not None):
             output.images = output.images[:1] # only the first frame derives from real latents; the rest decode from padding
-        if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image):
+        if (output.images is not None) and (len(output.images) > 0) and isinstance(output.images[0], Image.Image):
             sd_models.offload_ondemand(shared.sd_model) # in-pipe decode paths return materialized frames; the vae seam in processing_vae never runs
             return attach_audio(output.images, audio)
         model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner
@@ -622,7 +624,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         images = shared.history.last_latent
         output = SimpleNamespace(images=images) if images is not None else None
 
-    if (output is None or (hasattr(output, 'images') and len(output.images) == 0)) and has_images:
+    if (output is None or (hasattr(output, 'images') and (output.images is None or len(output.images) == 0))) and has_images:
         if output is not None:
             log.debug('Processing: using input as base output')
             output.images = p.init_images

modules/processing_prompt.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import os
 from typing import TYPE_CHECKING
 import torch
-from modules import shared, errors, timer, prompt_parser_diffusers
+from modules import shared, errors, timer, prompt_parser_diffusers, processing_helpers
 from modules.logger import log
 
 if TYPE_CHECKING:
@@ -189,6 +189,9 @@ def set_prompt(p: StableDiffusionProcessing,
         args = set_fallback_prompt(args, possible, prompts=prompts, negative_prompts=negative_prompts, prompts_2=None, negative_prompts_2=None)
         prompt_attention = 'fixed'
 
+    if processing_helpers.is_modular():
+        return prompt_attention, args
+
     if 'prompt_embeds' not in args and 'negative_prompt_embeds' not in args: # pass secondary prompts as-in
         args = set_fallback_prompt(args, possible, prompts=None, negative_prompts=None, prompts_2=prompts_2, negative_prompts_2=negative_prompts_2)
 

modules/ui_guidance.py

@@ -1,9 +1,7 @@
 import gradio as gr
 from modules import shared
 from modules import ui_symbols, ui_components
-
-
-guiders = ['Default', 'CFG', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG']
+from modules.modular_guiders import guiders
 
 
 def create_guidance_inputs(tab):
@@ -11,7 +9,7 @@ def create_guidance_inputs(tab):
         with gr.Group():
 
             with gr.Row(elem_id=f"{tab}_guider_row", elem_classes=['flexbox'], visible=shared.opts.model_modular_enable):
-                guidance_name = gr.Dropdown(choices=guiders, value='Default', label='Guider', elem_id=f"{tab}_guider")
+                guidance_name = gr.Dropdown(choices=guiders.keys(), value='Default', label='Guider', elem_id=f"{tab}_guider")
                 guidance_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_guider_docs")
                 guidance_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_name], outputs=[])
             with gr.Row(visible=shared.opts.model_modular_enable):
@@ -123,6 +121,6 @@ def create_guidance_inputs(tab):
                 cfg_true = gr.Slider(minimum=-1.0, maximum=30.0, step=0.05, label='Attention guidance', value=-1.0, elem_id=f"{tab}_cfg_true")
                 cfg_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_cfg_adaptive")
 
-    _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args
+    _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args # TODO modular: guidance args are not implemented
     standard_args = [cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end]
     return guidance_args + standard_args

scripts/xyz/xyz_grid_classes.py

@@ -264,7 +264,6 @@ axis_options = [
     AxisOption("[Guidance] End", float, apply_field("cfg_end")),
     AxisOption("[Guidance] Image scale", float, apply_field("cfg_image")),
     AxisOption("[Guidance] Rescale", float, apply_field("cfg_rescale")),
-    AxisOption("[Guidance] Modular name", str, apply_guidance, choices=lambda: ['Default', 'CFG', 'Auto', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG']),
     AxisOption("[Refine] Upscaler", str, apply_field("hr_upscaler"), cost=0.3, choices=lambda: [x.name for x in shared.sd_upscalers]),
     AxisOption("[Refine] Sampler", str, apply_hr_sampler_name, fmt=format_value_add_label, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.visible_samplers()]),
     AxisOption("[Refine] Denoising strength", float, apply_field("denoising_strength")),