From a0dceebd573fe28d754ef5e2ba1eb26f78ffc292 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Aug 2026 15:33:47 +0200 Subject: [PATCH] proto modular guiders Signed-off-by: Vladimir Mandic --- modules/modular.py | 4 ++- modules/modular_guiders.py | 61 ++++++++++++++++++++++---------------- modules/ui_guidance.py | 45 ++++++++++++++++++++-------- 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/modules/modular.py b/modules/modular.py index 193ac76da..32a35d1a8 100644 --- a/modules/modular.py +++ b/modules/modular.py @@ -1,6 +1,6 @@ import os import diffusers -from modules import shared +from modules import shared, sd_hijack_modular from modules.logger import log @@ -46,6 +46,8 @@ def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffu except Exception as e: log.error(f'Modular: {e}') raise e + sd_hijack_modular.install_state_hook(modular_pipe) + sd_hijack_modular.register_callbacks(modular_pipe) return modular_pipe diff --git a/modules/modular_guiders.py b/modules/modular_guiders.py index b39e26f70..5ce8442e0 100644 --- a/modules/modular_guiders.py +++ b/modules/modular_guiders.py @@ -1,27 +1,21 @@ +import inspect import diffusers -from modules import shared, errors, processing +from modules import errors, shared, processing from modules.logger import log -# ['Default', 'CFG', 'Zero', 'PAG', 'APG', 'SLG', 'SEG', 'TCFG', 'FDG'] guiders = { # 'None': { 'cls': None, 'args': {}, }, 'Default': { 'cls': None, 'args': {}, }, 'CFG: ClassifierFreeGuidance': { 'cls': diffusers.ClassifierFreeGuidance, 'args': {} }, - 'Auto: AutoGuidance': { 'cls': diffusers.AutoGuidance, 'args': { 'dropout': 1.0, 'auto_guidance_layers': [7, 8, 9], 'auto_guidance_config': None } }, - 'Zero: ClassifierFreeZeroStar': { 'cls': diffusers.ClassifierFreeZeroStarGuidance, 'args': { 'zero_init_steps': 1 } }, - 'PAG: PerturbedAttentionGuidance': { 'cls': diffusers.PerturbedAttentionGuidance, 'args': { 'perturbed_guidance_scale': 2.8, 'perturbed_guidance_start': 0.01, 'perturbed_guidance_stop': 0.2, 'perturbed_guidance_layers': [7, 8, 9], 'perturbed_guidance_config': None } }, - 'APG: AdaptiveProjectedGuidance': { 'cls': diffusers.AdaptiveProjectedGuidance, 'args': { 'adaptive_projected_guidance_momentum': -1, 'adaptive_projected_guidance_rescale': 15.0 } }, - 'SLG: SkipLayerGuidance': { 'cls': diffusers.SkipLayerGuidance, 'args': { 'skip_layer_guidance_scale': 2.8, 'skip_layer_guidance_start': 0.01, 'skip_layer_guidance_stop': 0.2, 'skip_layer_guidance_layers': [7, 8, 9], 'skip_layer_config': None } }, - 'SEG: SmoothedEnergyGuidance': { 'cls': diffusers.SmoothedEnergyGuidance, 'args': { 'seg_guidance_scale': 3.0, 'seg_blur_sigma': 9999999.0, 'seg_blur_threshold_inf': 9999.0, 'seg_guidance_start': 0.0, 'seg_guidance_stop': 1.0, 'seg_guidance_layers': [7, 8, 9], 'seg_guidance_config': None } }, + 'Auto: AutoGuidance': { 'cls': diffusers.AutoGuidance, 'args': {} }, + 'Zero: ClassifierFreeZeroStar': { 'cls': diffusers.ClassifierFreeZeroStarGuidance, 'args': {} }, + 'PAG: PerturbedAttentionGuidance': { 'cls': diffusers.PerturbedAttentionGuidance, 'args': {} }, + 'APG: AdaptiveProjectedGuidance': { 'cls': diffusers.AdaptiveProjectedGuidance, 'args': {} }, + 'SLG: SkipLayerGuidance': { 'cls': diffusers.SkipLayerGuidance, 'args': {} }, + 'SEG: SmoothedEnergyGuidance': { 'cls': diffusers.SmoothedEnergyGuidance, 'args': {} }, 'TCFG: TangentialClassifierFreeGuidance': { 'cls': diffusers.TangentialClassifierFreeGuidance, 'args': {} }, - 'FDG: FrequencyDecoupledGuidance': { 'cls': diffusers.FrequencyDecoupledGuidance, 'args': { 'guidance_scales': [10.0, 5.0], 'parallel_weights': 1.0, 'guidance_rescale_space': "data" } }, -} -base_args = { - 'guidance_scale': 6.0, - 'guidance_rescale': 0.0, - 'start': 0.0, - 'stop': 1.0, + 'FDG: FrequencyDecoupledGuidance': { 'cls': diffusers.FrequencyDecoupledGuidance, 'args': {} }, } @@ -50,20 +44,35 @@ def set_guider(p: processing.StableDiffusionProcessing): guider_args = {k: v for k, v in guider_info.config.items() if not k.startswith('_') and v is not None} else: guider_args = {} - log.info(f'Guider: name={guidance_name} cls={guider_cls.__name__ if guider_cls is not None else None} args={guider_args}') + log.info(f'Guider: name="{guidance_name}" cls={guider_cls.__name__ if guider_cls is not None else None} args={guider_args}') return if guidance_name == 'None': shared.sd_model.update_components(guider=None) # breaks the pipeline - log.info(f'Guider: name={guidance_name}') + log.info(f'Guider: name="{guidance_name}"') return guider_info = guiders[guidance_name] guider_cls = guider_info['cls'] + guider_args = {} - for k, v in base_args.items(): - if isinstance(v, float): - guider_args[k] = v - log.warning('Guiders: partially implemented') # TODO: guiders + possible = list(inspect.signature(guider_cls.__init__).parameters) if guider_cls is not None else [] + if p.guidance_scale >= 0.0 and 'guidance_scale' in possible: + guider_args['guidance_scale'] = float(p.guidance_scale) + if p.guidance_rescale >= 0.0 and 'guidance_rescale' in possible: + guider_args['guidance_rescale'] = float(p.guidance_rescale) + if p.guidance_start >= 0.0 and 'start' in possible: + guider_args['start'] = float(p.guidance_start) + if p.guidance_stop >= 0.0 and 'stop' in possible: + guider_args['stop'] = float(p.guidance_stop) + + """ + import modules.ui_guidance + for k, v in modules.ui_guidance.get_modular_args().items(): + log.trace(f'Guiders: arg={k} value={v}') + """ + + log.warning('Guiders: advanced parameters are not yet implemented') # TODO: guiders + """ for k, v in guider_info['args'].items(): try: if k is None: @@ -86,13 +95,15 @@ def set_guider(p: processing.StableDiffusionProcessing): log.error(f'Guiders: arg={k} value={v} error={e}') errors.display(e, 'Guiders') # guider_args.update(guider_info['args']) + """ if guider_cls is not None: try: - guider_instance = guider_cls(**guider_args) - log.info(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args}') + guider_instance: diffusers.BaseGuidance = guider_cls(**guider_args) + log.info(f'Guider: name="{guidance_name}" cls={guider_cls.__name__} args={guider_args}') shared.sd_model.update_components(guider=guider_instance) except Exception as e: - log.error(f'Guider: name={guidance_name} cls={guider_cls.__name__} args={guider_args} {e}') + log.error(f'Guider: name="{guidance_name}" cls={guider_cls.__name__} args={guider_args} {e}') + errors.display(e, 'Guiders') return else: - log.warning(f'Guider: name={guidance_name} cls=None args={guider_args}') + log.warning(f'Guider: name="{guidance_name}" cls=None args={guider_args}') diff --git a/modules/ui_guidance.py b/modules/ui_guidance.py index d9d7670d3..4fffb1e2a 100644 --- a/modules/ui_guidance.py +++ b/modules/ui_guidance.py @@ -1,23 +1,34 @@ +from functools import partial import gradio as gr from modules import shared from modules import ui_symbols, ui_components from modules.modular_guiders import guiders +_stored_args = {} + + +def get_modular_args(): + return _stored_args + + def create_guidance_inputs(tab): with gr.Accordion(open=False, label='Guidance', elem_id=f"{tab}_guidance", elem_classes=["small-accordion"]): with gr.Group(): with gr.Row(elem_id=f"{tab}_guider_row", elem_classes=['flexbox'], visible=shared.opts.model_modular_enable): guidance_name = gr.Dropdown(choices=guiders.keys(), value='Default', label='Guider', elem_id=f"{tab}_guider") - guidance_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_guider_docs") + guidance_btn = ui_components.ToolButton(value=ui_symbols.info, elem_id=f"{tab}_guider_docs") guidance_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_name], outputs=[]) - with gr.Row(visible=shared.opts.model_modular_enable): - guidance_scale = gr.Slider(minimum=-1.0, maximum=30.0, step=0.1, label='_Guidance scale', value=-1.0, elem_id=f"{tab}_guidance_scale") - guidance_rescale = gr.Slider(minimum=-1.0, maximum=1.0, step=0.05, label='_Guidance rescale', value=-1.0, elem_id=f"{tab}_guidance_rescale") - with gr.Row(visible=shared.opts.model_modular_enable): - guidance_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='_Guidance start', value=0.0, elem_id=f"{tab}_guidance_start") - guidance_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='_Guidance stop', value=1.0, elem_id=f"{tab}_guidance_stop") + + base_group = gr.Group(visible=False) # default inherits from model + with base_group: + with gr.Row(visible=shared.opts.model_modular_enable): + guidance_scale = gr.Slider(minimum=-1.0, maximum=30.0, step=0.1, label='Guidance scale', value=-1.0, elem_id=f"{tab}_guidance_scale") + guidance_rescale = gr.Slider(minimum=-1.0, maximum=1.0, step=0.05, label='Guidance rescale', value=-1.0, elem_id=f"{tab}_guidance_rescale") + with gr.Row(visible=shared.opts.model_modular_enable): + guidance_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance start', value=0.0, elem_id=f"{tab}_guidance_start") + guidance_stop = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance stop', value=1.0, elem_id=f"{tab}_guidance_stop") guidance_args = [guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop] lsc_group = gr.Accordion(open=False, label='Layer skip guidance', elem_classes=["small-accordion"], visible=shared.opts.model_modular_enable) @@ -25,7 +36,7 @@ def create_guidance_inputs(tab): with gr.Row(): guidance_lsc_enabled = gr.Checkbox(label='Enable LayerSkipConfig', value=False) guidance_lsc_label = gr.Label(value='LSC: LayerSkipConfig', elem_id=f"{tab}_lsc_label", visible=False) - guidance_lsc_btn = ui_components.ToolButton(value=ui_symbols.book, elem_id=f"{tab}_lsc_docs", elem_classes=["guidance-docs"]) + guidance_lsc_btn = ui_components.ToolButton(value=ui_symbols.info, elem_id=f"{tab}_lsc_docs", elem_classes=["guidance-docs"]) guidance_lsc_btn.click(fn=None, _js='getGuidanceDocs', inputs=[guidance_lsc_label], outputs=[]) with gr.Row(): guidance_lsc_indices = gr.Textbox(label='LSC layer indices', value='1, 2, 3', placeholder='Comma-separated layer indices to skip') @@ -99,6 +110,7 @@ def create_guidance_inputs(tab): def adv_visibility(guidance_name): return [ + gr.update(visible=guidance_name != 'Default' and guidance_name != 'None'), gr.update(visible=guidance_name.startswith('Auto')), gr.update(visible=guidance_name.startswith('Zero')), gr.update(visible=guidance_name.startswith('PAG')), @@ -108,19 +120,26 @@ def create_guidance_inputs(tab): gr.update(visible=guidance_name.startswith('TCFG')), gr.update(visible=guidance_name.startswith('FDG')), ] - guidance_name.change(fn=adv_visibility, inputs=[guidance_name], outputs=[auto_group, zero_group, pag_group, apg_group, slg_group, seg_group, tcfg_group, fdg_group]) + guidance_name.change(fn=adv_visibility, inputs=[guidance_name], outputs=[base_group, auto_group, zero_group, pag_group, apg_group, slg_group, seg_group, tcfg_group, fdg_group]) - gr.HTML(value='

Fallback guidance

', visible=shared.opts.model_modular_enable, elem_id=f"{tab}_guidance_note") - with gr.Row(elem_id=f"{tab}_cfg_row", elem_classes=['flexbox']): + with gr.Row(elem_id=f"{tab}_cfg_row", elem_classes=['flexbox'], visible=not shared.opts.model_modular_enable): cfg_scale = gr.Slider(minimum=-1.0, maximum=30.0, step=0.1, label='Guidance scale', value=-1.0, elem_id=f"{tab}_cfg_scale") cfg_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='Guidance end', value=1.0, elem_id=f"{tab}_cfg_end") - with gr.Row(): + with gr.Row(visible=not shared.opts.model_modular_enable): cfg_rescale = gr.Slider(minimum=-1.0, maximum=1.0, step=0.05, label='Guidance rescale', value=-1.0, elem_id=f"{tab}_image_cfg_rescale") cfg_image = gr.Slider(minimum=-1.0, maximum=30.0, step=0.1, label='Refine guidance', value=-1.0, elem_id=f"{tab}_cfg_image") - with gr.Row(): + with gr.Row(visible=not shared.opts.model_modular_enable): cfg_true = gr.Slider(minimum=-1.0, maximum=30.0, step=0.05, label='Attention guidance', value=-1.0, elem_id=f"{tab}_cfg_true") cfg_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_cfg_adaptive") _modular_args = guidance_args + lsc_args + guidance_auto_args + guidance_zero_args + guidance_pag_args + guidance_apg_args + guidance_slg_args + guidance_seg_args + guidance_fdg_args # TODO modular: guidance args are not implemented + def update_stored(component, label): + _stored_args[label] = component + for component in _modular_args: + label = getattr(component, 'label', None) + value = getattr(component, 'value', None) + _stored_args[label] = value + component.change(fn=partial(update_stored, label=label), inputs=[component], outputs=[]) + standard_args = [cfg_scale, cfg_image, cfg_rescale, cfg_true, cfg_adaptive, cfg_end] return guidance_args + standard_args