diff --git a/CHANGELOG.md b/CHANGELOG.md index 188a023fc..47a7d77eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,10 @@ - implement progress and preview - intercept and profiling hooks - on-demand convert standard model on-demand +- **Other** + - new optional transformer hooks: *settings -> compute add-ons* + *PAG: Perturbed attention guidance, PAB: Pyramid attention broadcast, FBC: First Block Cache, FC: Faster Cache, LS: Layer Skip, MC: Mag Cache, TS: TaylorSeer* + *note*: compatibility of different methods varies across different models - **Fixes** - unnecessary secondary prompt if same - js fetch exception handling diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 7597a81c6..8dec87411 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 7597a81c631b9fcdd519d4fa7c9c2d658b365d98 +Subproject commit 8dec87411813d6cb413efb4f114c4d96388223ae diff --git a/modules/modular.py b/modules/modular.py index f0b4b2d3c..d17f6aba8 100644 --- a/modules/modular.py +++ b/modules/modular.py @@ -31,12 +31,20 @@ def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: return compatible +def is_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: + if diffusion_pipeline is None: + return False + return isinstance(diffusion_pipeline, diffusers.ModularPipeline) or 'Modular' in diffusion_pipeline.__class__.__name__ + + def is_guider(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: guider = getattr(diffusion_pipeline, 'guider', None) return guider is not None def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffusers.ModularPipeline: + if is_modular(diffusion_pipeline): + return diffusion_pipeline modular_pipe = None try: modular_cls = get_modular_class(diffusion_pipeline) @@ -47,7 +55,7 @@ def convert_to_modular(diffusion_pipeline: diffusers.DiffusionPipeline) -> diffu components_dct = {k: v for k, v in diffusion_pipeline.components.items() if v is not None} modular_pipe.update_components(**components_dct, **diffusion_pipeline.parameters) modular_pipe.original_pipe = diffusion_pipeline - log.debug(f'Modular convert: source={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__}') + log.debug(f'Modular: convert={diffusion_pipeline.__class__.__name__} target={modular_pipe.__class__.__name__}') except Exception as e: log.error(f'Modular: {e}') raise e diff --git a/modules/modular_cache.py b/modules/modular_cache.py deleted file mode 100644 index f19cb2e45..000000000 --- a/modules/modular_cache.py +++ /dev/null @@ -1,12 +0,0 @@ -from modules import processing -from modules.logger import log - - -def set_cache(p: processing.StableDiffusionProcessing, phase: str | None = None): # pylint: disable=unused-argument - import modules.ui_cache - inputs = modules.ui_cache.get_modular_args() - method = inputs.get('cache_method', 'None') - if method == 'None': - return - args = {} - log.debug(f'Pipeline: cache={method} args={args}') diff --git a/modules/modular_guiders.py b/modules/modular_guiders.py index 92bb43a97..3228bcabd 100644 --- a/modules/modular_guiders.py +++ b/modules/modular_guiders.py @@ -80,7 +80,6 @@ def set_args(guidance_name: str): args['guidance_scales'] = [float(x.strip()) for x in inputs.get('fdg_scales', '5.0').split(',')] args['parallel_weights'] = float(inputs.get('fdg_weights', 1.0)) args['guidance_rescale_space'] = inputs.get('fdg_rescale_space', 'data') - log.trace(f'Guiders: args={args}') return args diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index fd5387d44..e46aa785c 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -75,9 +75,8 @@ def process_pre(p: processing.StableDiffusionProcessing, phase: str | None = Non return if is_modular(shared.sd_model): if modular.is_guider(shared.sd_model): - from modules import modular_guiders, modular_cache + from modules import modular_guiders modular_guiders.set_guider(p, phase) - modular_cache.set_cache(p, phase) else: try: log.info(f'Processing modifiers: phase={phase} apply') @@ -95,7 +94,7 @@ def process_pre(p: processing.StableDiffusionProcessing, phase: str | None = Non ipadapter.apply(shared.sd_model, p) # apply-only sd_hijack_freeu.apply_freeu(p) - transformer_cache.set_cache() + transformer_cache.set_cache(p) para_attention.apply_first_block_cache() teacache.apply_teacache(p) except Exception as e: @@ -221,7 +220,8 @@ def process_base(p: processing.StableDiffusionProcessing): for k, v in base_args.items(): if isinstance(v, torch.Tensor): err_args[k] = f'{v.device}:{v.dtype}:{v.shape}' - log.error(f'Processing: step=base args={err_args} {e}') + log.error(f'Processing: step=base args={err_args}') + log.error(f'Processing: {e}') errors.display(e, 'Processing') modelstats.analyze() finally: diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 778407fd6..4b36e4788 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -82,6 +82,8 @@ def full_vqgan_decode(latents, model): def full_vae_decode(latents, model): t0 = time.time() + if latents.ndim == 4 and latents.shape[1] == 3: # already decoded + return latents if not hasattr(model, 'vae') and hasattr(model, 'pipe'): model = model.pipe if model is None or not hasattr(model, 'vae'): diff --git a/modules/sd_hijack_modular.py b/modules/sd_hijack_modular.py index 2aa77fcdf..fa751f33d 100644 --- a/modules/sd_hijack_modular.py +++ b/modules/sd_hijack_modular.py @@ -162,7 +162,7 @@ def install_state_hook(pipe): if shared.state.interrupted or shared.state.skipped: # fires per tile, so tiled encodes abort promptly raise AssertionError('Interrupted...') - for name in ('unet', 'transformer', 'transformer_ref'): + for name in ('unet', 'transformer', 'transformer_2', 'transformer_ref'): module = getattr(pipe, name, None) if module is not None: target = getattr(module, 'model', module) # conditioning calls the inner model directly diff --git a/modules/transformer_cache.py b/modules/transformer_cache.py index 0c204ff9f..6b6b9842f 100644 --- a/modules/transformer_cache.py +++ b/modules/transformer_cache.py @@ -1,52 +1,130 @@ import os import diffusers -from modules import shared, errors +from modules import shared, errors, processing, devices +from modules.sd_offload_utils import get_module_names from modules.logger import log debug = log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None -def set_cache(faster_cache=None, pyramid_attention_broadcast=None): - if not shared.sd_loaded or not hasattr(shared.sd_model, 'transformer'): +def get_transformers(): + if not shared.sd_loaded: + return None + for module_name in get_module_names(shared.sd_model): + module = getattr(shared.sd_model, module_name, None) + if (module is not None) and ('transformer' in module_name or 'Transformer' in module.__class__.__name__): + yield module + + +def set_cache(p: processing.StableDiffusionProcessing): + if not shared.sd_loaded: return - faster_cache = faster_cache if faster_cache is not None else shared.opts.faster_cache_enabled - pyramid_attention_broadcast = pyramid_attention_broadcast if pyramid_attention_broadcast is not None else shared.opts.pab_enabled - if (not faster_cache) and (not pyramid_attention_broadcast): - return - if (not hasattr(shared.sd_model.transformer, 'enable_cache')) or (not hasattr(shared.sd_model.transformer, 'disable_cache')): - log.debug(f'Transformer cache: cls={shared.sd_model.transformer.__class__.__name__} fc={faster_cache} pab={pyramid_attention_broadcast} not supported') - return - try: - if faster_cache: # https://github.com/huggingface/diffusers/pull/10163 - distilled = shared.opts.fc_guidance_distilled - config = diffusers.FasterCacheConfig( - spatial_attention_block_skip_range=shared.opts.fc_spacial_skip_range, - spatial_attention_timestep_skip_range=(int(shared.opts.fc_spacial_skip_start), int(shared.opts.fc_spacial_skip_end)), - unconditional_batch_skip_range=shared.opts.fc_uncond_skip_range, - unconditional_batch_timestep_skip_range=(int(shared.opts.fc_uncond_skip_start), int(shared.opts.fc_uncond_skip_end)), - attention_weight_callback=lambda _: shared.opts.fc_attention_weight, - tensor_format=shared.opts.fc_tensor_format, - is_guidance_distilled=distilled, - current_timestep_callback=lambda: shared.sd_model.current_timestep, - ) - shared.sd_model.transformer.disable_cache() - shared.sd_model.transformer.enable_cache(config) - log.debug(f'Transformer cache: type={config.__class__.__name__}') - debug(f'Transformer cache: {vars(config)}') - elif pyramid_attention_broadcast: # https://github.com/huggingface/diffusers/pull/9562 - config = diffusers.PyramidAttentionBroadcastConfig( - spatial_attention_block_skip_range=shared.opts.pab_spacial_skip_range, - spatial_attention_timestep_skip_range=(int(shared.opts.pab_spacial_skip_start), int(shared.opts.pab_spacial_skip_end)), - current_timestep_callback=lambda: shared.sd_model.current_timestep, - ) - shared.sd_model.transformer.disable_cache() - shared.sd_model.transformer.enable_cache(config) - log.debug(f'Transformer cache: type={config.__class__.__name__}') - debug(f'Transformer cache: {vars(config)}') - else: - debug('Transformer cache: not enabled') - shared.sd_model.transformer.disable_cache() - except Exception as e: - log.error(f'Transformer cache: {e}') - errors.display(e, 'Transformer cache') + for module in get_transformers(): + try: + + if shared.opts.fc_enabled: + config = diffusers.hooks.FasterCacheConfig( + spatial_attention_block_skip_range=int(shared.opts.fc_spacial_skip_range), + spatial_attention_timestep_skip_range=(int(shared.opts.fc_spacial_skip_start), int(shared.opts.fc_spacial_skip_end)), + unconditional_batch_skip_range=int(shared.opts.fc_uncond_skip_range), + unconditional_batch_timestep_skip_range=(int(shared.opts.fc_uncond_skip_start), int(shared.opts.fc_uncond_skip_end)), + attention_weight_callback=lambda _: float(shared.opts.fc_attention_weight), + tensor_format=str(shared.opts.fc_tensor_format), + is_guidance_distilled=bool(shared.opts.fc_guidance_distilled), + current_timestep_callback=lambda: shared.sd_model.current_timestep, + ) + if getattr(shared.sd_model, 'cache_applied', None) == config: + return + if hasattr(module, 'disable_cache'): + module.disable_cache() + shared.sd_model.cache_applied = config + if not hasattr(shared.sd_model, 'current_timestep'): + log.warning(f'Transformer cache: method=FasterCache cls={shared.sd_model.__class__.__name__} not compatible') + else: + diffusers.hooks.apply_faster_cache(module, config) + log.debug(f'Transformer cache: method=FasterCache module={module.__class__.__name__} config={config}') + + if shared.opts.pab_enabled: + config = diffusers.hooks.PyramidAttentionBroadcastConfig( + spatial_attention_block_skip_range=int(shared.opts.pab_spacial_skip_range) if shared.opts.pab_spacial_skip_range > 0 else None, + spatial_attention_timestep_skip_range=(int(shared.opts.pab_spacial_skip_start), int(shared.opts.pab_spacial_skip_end)), + current_timestep_callback=lambda: shared.sd_model.current_timestep, + ) + if getattr(shared.sd_model, 'cache_applied', None) == config: + return + if hasattr(module, 'disable_cache'): + module.disable_cache() + shared.sd_model.cache_applied = config + if not hasattr(shared.sd_model, 'current_timestep'): + log.warning(f'Transformer cache: method=PyramidAttentionBroadcast cls={shared.sd_model.__class__.__name__} not compatible') + else: + diffusers.hooks.apply_pyramid_attention_broadcast(module, config) + log.debug(f'Transformer cache: method=PyramidAttentionBroadcast module={module.__class__.__name__} config={config}') + + if shared.opts.ls_enabled: + config = diffusers.hooks.LayerSkipConfig( + indices=[int(i.strip()) for i in shared.opts.ls_indices.split(',') if i.strip().isnumeric()], + fqn=str(shared.opts.ls_fqn), + skip_attention=bool(shared.opts.ls_skip_attention), + skip_attention_scores=bool(shared.opts.ls_skip_attention_scores), + skip_ff=bool(shared.opts.ls_skip_ff), + dropout=float(shared.opts.ls_dropout), + ) + if getattr(shared.sd_model, 'cache_applied', None) == config: + return + if hasattr(module, 'disable_cache'): + module.disable_cache() + shared.sd_model.cache_applied = config + diffusers.hooks.apply_layer_skip(module, config) + log.debug(f'Transformer cache: method=LayerSkip module={module.__class__.__name__} config={config}') + + if shared.opts.mc_enabled: + config = diffusers.hooks.MagCacheConfig( + threshold=float(shared.opts.mc_threshold), + max_skip_steps=int(shared.opts.mc_max_skip_steps), + retention_ratio=float(shared.opts.mc_retention_ratio), + num_inference_steps=int(p.steps) + ) + if getattr(shared.sd_model, 'cache_applied', None) == config: + return + if hasattr(module, 'disable_cache'): + module.disable_cache() + shared.sd_model.cache_applied = config + diffusers.hooks.apply_mag_cache(module, config) + log.debug(f'Transformer cache: method=MagCache module={module.__class__.__name__} config={config}') + + if shared.opts.ts_enabled: + config = diffusers.hooks.TaylorSeerCacheConfig( + cache_interval=int(shared.opts.ts_cache_interval), + disable_cache_before_step=int(shared.opts.ts_disable_cache_before_step), + disable_cache_after_step=int(shared.opts.ts_disable_cache_after_step), + max_order=int(shared.opts.ts_max_order), + taylor_factors_dtype=devices.dtype, + skip_predict_identifiers=[i.strip() for i in shared.opts.ts_skip_predict_identifiers.split(',') if i.strip()], + cache_identifiers=[i.strip() for i in shared.opts.ts_cache_identifiers.split(',') if i.strip()], + use_lite_mode=bool(shared.opts.ts_use_lite_mode), + ) + if getattr(shared.sd_model, 'cache_applied', None) == config: + return + if hasattr(module, 'disable_cache'): + module.disable_cache() + shared.sd_model.cache_applied = config + diffusers.hooks.apply_taylorseer_cache(module, config) + log.debug(f'Transformer cache: method=TaylorSeerCache module={module.__class__.__name__} config={config}') + + if shared.opts.fb_enabled: + config = diffusers.hooks.FirstBlockCacheConfig( + threshold=float(shared.opts.fb_threshold), + ) + if getattr(shared.sd_model, 'cache_applied', None) == config: + return + if hasattr(module, 'disable_cache'): + module.disable_cache() + shared.sd_model.cache_applied = config + diffusers.hooks.apply_first_block_cache(module, config) + log.debug(f'Transformer cache: method=FirstBlockCache module={module.__class__.__name__} config={config}') + + except Exception as e: + log.error(f'Transformer cache: {e}') + errors.display(e, 'Transformer cache') diff --git a/modules/ui_cache.py b/modules/ui_cache.py deleted file mode 100644 index 149348f90..000000000 --- a/modules/ui_cache.py +++ /dev/null @@ -1,106 +0,0 @@ -from functools import partial -import gradio as gr -import diffusers.hooks # pylint: disable=unused-import -from modules import ui_common - - -_stored_args = {} -methods = { - 'None': {}, - 'Context Parallel': {}, - 'Faster Cache': {}, - 'First Block Cache': {}, - 'Layer Skip': {}, - 'Mag Cache': {}, - 'Pyramid Attention Broadcast': {}, - 'TaylorSeer Cache': {}, - 'Text KV Cache': {}, -} - - -def get_modular_args(): - return _stored_args - - -def get_cache_methods(): - from modules.processing_helpers import is_modular - if is_modular(): - return list(methods.keys()) - return ['None'] - - -def create_cache_inputs(tab): - with gr.Accordion(open=False, label='Cache', elem_id=f"{tab}_cache", elem_classes=["small-accordion"]): - with gr.Group(): - with gr.Row(elem_id=f"{tab}_cache_row", elem_classes=['flexbox']): - cache_name = gr.Dropdown(choices=get_cache_methods(), value='None', label='Method', elem_id=f"{tab}_cache") - _cache_check = ui_common.create_refresh_button(cache_name, get_cache_methods) - - acc_context_parallel = gr.Accordion(open=True, label='Context Parallel', elem_classes=["small-accordion"], visible=False) - with acc_context_parallel: - gr.HTML(value="

TODO: Context Parallel

") - args_context_parallel = [] - - acc_faster_cache = gr.Accordion(open=True, label='Faster Cache', elem_classes=["small-accordion"], visible=False) - with acc_faster_cache: - gr.HTML(value="

TODO: Faster Cache

") - args_faster_cache = [] - - acc_first_block_cache = gr.Accordion(open=True, label='First Block Cache', elem_classes=["small-accordion"], visible=False) - with acc_first_block_cache: - gr.HTML(value="

TODO: First Block Cache

") - args_first_block_cache = [] - - acc_layer_skip = gr.Accordion(open=True, label='Layer Skip', elem_classes=["small-accordion"], visible=False) - with acc_layer_skip: - gr.HTML(value="

TODO: Layer Skip

") - args_layer_skip = [] - - acc_mag_cache = gr.Accordion(open=True, label='Mag Cache', elem_classes=["small-accordion"], visible=False) - with acc_mag_cache: - gr.HTML(value="

TODO: Mag Cache

") - args_mag_cache = [] - - acc_pyramid_attention_broadcast = gr.Accordion(open=True, label='Pyramid Attention Broadcast', elem_classes=["small-accordion"], visible=False) - with acc_pyramid_attention_broadcast: - gr.HTML(value="

TODO: Pyramid Attention Broadcast

") - args_pyramid_attention_broadcast = [] - - acc_taylorseer_cache = gr.Accordion(open=True, label='TaylorSeer Cache', elem_classes=["small-accordion"], visible=False) - with acc_taylorseer_cache: - gr.HTML(value="

TODO: TaylorSeer Cache

") - args_taylorseer_cache = [] - - acc_text_kv_cache = gr.Accordion(open=True, label='Text KV Cache', elem_classes=["small-accordion"], visible=False) - with acc_text_kv_cache: - gr.HTML(value="

TODO: Text KV Cache

") - args_text_kv_cache = [] - - def adv_visibility(guidance_name): - _stored_args['cache_name'] = guidance_name - return [ - gr.update(visible=guidance_name == 'Context Parallel'), - gr.update(visible=guidance_name == 'Faster Cache'), - gr.update(visible=guidance_name == 'First Block Cache'), - gr.update(visible=guidance_name == 'Layer Skip'), - gr.update(visible=guidance_name == 'Mag Cache'), - gr.update(visible=guidance_name == 'Pyramid Attention Broadcast'), - gr.update(visible=guidance_name == 'TaylorSeer Cache'), - gr.update(visible=guidance_name == 'Text KV Cache') - ] - cache_name.change(fn=adv_visibility, - inputs=[cache_name], - outputs=[acc_context_parallel, acc_faster_cache, acc_first_block_cache, acc_layer_skip, acc_mag_cache, acc_pyramid_attention_broadcast, acc_taylorseer_cache, acc_text_kv_cache], - ) - - modular_args = args_context_parallel + args_faster_cache + args_first_block_cache + args_layer_skip + args_mag_cache + args_pyramid_attention_broadcast + args_taylorseer_cache + args_text_kv_cache - def update_stored(component, name): - _stored_args[name] = component - for component in modular_args: - label = getattr(component, 'label', None) - value = getattr(component, 'value', None) - name = label.lower().replace(' ', '_') if label is not None else None - _stored_args[name] = value - component.change(fn=partial(update_stored, name=name), inputs=[component], outputs=[]) - - return [cache_name] diff --git a/modules/ui_control.py b/modules/ui_control.py index 404db40e7..269e00aad 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -4,7 +4,7 @@ import asyncio import gradio as gr from modules.control import unit from modules import errors, shared, progress, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports -from modules import ui_common, ui_sections, ui_guidance, ui_cache +from modules import ui_common, ui_sections, ui_guidance from modules import ui_control_helpers as helpers from modules.logger import log from modules.memstats import ram_stats @@ -202,7 +202,6 @@ def create_ui(_blocks: gr.Blocks=None): mask_controls = masking.create_segment_ui() cfg_name, cfg_scale, cfg_image, cfg_rescale, cfg_start, cfg_stop, cfg_true, cfg_adaptive = ui_guidance.create_guidance_inputs('control') - _cache_name = ui_cache.create_cache_inputs('control') vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('control') grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, grading_shadows_tint, grading_highlights_tint, grading_split_tone_balance, grading_vignette, grading_grain, grading_lut_file, grading_lut_strength = ui_sections.create_color_inputs('control') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires = ui_sections.create_latent_inputs('control') diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index 991e46385..cad753838 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -342,20 +342,16 @@ def create_settings(cmd_opts): "pab_sep": OptionInfo("

PAB: Pyramid attention broadcast

", "", gr.HTML), "pab_enabled": OptionInfo(False, "PAB cache enabled"), - "pab_spacial_skip_range": OptionInfo(2, "PAB spacial skip range", gr.Slider, {"minimum": 1, "maximum": 4, "step": 1}), + "pab_spacial_skip_range": OptionInfo(0, "PAB spacial skip range", gr.Slider, {"minimum": 0, "maximum": 4, "step": 1}), "pab_spacial_skip_start": OptionInfo(100, "PAB spacial skip start", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), "pab_spacial_skip_end": OptionInfo(800, "PAB spacial skip end", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), - "cache_dit_sep": OptionInfo("

Cache-DiT

", "", gr.HTML), - "cache_dit_enabled": OptionInfo(False, "Cache-DiT enabled"), - "cache_dit_calibrator": OptionInfo("None", "Cache-DiT calibrator", gr.Radio, {"choices": ["None", "TaylorSeer", "FoCa"]}), - "cache_dit_fcompute": OptionInfo(-1, "Cache-DiT F-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), - "cache_dit_bcompute": OptionInfo(-1, "Cache-DiT B-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), - "cache_dit_threshold": OptionInfo(-1, "Cache-DiT residual diff threshold", gr.Slider, {"minimum": -1.0, "maximum": 1.0, "step": 0.01}), - "cache_dit_warmup": OptionInfo(-1, "Cache-DiT warmup steps", gr.Slider, {"minimum": -1, "maximum": 50, "step": 1}), + "fb_sep": OptionInfo("

FBC: First Block Cache

", "", gr.HTML), + "fb_enabled": OptionInfo(False, "First Block cache enabled"), + "fb_threshold": OptionInfo(0.05, "First Block cache threshold", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "faster_cache__sep": OptionInfo("

Faster Cache

", "", gr.HTML), - "faster_cache_enabled": OptionInfo(False, "FasterCache cache enabled"), + "fc_sep": OptionInfo("

FC: Faster Cache

", "", gr.HTML), + "fc_enabled": OptionInfo(False, "FasterCache cache enabled"), "fc_spacial_skip_range": OptionInfo(2, "FasterCache spacial skip range", gr.Slider, {"minimum": 1, "maximum": 4, "step": 1}), "fc_spacial_skip_start": OptionInfo(0, "FasterCache spacial skip start", gr.Slider, {"minimum": 0, "maximum": 1000, "step": 1}), "fc_spacial_skip_end": OptionInfo(681, "FasterCache spacial skip end", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.01}), @@ -366,6 +362,39 @@ def create_settings(cmd_opts): "fc_tensor_format": OptionInfo("BCFHW", "FasterCache tensor format", gr.Radio, {"choices": ["BCFHW", "BFCHW", "BCHW"]}), "fc_guidance_distilled": OptionInfo(False, "FasterCache guidance distilled", gr.Checkbox), + "ls_sep": OptionInfo("

LS: Layer Skip

", "", gr.HTML), + "ls_enabled": OptionInfo(False, "Layer Skip enabled"), + "ls_indices": OptionInfo('', "Layer Skip indices", gr.Textbox, {"placeholder": "e.g. 0,1,2-5"}), + "ls_fqn": OptionInfo("auto", "Layer Skip FQN", gr.Textbox, {"placeholder": "e.g. model.diffusion_model.input_blocks.0"}), + "ls_skip_attention": OptionInfo(True, "Layer Skip skip attention", gr.Checkbox), + "ls_skip_attention_scores": OptionInfo(False, "Layer Skip skip attention scores", gr.Checkbox), + "ls_skip_ff": OptionInfo(True, "Layer Skip skip feedforward", gr.Checkbox), + "ls_dropout": OptionInfo(1.0, "Layer Skip dropout", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + + "mc_sep": OptionInfo("

MC: Mag Cache

", "", gr.HTML), + "mc_enabled": OptionInfo(False, "Mag Cache enabled"), + "mc_threshold": OptionInfo(0.06, "Mag Cache threshold", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "mc_max_skip_steps": OptionInfo(3, "Mag Cache max skip steps", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + "mc_retention_ratio": OptionInfo(0.2, "Mag Cache retention ratio", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + + "ts_sep": OptionInfo("

TS: TaylorSeer

", "", gr.HTML), + "ts_enabled": OptionInfo(False, "TaylorSeer enabled"), + "ts_cache_interval": OptionInfo(5, "TaylorSeer cache interval", gr.Slider, {"minimum": 1, "maximum": 20, "step": 1}), + "ts_disable_cache_before_step": OptionInfo(3, "TaylorSeer disable cache before step", gr.Slider, {"minimum": 0, "maximum": 20, "step": 1}), + "ts_disable_cache_after_step": OptionInfo(-1, "TaylorSeer disable cache after step", gr.Slider, {"minimum": -1, "maximum": 20, "step": 1}), + "ts_max_order": OptionInfo(1, "TaylorSeer max order", gr.Slider, {"minimum": 1, "maximum": 5, "step": 1}), + "ts_skip_predict_identifiers": OptionInfo("", "TaylorSeer skip predict identifiers", gr.Textbox, {"placeholder": "e.g. model.diffusion_model.input_blocks.0,model.diffusion_model.middle_block"}), + "ts_cache_identifiers": OptionInfo("", "TaylorSeer cache identifiers", gr.Textbox, {"placeholder": "e.g. model.diffusion_model.input_blocks.0,model.diffusion_model.middle_block"}), + "ts_use_lite_mode": OptionInfo(False, "TaylorSeer use lite mode", gr.Checkbox), + + "cache_dit_sep": OptionInfo("

Cache-DiT

", "", gr.HTML), + "cache_dit_enabled": OptionInfo(False, "Cache-DiT enabled"), + "cache_dit_calibrator": OptionInfo("None", "Cache-DiT calibrator", gr.Radio, {"choices": ["None", "TaylorSeer", "FoCa"]}), + "cache_dit_fcompute": OptionInfo(-1, "Cache-DiT F-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), + "cache_dit_bcompute": OptionInfo(-1, "Cache-DiT B-compute blocks", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), + "cache_dit_threshold": OptionInfo(-1, "Cache-DiT residual diff threshold", gr.Slider, {"minimum": -1.0, "maximum": 1.0, "step": 0.01}), + "cache_dit_warmup": OptionInfo(-1, "Cache-DiT warmup steps", gr.Slider, {"minimum": -1, "maximum": 50, "step": 1}), + "para_sep": OptionInfo("

Para-attention

", "", gr.HTML), "para_cache_enabled": OptionInfo(False, "ParaAttention first-block cache enabled"), "para_diff_threshold": OptionInfo(0.1, "ParaAttention residual diff threshold", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),