From f90bb1823bed3128b1173c373bc18b7766c3d912 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 30 Aug 2026 20:41:59 +0200 Subject: [PATCH] modular cache framework Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 17 +++-- TODO.md | 3 +- extensions-builtin/sdnq | 2 +- modules/modular_cache.py | 12 ++++ modules/processing_diffusers.py | 98 ++++++++++++++--------------- modules/processing_helpers.py | 12 ++-- modules/ui_cache.py | 106 ++++++++++++++++++++++++++++++++ modules/ui_control.py | 3 +- modules/ui_guidance.py | 3 +- wiki | 2 +- 10 files changed, 194 insertions(+), 64 deletions(-) create mode 100644 modules/modular_cache.py create mode 100644 modules/ui_cache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 68ad17ed5..5af3c0fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2026-08-29 +## Update for 2026-08-30 - **LoRA** - *TODO*: see [LoRA docs](https://vladmandic.github.io/sdnext-docs/LoRA) for all of the improvements and usage instructions @@ -20,16 +20,21 @@ *note*: this is a breaking change - if you had custom attention settings in previous releases, you will need to re-apply them in the new settings section - new `sparse-attention` method that can be combined with other attention methods to reduce memory usage and improve performance on large models + - new attention mechanisms decision tree and apply method refactor - **Modular Pipelines** - - *TODO*: see [Modular Pipelines docs](https://vladmandic.github.io/sdnext-docs/Modular-Pipelines) for details and usage instructions + - *TODO*: short note... + see [Modular Pipelines docs](https://vladmandic.github.io/sdnext-docs/Modular-Pipelines) for details and usage instructions + - new model **Guidance** stack for modular pipelines + includes: *CFG, PAG, Auto, Zero, APG, SLG, SEG, TCFG, FDG* + *TODO*: see [Guidance docs](https://vladmandic.github.io/sdnext-docs/Guidance) for details and usage instructions + - new model **Caching** stack for modular pipelines + includes: *FasterCache, FirstBlockCache, LayerSkip, MagCache, PyramidAttentionBroadcast, TaylorSeerCache, TextKVCache* + *TODO*: see [Caching docs](https://vladmandic.github.io/sdnext-docs/Caching) for details and usage instructions - implement progress and preview - intercept and profiling hooks - - full modular guidance - on-demand convert standard model on-demand -- **Internal** - - attention mechanisms decision tree and apply method refactor - **Fixes** - - unnecessary secondary prompt + - unnecessary secondary prompt if same - js fetch exception handling ## Update for 2026-08-26 diff --git a/TODO.md b/TODO.md index 798cef684..532410bc5 100644 --- a/TODO.md +++ b/TODO.md @@ -14,7 +14,8 @@ ## Issues -- Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322, @vladmandic +- [Inpaint](https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322), @vladmandic +- [OpenVINO](https://github.com/vladmandic/sdnext/issues/4939), @vladmandic ## Features diff --git a/extensions-builtin/sdnq b/extensions-builtin/sdnq index 0d0b5d204..06c83b387 160000 --- a/extensions-builtin/sdnq +++ b/extensions-builtin/sdnq @@ -1 +1 @@ -Subproject commit 0d0b5d20477caf0b56ea97eaed50e7ac2561b018 +Subproject commit 06c83b3878d7240da5d8b8c94ae798af9a047e6b diff --git a/modules/modular_cache.py b/modules/modular_cache.py new file mode 100644 index 000000000..f19cb2e45 --- /dev/null +++ b/modules/modular_cache.py @@ -0,0 +1,12 @@ +from modules import processing +from modules.logger import log + + +def set_cache(p: processing.StableDiffusionProcessing, phase: str | None = None): # pylint: disable=unused-argument + import modules.ui_cache + inputs = modules.ui_cache.get_modular_args() + method = inputs.get('cache_method', 'None') + if method == 'None': + return + args = {} + log.debug(f'Pipeline: cache={method} args={args}') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index ec172aea3..fd5387d44 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -7,7 +7,7 @@ import diffusers from PIL import Image from modules import shared, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks, attention, modular from modules.logger import log -from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name +from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name, is_modular from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed from modules.lora import lora_common @@ -71,59 +71,61 @@ def restore_state(p: processing.StableDiffusionProcessing): def process_pre(p: processing.StableDiffusionProcessing, phase: str | None = None): - from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion, cachedit - if shared.sd_model is None: - log.warning('Processing modifiers: model not loaded') + if not shared.sd_loaded: return - log.info(f'Processing modifiers: phase={phase} apply') - try: - # apply-with-unapply - # sd_hijack_compile.install() - sd_models_compile.check_deepcache(enable=True) - ipadapter.apply(shared.sd_model, p) - token_merge.apply_token_merging(shared.sd_model) - hidiffusion.apply(p, shared.sd_model_type) - ras.apply(shared.sd_model, p) - pag.apply(p) - cfgzero.apply(p) - linfusion.apply(shared.sd_model) - cachedit.apply_cache_dit(shared.sd_model) - # apply-only - sd_hijack_freeu.apply_freeu(p) - transformer_cache.set_cache() - para_attention.apply_first_block_cache() - teacache.apply_teacache(p) - - except Exception as e: - log.error(f'Processing apply: {e}') - errors.display(e, 'apply') - + if is_modular(shared.sd_model): + if modular.is_guider(shared.sd_model): + from modules import modular_guiders, modular_cache + modular_guiders.set_guider(p, phase) + modular_cache.set_cache(p, phase) + else: + try: + log.info(f'Processing modifiers: phase={phase} apply') + from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion, cachedit + # apply-with-unapply + # sd_hijack_compile.install() + sd_models_compile.check_deepcache(enable=True) + token_merge.apply_token_merging(shared.sd_model) + hidiffusion.apply(p, shared.sd_model_type) + ras.apply(shared.sd_model, p) + pag.apply(p) + cfgzero.apply(p) + linfusion.apply(shared.sd_model) + cachedit.apply_cache_dit(shared.sd_model) + ipadapter.apply(shared.sd_model, p) + # apply-only + sd_hijack_freeu.apply_freeu(p) + transformer_cache.set_cache() + para_attention.apply_first_block_cache() + teacache.apply_teacache(p) + except Exception as e: + log.error(f'Processing apply: {e}') + errors.display(e, 'apply') shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - - if modular.is_guider(shared.sd_model): - from modules import modular_guiders - modular_guiders.set_guider(p, phase) - timer.process.record('pre') def process_post(p: processing.StableDiffusionProcessing): - from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion, cachedit - log.info('Processing modifiers: unapply') - - try: - sd_models_compile.check_deepcache(enable=False) - ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False)) - token_merge.remove_token_merging(shared.sd_model) - hidiffusion.unapply() - ras.unapply(shared.sd_model) - pag.unapply() - cfgzero.unapply() - linfusion.unapply(shared.sd_model) - cachedit.unapply_cache_dir(shared.sd_model) - except Exception as e: - log.error(f'Processing unapply: {e}') - errors.display(e, 'unapply') + if not shared.sd_loaded: + return + if is_modular(shared.sd_model): + pass + else: + try: + from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion, cachedit + log.info('Processing modifiers: unapply') + sd_models_compile.check_deepcache(enable=False) + ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False)) + token_merge.remove_token_merging(shared.sd_model) + hidiffusion.unapply() + ras.unapply(shared.sd_model) + pag.unapply() + cfgzero.unapply() + linfusion.unapply(shared.sd_model) + cachedit.unapply_cache_dir(shared.sd_model) + except Exception as e: + log.error(f'Processing unapply: {e}') + errors.display(e, 'unapply') timer.process.record('post') diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index c31adc4c3..dfaabc65d 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -17,12 +17,16 @@ debug_steps = log.trace if os.environ.get('SD_STEPS_DEBUG', None) is not None el debug_steps('Trace: STEPS') -def is_modular(): - return sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.MODULAR +def is_modular(pipe = None): + if not shared.sd_loaded: + return False + pipe = pipe or shared.sd_model + return sd_models.get_diffusers_task(pipe) == sd_models.DiffusersTaskType.MODULAR -def is_txt2img(): - return sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE +def is_txt2img(pipe = None): + pipe = pipe or shared.sd_model + return sd_models.get_diffusers_task(pipe) == sd_models.DiffusersTaskType.TEXT_2_IMAGE def is_refiner_enabled(p): diff --git a/modules/ui_cache.py b/modules/ui_cache.py new file mode 100644 index 000000000..149348f90 --- /dev/null +++ b/modules/ui_cache.py @@ -0,0 +1,106 @@ +from functools import partial +import gradio as gr +import diffusers.hooks # pylint: disable=unused-import +from modules import ui_common + + +_stored_args = {} +methods = { + 'None': {}, + 'Context Parallel': {}, + 'Faster Cache': {}, + 'First Block Cache': {}, + 'Layer Skip': {}, + 'Mag Cache': {}, + 'Pyramid Attention Broadcast': {}, + 'TaylorSeer Cache': {}, + 'Text KV Cache': {}, +} + + +def get_modular_args(): + return _stored_args + + +def get_cache_methods(): + from modules.processing_helpers import is_modular + if is_modular(): + return list(methods.keys()) + return ['None'] + + +def create_cache_inputs(tab): + with gr.Accordion(open=False, label='Cache', elem_id=f"{tab}_cache", elem_classes=["small-accordion"]): + with gr.Group(): + with gr.Row(elem_id=f"{tab}_cache_row", elem_classes=['flexbox']): + cache_name = gr.Dropdown(choices=get_cache_methods(), value='None', label='Method', elem_id=f"{tab}_cache") + _cache_check = ui_common.create_refresh_button(cache_name, get_cache_methods) + + acc_context_parallel = gr.Accordion(open=True, label='Context Parallel', elem_classes=["small-accordion"], visible=False) + with acc_context_parallel: + gr.HTML(value="

TODO: Context Parallel

") + args_context_parallel = [] + + acc_faster_cache = gr.Accordion(open=True, label='Faster Cache', elem_classes=["small-accordion"], visible=False) + with acc_faster_cache: + gr.HTML(value="

TODO: Faster Cache

") + args_faster_cache = [] + + acc_first_block_cache = gr.Accordion(open=True, label='First Block Cache', elem_classes=["small-accordion"], visible=False) + with acc_first_block_cache: + gr.HTML(value="

TODO: First Block Cache

") + args_first_block_cache = [] + + acc_layer_skip = gr.Accordion(open=True, label='Layer Skip', elem_classes=["small-accordion"], visible=False) + with acc_layer_skip: + gr.HTML(value="

TODO: Layer Skip

") + args_layer_skip = [] + + acc_mag_cache = gr.Accordion(open=True, label='Mag Cache', elem_classes=["small-accordion"], visible=False) + with acc_mag_cache: + gr.HTML(value="

TODO: Mag Cache

") + args_mag_cache = [] + + acc_pyramid_attention_broadcast = gr.Accordion(open=True, label='Pyramid Attention Broadcast', elem_classes=["small-accordion"], visible=False) + with acc_pyramid_attention_broadcast: + gr.HTML(value="

TODO: Pyramid Attention Broadcast

") + args_pyramid_attention_broadcast = [] + + acc_taylorseer_cache = gr.Accordion(open=True, label='TaylorSeer Cache', elem_classes=["small-accordion"], visible=False) + with acc_taylorseer_cache: + gr.HTML(value="

TODO: TaylorSeer Cache

") + args_taylorseer_cache = [] + + acc_text_kv_cache = gr.Accordion(open=True, label='Text KV Cache', elem_classes=["small-accordion"], visible=False) + with acc_text_kv_cache: + gr.HTML(value="

TODO: Text KV Cache

") + args_text_kv_cache = [] + + def adv_visibility(guidance_name): + _stored_args['cache_name'] = guidance_name + return [ + gr.update(visible=guidance_name == 'Context Parallel'), + gr.update(visible=guidance_name == 'Faster Cache'), + gr.update(visible=guidance_name == 'First Block Cache'), + gr.update(visible=guidance_name == 'Layer Skip'), + gr.update(visible=guidance_name == 'Mag Cache'), + gr.update(visible=guidance_name == 'Pyramid Attention Broadcast'), + gr.update(visible=guidance_name == 'TaylorSeer Cache'), + gr.update(visible=guidance_name == 'Text KV Cache') + ] + cache_name.change(fn=adv_visibility, + inputs=[cache_name], + outputs=[acc_context_parallel, acc_faster_cache, acc_first_block_cache, acc_layer_skip, acc_mag_cache, acc_pyramid_attention_broadcast, acc_taylorseer_cache, acc_text_kv_cache], + ) + + modular_args = args_context_parallel + args_faster_cache + args_first_block_cache + args_layer_skip + args_mag_cache + args_pyramid_attention_broadcast + args_taylorseer_cache + args_text_kv_cache + def update_stored(component, name): + _stored_args[name] = component + for component in modular_args: + label = getattr(component, 'label', None) + value = getattr(component, 'value', None) + name = label.lower().replace(' ', '_') if label is not None else None + _stored_args[name] = value + component.change(fn=partial(update_stored, name=name), inputs=[component], outputs=[]) + + return [cache_name] diff --git a/modules/ui_control.py b/modules/ui_control.py index 269e00aad..404db40e7 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -4,7 +4,7 @@ import asyncio import gradio as gr from modules.control import unit from modules import errors, shared, progress, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports -from modules import ui_common, ui_sections, ui_guidance +from modules import ui_common, ui_sections, ui_guidance, ui_cache from modules import ui_control_helpers as helpers from modules.logger import log from modules.memstats import ram_stats @@ -202,6 +202,7 @@ def create_ui(_blocks: gr.Blocks=None): mask_controls = masking.create_segment_ui() cfg_name, cfg_scale, cfg_image, cfg_rescale, cfg_start, cfg_stop, cfg_true, cfg_adaptive = ui_guidance.create_guidance_inputs('control') + _cache_name = ui_cache.create_cache_inputs('control') vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('control') grading_brightness, grading_contrast, grading_saturation, grading_hue, grading_gamma, grading_sharpness, grading_color_temp, grading_shadows, grading_midtones, grading_highlights, grading_clahe_clip, grading_clahe_grid, grading_shadows_tint, grading_highlights_tint, grading_split_tone_balance, grading_vignette, grading_grain, grading_lut_file, grading_lut_strength = ui_sections.create_color_inputs('control') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, hdr_apply_hires = ui_sections.create_latent_inputs('control') diff --git a/modules/ui_guidance.py b/modules/ui_guidance.py index 27bd82545..820080cf1 100644 --- a/modules/ui_guidance.py +++ b/modules/ui_guidance.py @@ -15,7 +15,6 @@ def get_modular_args(): def create_guidance_inputs(tab): with gr.Accordion(open=False, label='Guidance', elem_id=f"{tab}_guidance", elem_classes=["small-accordion"]): with gr.Group(): - with gr.Row(elem_id=f"{tab}_guider_row", elem_classes=['flexbox'], visible=shared.opts.model_modular_enable): cfg_name = gr.Dropdown(choices=guiders.keys(), value='Default', label='Guider', elem_id=f"{tab}_guider") cfg_name_btn = ui_components.ToolButton(value=ui_symbols.info, elem_id=f"{tab}_guider_docs") @@ -89,7 +88,7 @@ def create_guidance_inputs(tab): def adv_visibility(guidance_name): return [ - gr.update(visible=guidance_name != 'Default' and guidance_name != 'None'), + gr.update(visible=True), gr.update(visible=guidance_name.startswith('Auto')), gr.update(visible=guidance_name.startswith('Zero')), gr.update(visible=guidance_name.startswith('PAG')), diff --git a/wiki b/wiki index 47f645c59..1f45008c0 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 47f645c59c21c5042f88bbe4c9ca550fe2b9300b +Subproject commit 1f45008c0de4e929211f182febe9666d3390a34e