diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ddbec6f8..09ac15643 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,24 @@ # Change Log for SD.Next -## Update for 2026-08-28 +## Update for 2026-08-29 - **LoRA** + - *TODO*: see [LoRA docs](https://vladmandic.github.io/sdnext-docs/LoRA) for all of the improvements and usage instructions + *note*: lora now has its own settings section in *settings -> lora* - new apply engine that allows lora to be applied much faster - new calibration engine that allows lora to be applied with far smaller error when dealing with highly quantized models - - *TODO*: see [LoRA docs](https://vladmandic.github.io/sdnext-docs/LoRA) for details and usage instructions - *note*: calibration data is stored once calculated so it can be reused for future runs location is `models/calibration` folder - new factor cache that allows lora effects to be pre-calculated and persistently cached for future runs location is `models/lora-factor-cache` folder + - multi-network stack modes + can significantly improve lora quality when using multiple loras at once + - per-block strength - **Attention** + - *TODO*: see [Attention docs](https://vladmandic.github.io/sdnext-docs/Attention) for details and usage instructions + *note*: attention now has its own settings section in *settings -> cross attention* - new `sparse-attention` method that can be combined with other attention methods to reduce memory usage and improve performance on large models - *TODO*: see [Attention docs](https://vladmandic.github.io/sdnext-docs/Attention) for details and usage instructions - **Internal** - modular pipelines intercept and profiling hooks - attention mechanisms decision tree and apply method refactor @@ -1918,7 +1923,7 @@ And check out new **history** tab in the right panel, it now shows visualization *note*: this does not impact the actual image resolution, only the resolution at which detailer internally operates - refactor reuse-seed and add functionality to all tabs - refactor modernui js codebase - - move zluda flash attenion to *Triton Flash attention* option + - move zluda flash attenion to *Triton AMD Flash attention* option - remove samplers filtering - allow both flow-matching and discrete samplers for sdxl models - cleanup command line parameters diff --git a/TODO.md b/TODO.md index 032000612..a12d3234c 100644 --- a/TODO.md +++ b/TODO.md @@ -2,8 +2,6 @@ ## Short-term -- LoRA: merge new handler, @CalamitousFelicitousness -- Attn: merge refactor, @CalamitousFelicitousness - MiniMax LoRA: native loader for MiniMax-H3: fl2va, ref2va, pruned - MiniMax TAESD: - MiniMax: Create pre-quant for MiniMax-H3-Turbo diff --git a/cli/sdnq-attention-benchmark.py b/cli/sdnq-attention-benchmark.py index c0959bba1..7e3ddf354 100755 --- a/cli/sdnq-attention-benchmark.py +++ b/cli/sdnq-attention-benchmark.py @@ -992,7 +992,7 @@ def print_environment(fp8_result, prep_status, prep_detail, weight_dequant_resul lines.append(f"float8_e4m3fn matmul: [red]not supported on this gpu, selecting it fails generation[/red] [dim]({escape(fp8_result['qk'][1])})[/dim]") else: lines.append(f"float8_e4m3fn matmul: [red]failed to compile in this environment, selecting it fails generation[/red]; the error is not the hardware-capability signature, a torch or triton issue is more likely than the gpu [dim]({escape(fp8_result['qk'][1])})[/dim]") - lines.append(f"sdnq attention enabled in current config: {'[green]yes[/green]' if 'SDNQ attention' in shared.opts.sdp_overrides else '[yellow]no, enable via Compute Settings -> SDP overrides (requires restart)[/yellow]'}") + lines.append(f"sdnq attention enabled in current config: {'[green]yes[/green]' if 'SDNQ attention' in shared.opts.cross_attention_optimization else '[yellow]no, enable via Compute Settings -> Cross Attention (requires restart)[/yellow]'}") if prep_status == "disabled": lines.append("compiled input prep: torch.compile disabled in config, input prep runs eager") elif prep_status == "working": @@ -2849,7 +2849,7 @@ def bench_block_geometry(iters, warmup, config_timeout=300, selected=None): weights_mode = str(getattr(shared.opts, "sdnq_quantize_weights_mode", "")) current_id = None if weights_mode == "int8" and getattr(shared.opts, "sdnq_quantize_matmul_mode", "disabled") != "disabled": - current_id = "int8-mm-atten" if "SDNQ attention" in shared.opts.sdp_overrides else "int8-mm" + current_id = "int8-mm-atten" if "SDNQ attention" in shared.opts.cross_attention_optimization else "int8-mm" if current_id and results.get(current_id, {}).get("ms"): notes.append(f"current config runs the {results[current_id]['label']} row for int8-quantized models") if any(entry.get("ms") for config_id, entry in results.items() if config_id.endswith("sagefp16")): diff --git a/extensions-builtin/sdnq b/extensions-builtin/sdnq index 530f4d96c..f18fbf08a 160000 --- a/extensions-builtin/sdnq +++ b/extensions-builtin/sdnq @@ -1 +1 @@ -Subproject commit 530f4d96c0ccd755246d894e474202a1b16cdf5a +Subproject commit f18fbf08a5987e9f3f166cd837298e738d2bb881 diff --git a/installer.py b/installer.py index b02558659..0a5521d38 100644 --- a/installer.py +++ b/installer.py @@ -57,6 +57,7 @@ args = Dot({ 'use_ipex': False, 'use_cuda': False, 'use_rocm': False, + 'use_openvino': False, 'experimental': False, 'test': False, 'tls_selfsign': False, @@ -555,6 +556,8 @@ def check_python(supported_minors=None, experimental_minors=None, reason=None): def register_sdnq(): t_start = time.time() os.environ.setdefault('SDNQ_LOGGER_NAME', 'sd') + if not args.use_openvino: + os.environ.setdefault('SDNQ_USE_OPENVINO_MM', '0') fn = os.path.join('extensions-builtin', 'sdnq', 'src', 'sdnq', '__init__.py') name = "sdnq" spec = importlib.util.spec_from_file_location(name, fn) diff --git a/modules/attention/backends/flash_ck.py b/modules/attention/backends/flash_ck.py index 09bd86e75..ebd0b27c0 100644 --- a/modules/attention/backends/flash_ck.py +++ b/modules/attention/backends/flash_ck.py @@ -5,13 +5,17 @@ from modules.attention.registry import AttentionBackend, Constraints, Platform def prepare(platform: Platform, original): # pylint: disable=unused-argument + try: + import flash_attn # pylint: disable=unused-import + except ImportError: + log.warning('Attention: type="Flash attention" not installed: starting build, this may take a while...') if platform.backend == 'rocm': if not installed('flash-attn'): - log.info('Torch attention: type="Flash attention" building...') + log.info('Attention: type="Flash attention" building...') agent = rocm.Agent(platform.device) install(rocm.get_flash_attention_command(agent), reinstall=True) else: - install('flash-attn') + install('--no-build-isolation flash-attn') from flash_attn import flash_attn_func def call(query, key, value, attn_mask, dropout_p, is_causal, scale, enable_gqa): # pylint: disable=unused-argument @@ -34,7 +38,7 @@ def prepare(platform: Platform, original): # pylint: disable=unused-argument attn_output = attn_output.squeeze(0) return attn_output - log.debug('Torch attention: type="Flash attention"') + log.debug('Attention: type="Flash attention"') return call diff --git a/modules/attention/backends/flex.py b/modules/attention/backends/flex.py index c5e6a9d25..2bf283cea 100644 --- a/modules/attention/backends/flex.py +++ b/modules/attention/backends/flex.py @@ -35,7 +35,7 @@ def prepare(platform: Platform, original): # pylint: disable=unused-argument block_mask = create_block_mask(causal_mask, query.shape[0], query.shape[1], query.shape[-2], key.shape[-2], device=query.device) return flex_attention(query, key, value, score_mod=score_mod, block_mask=block_mask, scale=scale, enable_gqa=enable_gqa) - log.debug('Torch attention: type="Flex attention"') + log.debug('Attention: type="Flex attention"') return call diff --git a/modules/attention/backends/sage.py b/modules/attention/backends/sage.py index d51055970..19b01992e 100644 --- a/modules/attention/backends/sage.py +++ b/modules/attention/backends/sage.py @@ -1,11 +1,13 @@ import torch -from installer import install +from installer import install, installed from modules.logger import log from modules.attention.registry import AttentionBackend, Constraints, Platform def prepare(platform: Platform, original): # pylint: disable=unused-argument - install('sageattention') + if not installed('sageattention'): + log.warning('Attention: type="Sage attention" not installed: starting build, this may take a while...') + install('--no-build-isolation git+http://github.com/thu-ml/SageAttention.git', 'sageattention') use_cuda_backend = False if platform.backend == 'cuda' and torch.cuda.get_device_capability(platform.device) == (8, 6): @@ -43,7 +45,7 @@ def prepare(platform: Platform, original): # pylint: disable=unused-argument value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) return sage_attn_impl(query, key, value, is_causal, scale) - log.debug(f'Torch attention: type="Sage attention" backend={"cuda" if use_cuda_backend else "auto"}') + log.debug(f'Attention: type="Sage attention" backend={"cuda" if use_cuda_backend else "auto"}') return call diff --git a/modules/attention/backends/sdnq.py b/modules/attention/backends/sdnq.py index 3769da964..4422ce6f1 100644 --- a/modules/attention/backends/sdnq.py +++ b/modules/attention/backends/sdnq.py @@ -33,7 +33,7 @@ def prepare(platform: Platform, original): # pylint: disable=unused-argument call.caps = backend.caps if block_mask else frozenset() if not block_mask and getattr(shared.opts, 'sparse_attention_enabled', False): log.warning('SDNQ attention: the installed sdnq has no block mask input, sparse attention cannot use it; update the sdnq submodule') - log.debug(f'Torch attention: type="SDNQ attention" matmul={options["matmul_dtype"]}:{options["pv_matmul_dtype"]} smooth={options["smooth_k"]} hadamard={options["use_hadamard"]} quantize_fp32={options["quantize_fp32"]} fp16_accum={options["use_fp16_accum"]} block_mask={block_mask}') + log.debug(f'Attention: type="SDNQ attention" matmul={options["matmul_dtype"]}:{options["pv_matmul_dtype"]} smooth={options["smooth_k"]} hadamard={options["use_hadamard"]} quantize_fp32={options["quantize_fp32"]} fp16_accum={options["use_fp16_accum"]} block_mask={block_mask}') return call diff --git a/modules/attention/backends/triton_amd.py b/modules/attention/backends/triton_amd.py index a1f3be7df..45a655ede 100644 --- a/modules/attention/backends/triton_amd.py +++ b/modules/attention/backends/triton_amd.py @@ -21,12 +21,12 @@ def prepare(platform: Platform, original): # pylint: disable=unused-argument interface_fa.fwd(query, key, value, out_padded, dropout_p, scale, is_causal) return out_padded[..., :head_size_og].transpose(1, 2) - log.debug('Torch attention: type="Triton Flash attention"') + log.debug('Attention: type="Triton AMD Flash attention"') return call backend = AttentionBackend( - name='triton', label='Triton Flash attention', priority=30, prepare=prepare, + name='triton', label='Triton AMD Flash attention', priority=30, prepare=prepare, constraints=Constraints(max_head_dim=128, allow_mask=False, same_device=True), platforms=frozenset({'rocm', 'zluda'}), ) diff --git a/modules/attention/dispatcher.py b/modules/attention/dispatcher.py index ea7466373..ad5082d2c 100644 --- a/modules/attention/dispatcher.py +++ b/modules/attention/dispatcher.py @@ -3,32 +3,52 @@ from modules.logger import log from installer import install, torch_info -def set_diffusers_attention(pipe, quiet = False): - from modules import shared, devices - - log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"') - if shared.opts.cross_attention_optimization == "Disabled": - torch_info.set(attention="disabled") - elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers - devices.set_sdpa_params() - elif shared.opts.cross_attention_optimization == "xFormers": - if hasattr(pipe, 'enable_xformers_memory_efficient_attention'): - torch_info.set(attention="xformers") - pipe.enable_xformers_memory_efficient_attention() - else: - log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") +def set_xformers_attention(pipe): + try: + # install('xformers') + import xformers + log.debug(f'Attention: xFormers={xformers.__version__}') + import diffusers.utils.import_utils + diffusers.utils.import_utils._xformers_available = True # pylint: disable=protected-access + diffusers.utils.import_utils._xformers_version = xformers.__version__ # pylint: disable=protected-access + import diffusers.models.attention_processor + import importlib + importlib.reload(diffusers.models.attention_processor) + # diffusers.models.attention_processor.xformers = xformers + except Exception as e: + log.error(f'Attention: xFormers {e}') + return + if hasattr(pipe, 'enable_xformers_memory_efficient_attention'): + torch_info.set(attention="xformers") + pipe.enable_xformers_memory_efficient_attention() else: - log.warning(f'Torch attention: method="{shared.opts.cross_attention_optimization}" unknown, pipe={pipe.__class__.__name__} keeps its own attention processor') + log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") + + +def set_diffusers_attention(pipe, quiet = False): + from modules import shared, attention + log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"') + + attention.reapply() + plan = attention.get_plan() + if plan is not None and (plan.entries or plan.terminal): + pass # already set by router + elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers + pass # attention.reapply already called devices.set_sdpa_params + elif shared.opts.cross_attention_optimization == "xFormers": + set_xformers_attention(pipe) + elif shared.opts.cross_attention_optimization == "Disabled" or shared.opts.cross_attention_optimization == "Default": + torch_info.set(attention="default") + else: + log.warning(f'Attention: cls={pipe.__class__.__name__} method="{shared.opts.cross_attention_optimization}" not applied') if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"): if shared.opts.attention_slicing == "Enabled": pipe.enable_attention_slicing() else: pipe.disable_attention_slicing() - log.debug(f"Torch attention: slicing={shared.opts.attention_slicing}") - + log.debug(f"Attention: slicing={shared.opts.attention_slicing}") pipe.current_attn_name = shared.opts.cross_attention_optimization - pipe.current_attn_overrides = list(shared.opts.sdp_overrides) orig_get_kernel = None diff --git a/modules/attention/registry.py b/modules/attention/registry.py index 69f9cc231..2403cde85 100644 --- a/modules/attention/registry.py +++ b/modules/attention/registry.py @@ -57,7 +57,7 @@ class Constraints: class AttentionBackend: """One attention implementation: how to prepare it once and which calls it serves.""" name: str - label: str # the sdp_overrides choice that enables it + label: str # the cross_attention_optimization choice that enables it priority: int # higher priority entries are tried first prepare: Callable[[Platform, AttentionCall], AttentionCall | None] # imports and configures the implementation, returns its call or None constraints: Constraints = field(default_factory=Constraints) diff --git a/modules/attention/router.py b/modules/attention/router.py index a99009959..e1e7d80a1 100644 --- a/modules/attention/router.py +++ b/modules/attention/router.py @@ -42,12 +42,12 @@ def build_plan(labels, platform: Platform, original: AttentionCall, reg: Registr if backend.label not in labels: continue if not backend.available_on(platform): - log.warning(f'Torch attention: type="{backend.label}" not available on backend={platform.backend}') + log.warning(f'Attention: type="{backend.label}" not available on backend={platform.backend}') continue try: call = backend.prepare(platform, original) except Exception as err: - log.error(f'Torch attention: type="{backend.label}" {err}') + log.error(f'Attention: type="{backend.label}" {err}') continue if call is None: continue @@ -100,7 +100,7 @@ def install_router(labels, platform: Platform, original: AttentionCall, reg: Reg torch.nn.functional.scaled_dot_product_attention = make_router(plan, observer, stage) if (plan.entries or plan.terminal is not None) else original current_plan = plan torch_info.set(attention='>'.join(plan.chain())) - log.debug(f'Torch attention: chain={">".join(plan.chain())} overrides={list(labels)} backend={platform.backend} sparse={stage is not None}') + log.debug(f'Attention: chain={">".join(plan.chain())} backend={platform.backend} sparse={stage is not None}') return plan @@ -116,7 +116,7 @@ def build_sparse_stage(plan: Plan): capable = [entry.backend.name for entry in plan.entries if 'block_mask' in entry.caps] if not capable: names = [backend.label for backend in default_registry.with_cap('block_mask')] - log.warning(f'Sparse attention: enabled but no active backend consumes a block mask, enable one of {names} in sdp overrides; attention stays dense') + log.warning(f'Attention: sparse=True compatible={names} not set') return None built = sparse_stage.make_stage(options) if built is not None: @@ -132,7 +132,7 @@ def reapply_options(reg: Registry | None = None) -> list[str]: """Settings whose change rebuilds the chain: the override set, the torch kernel flags, every option a backend captures, and the sparse stage.""" from modules.attention.sparse import stage as sparse_stage reg = reg if reg is not None else default_registry - return ['sdp_options', 'sdp_overrides', *reg.options(), *sparse_stage.OPTION_NAMES] + return ['sdp_options', 'cross_attention_optimization', *reg.options(), *sparse_stage.OPTION_NAMES] def reapply() -> None: @@ -142,7 +142,7 @@ def reapply() -> None: compiled = getattr(shared, 'compiled_model_state', None) if compiled is not None and getattr(compiled, 'is_compiled', False): torch._dynamo.reset() # pylint: disable=protected-access - log.debug('Torch attention: dynamo reset, compiled model resident') + log.debug('Attention: dynamo reset, compiled model resident') def report() -> dict: diff --git a/modules/attention/sparse/selector.py b/modules/attention/sparse/selector.py index dc3cb19fc..815494882 100644 --- a/modules/attention/sparse/selector.py +++ b/modules/attention/sparse/selector.py @@ -27,7 +27,8 @@ class BlockSelection: @property def shape(self) -> tuple[int, int, int, int]: - return tuple(self.keep.shape) + b, h, nq, nk = self.keep.shape + return (b, h, nq, nk) def density(self) -> float: """Fraction of tiles kept. Reads back from the accelerator, so this is for reporting and tests, never the hot path.""" diff --git a/modules/devices.py b/modules/devices.py index b29b0213c..001bd0eed 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -493,17 +493,45 @@ def override_ipex_math(): log.warning(f'Torch ipex: {e}') +def report_attention(): + from importlib.metadata import version + try: + flash = version('flash-attn') + except Exception: + flash = False + try: + sage = version('sageattention') + except Exception: + sage = False + try: + xformers = version('xformers') + except Exception: + xformers = False + try: + kernels = version('kernels') + except Exception: + kernels = False + from diffusers.models import attention_dispatch as a + try: + import sdnq + sdnq_ver = sdnq.__version__ + except Exception: + sdnq_ver = False + # log.debug(f'Attention available: flash={a._CAN_USE_FLASH_ATTN} flash3={a._CAN_USE_FLASH_ATTN_3} sage={a._CAN_USE_SAGE_ATTN} flex={a._CAN_USE_FLEX_ATTN} npu={a._CAN_USE_NPU_ATTN} xla={a._CAN_USE_XLA_ATTN} xformers={a._CAN_USE_XFORMERS_ATTN} kernels={a.is_kernels_available()} sdnq=True') # pylint: disable=protected-access + log.debug(f'Attention available: sdnq={sdnq_ver} flash={flash} sage={sage} flex={a._CAN_USE_FLEX_ATTN} xformers={xformers} npu={a._CAN_USE_NPU_ATTN} xla={a._CAN_USE_XLA_ATTN} kernels={kernels}') # pylint: disable=protected-access + + def set_sdpa_params(): try: + global sdpa_original # pylint: disable=global-statement + report = sdpa_original is None try: - global sdpa_original # pylint: disable=global-statement if sdpa_original is not None: torch.nn.functional.scaled_dot_product_attention = sdpa_original else: sdpa_original = torch.nn.functional.scaled_dot_product_attention except Exception as err: - log.warning(f'Torch attention: type="sdpa" {err}') - + log.warning(f'Attention: type="sdpa" {err}') try: torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options or 'Flash attention' in opts.sdp_options) torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options or 'Memory attention' in opts.sdp_options) @@ -511,27 +539,12 @@ def set_sdpa_params(): if hasattr(torch.backends.cuda, "allow_fp16_bf16_reduction_math_sdp"): # only valid for torch >= 2.5 torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True) torch_info.set(attention="sdpa") - log.debug(f'Torch attention: type="sdpa" kernels={opts.sdp_options} overrides={opts.sdp_overrides}') + log.debug(f'Attention: type="sdpa" kernels={opts.sdp_options}') except Exception as err: - log.warning(f'Torch attention: type="sdpa" {err}') - - attention.install_router(opts.sdp_overrides, attention.Platform(backend=backend, device=device), sdpa_original) - - from importlib.metadata import version - try: - flash = version('flash-attn') - except Exception: - flash = False - try: - sage = version('sageattention') - except Exception: - sage = False - if flash or sage: - log.debug(f'Torch attention installed: flashattn={flash} sageattention={sage}') - - from diffusers.models import attention_dispatch as a - log.debug(f'Torch attention available: flash={a._CAN_USE_FLASH_ATTN} flash3={a._CAN_USE_FLASH_ATTN_3} sage={a._CAN_USE_SAGE_ATTN} flex={a._CAN_USE_FLEX_ATTN} npu={a._CAN_USE_NPU_ATTN} xla={a._CAN_USE_XLA_ATTN} xformers={a._CAN_USE_XFORMERS_ATTN} kernels={a.is_kernels_available()} sdnq=True') # pylint: disable=protected-access - + log.warning(f'Attention: type="sdpa" {err}') + attention.install_router([opts.cross_attention_optimization], attention.Platform(backend=backend, device=device), sdpa_original) + if report: + report_attention() except Exception as e: log.warning(f'Torch SDPA: {e}') diff --git a/modules/framepack/framepack_install.py b/modules/framepack/framepack_install.py index 121b6c680..404e89652 100644 --- a/modules/framepack/framepack_install.py +++ b/modules/framepack/framepack_install.py @@ -18,15 +18,6 @@ def rename(src:str, dst:str): def install_requirements(attention:str='SDPA'): install('av') - if attention == 'Xformers': - log.debug('FramePack install: xformers') - install('xformers') - elif attention == 'FlashAttention': - log.debug('FramePack install: flash-attn') - install('flash-attn') - elif attention == 'SageAttention': - log.debug('FramePack install: sageattention') - install('sageattention') def git_clone(git_repo:str, git_dir:str, tmp_dir:str): diff --git a/modules/lora/lora_stack.py b/modules/lora/lora_stack.py index 85818c8b9..b0fa08910 100644 --- a/modules/lora/lora_stack.py +++ b/modules/lora/lora_stack.py @@ -248,7 +248,7 @@ def clear(): state['reported'] = None -def register(layer_name, module, kind, scores, segments=None, nets=None, abs_sums=None): +def register(layer_name, module, kind, scores, segments: tuple[tuple[int, int], tuple[int, int], bool] | None = None, nets=None, abs_sums=None): """Record a select-mode layer for schedule finalization. kind 'factor': segments = ((s0, s1), (t0, t1), transposed) column ranges on the svd @@ -258,6 +258,8 @@ def register(layer_name, module, kind, scores, segments=None, nets=None, abs_sum """ entry = {'layer': layer_name, 'module': weakref.ref(module), 'kind': kind, 'segments': segments, 'scores': scores, 'nets': nets, 'abs_sums': abs_sums, 'stash': None} if kind == 'factor': + if segments is None: + raise ValueError("segments is required when kind='factor'") (s0, s1), (t0, t1), transposed = segments up = module.svd_up.data entry['stash'] = (segment_view(up, s0, s1, transposed).clone(), segment_view(up, t0, t1, transposed).clone()) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index a7cad820a..b8ae288b0 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -546,9 +546,8 @@ def update_pipeline(sd_model, p: processing.StableDiffusionProcessing): global orig_pipeline # pylint: disable=global-statement orig_pipeline = updated_model # processed ONNX pipeline should not be replaced with original pipeline. current_attn = getattr(updated_model, "current_attn_name", None) - current_overrides = getattr(updated_model, "current_attn_overrides", None) - if current_attn != shared.opts.cross_attention_optimization or current_overrides != list(shared.opts.sdp_overrides): - log.info(f"Setting attention optimization: {shared.opts.cross_attention_optimization} overrides={shared.opts.sdp_overrides}") + if current_attn != shared.opts.cross_attention_optimization: + # log.info(f"Setting attention optimization: {shared.opts.cross_attention_optimization}") attention.set_diffusers_attention(updated_model) return updated_model diff --git a/modules/sd_models.py b/modules/sd_models.py index 6140571e9..a21877dc8 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1261,7 +1261,6 @@ def copy_diffuser_options(new_pipe, orig_pipe): new_pipe.sd_model_hash = getattr(orig_pipe, 'sd_model_hash', None) new_pipe.has_accelerate = getattr(orig_pipe, 'has_accelerate', False) new_pipe.current_attn_name = getattr(orig_pipe, 'current_attn_name', None) - new_pipe.current_attn_overrides = getattr(orig_pipe, 'current_attn_overrides', None) new_pipe.default_scheduler = getattr(orig_pipe, 'default_scheduler', None) new_pipe.image_encoder = getattr(orig_pipe, 'image_encoder', None) new_pipe.feature_extractor = getattr(orig_pipe, 'feature_extractor', None) @@ -1289,7 +1288,6 @@ def backup_pipe_components(pipe): 'sd_model_hash': getattr(pipe, "sd_model_hash", None), 'has_accelerate': getattr(pipe, "has_accelerate", None), 'current_attn_name': getattr(pipe, "current_attn_name", None), - 'current_attn_overrides': getattr(pipe, "current_attn_overrides", None), 'default_scheduler': getattr(pipe, "default_scheduler", None), 'image_encoder': getattr(pipe, "image_encoder", None), 'feature_extractor': getattr(pipe, "feature_extractor", None), @@ -1311,7 +1309,6 @@ def restore_pipe_components(pipe, components): pipe.sd_model_hash = components['sd_model_hash'] pipe.has_accelerate = components['has_accelerate'] pipe.current_attn_name = components['current_attn_name'] - pipe.current_attn_overrides = components.get('current_attn_overrides') pipe.default_scheduler = components['default_scheduler'] if components['image_encoder'] is not None: diff --git a/modules/shared_defaults.py b/modules/shared_defaults.py index 0530fb6a3..46ee0af83 100644 --- a/modules/shared_defaults.py +++ b/modules/shared_defaults.py @@ -44,20 +44,15 @@ def get_default_modes(cmd_opts, mem_stat): default_sdp_choices = ['Flash', 'Memory', 'Math'] default_sdp_options = ['Flash', 'Memory', 'Math'] - default_sdp_override_choices = ['Dynamic attention', 'Flex attention', 'Flash attention', 'Sage attention', 'SDNQ attention'] - default_sdp_override_options = [] - if devices.backend == "zluda": default_sdp_options = ['Math'] - default_sdp_override_options = ['Dynamic attention'] - default_sdp_override_choices.append('Triton Flash attention') + default_cross_attention = ['Dynamic attention'] elif devices.backend == "rocm": - default_sdp_override_choices.append('Triton Flash attention') agent = devices.get_hip_agent() if agent.gfx_version < 0x1100: - default_sdp_override_options = ['Dynamic attention'] # only RDNA2 and older GPUs needs this + default_cross_attention = ['Dynamic attention'] # only RDNA2 and older GPUs needs this elif devices.backend in {"cpu", "mps"}: - default_sdp_override_options = ['Dynamic attention'] + default_cross_attention = ['Dynamic attention'] if devices.get_optimal_device_name() != "cpu": os.environ.setdefault('SDNQ_USE_OPENVINO_MM', '0') # TODO sdnq openvino: this is too late as sdnq already initialized it @@ -69,8 +64,6 @@ def get_default_modes(cmd_opts, mem_stat): default_cross_attention, default_sdp_options, default_sdp_choices, - default_sdp_override_options, - default_sdp_override_choices, default_diffusers_offload_always, default_diffusers_offload_never, ) diff --git a/modules/shared_items.py b/modules/shared_items.py index ad37125db..75efe9dba 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -139,12 +139,17 @@ def refresh_te_list(): def list_crossattention(): return [ - "Disabled", - "Scaled-Dot-Product", - "xFormers", + 'Default', + 'Scaled-Dot-Product', + 'SDNQ attention', + 'xFormers', + 'Flex attention', + 'Flash attention', + 'Sage attention', + 'Dynamic attention', + 'Triton AMD Flash attention' ] - def get_pipelines(): from modules.logger import log """ diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index 3d6f02a08..0ea9bd9db 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -63,7 +63,7 @@ def create_settings(cmd_opts): # Calculate default modes mem_stat = memory_stats() - startup_offload_mode, startup_offload_min_gpu, startup_offload_max_gpu, startup_cross_attention, startup_sdp_options, startup_sdp_choices, startup_sdp_override_options, startup_sdp_override_choices, startup_offload_always, startup_offload_never = get_default_modes(cmd_opts=cmd_opts, mem_stat=mem_stat) + startup_offload_mode, startup_offload_min_gpu, startup_offload_max_gpu, startup_cross_attention, startup_sdp_options, startup_sdp_choices, startup_offload_always, startup_offload_never = get_default_modes(cmd_opts=cmd_opts, mem_stat=mem_stat) # System variables gpu_memory = round(mem_stat['gpu']['total'] if "gpu" in mem_stat else 0) @@ -155,7 +155,7 @@ def create_settings(cmd_opts): "group_offload_type": OptionInfo("leaf_level", "Group offload type", gr.Radio, {"choices": ['leaf_level', 'block_level']}), "group_offload_stream": OptionInfo(False, "Prefetch with streams", gr.Checkbox), 'group_offload_record': OptionInfo(False, "Overlap stream transfers", gr.Checkbox), - 'group_offload_pin': OptionInfo(True, "Pin offload memory", gr.Checkbox), + 'group_offload_pin': OptionInfo(False, "Pin offload memory", gr.Checkbox), 'group_offload_blocks': OptionInfo(1, "Group offload blocks", gr.Number), "caption_offload_sep": OptionInfo("

Caption Model Offloading

", "", gr.HTML), "caption_offload": OptionInfo(True, "Offload caption models"), @@ -206,6 +206,7 @@ def create_settings(cmd_opts): "trt_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model"]}), "trt_quantization_type": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int4", "fp8", "nf4", "nvfp4"]}), })) + # --- VAE & Text Encoder --- options_templates.update(options_section(('vae_encoder', "Variational Auto Encoder"), { "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), @@ -234,25 +235,18 @@ def create_settings(cmd_opts): })) # --- Compute Settings --- - options_templates.update(options_section(('cuda', "Compute Settings"), { - "math_sep": OptionInfo("

Execution Precision

", "", gr.HTML), - "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"], "visible": False}), - "cuda_dtype": OptionInfo("Auto", "Device precision type", gr.Radio, {"choices": ["Auto", "FP32", "FP16", "BF16"]}), - "force_dtype": OptionInfo(False, "Force dtype on load", None, None, None), - "no_half": OptionInfo(False, "Force full precision (--no-half)", None, None, None), - "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling", gr.Checkbox, {"visible": False}), - - "generator_sep": OptionInfo("

Noise Options

", "", gr.HTML), - "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), - - "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), + options_templates.update(options_section(('cuda', "Cross Attention"), { "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}), - "sdp_options": OptionInfo(startup_sdp_options, "SDP kernels", gr.CheckboxGroup, {"choices": startup_sdp_choices}), - "sdp_overrides": OptionInfo(startup_sdp_override_options, "SDP overrides", gr.CheckboxGroup, {"choices": startup_sdp_override_choices}), + # "sdp_overrides": OptionInfo(startup_sdp_override_options, "SDP overrides", gr.CheckboxGroup, {"choices": startup_sdp_override_choices}), + + "attention_slicing_sep": OptionInfo("

Attention Slicing

", "", gr.HTML), "attention_slicing": OptionInfo('Default', "Attention slicing", gr.Radio, {"choices": ['Default', 'Enabled', 'Disabled']}), "dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}), "dynamic_attention_trigger_rate": OptionInfo(1, "Dynamic Attention trigger rate", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4)*2, "step": 0.01}), + "sdp_attention_sep": OptionInfo("

SDP Attention

", "", gr.HTML), + "sdp_options": OptionInfo(startup_sdp_options, "SDP kernels", gr.CheckboxGroup, {"choices": startup_sdp_choices}), + "sdnq_attention_sep": OptionInfo("

SDNQ Attention

", "", gr.HTML), "sdnq_attention_smooth_k": OptionInfo(True, "SDNQ Attention use Smooth K", gr.Checkbox), "sdnq_attention_use_hadamard": OptionInfo(False, "SDNQ Attention use Hadamard", gr.Checkbox), @@ -275,16 +269,18 @@ def create_settings(cmd_opts): "hf_attention": OptionInfo('', "Attention dispatcher kernel", gr.Textbox), })) - # --- Server Settings --- - options_templates.update(options_section(('server', "Server Settings"), { - "server_listen": OptionInfo(False, "Listen on all interfaces", gr.Checkbox), - "server_status": OptionInfo(120, "Automatic server status monitor rate", gr.Number, {"minimum": 0, "maximum": 1000, "step": 1}), - "server_monitor": OptionInfo(0, "Automatic server memory monitor rate", gr.Number, {"minimum": 0, "maximum": 1000, "step": 1}), - "server_rate_limit": OptionInfo(300, "API base rate limit rate", gr.Number, {"minimum": 0, "maximum": 1000, "step": 1}), - })) - # --- Backend Settings --- - options_templates.update(options_section(('backends', "Backend Settings"), { + options_templates.update(options_section(('backends', "Compute Settings"), { + "math_sep": OptionInfo("

Execution Precision

", "", gr.HTML), + "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"], "visible": False}), + "cuda_dtype": OptionInfo("Auto", "Device precision type", gr.Radio, {"choices": ["Auto", "FP32", "FP16", "BF16"]}), + "force_dtype": OptionInfo(False, "Force dtype on load", None, None, None), + "no_half": OptionInfo(False, "Force full precision (--no-half)", None, None, None), + "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling", gr.Checkbox, {"visible": False}), + + "generator_sep": OptionInfo("

Noise Options

", "", gr.HTML), + "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), + "other_sep": OptionInfo("

Torch Options

", "", gr.HTML), "opt_channelslast": OptionInfo(False, "Channels last "), "cudnn_deterministic": OptionInfo(False, "Deterministic mode"), @@ -305,7 +301,7 @@ def create_settings(cmd_opts): "onnx_execution_provider": OptionInfo(default_onnx_execution_provider, 'ONNX Execution Provider', gr.Dropdown, lambda: {"choices": default_onnx_execution_providers}), "onnx_cpu_fallback": OptionInfo(True, 'ONNX allow fallback to CPU'), "onnx_cache_converted": OptionInfo(True, 'ONNX cache converted models'), - "onnx_unload_base": OptionInfo(False, 'ONNX unload base model when processing refiner'), + "onnx_unload_base": OptionInfo(False, 'ONNX unload base model when processing refiner', gr.Checkbox, {"visible": False}), "olive_sep": OptionInfo("

Olive

", "", gr.HTML), "olive_float16": OptionInfo(True, 'Olive use FP16 on optimization'), @@ -324,8 +320,8 @@ def create_settings(cmd_opts): "openvino_disable_memory_cleanup": OptionInfo(True, "OpenVINO disable memory cleanup", gr.Checkbox, {"visible": cmd_opts.use_openvino}), })) - # --- Pipeline Modifiers --- - options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { + # --- Compute Add-ons --- + options_templates.update(options_section(('advanced', "Compute Add-ons"), { "clip_skip_sep": OptionInfo("

CLiP Skip

", "", gr.HTML), "clip_skip_enabled": OptionInfo(False, "CLiP skip enabled"), @@ -430,6 +426,14 @@ def create_settings(cmd_opts): "pruna_pruners": OptionInfo([], "Pruna pruners", gr.CheckboxGroup, {"choices": ["kvpress", "padding_pruning", "token_merging", "torch_structured", "torch_unstructured"]}), })) + # --- Server Settings --- + options_templates.update(options_section(('server', "Server Settings"), { + "server_listen": OptionInfo(False, "Listen on all interfaces", gr.Checkbox), + "server_status": OptionInfo(120, "Automatic server status monitor rate", gr.Number, {"minimum": 0, "maximum": 1000, "step": 1}), + "server_monitor": OptionInfo(0, "Automatic server memory monitor rate", gr.Number, {"minimum": 0, "maximum": 1000, "step": 1}), + "server_rate_limit": OptionInfo(300, "API base rate limit rate", gr.Number, {"minimum": 0, "maximum": 1000, "step": 1}), + })) + # --- System Paths --- options_templates.update(options_section(('system-paths', "System Paths"), { "models_paths_sep_options": OptionInfo("

Models Paths

", "", gr.HTML), diff --git a/modules/ui_settings.py b/modules/ui_settings.py index c1f8938a7..216895788 100644 --- a/modules/ui_settings.py +++ b/modules/ui_settings.py @@ -268,7 +268,7 @@ def create_ui(disabled_tabs=None): item for item in shared.opts.data_labels.items() if item[1].section is not None and item[1].section[0] == section_id ] # find all items in this section - hidden = section_id is None or 'hidden' in section_id.lower() or 'hidden' in section_text.lower() + hidden = (section_id is None) or ('hidden' in section_id.lower()) or ('hidden' in section_text.lower()) or ('legacy' in section_id.lower()) or ('legacy' in section_text.lower()) # log.trace(f'Settings: section="{section_id}" title="{section_text}" items={len(items)} hidden={hidden}') if hidden: for (key, _item) in items: @@ -282,7 +282,7 @@ def create_ui(disabled_tabs=None): quicksettings_list.append((key, item)) components.append(dummy_component) else: - with gr.Row(elem_id=f"settings_section_row_{section_id}", elem_classes=["settings_section"]): # only so we can add dirty indicator at the start of the row + with gr.Row(elem_id=f"settings_section_row_{section_id}", elem_classes=["settings_section"]): component = create_setting_component(key) shared.settings_components[key] = component current_items.append(key) diff --git a/scripts/xyz/xyz_grid_classes.py b/scripts/xyz/xyz_grid_classes.py index f8ff5d05c..2dd9ed356 100644 --- a/scripts/xyz/xyz_grid_classes.py +++ b/scripts/xyz/xyz_grid_classes.py @@ -4,9 +4,7 @@ from scripts.xyz.xyz_grid_shared import ( # pylint: disable=no-name-in-module, u apply_task_args, apply_setting, apply_attention, - apply_attention_overrides, apply_attention_dispatcher, - list_sdp_overrides, save_attention, restore_attention, apply_prompt_primary, @@ -280,7 +278,6 @@ axis_options = [ AxisOption("[Quant] SDNQ quant mode", str, apply_sdnq_quant, cost=0.9, fmt=format_value_add_label, choices=lambda: ['none'] + sorted(shared_items.sdnq_quant_modes)), AxisOption("[Quant] SDNQ quant mode TE", str, apply_sdnq_quant_te, cost=0.9, fmt=format_value_add_label, choices=lambda: ['none'] + sorted(shared_items.sdnq_quant_modes)), AxisOption("[Attention] Method", str, apply_setting('cross_attention_optimization'), cost=0.2, choices=shared_items.list_crossattention), - AxisOption("[Attention] SDP override", str, apply_attention_overrides, cost=0.2, choices=list_sdp_overrides), AxisOption("[Attention] Dispatcher", str, apply_attention_dispatcher, cost=0.2, choices=lambda: ['None'] + attention.list_dispatcher_backends()), AxisOption("[Attention] SDNQ matmul", str, apply_attention('sdnq_attention_matmul_type'), cost=0.2, choices=lambda: list(shared_items.sdnq_matmul_modes)), AxisOption("[Attention] SDNQ PV matmul", str, apply_attention('sdnq_attention_pv_matmul_type'), cost=0.2, choices=lambda: list(shared_items.sdnq_matmul_modes)), diff --git a/scripts/xyz/xyz_grid_shared.py b/scripts/xyz/xyz_grid_shared.py index bb6deabdb..34e9d971f 100644 --- a/scripts/xyz/xyz_grid_shared.py +++ b/scripts/xyz/xyz_grid_shared.py @@ -87,13 +87,6 @@ def attention_options() -> list: return ['cross_attention_optimization', 'hf_attention', *attention.reapply_options()] -def list_sdp_overrides() -> list: - item = shared.opts.data_labels.get('sdp_overrides', None) - args = item.component_args if item is not None else None - args = args() if callable(args) else args - return ['None'] + list((args or {}).get('choices', None) or []) - - def apply_attention(field): def fun(p, x, xs): from modules import attention @@ -106,17 +99,6 @@ def apply_attention(field): return fun -def apply_attention_overrides(p, x, xs): - from modules import attention - labels = [label.strip() for label in str(x).split('+') if len(label.strip()) > 0 and label.strip().lower() != 'none'] - unknown = [label for label in labels if attention.registry.by_label(label) is None] - if len(unknown) > 0: - log.warning(f'XYZ grid apply attention: unknown overrides={unknown} available={attention.registry.labels()}') - shared.opts.data['sdp_overrides'] = labels - attention.reapply() - log.debug(f'XYZ grid apply attention: overrides={labels}') - - def apply_attention_dispatcher(p, x, xs): from modules import attention value = '' if str(x).strip().lower() in ['none', 'default'] else str(x).strip() diff --git a/test/test-attention-router.py b/test/test-attention-router.py index 0bf20e3c0..f51b4898c 100644 --- a/test/test-attention-router.py +++ b/test/test-attention-router.py @@ -110,17 +110,17 @@ def run_test(cat: str, fn): # devices.set_sdpa_params applied the hijacks in this order; each wrapped the previous, so the # last applied was tried first. Dynamic replaced the chain end instead of wrapping it; flex did # too, which left everything stacked before it unreachable, so it is an ordinary entry now. -OLD_ORDER = ['Dynamic attention', 'Flex attention', 'Triton Flash attention', 'Flash attention', 'Sage attention', 'SDNQ attention'] +OLD_ORDER = ['Dynamic attention', 'Flex attention', 'Triton AMD Flash attention', 'Flash attention', 'Sage attention', 'SDNQ attention'] OLD_TERMINALS = {'Dynamic attention'} OLD_NAMES = { 'Dynamic attention': 'dynamic', 'Flex attention': 'flex', - 'Triton Flash attention': 'triton', + 'Triton AMD Flash attention': 'triton', 'Flash attention': 'flash', 'Sage attention': 'sage', 'SDNQ attention': 'sdnq', } -# mirrors shared_defaults.get_default_modes: five choices everywhere, Triton Flash attention added on rocm and zluda +# mirrors shared_defaults.get_default_modes: five choices everywhere, Triton AMD Flash attention added on rocm and zluda CHOICES = OLD_ORDER TRITON_PLATFORMS = {'rocm', 'zluda'} @@ -137,7 +137,7 @@ GATES = { def oracle_chain(labels, platform_backend): enabled = [label for label in OLD_ORDER if label in labels] if platform_backend not in TRITON_PLATFORMS: - enabled = [label for label in enabled if label != 'Triton Flash attention'] + enabled = [label for label in enabled if label != 'Triton AMD Flash attention'] terminal = None entries = [] for label in enabled: @@ -465,7 +465,7 @@ def test_attention_slicing_follows_the_choice(): saved = {key: shared.opts.data.get(key, None) for key in ['attention_slicing', 'cross_attention_optimization']} try: - shared.opts.data['cross_attention_optimization'] = 'Disabled' # the branch under test is the only one that should act + shared.opts.data['cross_attention_optimization'] = 'Default' # the branch under test is the only one that should act for choice, expected in [('Default', []), ('Enabled', ['enable']), ('Disabled', ['disable'])]: shared.opts.data['attention_slicing'] = choice pipe = Pipe() @@ -533,7 +533,7 @@ def test_escape_hatch_bypasses_the_router(): def test_reapply_options_cover_declared_backend_options(): from modules import shared names = attention.reapply_options() - assert names[:2] == ['sdp_options', 'sdp_overrides'], names + assert names[:2] == ['sdp_options'], names declared = attention.registry.options() assert set(declared) <= set(names), (declared, names) assert attention.registry.backends['sdnq'].options and set(attention.registry.backends['sdnq'].options) <= set(declared) diff --git a/test/test-xyz-attention.py b/test/test-xyz-attention.py index 3a16754f2..2f3f65914 100644 --- a/test/test-xyz-attention.py +++ b/test/test-xyz-attention.py @@ -93,7 +93,7 @@ def sample_value(axis): """A value the axis accepts that is not the current one, avoiding backends whose prepare installs a package.""" if axis.choices is None: return int(shared.opts.get(option_of(axis) or 'sparse_attention_budget') or 0) + 5 - choices = [choice for choice in axis.choices() if choice not in ['Sage attention', 'Flash attention', 'Triton Flash attention']] + choices = [choice for choice in axis.choices() if choice not in ['Sage attention', 'Flash attention', 'Triton AMD Flash attention']] current = str(shared.opts.get(option_of(axis)) if option_of(axis) else '') return next((choice for choice in choices if str(choice) != current), choices[0]) @@ -162,32 +162,6 @@ def test_bool_axes_coerce_the_string_the_dropdown_sends(): return True -def test_override_axis_parses_labels(): - saved = xyz.save_attention() - try: - xyz.apply_attention_overrides(None, 'None', []) - assert shared.opts.data['sdp_overrides'] == [], shared.opts.data['sdp_overrides'] - xyz.apply_attention_overrides(None, 'Flex attention', []) - assert shared.opts.data['sdp_overrides'] == ['Flex attention'], shared.opts.data['sdp_overrides'] - xyz.apply_attention_overrides(None, 'Flex attention+SDNQ attention', []) - assert shared.opts.data['sdp_overrides'] == ['Flex attention', 'SDNQ attention'], shared.opts.data['sdp_overrides'] - finally: - xyz.restore_attention(saved) - return True - - -def test_override_axis_rebuilds_the_chain(): - saved = xyz.save_attention() - try: - xyz.apply_attention_overrides(None, 'Flex attention', []) - assert 'flex' in attention.get_plan().chain(), attention.get_plan().chain() - xyz.apply_attention_overrides(None, 'None', []) - assert 'flex' not in attention.get_plan().chain(), attention.get_plan().chain() - finally: - xyz.restore_attention(saved) - return True - - def test_dispatcher_axis_clears_on_none(): saved = xyz.save_attention() try: @@ -224,8 +198,6 @@ def run_all(): for fn in [ test_axes_write_and_restore_exactly, test_bool_axes_coerce_the_string_the_dropdown_sends, - test_override_axis_parses_labels, - test_override_axis_rebuilds_the_chain, test_dispatcher_axis_clears_on_none, test_restore_set_covers_every_attention_setting, ]: diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index 07665ddb2..4e84f1f34 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -1465,7 +1465,7 @@ {"id":"","label":"Sections","localized":"","hint":"","ui":"video"}, {"id":"","label":"Samplers","localized":"","hint":"Samplers/schedulers advanced settings","ui":"tab_txt2img"}, {"id":"","label":"SDP kernels","localized":"","hint":"Which of torch's built-in attention kernels torch is allowed to choose from. These are permissions rather than a selection: torch picks one per call from whatever is left enabled, preferring Flash, dropping to Memory for calls flash cannot serve such as those carrying an arbitrary attention mask, and to Math when neither fits. Clearing a box removes a candidate; it never pins the remaining one to every call.

Flash is torch's own build of the FlashAttention kernel. It is not the same thing as the Flash attention entry in SDP overrides, which calls the separately installed flash-attn package and bypasses torch entirely.
Memory is the memory-efficient kernel, which accepts arbitrary masks that flash does not.
Math is the unfused reference path, the widest in what it accepts and the least optimized. Leaving it enabled keeps a fallback for calls the other two decline.

Applies while Attention method is Scaled-Dot-Product, and continues to govern the calls that an enabled override declines.

All three by default. ZLUDA starts with Math alone.","ui":"settings_cuda"}, - {"id":"","label":"SDP overrides","localized":"","hint":"Replaces torch attention with another implementation. Each entry declares the shapes, dtypes and mask conditions it can serve; a call that fails them moves to the next entry and finally back to torch, so several can be enabled together and the chain resolves per call.

Flash attention installs and calls the flash-attn package directly, for calls with no attention mask, half precision inputs and a head dimension of 128 or less.
Sage attention computes attention with quantized matmuls, for head dimensions of 64, 96 or 128 and no attention mask.
SDNQ attention is SD.Next's own quantized Triton kernel, configured in the section below. It takes attention masks, which the other quantized backends do not, and it is one of the two backends Sparse Attention can drive.
Flex attention uses torch's compiled flex_attention, the other backend Sparse Attention can drive.
Dynamic attention slices attention to fit available memory and serves every call the others decline, standing in for the torch fallback.
Triton Flash attention is a Triton implementation for ROCm and ZLUDA, listed only on those backends.

The quantized and compiled backends trade some numerical accuracy for throughput. How much of each arrives depends on the model, the sequence length and the GPU, so comparing them on the actual workload settles it faster than picking by reputation.

None by default on CUDA. ZLUDA, CPU and MPS start with Dynamic attention, as do ROCm GPUs older than RDNA3.","ui":"settings_cuda"}, + {"id":"","label":"SDP overrides","localized":"","hint":"Replaces torch attention with another implementation. Each entry declares the shapes, dtypes and mask conditions it can serve; a call that fails them moves to the next entry and finally back to torch, so several can be enabled together and the chain resolves per call.

Flash attention installs and calls the flash-attn package directly, for calls with no attention mask, half precision inputs and a head dimension of 128 or less.
Sage attention computes attention with quantized matmuls, for head dimensions of 64, 96 or 128 and no attention mask.
SDNQ attention is SD.Next's own quantized Triton kernel, configured in the section below. It takes attention masks, which the other quantized backends do not, and it is one of the two backends Sparse Attention can drive.
Flex attention uses torch's compiled flex_attention, the other backend Sparse Attention can drive.
Dynamic attention slices attention to fit available memory and serves every call the others decline, standing in for the torch fallback.
Triton AMD Flash attention is a Triton implementation for ROCm and ZLUDA, listed only on those backends.

The quantized and compiled backends trade some numerical accuracy for throughput. How much of each arrives depends on the model, the sequence length and the GPU, so comparing them on the actual workload settles it faster than picking by reputation.

None by default on CUDA. ZLUDA, CPU and MPS start with Dynamic attention, as do ROCm GPUs older than RDNA3.","ui":"settings_cuda"}, {"id":"","label":"SDNQ Attention","localized":"","hint":"Settings for the SDNQ attention entry in SDP overrides. They do nothing until that override is enabled.

The kernel quantizes the two matmuls inside attention, computing them on lower precision operands and rescaling the result. It is written in Triton, so it needs a working Triton for the active device.
It takes an attention mask and a block mask together, which is what lets Sparse Attention use it.
Short sequences and single-head calls are left to the rest of the chain, so text encoders and the VAE keep ordinary attention.","ui":"settings_cuda"}, {"id":"","label":"SDNQ Attention use Smooth K","localized":"","hint":"Subtracts the mean of the keys before quantizing them. Keys carry a large offset that is shared across the sequence, which spends most of the quantized range representing a value identical for every key and leaves little of it for the differences that decide the attention.
Softmax ignores a constant shift applied to every score in a row, so removing that offset changes the quantization error and not the attention.

Costs one mean and one subtraction per call.
Enabled by default.","ui":"settings_cuda"}, {"id":"","label":"SDNQ Attention use Hadamard","localized":"","hint":"Rotates queries and keys by a Hadamard transform before quantizing them. The rotation spreads a few oversized channels across all of them, which is the error shape quantization handles worst. The transform is orthogonal, so the scores it produces are the ones the unrotated tensors would produce, minus the quantization error it removes.
With SDNQ Attention PV MatMul type also set, the values are rotated as well and the output is rotated back.

Costs a rotation pass on every attention call, so it is worth enabling where a model shows quantization artifacts without it.
Idle while SDNQ Attention MatMul type is disabled, since nothing is quantized then.

Disabled by default.","ui":"settings_cuda"},