diff --git a/modules/attention/context.py b/modules/attention/context.py index 857f8bf9c..71442e678 100644 --- a/modules/attention/context.py +++ b/modules/attention/context.py @@ -27,12 +27,31 @@ def denoiser_name(pipe) -> str | None: return None +def install_layout_hook(pipe) -> None: + """Let a classic pipeline's denoiser publish its own packing: the modular path has its own hook, this is the rest.""" + from modules import shared + if pipe is None or not getattr(shared.opts, 'sparse_attention_enabled', False): + return + module = getattr(pipe, 'transformer', None) + if module is None: + module = getattr(pipe, 'unet', None) + if module is None or getattr(module, 'sdnext_layout_hook', None) is not None or getattr(module, 'sdnext_state_hook', None) is not None: + return + from modules.attention.sparse import layout as sparse_layout + + def publish(denoiser, args, kwargs): # pylint: disable=unused-argument + set_layout(sparse_layout.layout_from_kwargs(kwargs, denoiser.__class__.__name__)) + + module.sdnext_layout_hook = module.register_forward_pre_hook(publish, with_kwargs=True) + + def begin(pipe, steps: int = 0) -> None: from modules import devices current.active = True current.role = 'transformer' current.layout = None current.model_key = (pipe.__class__.__name__, denoiser_name(pipe)) if pipe is not None else None + install_layout_hook(pipe) device = devices.device if devices.device is not None else torch.device('cpu') if current.step_buffer is None or current.step_buffer.device != device: current.step_buffer = torch.zeros((), dtype=torch.int64, device=device) diff --git a/modules/attention/sparse/__init__.py b/modules/attention/sparse/__init__.py index f8d0cdf9b..741692aa4 100644 --- a/modules/attention/sparse/__init__.py +++ b/modules/attention/sparse/__init__.py @@ -1,8 +1,8 @@ """Block-sparse attention: the selector, the token layout it respects, and the consumers that apply it.""" from modules.attention.sparse.selector import BlockSelection, SparseSpec, block_count, radial_blocks, schedule, select_blocks -from modules.attention.sparse.layout import Span, TokenLayout, block_pins, layout_from_index_kwargs, layout_from_prefix, layout_from_segments, publish_segments, segments_from_live +from modules.attention.sparse.layout import Span, TokenLayout, block_pins, layout_from_index_kwargs, layout_from_kwargs, layout_from_prefix, layout_from_segments, publish_segments, segments_from_live __all__ = [ 'BlockSelection', 'SparseSpec', 'block_count', 'radial_blocks', 'schedule', 'select_blocks', - 'Span', 'TokenLayout', 'block_pins', 'layout_from_index_kwargs', 'layout_from_prefix', 'layout_from_segments', 'publish_segments', 'segments_from_live', + 'Span', 'TokenLayout', 'block_pins', 'layout_from_index_kwargs', 'layout_from_kwargs', 'layout_from_prefix', 'layout_from_segments', 'publish_segments', 'segments_from_live', ] diff --git a/modules/attention/sparse/layout.py b/modules/attention/sparse/layout.py index c6c3de813..1f8eeac2e 100644 --- a/modules/attention/sparse/layout.py +++ b/modules/attention/sparse/layout.py @@ -71,6 +71,30 @@ def layout_from_index_kwargs(kwargs: dict, length: int | None = None) -> TokenLa return TokenLayout(spans=tuple(spans), length=length if length is not None else spans[-1].end, source='indices') +# how an architecture orders its joint sequence, which the call itself does not reveal. Verified against the +# diffusers transformers that take txt_ids and img_ids; HiDream packs image first and is deliberately absent, so +# it falls back rather than being pinned backwards. An unlisted class publishes nothing. +JOINT_TEXT_FIRST = frozenset({ + 'FluxTransformer2DModel', 'Flux2Transformer2DModel', 'ChromaTransformer2DModel', 'BriaTransformer2DModel', + 'BriaFiboTransformer2DModel', 'LongCatImageTransformer2DModel', 'OvisImageTransformer2DModel', +}) + + +def layout_from_stream_ids(kwargs: dict, cls_name: str | None) -> TokenLayout | None: + """Read the stream lengths off the rotary id tensors a joint transformer is given by name.""" + if cls_name not in JOINT_TEXT_FIRST: + return None + text, image = kwargs.get('txt_ids'), kwargs.get('img_ids') + if not torch.is_tensor(text) or not torch.is_tensor(image) or text.dim() < 2 or image.dim() < 2: + return None + return layout_from_segments((('text', text.shape[-2]), ('image', image.shape[-2])), source='stream-ids') + + +def layout_from_kwargs(kwargs: dict, cls_name: str | None = None) -> TokenLayout | None: + """Whatever the denoiser says about its own packing, by whichever convention it uses.""" + return layout_from_index_kwargs(kwargs or {}) or layout_from_stream_ids(kwargs or {}, cls_name) + + def layout_from_segments(segments, length: int | None = None, source: str = 'segments') -> TokenLayout: """Build a layout from ordered (kind, count) pairs, the form a transformer knows at its packing site.""" spans: list[Span] = [] diff --git a/test/test-attention-sparse.py b/test/test-attention-sparse.py index ddcf98316..59e571e18 100644 --- a/test/test-attention-sparse.py +++ b/test/test-attention-sparse.py @@ -258,6 +258,55 @@ def test_segments_from_live_splits_interior_padding(): return True +def test_layout_from_stream_ids_reads_the_joint_convention(): + from modules.attention.sparse import layout as layout_mod + flat = (torch.zeros(512, 3, device=device), torch.zeros(4096, 3, device=device)) # flux1 passes 2d ids + batched = (torch.zeros(1, 512, 4, device=device), torch.zeros(1, 4096, 4, device=device)) # flux2 passes 3d + for text, image in (flat, batched): + token_layout = layout_mod.layout_from_stream_ids({'txt_ids': text, 'img_ids': image}, 'FluxTransformer2DModel') + assert token_layout is not None and token_layout.length == 4608, token_layout + assert [(s.kind, s.start, s.end) for s in token_layout.spans] == [('text', 0, 512), ('image', 512, 4608)], token_layout.spans + # an architecture whose packing order is not verified publishes nothing rather than pinning the wrong half dense + assert layout_mod.layout_from_stream_ids({'txt_ids': flat[0], 'img_ids': flat[1]}, 'HiDreamImageTransformer2DModel') is None + assert layout_mod.layout_from_stream_ids({'txt_ids': flat[0], 'img_ids': flat[1]}, None) is None + assert layout_mod.layout_from_stream_ids({}, 'FluxTransformer2DModel') is None + indices = {'video_indices': torch.arange(0, 64, device=device), 'txt_ids': flat[0], 'img_ids': flat[1]} + assert layout_mod.layout_from_kwargs(indices, 'FluxTransformer2DModel').source == 'indices', 'the index form wins when both are present' + return True + + +def test_layout_hook_publishes_from_the_denoiser_kwargs(): + from modules.attention import context as ctx + + class FluxTransformer2DModel(torch.nn.Module): # the reader keys on the class name, so the fake carries a real one + def forward(self, hidden_states=None, txt_ids=None, img_ids=None): # pylint: disable=unused-argument + return hidden_states + + class Pipe: + def __init__(self, transformer): + self.transformer = transformer + + denoiser = FluxTransformer2DModel() + pipe = Pipe(denoiser) + previous = getattr(shared.opts, 'sparse_attention_enabled', False) + try: + shared.opts.data['sparse_attention_enabled'] = False + ctx.install_layout_hook(pipe) + assert getattr(denoiser, 'sdnext_layout_hook', None) is None, 'nothing is hooked while the feature is off' + shared.opts.data['sparse_attention_enabled'] = True + ctx.install_layout_hook(pipe) + ctx.install_layout_hook(pipe) + assert getattr(denoiser, 'sdnext_layout_hook', None) is not None, 'the denoiser is hooked once' + ctx.set_layout(None) + denoiser(hidden_states=torch.zeros(1, 4096, 4, device=device), txt_ids=torch.zeros(512, 3, device=device), img_ids=torch.zeros(4096, 3, device=device)) + published = ctx.current.layout + assert published is not None and published.length == 4608 and published.source == 'stream-ids', published + finally: + shared.opts.data['sparse_attention_enabled'] = previous + ctx.set_layout(None) + return True + + def test_block_pins_are_cached_per_geometry(): layout = sparse.layout_from_segments([('text', 128), ('video', 1024)]) first = sparse.block_pins(layout, 1152, 1152, 128, 64, device) @@ -479,6 +528,8 @@ def run_all(): test_block_pins_pin_a_boundary_tile, test_block_pins_are_cached_per_geometry, test_segments_from_live_splits_interior_padding, + test_layout_from_stream_ids_reads_the_joint_convention, + test_layout_hook_publishes_from_the_denoiser_kwargs, ]: run_test(cat, fn)