From e9dcceac0e1d92b1725a3fb1116a72a262f0bc34 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 26 Apr 2024 11:06:08 -0400 Subject: [PATCH] fix pixart-sigma --- CHANGELOG.md | 4 ++-- modules/sd_hijack_accelerate.py | 21 +++++++++++++++++++-- modules/sd_hijack_pixart.py | 4 +++- modules/sd_models.py | 4 ++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e17d6efae..691f13029 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ ### Note *Note*: [SD.Next](https://github.com/vladmandic/automatic) is no longer marked as a fork of [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui/) and github project has been fully detached -Given huge number of changes with *+3443/-3342 commits diff over the past year and complete focus on different backend/engine, +Given huge number of changes with *+3443/-3342 commits diff over the past year, a completely different backend/engine and a change of focus, it is time to give credit to original [author](https://github.com/auTOMATIC1111), and move on! ## Update for 2024-04-26 @@ -57,7 +57,7 @@ it is time to give credit to original [author](https://github.com/auTOMATIC1111) pixart-Σ is a high end diffusion Transformer model (DiT) with a T5 encoder/decoder capable of directly generating images at 4K resolution to use, simply select from *networks -> models -> PixArt-Σ* *note*: this is a very large model at ~22GB - set parameters: *precision: fp32*, *sampler: Default* + set parameters: *sampler: Default* - [SDXS](https://github.com/IDKiro/sdxs) sdxs is an extremely fast 1-step generation consistency model that also uses TAESD as quick VAE out-of-the-box to use, simply select from *networks -> models -> SDXS* diff --git a/modules/sd_hijack_accelerate.py b/modules/sd_hijack_accelerate.py index c1d8429b5..bddd6e937 100644 --- a/modules/sd_hijack_accelerate.py +++ b/modules/sd_hijack_accelerate.py @@ -2,12 +2,15 @@ from typing import Optional, Union import time import torch import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import _pair import accelerate.utils.modeling from modules import devices tensor_to_timer = 0 -orig_method = accelerate.utils.set_module_tensor_to_device +orig_set_module = accelerate.utils.set_module_tensor_to_device +orig_torch_conv = torch.nn.modules.conv.Conv2d._conv_forward # pylint: disable=protected-access def check_device_same(d1, d2): @@ -57,10 +60,24 @@ def hijack_accelerate(): def restore_accelerate(): - accelerate.utils.set_module_tensor_to_device = orig_method + accelerate.utils.set_module_tensor_to_device = orig_set_module def hijack_hfhub(): import contextlib import huggingface_hub.file_download huggingface_hub.file_download.FileLock = contextlib.nullcontext + + +def torch_conv_forward(self, input, weight, bias): # pylint: disable=redefined-builtin + if self.padding_mode != 'zeros': + return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode), weight, bias, self.stride, _pair(0), self.dilation, self.groups) # pylint: disable=protected-access + if weight.dtype != bias.dtype: + bias.to(weight.dtype) + return F.conv2d(input, weight, bias, self.stride, self.padding, self.dilation, self.groups) + +def hijack_torch_conv(): + torch.nn.modules.conv.Conv2d._conv_forward = torch_conv_forward # pylint: disable=protected-access + +def restore_torch_conv(): + torch.nn.modules.conv.Conv2d._conv_forward = orig_torch_conv # pylint: disable=protected-access diff --git a/modules/sd_hijack_pixart.py b/modules/sd_hijack_pixart.py index 320ab7668..2895f9ec2 100644 --- a/modules/sd_hijack_pixart.py +++ b/modules/sd_hijack_pixart.py @@ -168,7 +168,7 @@ ASPECT_RATIO_512_BIN = { def pipeline_pixart_alpha_call( self, prompt: Union[str, List[str]] = None, - negative_prompt: str = "", + negative_prompt: Union[str, List[str]] = None, num_inference_steps: int = 20, timesteps: List[int] = None, guidance_scale: float = 4.5, @@ -463,6 +463,8 @@ class PixArtSigmaPipeline(PixArtAlphaPipeline): self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) +PixArtSigmaPipeline.__call__ = pipeline_pixart_alpha_call + def pixart_sigma_init_patched_inputs(self, norm_type): assert self.config.sample_size is not None, "Transformer2DModel over patched input must provide sample_size" diff --git a/modules/sd_models.py b/modules/sd_models.py index b4dc870df..d72376db7 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -125,6 +125,7 @@ class NoWatermark: def setup_model(): list_models() sd_hijack_accelerate.hijack_hfhub() + # sd_hijack_accelerate.hijack_torch_conv() if shared.backend == shared.Backend.ORIGINAL: enable_midas_autodownload() @@ -983,6 +984,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No elif model_type in ['PixArt-Sigma']: # forced pipeline try: from modules.sd_hijack_pixart import PixArtSigmaPipeline, patch_pixart_sigma_transformer + shared.opts.data['cuda_dtype'] = 'FP32' # override + shared.opts.data['set_diffuser_options'] = True # override + devices.set_cuda_params() sd_model = PixArtSigmaPipeline.from_pretrained( checkpoint_info.path, transformer=patch_pixart_sigma_transformer(),