fix pixart-sigma

CHANGELOG.md

@@ -11,7 +11,7 @@
 ### Note
 
 *Note*: [SD.Next](https://github.com/vladmandic/automatic) is no longer marked as a fork of [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui/) and github project has been fully detached  
-Given huge number of changes with *+3443/-3342 commits diff over the past year and complete focus on different backend/engine,  
+Given huge number of changes with *+3443/-3342 commits diff over the past year, a completely different backend/engine and a change of focus,  
 it is time to give credit to original [author](https://github.com/auTOMATIC1111),  and move on!  
 
 ## Update for 2024-04-26
@@ -57,7 +57,7 @@ it is time to give credit to original [author](https://github.com/auTOMATIC1111)
     pixart-Σ is a high end diffusion Transformer model (DiT) with a T5 encoder/decoder capable of directly generating images at 4K resolution  
     to use, simply select from *networks -> models -> PixArt-Σ*  
     *note*: this is a very large model at ~22GB  
-    set parameters: *precision: fp32*, *sampler: Default*  
+    set parameters: *sampler: Default*  
   - [SDXS](https://github.com/IDKiro/sdxs)
     sdxs is an extremely fast 1-step generation consistency model that also uses TAESD as quick VAE out-of-the-box  
     to use, simply select from *networks -> models -> SDXS*  

modules/sd_hijack_accelerate.py

@@ -2,12 +2,15 @@ from typing import Optional, Union
 import time
 import torch
 import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.modules.utils import _pair
 import accelerate.utils.modeling
 from modules import devices
 
 
 tensor_to_timer = 0
-orig_method = accelerate.utils.set_module_tensor_to_device
+orig_set_module = accelerate.utils.set_module_tensor_to_device
+orig_torch_conv = torch.nn.modules.conv.Conv2d._conv_forward # pylint: disable=protected-access
 
 
 def check_device_same(d1, d2):
@@ -57,10 +60,24 @@ def hijack_accelerate():
 
 
 def restore_accelerate():
-    accelerate.utils.set_module_tensor_to_device = orig_method
+    accelerate.utils.set_module_tensor_to_device = orig_set_module
 
 
 def hijack_hfhub():
     import contextlib
     import huggingface_hub.file_download
     huggingface_hub.file_download.FileLock = contextlib.nullcontext
+
+
+def torch_conv_forward(self, input, weight, bias): # pylint: disable=redefined-builtin
+    if self.padding_mode != 'zeros':
+        return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode), weight, bias, self.stride, _pair(0), self.dilation, self.groups) # pylint: disable=protected-access
+    if weight.dtype != bias.dtype:
+        bias.to(weight.dtype)
+    return F.conv2d(input, weight, bias, self.stride, self.padding, self.dilation, self.groups)
+
+def hijack_torch_conv():
+    torch.nn.modules.conv.Conv2d._conv_forward = torch_conv_forward # pylint: disable=protected-access
+
+def restore_torch_conv():
+    torch.nn.modules.conv.Conv2d._conv_forward = orig_torch_conv # pylint: disable=protected-access

modules/sd_hijack_pixart.py

@@ -168,7 +168,7 @@ ASPECT_RATIO_512_BIN = {
 def pipeline_pixart_alpha_call(
         self,
         prompt: Union[str, List[str]] = None,
-        negative_prompt: str = "",
+        negative_prompt: Union[str, List[str]] = None,
         num_inference_steps: int = 20,
         timesteps: List[int] = None,
         guidance_scale: float = 4.5,
@@ -463,6 +463,8 @@ class PixArtSigmaPipeline(PixArtAlphaPipeline):
         self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
         self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
 
+PixArtSigmaPipeline.__call__ = pipeline_pixart_alpha_call
+
 
 def pixart_sigma_init_patched_inputs(self, norm_type):
     assert self.config.sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

modules/sd_models.py

@@ -125,6 +125,7 @@ class NoWatermark:
 def setup_model():
     list_models()
     sd_hijack_accelerate.hijack_hfhub()
+    # sd_hijack_accelerate.hijack_torch_conv()
     if shared.backend == shared.Backend.ORIGINAL:
         enable_midas_autodownload()
 
@@ -983,6 +984,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
             elif model_type in ['PixArt-Sigma']: # forced pipeline
                 try:
                     from modules.sd_hijack_pixart import PixArtSigmaPipeline, patch_pixart_sigma_transformer
+                    shared.opts.data['cuda_dtype'] = 'FP32' # override
+                    shared.opts.data['set_diffuser_options'] = True # override
+                    devices.set_cuda_params()
                     sd_model = PixArtSigmaPipeline.from_pretrained(
                         checkpoint_info.path,
                         transformer=patch_pixart_sigma_transformer(),