add modelscope

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 - **Diffusers**
   - **IP Adapter** add support for `ip-adapter-plus_sd15` and `ip-adapter-plus-face_sd15`  
+  - added support for basic [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) model  
+    - simply select from *networks -> reference* and use from *txt2img* tab
 - **General**
   - **LoRA** add support for block weights, thanks @AI-Casanova  
     example `<lora:SDXL_LCM_LoRA:1.0:in=0:mid=1:out=0>`  

html/reference.json

@@ -88,5 +88,10 @@
     "path": "thu-ml/unidiffuser-v1",
     "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.",
     "preview": "thu-ml--unidiffuser-v1.jpg"
+  },
+  "ModelScope T2V": {
+    "path": "damo-vilab/text-to-video-ms-1.7b",
+    "desc": "The text-to-video generation diffusion model consists of three sub-networks: text feature extraction model, text feature-to-video latent space diffusion model, and video latent space to video visual space model. The overall model parameters are about 1.7 billion. Currently, it only supports English input. The diffusion model adopts a UNet3D structure, and implements video generation through the iterative denoising process from the pure Gaussian noise video.",
+    "preview": "damo-vilab--text-to-video-ms-1.7b.jpg"
   }
 }

modules/processing_diffusers.py

@@ -164,7 +164,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             decoded = taesd_vae_decode(latents=latents)
         # TODO validate decoded sample diffusers
         # decoded = validate_sample(decoded)
-        imgs = model.image_processor.postprocess(decoded, output_type=output_type)
+        if hasattr(model, 'image_processor'):
+            imgs = model.image_processor.postprocess(decoded, output_type=output_type)
+        else:
+            import diffusers
+            image_processor = diffusers.image_processor.VaeImageProcessor()
+            imgs = image_processor.postprocess(decoded, output_type=output_type)
         shared.state.job = prev_job
         if shared.cmd_opts.profile:
             t1 = time.time()
@@ -345,7 +350,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 args[arg] = task_kwargs[arg]
         task_args = getattr(p, 'task_args', {})
         for k, v in task_args.items():
-            args[k] = v
+            if k in possible:
+                args[k] = v
 
         hypertile_set(p, hr=len(getattr(p, 'init_images', [])))
         clean = args.copy()

scripts/animatediff.py

@@ -152,6 +152,8 @@ class Script(scripts.Script):
             p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}'
         p.extra_generation_params['AnimateDiff'] = loaded_adapter
         p.do_not_save_grid = True
+        if 'animatediff' not in p.ops:
+            p.ops.append('animatediff')
         p.task_args['num_frames'] = frames
         p.task_args['num_inference_steps'] = p.steps
         if not latent_mode:

scripts/stablevideodiffusion.py

@@ -44,7 +44,8 @@ class Script(scripts.Script):
         return [num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
 
     def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
-        if shared.sd_model is None or shared.sd_model.__class__.__name__ != 'StableVideoDiffusionPipeline':
+        c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else ''
+        if c != 'StableVideoDiffusionPipeline' and c != 'TextToVideoSDPipeline':
             return None
         if hasattr(p, 'init_images') and len(p.init_images) > 0:
             if override_resolution:
@@ -53,9 +54,13 @@ class Script(scripts.Script):
                 p.task_args['image'] = images.resize_image(resize_mode=2, im=p.init_images[0], width=p.width, height=p.height, upscaler_name=None, output_type='pil')
             else:
                 p.task_args['image'] = p.init_images[0]
-            p.ops.append('svd')
+            p.ops.append('stablevideo')
             p.do_not_save_grid = True
-            p.sampler_name = 'Default' # svd does not support non-default sampler
+            if c == 'StableVideoDiffusionPipeline':
+                p.sampler_name = 'Default' # svd does not support non-default sampler
+                p.task_args['output_type'] = 'np'
+            else:
+                p.task_args['output_type'] = 'pil'
             p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator
             p.task_args['width'] = p.width
             p.task_args['height'] = p.height
@@ -66,7 +71,6 @@ class Script(scripts.Script):
             p.task_args['num_inference_steps'] = p.steps
             p.task_args['min_guidance_scale'] = min_guidance_scale
             p.task_args['max_guidance_scale'] = max_guidance_scale
-            p.task_args['output_type'] = 'np'
             shared.log.debug(f'StableVideo: args={p.task_args}')
             shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
             processed = processing.process_images(p)