diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e57de16e..481507267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - **Diffusers** - **IP Adapter** add support for `ip-adapter-plus_sd15` and `ip-adapter-plus-face_sd15` + - added support for basic [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) model + - simply select from *networks -> reference* and use from *txt2img* tab - **General** - **LoRA** add support for block weights, thanks @AI-Casanova example `` diff --git a/html/reference.json b/html/reference.json index a3ff809ed..44a4b15e2 100644 --- a/html/reference.json +++ b/html/reference.json @@ -88,5 +88,10 @@ "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", "preview": "thu-ml--unidiffuser-v1.jpg" + }, + "ModelScope T2V": { + "path": "damo-vilab/text-to-video-ms-1.7b", + "desc": "The text-to-video generation diffusion model consists of three sub-networks: text feature extraction model, text feature-to-video latent space diffusion model, and video latent space to video visual space model. The overall model parameters are about 1.7 billion. Currently, it only supports English input. The diffusion model adopts a UNet3D structure, and implements video generation through the iterative denoising process from the pure Gaussian noise video.", + "preview": "damo-vilab--text-to-video-ms-1.7b.jpg" } } \ No newline at end of file diff --git a/models/Reference/damo-vilab--text-to-video-ms-1.7b.jpg b/models/Reference/damo-vilab--text-to-video-ms-1.7b.jpg new file mode 100644 index 000000000..39ad83eea Binary files /dev/null and b/models/Reference/damo-vilab--text-to-video-ms-1.7b.jpg differ diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index e90cd0126..95b2c1c41 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -164,7 +164,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro decoded = taesd_vae_decode(latents=latents) # TODO validate decoded sample diffusers # decoded = validate_sample(decoded) - imgs = model.image_processor.postprocess(decoded, output_type=output_type) + if hasattr(model, 'image_processor'): + imgs = model.image_processor.postprocess(decoded, output_type=output_type) + else: + import diffusers + image_processor = diffusers.image_processor.VaeImageProcessor() + imgs = image_processor.postprocess(decoded, output_type=output_type) shared.state.job = prev_job if shared.cmd_opts.profile: t1 = time.time() @@ -345,7 +350,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args[arg] = task_kwargs[arg] task_args = getattr(p, 'task_args', {}) for k, v in task_args.items(): - args[k] = v + if k in possible: + args[k] = v hypertile_set(p, hr=len(getattr(p, 'init_images', []))) clean = args.copy() diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 4b0ef5d9c..531335761 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -152,6 +152,8 @@ class Script(scripts.Script): p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}' p.extra_generation_params['AnimateDiff'] = loaded_adapter p.do_not_save_grid = True + if 'animatediff' not in p.ops: + p.ops.append('animatediff') p.task_args['num_frames'] = frames p.task_args['num_inference_steps'] = p.steps if not latent_mode: diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 16077cc2c..116bf9636 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -44,7 +44,8 @@ class Script(scripts.Script): return [num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - if shared.sd_model is None or shared.sd_model.__class__.__name__ != 'StableVideoDiffusionPipeline': + c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else '' + if c != 'StableVideoDiffusionPipeline' and c != 'TextToVideoSDPipeline': return None if hasattr(p, 'init_images') and len(p.init_images) > 0: if override_resolution: @@ -53,9 +54,13 @@ class Script(scripts.Script): p.task_args['image'] = images.resize_image(resize_mode=2, im=p.init_images[0], width=p.width, height=p.height, upscaler_name=None, output_type='pil') else: p.task_args['image'] = p.init_images[0] - p.ops.append('svd') + p.ops.append('stablevideo') p.do_not_save_grid = True - p.sampler_name = 'Default' # svd does not support non-default sampler + if c == 'StableVideoDiffusionPipeline': + p.sampler_name = 'Default' # svd does not support non-default sampler + p.task_args['output_type'] = 'np' + else: + p.task_args['output_type'] = 'pil' p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator p.task_args['width'] = p.width p.task_args['height'] = p.height @@ -66,7 +71,6 @@ class Script(scripts.Script): p.task_args['num_inference_steps'] = p.steps p.task_args['min_guidance_scale'] = min_guidance_scale p.task_args['max_guidance_scale'] = max_guidance_scale - p.task_args['output_type'] = 'np' shared.log.debug(f'StableVideo: args={p.task_args}') shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) processed = processing.process_images(p)