add vgen

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Update for 2024-02-24
 
+- **Image2Video**
+  - new module for creating videos from images  
+  - simply enable from *img2img -> scripts -> image2video*  
+  - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl)  
 - **Improvements**
   - default theme updates
   - additional built-in theme *black-gray*

TODO.md

@@ -10,7 +10,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 - init latents: variations, tiling, img2img
 - x-adapter: <https://github.com/showlab/X-Adapter>
 - diffusers public callbacks  
-- image2video: pia and vgen pipelines  
 - video2video
 - async lowvram: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14855>
 - remove builtin: controlnet

scripts/image2video.py

@@ -6,7 +6,7 @@ from modules import scripts, processing, shared, images, sd_models, devices
 
 MODELS = [
     { 'name': 'None', 'info': '' },
-    { 'name': 'PIA', 'url': 'openmmlab/PIA-condition-adapter', 'info': '<a href="https://huggingface.co/docs/diffusers/main/en/api/pipelines/pia" target="_blank">Open MMLab Personalized Image Animator</a>' },
+    # { 'name': 'PIA', 'url': 'openmmlab/PIA-condition-adapter', 'info': '<a href="https://huggingface.co/docs/diffusers/main/en/api/pipelines/pia" target="_blank">Open MMLab Personalized Image Animator</a>' },
     { 'name': 'VGen', 'url': 'ali-vilab/i2vgen-xl', 'info': '<a href="https://huggingface.co/ali-vilab/i2vgen-xl" target="_blank">Alibaba VGen</a>' },
 ]
 
@@ -75,12 +75,12 @@ class Script(scripts.Script):
         shared.log.debug(f'Image2Video: model={model_name} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}')
         p.ops.append('image2video')
         p.do_not_save_grid = True
+        orig_pipeline = shared.sd_model
 
         if model_name == 'PIA':
             if shared.sd_model_type != 'sd':
                 shared.log.error('Image2Video PIA: base model must be SD15')
                 return
-            orig_pipeline = shared.sd_model
             shared.log.info(f'Image2Video PIA load: model={repo_id}')
             motion_adapter = diffusers.MotionAdapter.from_pretrained(repo_id)
             motion_adapter.to(devices.device, devices.dtype)
@@ -101,7 +101,6 @@ class Script(scripts.Script):
             shared.log.debug(f'Image2Video PIA: args={p.task_args}')
             processed = processing.process_images(p)
             shared.sd_model.motion_adapter = None
-            shared.sd_model = orig_pipeline
 
         if model_name == 'VGen':
             if not isinstance(shared.sd_model, diffusers.I2VGenXLPipeline):
@@ -121,6 +120,7 @@ class Script(scripts.Script):
             shared.log.debug(f'Image2Video VGen: args={p.task_args}')
             processed = processing.process_images(p)
 
+        shared.sd_model = orig_pipeline
         if video_type != 'None' and processed is not None:
             images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
         return processed