mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
improve animatediff
This commit is contained in:
Vladimir Mandic
+5
-4
@@ -30,14 +30,15 @@
|
||||
- *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions
|
||||
in the future, expect more optimizations, especially related to offloading/slicing/tiling,
|
||||
but at the moment this is pretty much experimental-only
|
||||
- **AnimateDiff**
|
||||
- [AnimateDiff](https://github.com/guoyww/animatediff/)
|
||||
- overall improved quality
|
||||
- can now be used with *second pass* - enhance, upscale and hires your videos!
|
||||
- **IP Adapter**
|
||||
- add support for `ip-adapter-plus_sd15`, `ip-adapter-plus-face_sd15` and `ip-adapter-full-face_sd15`
|
||||
- [IP Adapter](https://github.com/tencent-ailab/IP-Adapter)
|
||||
- add support for **ip-adapter-plus_sd15, ip-adapter-plus-face_sd15 and ip-adapter-full-face_sd15**
|
||||
- can now be used in *xyz-grid*
|
||||
- **Text-to-Video**
|
||||
- in text tab, select `text-to-video` script
|
||||
- supported models: ModelScope v1.7b, ZeroScope v1, ZeroScope v1.1, ZeroScope v2, ZeroScope v2 Dark, Potat v1
|
||||
- supported models: **ModelScope v1.7b, ZeroScope v1, ZeroScope v1.1, ZeroScope v2, ZeroScope v2 Dark, Potat v1**
|
||||
*if you know of any other t2v models you'd like to see supported, let me know!*
|
||||
- models are auto-downloaded on first use
|
||||
- *note*: current base model will be unloaded to free up resources
|
||||
|
||||
@@ -13,7 +13,5 @@ def restore_faces(np_image):
|
||||
face_restorers = [x for x in shared.face_restorers if x.name() == shared.opts.face_restoration_model or shared.opts.face_restoration_model is None]
|
||||
if len(face_restorers) == 0:
|
||||
return np_image
|
||||
|
||||
face_restorer = face_restorers[0]
|
||||
|
||||
return face_restorer.restore(np_image)
|
||||
|
||||
@@ -453,7 +453,10 @@ def process_diffusers(p: StableDiffusionProcessing):
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Profile: pipeline call: {t1-t0:.2f}')
|
||||
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
||||
shared.log.debug(f'Generated: frames={len(output.frames[0])}')
|
||||
if hasattr(output.frames[0], 'shape'):
|
||||
shared.log.debug(f'Generated: frames={output.frames[0].shape[1]}')
|
||||
else:
|
||||
shared.log.debug(f'Generated: frames={len(output.frames[0])}')
|
||||
output.images = output.frames[0]
|
||||
except AssertionError as e:
|
||||
shared.log.info(e)
|
||||
|
||||
+34
-9
@@ -10,6 +10,7 @@ TODO:
|
||||
- AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main
|
||||
"""
|
||||
|
||||
import os
|
||||
import gradio as gr
|
||||
import diffusers
|
||||
from modules import scripts, processing, shared, devices, sd_models
|
||||
@@ -18,12 +19,15 @@ from modules import scripts, processing, shared, devices, sd_models
|
||||
# config
|
||||
ADAPTERS = {
|
||||
'None': None,
|
||||
'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4',
|
||||
'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5',
|
||||
'Motion 1.5 v3' :'vladmandic/animatediff-v3',
|
||||
'Motion 1.5 v2' :'guoyww/animatediff-motion-adapter-v1-5-2',
|
||||
# 'Motion SD-XL Beta v1' :'vladmandic/animatediff-sdxl',
|
||||
'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5',
|
||||
'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4',
|
||||
'TemporalDiff': 'vladmandic/temporaldiff',
|
||||
'AnimateFace': 'vladmandic/animateface',
|
||||
# 'LongAnimateDiff 32': 'vladmandic/longanimatediff-32',
|
||||
# 'LongAnimateDiff 64': 'vladmandic/longanimatediff-64',
|
||||
# 'Motion SD-XL Beta v1' :'vladmandic/animatediff-sdxl',
|
||||
}
|
||||
LORAS = {
|
||||
'None': None,
|
||||
@@ -70,6 +74,10 @@ def set_adapter(adapter_name: str = 'None'):
|
||||
# shared.sd_model.image_encoder = None
|
||||
shared.sd_model.unet.set_default_attn_processor()
|
||||
shared.sd_model.unet.config.encoder_hid_dim_type = None
|
||||
if adapter_name.endswith('.ckpt') or adapter_name.endswith('.safetensors'):
|
||||
import huggingface_hub as hf
|
||||
folder, filename = os.path.split(adapter_name)
|
||||
adapter_name = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir)
|
||||
try:
|
||||
shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"')
|
||||
motion_adapter = None
|
||||
@@ -77,13 +85,14 @@ def set_adapter(adapter_name: str = 'None'):
|
||||
motion_adapter.to(shared.device)
|
||||
sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter')
|
||||
loaded_adapter = adapter_name
|
||||
|
||||
new_pipe = diffusers.AnimateDiffPipeline(
|
||||
vae=shared.sd_model.vae,
|
||||
text_encoder=shared.sd_model.text_encoder,
|
||||
tokenizer=shared.sd_model.tokenizer,
|
||||
unet=shared.sd_model.unet,
|
||||
scheduler=shared.sd_model.scheduler,
|
||||
feature_extractor=getattr(shared.sd_model, 'feature_extractor', None),
|
||||
image_encoder=getattr(shared.sd_model, 'image_encoder', None),
|
||||
motion_adapter=motion_adapter,
|
||||
)
|
||||
orig_pipe = shared.sd_model
|
||||
@@ -119,7 +128,9 @@ class Script(scripts.Script):
|
||||
with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'):
|
||||
with gr.Row():
|
||||
adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None')
|
||||
frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16)
|
||||
frames = gr.Slider(label='Frames', minimum=1, maximum=64, step=1, value=16)
|
||||
with gr.Row():
|
||||
override_scheduler = gr.Checkbox(label='Override sampler', value=True)
|
||||
with gr.Row():
|
||||
lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None')
|
||||
strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0)
|
||||
@@ -133,15 +144,29 @@ class Script(scripts.Script):
|
||||
mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False)
|
||||
mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
|
||||
video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate])
|
||||
return [adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
return [adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, override_scheduler]
|
||||
|
||||
def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, override_scheduler): # pylint: disable=arguments-differ, unused-argument
|
||||
adapter = ADAPTERS[adapter_index]
|
||||
lora = LORAS[lora_index]
|
||||
set_adapter(adapter)
|
||||
if motion_adapter is None:
|
||||
return
|
||||
shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} video={video_type}')
|
||||
if override_scheduler:
|
||||
p.sampler_name = 'Default'
|
||||
shared.sd_model.scheduler = diffusers.DDIMScheduler(
|
||||
beta_start=0.00085,
|
||||
beta_end=0.012,
|
||||
beta_schedule="linear",
|
||||
clip_sample=False,
|
||||
num_train_timesteps=1000,
|
||||
rescale_betas_zero_snr=False,
|
||||
set_alpha_to_one=True,
|
||||
steps_offset=0,
|
||||
timestep_spacing="linspace",
|
||||
trained_betas=None,
|
||||
)
|
||||
shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} video={video_type} scheduler={shared.sd_model.scheduler.__class__.__name__ if override_scheduler else p.sampler_name}')
|
||||
if lora is not None and lora != 'None':
|
||||
shared.sd_model.load_lora_weights(lora, adapter_name=lora)
|
||||
shared.sd_model.set_adapters([lora], adapter_weights=[strength])
|
||||
@@ -155,7 +180,7 @@ class Script(scripts.Script):
|
||||
if not latent_mode:
|
||||
p.task_args['output_type'] = 'np'
|
||||
|
||||
def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, override_scheduler): # pylint: disable=arguments-differ, unused-argument
|
||||
from modules.images import save_video
|
||||
if video_type != 'None':
|
||||
save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
|
||||
Reference in New Issue
Block a user