mirror of
https://github.com/vladmandic/automatic
synced 2026-09-10 06:48:43 +02:00
Vladimir Mandic
167 lines
7.7 KiB
Python
167 lines
7.7 KiB
Python
"""
|
|
Lightweight AnimateDiff implementation in Diffusers
|
|
Docs: <https://huggingface.co/docs/diffusers/api/pipelines/animatediff>
|
|
TODO:
|
|
- Use latents and update VAE decode
|
|
- SDXL
|
|
- MP4 with RIFE
|
|
- IP Adapter
|
|
- Custom models
|
|
- Custom LORAs
|
|
- Enable second pass
|
|
- TemporalDiff: https://huggingface.co/CiaraRowles/TemporalDiff/tree/main
|
|
- AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main
|
|
"""
|
|
|
|
import os
|
|
import gradio as gr
|
|
import diffusers
|
|
from modules import scripts, processing, shared, devices, sd_models
|
|
|
|
|
|
# config
|
|
ADAPTERS = {
|
|
'None': None,
|
|
'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4',
|
|
'Motion 1.5': 'guoyww/animatediff-motion-adapter-v1-5',
|
|
'Motion 1.5.2' :'guoyww/animatediff-motion-adapter-v1-5-2',
|
|
'TemporalDiff': 'vladmandic/temporaldiff',
|
|
'AnimateFace': 'vladmandic/animateface',
|
|
}
|
|
LORAS = {
|
|
'None': None,
|
|
'Zoom-in': 'guoyww/animatediff-motion-lora-zoom-in',
|
|
'Zoom-out': 'guoyww/animatediff-motion-lora-zoom-out',
|
|
'Pan-left': 'guoyww/animatediff-motion-lora-pan-left',
|
|
'Pan-right': 'guoyww/animatediff-motion-lora-pan-right',
|
|
'Tilt-up': 'guoyww/animatediff-motion-lora-tilt-up',
|
|
'Tilt-down': 'guoyww/animatediff-motion-lora-tilt-down',
|
|
'Roll-left': 'guoyww/animatediff-motion-lora-rolling-anticlockwise',
|
|
'Roll-right': 'guoyww/animatediff-motion-lora-rolling-clockwise',
|
|
}
|
|
|
|
# state
|
|
motion_adapter = None
|
|
loaded_adapter = None
|
|
orig_pipe = None
|
|
|
|
|
|
def set_adapter(name: str = None):
|
|
if shared.sd_model is None:
|
|
return
|
|
if shared.backend != shared.Backend.DIFFUSERS:
|
|
shared.log.warning('AnimateDiff: not in diffusers mode')
|
|
return
|
|
global motion_adapter, loaded_adapter, orig_pipe # pylint: disable=global-statement
|
|
adapter_name = name if name is not None and isinstance(name, str) else loaded_adapter
|
|
if adapter_name is None or adapter_name == 'None' or shared.sd_model is None:
|
|
motion_adapter = None
|
|
loaded_adapter = None
|
|
if orig_pipe is not None:
|
|
shared.log.debug(f'AnimateDiff restore pipeline: adapter="{loaded_adapter}"')
|
|
shared.sd_model = orig_pipe
|
|
orig_pipe = None
|
|
return
|
|
if shared.sd_model_type != 'sd':
|
|
shared.log.warning(f'AnimateDiff: unsupported model type: {shared.sd_model.__class__.__name__}')
|
|
return
|
|
if motion_adapter is not None and loaded_adapter == adapter_name:
|
|
shared.log.info(f'AnimateDiff cache: adapter="{adapter_name}"')
|
|
return
|
|
try:
|
|
shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"')
|
|
motion_adapter = diffusers.MotionAdapter.from_pretrained(adapter_name, cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, low_cpu_mem_usage=False, device_map=None)
|
|
motion_adapter.to(shared.device)
|
|
sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter')
|
|
loaded_adapter = adapter_name
|
|
|
|
new_pipe = diffusers.AnimateDiffPipeline(
|
|
vae=shared.sd_model.vae,
|
|
text_encoder=shared.sd_model.text_encoder,
|
|
tokenizer=shared.sd_model.tokenizer,
|
|
unet=shared.sd_model.unet,
|
|
scheduler=shared.sd_model.scheduler,
|
|
motion_adapter=motion_adapter,
|
|
)
|
|
orig_pipe = shared.sd_model
|
|
new_pipe.sd_checkpoint_info = shared.sd_model.sd_checkpoint_info
|
|
new_pipe.sd_model_hash = shared.sd_model.sd_model_hash
|
|
new_pipe.sd_model_checkpoint = shared.sd_model.sd_checkpoint_info.filename
|
|
new_pipe.is_sdxl = False
|
|
new_pipe.is_sd2 = False
|
|
new_pipe.is_sd1 = True
|
|
shared.sd_model = new_pipe
|
|
shared.sd_model.to(shared.device)
|
|
sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model')
|
|
shared.log.debug(f'AnimateDiff create pipeline: adapter="{loaded_adapter}"')
|
|
except Exception as e:
|
|
motion_adapter = None
|
|
loaded_adapter = None
|
|
shared.log.error(f'AnimateDiff load error: adapter="{adapter_name}" {e}')
|
|
|
|
|
|
class Script(scripts.Script):
|
|
def title(self):
|
|
return 'AnimateDiff'
|
|
|
|
def show(self, _is_img2img):
|
|
return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False
|
|
|
|
# return signature is array of gradio components
|
|
def ui(self, _is_img2img):
|
|
with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'):
|
|
with gr.Row():
|
|
adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None')
|
|
frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16)
|
|
with gr.Row():
|
|
lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None')
|
|
strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0)
|
|
with gr.Row():
|
|
override = gr.Checkbox(label='Override sampler', value=False)
|
|
with gr.Row():
|
|
create_gif = gr.Checkbox(label='Create GIF', value=False)
|
|
loop = gr.Checkbox(label='Loop', value=True)
|
|
duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2)
|
|
return [adapter_index, frames, lora_index, strength, override, create_gif, duration, loop]
|
|
|
|
def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, override, create_gif, duration, loop): # pylint: disable=arguments-differ, unused-argument
|
|
adapter = ADAPTERS[adapter_index]
|
|
lora = LORAS[lora_index]
|
|
set_adapter(adapter)
|
|
if motion_adapter is None:
|
|
return
|
|
shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} sampler={override} gif={create_gif}')
|
|
p.extra_generation_params['AnimateDiff'] = loaded_adapter
|
|
if override:
|
|
shared.sd_model.scheduler = diffusers.DDIMScheduler.from_pretrained('SG161222/Realistic_Vision_V5.1_noVAE', subfolder="scheduler", clip_sample=False, timestep_spacing="linspace", steps_offset=1)
|
|
if lora is not None and lora != 'None':
|
|
shared.sd_model.load_lora_weights(lora, adapter_name=lora)
|
|
shared.sd_model.set_adapters([lora], adapter_weights=[strength])
|
|
p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}'
|
|
p.do_not_save_grid = True
|
|
p.task_args['num_frames'] = frames
|
|
p.task_args['output_type'] = 'np' # TODO: AnimateDiff use latents and update vae_decode
|
|
p.task_args['num_inference_steps'] = p.steps
|
|
|
|
def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, override, create_gif, duration, loop): # pylint: disable=arguments-differ, unused-argument
|
|
if not create_gif or len(processed.images) < 2:
|
|
return
|
|
from modules.images import FilenameGenerator
|
|
image = processed.images[0]
|
|
namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image)
|
|
fn = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]")
|
|
fn = namegen.sanitize(os.path.join(shared.opts.outdir_save, fn))
|
|
fn = namegen.sequence(fn, shared.opts.outdir_save, '')
|
|
images = processed.images[1:]
|
|
if loop:
|
|
images += processed.images[::-1]
|
|
image.save(
|
|
f'{fn}.gif',
|
|
save_all = True,
|
|
append_images = images,
|
|
optimize = False,
|
|
duration = 1000.0 * duration / frames,
|
|
loop = 0 if loop else 1,
|
|
)
|
|
shared.log.info(f'AnimateDiff saved: file="{fn}" frames={len(images) + 1} duration={duration} loop={loop}')
|