diff --git a/CHANGELOG.md b/CHANGELOG.md index 54b002231..bd176b3ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,14 +30,15 @@ - *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions in the future, expect more optimizations, especially related to offloading/slicing/tiling, but at the moment this is pretty much experimental-only - - **AnimateDiff** + - [AnimateDiff](https://github.com/guoyww/animatediff/) + - overall improved quality - can now be used with *second pass* - enhance, upscale and hires your videos! - - **IP Adapter** - - add support for `ip-adapter-plus_sd15`, `ip-adapter-plus-face_sd15` and `ip-adapter-full-face_sd15` + - [IP Adapter](https://github.com/tencent-ailab/IP-Adapter) + - add support for **ip-adapter-plus_sd15, ip-adapter-plus-face_sd15 and ip-adapter-full-face_sd15** - can now be used in *xyz-grid* - **Text-to-Video** - in text tab, select `text-to-video` script - - supported models: ModelScope v1.7b, ZeroScope v1, ZeroScope v1.1, ZeroScope v2, ZeroScope v2 Dark, Potat v1 + - supported models: **ModelScope v1.7b, ZeroScope v1, ZeroScope v1.1, ZeroScope v2, ZeroScope v2 Dark, Potat v1** *if you know of any other t2v models you'd like to see supported, let me know!* - models are auto-downloaded on first use - *note*: current base model will be unloaded to free up resources diff --git a/modules/face_restoration.py b/modules/face_restoration.py index 4ae53d21b..55e1033c6 100644 --- a/modules/face_restoration.py +++ b/modules/face_restoration.py @@ -13,7 +13,5 @@ def restore_faces(np_image): face_restorers = [x for x in shared.face_restorers if x.name() == shared.opts.face_restoration_model or shared.opts.face_restoration_model is None] if len(face_restorers) == 0: return np_image - face_restorer = face_restorers[0] - return face_restorer.restore(np_image) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index da616b24a..66877a83a 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -453,7 +453,10 @@ def process_diffusers(p: StableDiffusionProcessing): t1 = time.time() shared.log.debug(f'Profile: pipeline call: {t1-t0:.2f}') if not hasattr(output, 'images') and hasattr(output, 'frames'): - shared.log.debug(f'Generated: frames={len(output.frames[0])}') + if hasattr(output.frames[0], 'shape'): + shared.log.debug(f'Generated: frames={output.frames[0].shape[1]}') + else: + shared.log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] except AssertionError as e: shared.log.info(e) diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 067e1b82d..074d31d6b 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -10,6 +10,7 @@ TODO: - AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main """ +import os import gradio as gr import diffusers from modules import scripts, processing, shared, devices, sd_models @@ -18,12 +19,15 @@ from modules import scripts, processing, shared, devices, sd_models # config ADAPTERS = { 'None': None, - 'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4', - 'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5', + 'Motion 1.5 v3' :'vladmandic/animatediff-v3', 'Motion 1.5 v2' :'guoyww/animatediff-motion-adapter-v1-5-2', - # 'Motion SD-XL Beta v1' :'vladmandic/animatediff-sdxl', + 'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5', + 'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4', 'TemporalDiff': 'vladmandic/temporaldiff', 'AnimateFace': 'vladmandic/animateface', + # 'LongAnimateDiff 32': 'vladmandic/longanimatediff-32', + # 'LongAnimateDiff 64': 'vladmandic/longanimatediff-64', + # 'Motion SD-XL Beta v1' :'vladmandic/animatediff-sdxl', } LORAS = { 'None': None, @@ -70,6 +74,10 @@ def set_adapter(adapter_name: str = 'None'): # shared.sd_model.image_encoder = None shared.sd_model.unet.set_default_attn_processor() shared.sd_model.unet.config.encoder_hid_dim_type = None + if adapter_name.endswith('.ckpt') or adapter_name.endswith('.safetensors'): + import huggingface_hub as hf + folder, filename = os.path.split(adapter_name) + adapter_name = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir) try: shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"') motion_adapter = None @@ -77,13 +85,14 @@ def set_adapter(adapter_name: str = 'None'): motion_adapter.to(shared.device) sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter') loaded_adapter = adapter_name - new_pipe = diffusers.AnimateDiffPipeline( vae=shared.sd_model.vae, text_encoder=shared.sd_model.text_encoder, tokenizer=shared.sd_model.tokenizer, unet=shared.sd_model.unet, scheduler=shared.sd_model.scheduler, + feature_extractor=getattr(shared.sd_model, 'feature_extractor', None), + image_encoder=getattr(shared.sd_model, 'image_encoder', None), motion_adapter=motion_adapter, ) orig_pipe = shared.sd_model @@ -119,7 +128,9 @@ class Script(scripts.Script): with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'): with gr.Row(): adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None') - frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16) + frames = gr.Slider(label='Frames', minimum=1, maximum=64, step=1, value=16) + with gr.Row(): + override_scheduler = gr.Checkbox(label='Override sampler', value=True) with gr.Row(): lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None') strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0) @@ -133,15 +144,29 @@ class Script(scripts.Script): mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) - return [adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + return [adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, override_scheduler] - def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, override_scheduler): # pylint: disable=arguments-differ, unused-argument adapter = ADAPTERS[adapter_index] lora = LORAS[lora_index] set_adapter(adapter) if motion_adapter is None: return - shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} video={video_type}') + if override_scheduler: + p.sampler_name = 'Default' + shared.sd_model.scheduler = diffusers.DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="linear", + clip_sample=False, + num_train_timesteps=1000, + rescale_betas_zero_snr=False, + set_alpha_to_one=True, + steps_offset=0, + timestep_spacing="linspace", + trained_betas=None, + ) + shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} video={video_type} scheduler={shared.sd_model.scheduler.__class__.__name__ if override_scheduler else p.sampler_name}') if lora is not None and lora != 'None': shared.sd_model.load_lora_weights(lora, adapter_name=lora) shared.sd_model.set_adapters([lora], adapter_weights=[strength]) @@ -155,7 +180,7 @@ class Script(scripts.Script): if not latent_mode: p.task_args['output_type'] = 'np' - def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, latent_mode, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, override_scheduler): # pylint: disable=arguments-differ, unused-argument from modules.images import save_video if video_type != 'None': save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)