diff --git a/CHANGELOG.md b/CHANGELOG.md index 113923d59..bf46c9b52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,6 @@ - items that require `diffusers==0.27.0.dev`: - EDM samplers for Playground 2.5 - Stable Cascade -- move models to script: - - StabilityAI SVD - - StabilityAI SVD XT 1.0 - - StabilityAI SVD XT 1.1 - fix reference models: - Warp Wuerstchen: pipeline does not have all components - Kandinsky 2.1: pipeline does not have all components @@ -44,10 +40,17 @@ - upscale can now run 0.1-8.0 scale and will also run if enabled at 1.0 to allow for upscalers that simply improve image quality - update ui section to reflect changes - *note*: behavior using backend:original is unchanged for backwards compatibilty -- **Image2Video** - - new module for creating videos from images - - simply enable from *img2img -> scripts -> image2video* - - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) +- **Video** + - **Image2Video** + - new module for creating videos from images + - simply enable from *img2img -> scripts -> image2video* + - model is auto-downloaded on first use + - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) + - **Stable Video Diffusion** + - updated with *SVD 1.0, SVD XT 1.0 and SVD XT 1.1* + - models are auto-downloaded on first use + - simply enable from *img2img -> scripts -> stable video diffusion* + - for svd 1.0, use frames=~14, for xt models use frames=~25 - **Composable LoRA**, thanks @AI-Casanova - control lora strength for each step for example: `` means strength=0.1 for step at 0% and intepolate towards strength=0.9 for step at 100% diff --git a/html/reference.json b/html/reference.json index ba53a7b5a..eca499bc8 100644 --- a/html/reference.json +++ b/html/reference.json @@ -92,28 +92,6 @@ "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0" }, - "StabilityAI SVD": { - "path": "stabilityai/stable-video-diffusion-img2vid", - "preview": "stabilityai--stable-video-diffusion-img2vid.jpg", - "desc": "(SVD) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 14 frames at resolution 576x1024 given a context frame of the same size. We also finetune the widely used f8-decoder for temporal consistency.", - "variant": "fp16", - "extras": "width: 1024, height: 576, sampler: Default, steps: 20" - }, - "StabilityAI SVD XT 1.0": { - "path": "stabilityai/stable-video-diffusion-img2vid-xt", - "preview": "stabilityai--stable-video-diffusion-img2vid-xt.jpg", - "desc": "(SVD) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 25 frames at resolution 576x1024 given a context frame of the same size, finetuned from SVD Image-to-Video [14 frames]. We also finetune the widely used f8-decoder for temporal consistency.", - "variant": "fp16", - "extras": "width: 1024, height: 576, sampler: Default, steps: 20" - }, - "StabilityAI SVD XT 1.1": { - "path": "stabilityai/stable-video-diffusion-img2vid-xt-1-1", - "preview": "stabilityai--stable-video-diffusion-img2vid-xt.jpg", - "desc": "(SVD 1.1) Image-to-Video is a latent diffusion model trained to generate short video clips from an image conditioning. This model was trained to generate 25 frames at resolution 1024x576 given a context frame of the same size, finetuned from SVD Image-to-Video [25 frames].", - "variant": "fp16", - "extras": "width: 1024, height: 576, sampler: Default, steps: 20" - }, - "Segmind Vega": { "path": "huggingface/segmind/Segmind-Vega", "preview": "segmind--Segmind-Vega.jpg", diff --git a/modules/modelloader.py b/modules/modelloader.py index ab5f989c6..d848c2a95 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -287,13 +287,13 @@ def get_reference_opts(name: str): model_opts = v break if not model_opts: - shared.log.error(f'Reference: model="{name}" not found') + # shared.log.error(f'Reference: model="{name}" not found') return {} shared.log.debug(f'Reference: model="{name}" {model_opts.get("extras", None)}') return model_opts -def load_reference(name: str): +def load_reference(name: str, variant: str = None, revision: str = None, mirror: str = None, custom_pipeline: str = None): found = [r for r in diffuser_repos if name == r['name'] or name == r['friendly'] or name == r['path']] if len(found) > 0: # already downloaded model_opts = get_reference_opts(found[0]['name']) @@ -306,10 +306,10 @@ def load_reference(name: str): model_dir = download_diffusers_model( hub_id=name, cache_dir=shared.opts.diffusers_dir, - variant=model_opts.get('variant', None), - revision=model_opts.get('revision', None), - mirror=model_opts.get('mirror', None), - custom_pipeline=model_opts.get('custom_pipeline', None) + variant=variant or model_opts.get('variant', None), + revision=revision or model_opts.get('revision', None), + mirror=mirror or model_opts.get('mirror', None), + custom_pipeline=custom_pipeline or model_opts.get('custom_pipeline', None) ) if model_dir is None: shared.log.error(f'Reference download: model="{name}"') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 747689dca..0c785cbb1 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -416,6 +416,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__ if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1: p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta + output = None try: t0 = time.time() sd_models_compile.check_deepcache(enable=True) @@ -450,7 +451,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0: # register used embeddings p.extra_generation_params['Embeddings'] = ', '.join(shared.sd_model.embedding_db.embeddings_used) - if hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: # replace input with output so it can be used by hires/refine + if hasattr(p, 'task_args') and p.task_args.get('image', None) is not None and output is not None: # replace input with output so it can be used by hires/refine p.task_args['image'] = output.images shared.state.nextjob() @@ -476,7 +477,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): save_intermediate(latents=output.images, suffix="-before-hires") shared.state.job = 'upscale' output.images = resize_hires(p, latents=output.images) - if hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: # replace input with output so it can be used by hires/refine + if hasattr(p, 'task_args') and p.task_args.get('image', None) is not None and output is not None: # replace input with output so it can be used by hires/refine p.task_args['image'] = output.images sd_hijack_hypertile.hypertile_set(p, hr=True) diff --git a/modules/sd_models.py b/modules/sd_models.py index af8f86bf7..381ec37bf 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1136,6 +1136,8 @@ class DiffusersTaskType(Enum): def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: + if pipe.__class__.__name__ == "StableVideoDiffusionPipeline": + return DiffusersTaskType.IMAGE_2_IMAGE if pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline": return DiffusersTaskType.INSTRUCT elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 76ec4e9af..0b7064e5c 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -2,11 +2,18 @@ Additional params for StableVideoDiffusion """ +import os import torch import gradio as gr -from modules import scripts, processing, shared, sd_models, images +from modules import scripts, processing, shared, sd_models, images, modelloader +models = { + "SVD 1.0": "stabilityai/stable-video-diffusion-img2vid", + "SVD XT 1.0": "stabilityai/stable-video-diffusion-img2vid-xt", + "SVD XT 1.1": "stabilityai/stable-video-diffusion-img2vid-xt-1-1", +} + class Script(scripts.Script): def title(self): return 'Stable Video Diffusion' @@ -26,6 +33,8 @@ class Script(scripts.Script): with gr.Row(): gr.HTML('  Stable Video Diffusion
') + with gr.Row(): + model = gr.Dropdown(label='Model', choices=list(models), value=list(models)[0]) with gr.Row(): num_frames = gr.Slider(label='Frames', minimum=1, maximum=50, step=1, value=14) min_guidance_scale = gr.Slider(label='Min guidance', minimum=0.0, maximum=10.0, step=0.1, value=1.0) @@ -44,41 +53,55 @@ class Script(scripts.Script): mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) - return [num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + return [model, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else '' - if c != 'StableVideoDiffusionPipeline' and c != 'TextToVideoSDPipeline': - shared.log.error(f'StableVideo: model selected={c} required=StableVideoDiffusion') + def run(self, p: processing.StableDiffusionProcessing, model, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + image = getattr(p, 'init_images', None) + if image is None or len(image) == 0: + shared.log.error('SVD: no init_images') return None - if hasattr(p, 'init_images') and len(p.init_images) > 0: - if override_resolution: - p.width = 1024 - p.height = 576 - p.task_args['image'] = images.resize_image(resize_mode=2, im=p.init_images[0], width=p.width, height=p.height, upscaler_name=None, output_type='pil') - else: - p.task_args['image'] = p.init_images[0] - p.ops.append('stablevideo') - p.do_not_save_grid = True - if c == 'StableVideoDiffusionPipeline': - p.sampler_name = 'Default' # svd does not support non-default sampler - p.task_args['output_type'] = 'pil' - p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator - p.task_args['width'] = p.width - p.task_args['height'] = p.height - p.task_args['num_frames'] = num_frames - p.task_args['decode_chunk_size'] = decode_chunk_size - p.task_args['motion_bucket_id'] = round(255 * motion_bucket_id) - p.task_args['noise_aug_strength'] = noise_aug_strength - p.task_args['num_inference_steps'] = p.steps - p.task_args['min_guidance_scale'] = min_guidance_scale - p.task_args['max_guidance_scale'] = max_guidance_scale - shared.log.debug(f'StableVideo: args={p.task_args}') - shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - processed = processing.process_images(p) - if video_type != 'None': - images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) - return processed else: - shared.log.error('StableVideo: no init_images') - return None + image = image[0] + + # load/download model on-demand + model_path = models[model] + model_name = os.path.basename(model_path) + has_checkpoint = sd_models.get_closet_checkpoint_match(model_path) + if has_checkpoint is None: + shared.log.error(f'SVD: no checkpoint for {model_name}') + modelloader.load_reference(model_path, variant='fp16') + c = shared.sd_model.__class__.__name__ + model_loaded = shared.sd_model.sd_checkpoint_info.model_name + if model_name != model_loaded or c != 'StableVideoDiffusionPipeline': + shared.opts.sd_model_checkpoint = model_path + sd_models.reload_model_weights() + model_loaded = sd_models.model_data.sd_model.sd_checkpoint_info + + # set params + if override_resolution: + p.width = 1024 + p.height = 576 + image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + p.ops.append('svd') + p.do_not_save_grid = True + p.init_images = [image] + p.sampler_name = 'Default' # svd does not support non-default sampler + p.task_args['output_type'] = 'pil' + p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator + p.task_args['image'] = image + p.task_args['width'] = p.width + p.task_args['height'] = p.height + p.task_args['num_frames'] = num_frames + p.task_args['decode_chunk_size'] = decode_chunk_size + p.task_args['motion_bucket_id'] = round(255 * motion_bucket_id) + p.task_args['noise_aug_strength'] = noise_aug_strength + p.task_args['num_inference_steps'] = p.steps + p.task_args['min_guidance_scale'] = min_guidance_scale + p.task_args['max_guidance_scale'] = max_guidance_scale + shared.log.debug(f'SVD: args={p.task_args}') + + # run processing + processed = processing.process_images(p) + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed