diff --git a/CHANGELOG.md b/CHANGELOG.md index c839cf403..3119b0ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Plus support for CogView-4, new CLiP models, improvements to remote VAE, additio ### Details for 2025-03-22 - **Video tab** + - see [Video Wiki](https://github.com/vladmandic/sdnext/wiki/Video) for details! - new top-level tab, replaces previous *video* script in text/image tabs old scripts are still present, but will be removed in the future - support for all latest models: diff --git a/TODO.md b/TODO.md index 267b1822b..02768136d 100644 --- a/TODO.md +++ b/TODO.md @@ -11,22 +11,21 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Video: API support is TBD - Video: Hunyuan Video I2V: transformers incompatibility - Video: Hunyuan Video I2V: add 16ch vs 33ch processing -- Video: WAN 2.1 14B I2V 480p/720p: broken offload -- Video: CogVideoX 1.5 5B T2V/I2V: requires pipeline update -- Video: LTXVideo 0.9.5 T2V/I2V: broken offload -- Video: LTXVideo 0.9.5 T2V/I2V: requires different params -- Video: LTXVideo 0.9.5 T2V/I2V: add support for conditioned input -- Video: LTXVideo 0.9.1 I2V: generator list mismatch - Video: Latte 1 T2V: dtype mismatch -- Video: Allegro T2V: all-gray output, requires vae-fp32 +- Video: WAN 2.1 14B I2V 480p/720p: broken offload +- Video: CogVideoX 1.5 5B T2V/I2V: all-gray output +- Video: Allegro T2V: all-gray output - Video: FasterCache and PyramidAttentionBroadcast granular config - Video: FasterCache and PyramidAttentionBroadcast for LTX and WAN ## Future Candidates -- Flux NF4 loader: -- IPAdapter negative: -- Control API enhance scripts compatibility +- Flux: NF4 loader: +- IPAdapter: negative guidance: +- Control: API enhance scripts compatibility +- Video: OponSora v2 https://huggingface.co/hpcai-tech/Open-Sora-v2 +- Video: STG: https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#spatiotemporal-skip-guidance +- Video: add generate context menu ## Code TODO diff --git a/modules/processing_args.py b/modules/processing_args.py index db8794214..058c8bdf1 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -21,6 +21,7 @@ disable_pbar = os.environ.get('SD_DISABLE_PBAR', None) is not None def task_specific_kwargs(p, model): task_args = {} is_img2img_model = bool('Zero123' in shared.sd_model.__class__.__name__) + print('HERE', sd_models.get_diffusers_task(model)) if len(getattr(p, 'init_images', [])) > 0: if isinstance(p.init_images[0], str): p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images] @@ -34,6 +35,7 @@ def task_specific_kwargs(p, model): 'height': 8 * math.ceil(p.height / 8), } elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: + print('HERE1', p.denoising_strength) if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) if 'hires' not in p.ops: diff --git a/modules/sd_models.py b/modules/sd_models.py index 8c4356793..9f66d9814 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -631,9 +631,12 @@ class DiffusersTaskType(Enum): def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType: - if pipe.__class__.__name__ in ["StableVideoDiffusionPipeline", "LEditsPPPipelineStableDiffusion", "LEditsPPPipelineStableDiffusionXL", "OmniGenPipeline"]: + cls = pipe.__class__.__name__ + if cls in ["LEditsPPPipelineStableDiffusion", "LEditsPPPipelineStableDiffusionXL", "OmniGenPipeline"]: # special case return DiffusersTaskType.IMAGE_2_IMAGE - elif pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline": + elif 'ImageToVideo' in cls or cls in ['LTXConditionPipeline', 'StableVideoDiffusionPipeline']: # i2v pipelines + return DiffusersTaskType.IMAGE_2_IMAGE + elif 'Instruct' in cls: return DiffusersTaskType.INSTRUCT elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values(): return DiffusersTaskType.IMAGE_2_IMAGE diff --git a/modules/ui_sections.py b/modules/ui_sections.py index d2e73bdcc..8d038ab13 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -219,7 +219,7 @@ def create_sampler_and_steps_selection(choices, tabname): sd_samplers.set_samplers() choices = [x for x in sd_samplers.samplers if not x.name == 'Same as primary'] with gr.Row(elem_classes=['flex-break']): - steps = gr.Slider(minimum=1, maximum=99, step=1, label="Steps", elem_id=f"{tabname}_steps", value=20) + steps = gr.Slider(minimum=1, maximum=100, step=1, label="Steps", elem_id=f"{tabname}_steps", value=20) sampler_index = gr.Dropdown(label='Sampling method', elem_id=f"{tabname}_sampling", choices=[x.name for x in choices], value='Default', type="index") return steps, sampler_index diff --git a/modules/ui_video.py b/modules/ui_video.py index cff8a54b0..c0495defe 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -110,6 +110,7 @@ def create_ui(): with gr.Accordion(open=False, label="Init image", elem_id='video_init_accordion'): gr.HTML("
  Init image") init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", height=512) + init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id="video_denoising_strength") with gr.Accordion(open=False, label="Accelerate", elem_id='video_accelerate_accordion'): faster_cache = gr.Checkbox(label='FasterCache', value=False, elem_id="video_faster_cache") pyramid_attention = gr.Checkbox(label='PyramidAttention', value=False, elem_id="video_pyramid_attention") @@ -163,7 +164,7 @@ def create_ui(): sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, - init_image, + init_image, init_strength, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index f5a7804d1..f7181b150 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -3,38 +3,6 @@ import diffusers import transformers -""" -# Model tests: download/load/generate - -- Hunyuan Video T2V: pass/pass/pass -- Hunyuan Video I2V: pass/pass/fail, transformers incompatibility -- SkyReels Hunyuan T2V: pass/pass/pass -- SkyReels Hunyuan I2V: pass/pass/pass -- Fast Hunyuan T2V: pass/pass/pass - -- LTXVideo 0.9.5 T2V: pass/pass/fail, completely broken offload, new pipeline -- LTXVideo 0.9.5 I2V: pass/pass/fail, completely broken offload, new pipeline -- LTXVideo 0.9.1 T2V: pass/pass/pass -- LTXVideo 0.9.1 I2V: pass/pass/fail, generator list mismatch -- LTXVideo 0.9.0 T2V: pass/pass/pass -- LTXVideo 0.9.0 I2V: pass/pass/pass - -- WAN 2.1 1.3B T2V: pass/pass/pass -- WAN 2.1 14B T2V: pass/pass/pass -- WAN 2.1 14B I2V 480p: pass/pass/fail, offloading cpu vs cuda -- WAN 2.1 14B I2V 720p: pass/pass/fail, offloading cpu vs cuda - -- CogVideoX 1.0 2B T2V: pass/pass/pass -- CogVideoX 1.0 5B T2V: pass/pass/pass -- CogVideoX 1.0 5B I2V: pass/pass/pass -- CogVideoX 1.5 5B T2V: download/load/fail, pipeline is tbd -- CogVideoX 1.5 5B I2V: download/load/fail, pipeline is tbd - -- Mochi 1 T2V: pass/pass/pass -- Latte 1 T2V: pass/pass/fail, float vs bfloat during generate -- Allegro T2V: pass/pass/fail, output is pure gray -""" - @dataclass class Model(): name: str @@ -104,13 +72,13 @@ models = { Model(name='None'), Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968 url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', - repo='YiYiXu/ltx-95', + repo='Lightricks/LTX-Video-0.9.5', repo_cls=diffusers.LTXPipeline, te_cls=transformers.T5EncoderModel, dit_cls=diffusers.LTXVideoTransformer3DModel), Model(name='LTXVideo 0.9.5 I2V', url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', - repo='YiYiXu/ltx-95', + repo='Lightricks/LTX-Video-0.9.5', repo_cls=diffusers.LTXConditionPipeline, te_cls=transformers.T5EncoderModel, dit_cls=diffusers.LTXVideoTransformer3DModel), @@ -214,7 +182,7 @@ models = { te_cls=transformers.T5EncoderModel, dit_cls=diffusers.CogVideoXTransformer3DModel), Model(name='CogVideoX 1.5 5B T2V', - url='https://huggingface.co/THUDM/THUDM/CogVideoX1.5-5B', + url='https://huggingface.co/THUDM/CogVideoX1.5-5B', repo='THUDM/CogVideoX1.5-5B', repo_cls=diffusers.CogVideoXPipeline, te_cls=transformers.T5EncoderModel, diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 2dbb2409e..e771f5254 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -16,10 +16,29 @@ def load_override(selected: Model): torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}') + if selected.name == 'LTXVideo 0.9.5 I2V': + kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, + subfolder="vae", + torch_dtype=torch.float32, + cache_dir=shared.opts.hfcache_dir) return kwargs def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model): + cls = shared.sd_model.__class__.__name__ + # Allegro + if selected.name == 'Allegro T2V': + shared.sd_model.vae.enable_tiling() + # Latte if selected.name == 'Latte 1 T2V': p.task_args['enable_temporal_attentions'] = False - debug(f'Video overrides: model="{selected.name}" args={p.task_args}') + p.task_args['video_length'] = p.frames + # LTX + if cls == 'LTXImageToVideoPipeline' or cls == 'LTXConditionPipeline': + p.task_args['generator'] = None + if cls == 'LTXConditionPipeline': + print('HERE2', p.denoising_strength) + p.task_args['strength'] = p.denoising_strength + if 'LTX' in shared.sd_model.__class__.__name__: + p.task_args['width'] = 32 * (p.width // 32) + p.task_args['height'] = 32 * (p.height // 32) diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index 57c2bee53..95d75e149 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -8,7 +8,7 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, faster_cache, pyramid_attention, override_settings = args + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, faster_cache, pyramid_attention, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': return video_utils.queue_err('model not selected') found = [model.name for model in models_def.models.get(engine, [])] @@ -36,6 +36,7 @@ def generate(*args, **kwargs): width=16 * int(width // 16), height=16 * int(height // 16), frames=int(frames), + denoising_strength=float(init_strength), init_image=init_image, cfg_scale=float(guidance_scale), diffusers_guidance_rescale=float(guidance_true), diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 8a69e19ec..bb971d9d5 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -1,6 +1,6 @@ import os import time -from modules import shared, sd_models, timer +from modules import shared, sd_models, timer, errors debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -30,7 +30,12 @@ def set_prompt(p): def hijack_encode_prompt(*args, **kwargs): t0 = time.time() - res = shared.sd_model.orig_encode_prompt(*args, **kwargs) + try: + res = shared.sd_model.orig_encode_prompt(*args, **kwargs) + except Exception as e: + shared.log.error(f'Video encode: {e}') + errors.display(e, 'Video encode') + res = None t1 = time.time() timer.process.add('te', t1-t0) debug(f'Video encode: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') diff --git a/modules/video_models/video_vae.py b/modules/video_models/video_vae.py index b609265cc..8adbc939e 100644 --- a/modules/video_models/video_vae.py +++ b/modules/video_models/video_vae.py @@ -1,6 +1,7 @@ import os import time -from modules import shared, sd_models, devices, timer +import torch +from modules import shared, sd_models, devices, timer, errors debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -55,8 +56,17 @@ def hijack_vae_decode(*args, **kwargs): pass if res is None: shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) - res = shared.sd_model.vae.orig_decode(*args, **kwargs) - print('HERE', shared.sd_model.vae.dtype) + try: + if torch.is_tensor(args[0]): + latent = args[0] + latent = latent.to(device=devices.device, dtype=shared.sd_model.vae.dtype) # upcast to vae dtype + res = shared.sd_model.vae.orig_decode(latent, *args[1:], **kwargs) + else: + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + except Exception as e: + shared.log.error(f'Video VAE: type={vae_type} {e}') + errors.display(e, 'Video VAE') + res = None t1 = time.time() timer.process.add('vae', t1-t0) debug(f'Video decode: type={vae_type} vae={shared.sd_model.vae.__class__.__name__} latents={args[0].shape} time={t1-t0:.2f}') diff --git a/wiki b/wiki index f58dcaaf6..ef3c65cb0 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f58dcaaf6a98aca2a6adf0a509ebb70a9bd3516a +Subproject commit ef3c65cb0eb3d023daae68d0817c1828cf16ea39