Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

update video

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-03-23 13:19:12 -04:00
parent 2c77a16195
commit d7bab01df0
12 changed files with 66 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -12,6 +12,7 @@ Plus support for CogView-4, new CLiP models, improvements to remote VAE, additio
### Details for 2025-03-22
- **Video tab**
- see [Video Wiki](https://github.com/vladmandic/sdnext/wiki/Video) for details!
- new top-level tab, replaces previous *video* script in text/image tabs
old scripts are still present, but will be removed in the future
- support for all latest models:
TODO.md
+9 -10
View File
@@ -11,22 +11,21 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
- Video: API support is TBD
- Video: Hunyuan Video I2V: transformers incompatibility <https://github.com/huggingface/diffusers/issues/11118>
- Video: Hunyuan Video I2V: add 16ch vs 33ch processing <https://github.com/huggingface/diffusers/pull/11066>
- Video: WAN 2.1 14B I2V 480p/720p: broken offload
- Video: CogVideoX 1.5 5B T2V/I2V: requires pipeline update
- Video: LTXVideo 0.9.5 T2V/I2V: broken offload
- Video: LTXVideo 0.9.5 T2V/I2V: requires different params
- Video: LTXVideo 0.9.5 T2V/I2V: add support for conditioned input
- Video: LTXVideo 0.9.1 I2V: generator list mismatch
- Video: Latte 1 T2V: dtype mismatch <https://github.com/huggingface/diffusers/issues/11137>
- Video: Allegro T2V: all-gray output, requires vae-fp32
- Video: WAN 2.1 14B I2V 480p/720p: broken offload
- Video: CogVideoX 1.5 5B T2V/I2V: all-gray output
- Video: Allegro T2V: all-gray output
- Video: FasterCache and PyramidAttentionBroadcast granular config
- Video: FasterCache and PyramidAttentionBroadcast for LTX and WAN <https://github.com/huggingface/diffusers/issues/11134>
## Future Candidates
- Flux NF4 loader: <https://github.com/huggingface/diffusers/issues/9996>
- IPAdapter negative: <https://github.com/huggingface/diffusers/discussions/7167>
- Control API enhance scripts compatibility
- Flux: NF4 loader: <https://github.com/huggingface/diffusers/issues/9996>
- IPAdapter: negative guidance: <https://github.com/huggingface/diffusers/discussions/7167>
- Control: API enhance scripts compatibility
- Video: OponSora v2 https://huggingface.co/hpcai-tech/Open-Sora-v2
- Video: STG: https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#spatiotemporal-skip-guidance
- Video: add generate context menu
## Code TODO
modules/processing_args.py
+2
View File
@@ -21,6 +21,7 @@ disable_pbar = os.environ.get('SD_DISABLE_PBAR', None) is not None
def task_specific_kwargs(p, model):
task_args = {}
is_img2img_model = bool('Zero123' in shared.sd_model.__class__.__name__)
print('HERE', sd_models.get_diffusers_task(model))
if len(getattr(p, 'init_images', [])) > 0:
if isinstance(p.init_images[0], str):
p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images]
@@ -34,6 +35,7 @@ def task_specific_kwargs(p, model):
'height': 8 * math.ceil(p.height / 8),
}
elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0:
print('HERE1', p.denoising_strength)
if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'):
model.register_to_config(requires_aesthetics_score = False)
if 'hires' not in p.ops:
modules/sd_models.py
+5 -2
View File
@@ -631,9 +631,12 @@ class DiffusersTaskType(Enum):
def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType:
if pipe.__class__.__name__ in ["StableVideoDiffusionPipeline", "LEditsPPPipelineStableDiffusion", "LEditsPPPipelineStableDiffusionXL", "OmniGenPipeline"]:
cls = pipe.__class__.__name__
if cls in ["LEditsPPPipelineStableDiffusion", "LEditsPPPipelineStableDiffusionXL", "OmniGenPipeline"]: # special case
return DiffusersTaskType.IMAGE_2_IMAGE
elif pipe.__class__.__name__ == "StableDiffusionXLInstructPix2PixPipeline":
elif 'ImageToVideo' in cls or cls in ['LTXConditionPipeline', 'StableVideoDiffusionPipeline']: # i2v pipelines
return DiffusersTaskType.IMAGE_2_IMAGE
elif 'Instruct' in cls:
return DiffusersTaskType.INSTRUCT
elif pipe.__class__ in diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.values():
return DiffusersTaskType.IMAGE_2_IMAGE
modules/ui_sections.py
+1 -1
View File
@@ -219,7 +219,7 @@ def create_sampler_and_steps_selection(choices, tabname):
sd_samplers.set_samplers()
choices = [x for x in sd_samplers.samplers if not x.name == 'Same as primary']
with gr.Row(elem_classes=['flex-break']):
steps = gr.Slider(minimum=1, maximum=99, step=1, label="Steps", elem_id=f"{tabname}_steps", value=20)
steps = gr.Slider(minimum=1, maximum=100, step=1, label="Steps", elem_id=f"{tabname}_steps", value=20)
sampler_index = gr.Dropdown(label='Sampling method', elem_id=f"{tabname}_sampling", choices=[x.name for x in choices], value='Default', type="index")
return steps, sampler_index
modules/ui_video.py
+2 -1
View File
@@ -110,6 +110,7 @@ def create_ui():
with gr.Accordion(open=False, label="Init image", elem_id='video_init_accordion'):
gr.HTML("<br>&nbsp Init image")
init_image = gr.Image(elem_id="video_image", show_label=False, type="pil", image_mode="RGB", height=512)
init_strength = gr.Slider(label='Init strength', minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id="video_denoising_strength")
with gr.Accordion(open=False, label="Accelerate", elem_id='video_accelerate_accordion'):
faster_cache = gr.Checkbox(label='FasterCache', value=False, elem_id="video_faster_cache")
pyramid_attention = gr.Checkbox(label='PyramidAttention', value=False, elem_id="video_pyramid_attention")
@@ -163,7 +164,7 @@ def create_ui():
sampler_shift, dynamic_shift,
seed,
guidance_scale, guidance_true,
init_image,
init_image, init_strength,
vae_type, vae_tile_frames,
save_frames,
video_type, video_duration, video_loop, video_pad, video_interpolate,
modules/video_models/models_def.py
+3 -35
View File
@@ -3,38 +3,6 @@ import diffusers
import transformers
"""
# Model tests: download/load/generate
- Hunyuan Video T2V: pass/pass/pass
- Hunyuan Video I2V: pass/pass/fail, transformers incompatibility
- SkyReels Hunyuan T2V: pass/pass/pass
- SkyReels Hunyuan I2V: pass/pass/pass
- Fast Hunyuan T2V: pass/pass/pass
- LTXVideo 0.9.5 T2V: pass/pass/fail, completely broken offload, new pipeline
- LTXVideo 0.9.5 I2V: pass/pass/fail, completely broken offload, new pipeline
- LTXVideo 0.9.1 T2V: pass/pass/pass
- LTXVideo 0.9.1 I2V: pass/pass/fail, generator list mismatch
- LTXVideo 0.9.0 T2V: pass/pass/pass
- LTXVideo 0.9.0 I2V: pass/pass/pass
- WAN 2.1 1.3B T2V: pass/pass/pass
- WAN 2.1 14B T2V: pass/pass/pass
- WAN 2.1 14B I2V 480p: pass/pass/fail, offloading cpu vs cuda
- WAN 2.1 14B I2V 720p: pass/pass/fail, offloading cpu vs cuda
- CogVideoX 1.0 2B T2V: pass/pass/pass
- CogVideoX 1.0 5B T2V: pass/pass/pass
- CogVideoX 1.0 5B I2V: pass/pass/pass
- CogVideoX 1.5 5B T2V: download/load/fail, pipeline is tbd
- CogVideoX 1.5 5B I2V: download/load/fail, pipeline is tbd
- Mochi 1 T2V: pass/pass/pass
- Latte 1 T2V: pass/pass/fail, float vs bfloat during generate
- Allegro T2V: pass/pass/fail, output is pure gray
"""
@dataclass
class Model():
name: str
@@ -104,13 +72,13 @@ models = {
Model(name='None'),
Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968
url='https://huggingface.co/Lightricks/LTX-Video-0.9.5',
repo='YiYiXu/ltx-95',
repo='Lightricks/LTX-Video-0.9.5',
repo_cls=diffusers.LTXPipeline,
te_cls=transformers.T5EncoderModel,
dit_cls=diffusers.LTXVideoTransformer3DModel),
Model(name='LTXVideo 0.9.5 I2V',
url='https://huggingface.co/Lightricks/LTX-Video-0.9.5',
repo='YiYiXu/ltx-95',
repo='Lightricks/LTX-Video-0.9.5',
repo_cls=diffusers.LTXConditionPipeline,
te_cls=transformers.T5EncoderModel,
dit_cls=diffusers.LTXVideoTransformer3DModel),
@@ -214,7 +182,7 @@ models = {
te_cls=transformers.T5EncoderModel,
dit_cls=diffusers.CogVideoXTransformer3DModel),
Model(name='CogVideoX 1.5 5B T2V',
url='https://huggingface.co/THUDM/THUDM/CogVideoX1.5-5B',
url='https://huggingface.co/THUDM/CogVideoX1.5-5B',
repo='THUDM/CogVideoX1.5-5B',
repo_cls=diffusers.CogVideoXPipeline,
te_cls=transformers.T5EncoderModel,
modules/video_models/video_overrides.py
+20 -1
View File
@@ -16,10 +16,29 @@ def load_override(selected: Model):
torch_dtype=torch.float32,
cache_dir=shared.opts.hfcache_dir)
debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}')
if selected.name == 'LTXVideo 0.9.5 I2V':
kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo,
subfolder="vae",
torch_dtype=torch.float32,
cache_dir=shared.opts.hfcache_dir)
return kwargs
def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model):
cls = shared.sd_model.__class__.__name__
# Allegro
if selected.name == 'Allegro T2V':
shared.sd_model.vae.enable_tiling()
# Latte
if selected.name == 'Latte 1 T2V':
p.task_args['enable_temporal_attentions'] = False
debug(f'Video overrides: model="{selected.name}" args={p.task_args}')
p.task_args['video_length'] = p.frames
# LTX
if cls == 'LTXImageToVideoPipeline' or cls == 'LTXConditionPipeline':
p.task_args['generator'] = None
if cls == 'LTXConditionPipeline':
print('HERE2', p.denoising_strength)
p.task_args['strength'] = p.denoising_strength
if 'LTX' in shared.sd_model.__class__.__name__:
p.task_args['width'] = 32 * (p.width // 32)
p.task_args['height'] = 32 * (p.height // 32)
modules/video_models/video_run.py
+2 -1
View File
@@ -8,7 +8,7 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e
def generate(*args, **kwargs):
task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, faster_cache, pyramid_attention, override_settings = args
task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, init_strength, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, faster_cache, pyramid_attention, override_settings = args
if engine is None or model is None or engine == 'None' or model == 'None':
return video_utils.queue_err('model not selected')
found = [model.name for model in models_def.models.get(engine, [])]
@@ -36,6 +36,7 @@ def generate(*args, **kwargs):
width=16 * int(width // 16),
height=16 * int(height // 16),
frames=int(frames),
denoising_strength=float(init_strength),
init_image=init_image,
cfg_scale=float(guidance_scale),
diffusers_guidance_rescale=float(guidance_true),
modules/video_models/video_utils.py
+7 -2
View File
@@ -1,6 +1,6 @@
import os
import time
from modules import shared, sd_models, timer
from modules import shared, sd_models, timer, errors
debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
@@ -30,7 +30,12 @@ def set_prompt(p):
def hijack_encode_prompt(*args, **kwargs):
t0 = time.time()
res = shared.sd_model.orig_encode_prompt(*args, **kwargs)
try:
res = shared.sd_model.orig_encode_prompt(*args, **kwargs)
except Exception as e:
shared.log.error(f'Video encode: {e}')
errors.display(e, 'Video encode')
res = None
t1 = time.time()
timer.process.add('te', t1-t0)
debug(f'Video encode: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}')
modules/video_models/video_vae.py
+13 -3
View File
@@ -1,6 +1,7 @@
import os
import time
from modules import shared, sd_models, devices, timer
import torch
from modules import shared, sd_models, devices, timer, errors
debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
@@ -55,8 +56,17 @@ def hijack_vae_decode(*args, **kwargs):
pass
if res is None:
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
res = shared.sd_model.vae.orig_decode(*args, **kwargs)
print('HERE', shared.sd_model.vae.dtype)
try:
if torch.is_tensor(args[0]):
latent = args[0]
latent = latent.to(device=devices.device, dtype=shared.sd_model.vae.dtype) # upcast to vae dtype
res = shared.sd_model.vae.orig_decode(latent, *args[1:], **kwargs)
else:
res = shared.sd_model.vae.orig_decode(*args, **kwargs)
except Exception as e:
shared.log.error(f'Video VAE: type={vae_type} {e}')
errors.display(e, 'Video VAE')
res = None
t1 = time.time()
timer.process.add('vae', t1-t0)
debug(f'Video decode: type={vae_type} vae={shared.sd_model.vae.__class__.__name__} latents={args[0].shape} time={t1-t0:.2f}')
wiki
+1 -1
Submodule wiki updated: f58dcaaf6a...ef3c65cb0e