From 46bc0834b11cd9bfaa2b782fe61745de7564dc00 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 21 Mar 2025 14:53:52 -0400 Subject: [PATCH] video tab major update Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 21 +++-- modules/processing_args.py | 15 +-- modules/processing_class.py | 6 +- modules/processing_info.py | 3 + modules/shared.py | 2 +- modules/ui_symbols.py | 1 + modules/ui_video.py | 84 ++++++++++------- modules/video_models/models_def.py | 31 +++++- modules/video_models/run_allegro.py | 91 ------------------ modules/video_models/run_cog.py | 91 ------------------ modules/video_models/run_hunyuan.py | 94 ------------------- modules/video_models/run_ltx.py | 91 ------------------ modules/video_models/run_mochi.py | 91 ------------------ modules/video_models/video_cache.py | 45 +++++++++ .../video_models/{run_wan.py => video_run.py} | 11 ++- modules/video_models/video_utils.py | 2 +- 16 files changed, 163 insertions(+), 516 deletions(-) delete mode 100644 modules/video_models/run_allegro.py delete mode 100644 modules/video_models/run_cog.py delete mode 100644 modules/video_models/run_hunyuan.py delete mode 100644 modules/video_models/run_ltx.py delete mode 100644 modules/video_models/run_mochi.py create mode 100644 modules/video_models/video_cache.py rename modules/video_models/{run_wan.py => video_run.py} (90%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc01e323..5bcc50a84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-03-19 +## Update for 2025-03-21 ### ToDo/Limitations @@ -12,15 +12,19 @@ - Video: HunyuanVideo-I2V incompatible with latest transformers - Video: LTXVideo-095 support for conditioned input - Video: LTXVideo-095 support for offloading - - Video: FasterCache: https://github.com/huggingface/diffusers/pull/10163 - - Video: PyramidAttention: https://github.com/huggingface/diffusers/pull/9562 + - Video: FasterCache and PyramidAttentionBroadcast granular config + - Video: FasterCache and PyramidAttentionBroadcast for LTX and WAN + - Video: HunyuanVideo-I2V-16ch + - Video: CogVideo-15 support -### Highlights for 2025-03-20 +### Highlights for 2025-03-21 -Brand new Video processing module with support for all latest models: **WAN21, Hunyuan, LTX, Cog, Allegro, Mochi1** and more! -Plus support for CogView-4, new CLiP models, improvements to remote VAE, additional docs/guides. +Brand new Video processing module with support for all latest models: **WAN21, Hunyuan, LTX, Cog, Allegro, Mochi1, Latte1** +And combined with *on-the-fly quantization*, support for *Local/Tiny/Remote* VAE, acceleration modules such as *FasterCache or PAB* and more! -### Details for 2025-03-20 +Plus support for CogView-4, new CLiP models, improvements to remote VAE, additional docs/guides + +### Details for 2025-03-21 - **Video tab** - initial release so consider this as alpha version @@ -39,6 +43,9 @@ Plus support for CogView-4, new CLiP models, improvements to remote VAE, additio - **Tiny VAE**: support for *Hunyuan, WAN, Mochi* - **Remote VAE**: support for *Hunyuan* - **LoRA**: support for *Hunyuan, LTX, WAN, Mochi, Cog* + - acceleration: + - [FasterCache](https://huggingface.co/papers/2410.19355): support for *Hunyuan, Mochi, Latte, Allegro, Cog* + - [PyramidAttentionBroadcast](https://huggingface.co/papers/2408.12588): support for *Hunyuan, Mochi, Latte, Allegro, Cog* - additional key points: - all models are auto-downloaded upon first use uses *system paths -> huggingface* folder diff --git a/modules/processing_args.py b/modules/processing_args.py index 261a996b0..a5f04eb7d 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -26,7 +26,7 @@ def task_specific_kwargs(p, model): p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images] if isinstance(p.init_images[0], Image.Image): p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images if i is not None] - if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model: + if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model and 'video' not in p.ops: p.ops.append('txt2img') if hasattr(p, 'width') and hasattr(p, 'height'): task_args = { @@ -238,13 +238,13 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'): model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values model.scheduler.noise_sampler_seed = p.seeds # some schedulers have internal noise generator and do not use pipeline generator - if 'seed' in possible: + if 'seed' in possible and p.seed is not None: args['seed'] = p.seed - if 'noise_sampler_seed' in possible: + if 'noise_sampler_seed' in possible and p.seeds is not None: args['noise_sampler_seed'] = p.seeds - if 'guidance_scale' in possible: + if 'guidance_scale' in possible and p.cfg_scale is not None and p.cfg_scale > 0: args['guidance_scale'] = p.cfg_scale - if 'img_guidance_scale' in possible and hasattr(p, 'image_cfg_scale'): + if 'img_guidance_scale' in possible and hasattr(p, 'image_cfg_scale') and p.image_cfg_scale is not None and p.image_cfg_scale > 0: args['img_guidance_scale'] = p.image_cfg_scale if 'generator' in possible: generator = get_generator(p) @@ -304,9 +304,10 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t # handle remaining args for arg in kwargs: if arg in possible: # add kwargs + if type(kwargs[arg]) == float or type(kwargs[arg]) == int: + if kwargs[arg] <= -1: # skip -1 as default value + continue args[arg] = kwargs[arg] - else: - pass task_kwargs = task_specific_kwargs(p, model) for arg in task_kwargs: diff --git a/modules/processing_class.py b/modules/processing_class.py index a0d2104b5..eb4e333ec 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -236,9 +236,13 @@ class StableDiffusionProcessing: self.height = firstphase_height self.sampler_name = sampler_name or processing_helpers.get_sampler_name(sampler_index, img=True) self.hr_sampler_name: str = hr_sampler_name if hr_sampler_name != 'Same as primary' else self.sampler_name - self.override_settings = {k: v for k, v in (override_settings or {}).items() if k not in shared.restricted_opts} self.inpaint_full_res = inpaint_full_res if isinstance(inpaint_full_res, bool) else self.inpaint_full_res self.inpaint_full_res = inpaint_full_res != 0 if isinstance(inpaint_full_res, int) else self.inpaint_full_res + try: + self.override_settings = {k: v for k, v in (override_settings or {}).items() if k not in shared.restricted_opts} + except Exception as e: + shared.log.error(f'Override: {override_settings} {e}') + self.override_settings = {} # null items initialized later self.prompts = None diff --git a/modules/processing_info.py b/modules/processing_info.py index 3b0e1bd24..4db8ccaa5 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -186,6 +186,9 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No for k, v in args.copy().items(): if v is None: del args[k] + if type(v) is float or type(v) is int: + if v <= -1: + del args[k] if isinstance(v, str): if len(v) == 0 or v == '0x0': del args[k] diff --git a/modules/shared.py b/modules/shared.py index 8b2b2b9b0..a4a448966 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -779,7 +779,7 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), 'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001, "visible": native}), 'schedulers_timesteps_range': OptionInfo(1000, "Timesteps range", gr.Slider, {"minimum": 250, "maximum": 4000, "step": 1, "visible": native}), 'schedulers_shift': OptionInfo(3, "Sampler shift", gr.Slider, {"minimum": 0.1, "maximum": 10, "step": 0.1, "visible": False}), - 'schedulers_dynamic_shift': OptionInfo(True, "Sampler dynamic shift", gr.Checkbox, {"visible": False}), + 'schedulers_dynamic_shift': OptionInfo(False, "Sampler dynamic shift", gr.Checkbox, {"visible": False}), # managed from ui.py for backend original k-diffusion "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching", gr.Checkbox, {"visible": not native}), diff --git a/modules/ui_symbols.py b/modules/ui_symbols.py index c92ddb8f6..ef426e7e7 100644 --- a/modules/ui_symbols.py +++ b/modules/ui_symbols.py @@ -20,6 +20,7 @@ reuse = '♻️' info = 'ℹ' # noqa reset = '🔄' upload = '⬆️' +loading = '↺' reuse = '⬅️' search = '🔍' preview = '🖼️' diff --git a/modules/ui_video.py b/modules/ui_video.py index 20edfc89e..5044d0c36 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -1,59 +1,69 @@ +import os import gradio as gr from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste from modules.ui_components import ToolButton -from modules.video_models import models_def, video_utils, video_load +from modules.video_models import models_def, video_utils + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None def engine_change(engine): + debug(f'Video change: engine="{engine}"') found = [model.name for model in models_def.models.get(engine, [])] return gr.update(choices=found, value=found[0] if len(found) > 0 else None) def model_change(engine, model): + debug(f'Video change: engine="{engine}" model="{model}"') found = [model.name for model in models_def.models.get(engine, [])] selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - yield ['Video model loading', - gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False), - video_utils.get_url(selected.url if selected else None), - ] + return video_utils.get_url(selected.url if selected else None) + + +def model_load(engine, model): + debug(f'Video load: engine="{engine}" model="{model}"') + found = [model.name for model in models_def.models.get(engine, [])] + selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + yield f'Video model loading: {selected.name}' if selected: if 'None' in selected.name: sd_models.unload_model_weights() msg = 'Video model unloaded' else: + from modules.video_models import video_load msg = video_load.load_model(selected) else: sd_models.unload_model_weights() msg = 'Video model unloaded' - return [msg, - video_utils.get_url(selected.url if selected else None), - ] + yield msg + return msg def run_video(*args): engine, model = args[2], args[3] + debug(f'Video run: engine="{engine}" model="{model}"') found = [model.name for model in models_def.models.get(engine, [])] selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + if not selected or engine is None or model is None or engine == 'None' or model == 'None': + return video_utils.queue_err('model not selected') + debug(f'Video run: {str(selected)}') + from modules.video_models import video_run if selected and 'Hunyuan' in selected.name: - from modules.video_models import run_hunyuan - return run_hunyuan.generate(*args) + return video_run.generate('Hunyuan', *args) elif selected and 'LTX' in selected.name: - from modules.video_models import run_ltx - return run_ltx.generate(*args) + return video_run.generate('LTX', *args) elif selected and 'Mochi' in selected.name: - from modules.video_models import run_mochi - return run_mochi.generate(*args) + return video_run.generate('Mochi', *args) elif selected and 'Cog' in selected.name: - from modules.video_models import run_cog - return run_cog.generate(*args) + return video_run.generate('Cog', *args) elif selected and 'Allegro' in selected.name: - from modules.video_models import run_allegro - return run_allegro.generate(*args) + return video_run.generate('Allegro', *args) elif selected and 'WAN' in selected.name: - from modules.video_models import run_wan - return run_wan.generate(*args) - shared.log.error(f'Video model not found: args={args}') - return [], None, '', '', f'Video model not found: engine={engine} model={model}' + return video_run.generate('Wan', *args) + elif selected and 'Latte' in selected.name: + return video_run.generate('Latte', *args) + return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"') def create_ui(): @@ -74,23 +84,25 @@ def create_ui(): with gr.Row(): engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine") model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model") + btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load", label='Load model') with gr.Row(): - url = gr.HTML(label='Model URL', elem_id='video_model_url', value='') - with gr.Row(): - width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480) - with gr.Row(): - frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames") - seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) - random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed') - reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed') + url = gr.HTML(label='Model URL', elem_id='video_model_url', value='

') + with gr.Accordion(open=True, label="Size", elem_id='video_size_accordion'): + with gr.Row(): + width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480) + with gr.Row(): + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames") + seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) + random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed", label='Random seed') + reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed", label='Reuse seed') with gr.Accordion(open=True, label="Parameters", elem_id='video_parameters_accordion'): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") with gr.Row(): - sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift") - dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift", interactive=False) # TODO video: dynamic shift + sampler_shift = gr.Slider(label='Sampler shift', minimum=-1.0, maximum=20.0, step=0.1, value=-1.0, elem_id="video_scheduler_shift") + dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift") with gr.Row(): - guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale") - guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true") + guidance_scale = gr.Slider(label='Guidance scale', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_scale") + guidance_true = gr.Slider(label='True guidance', minimum=-1.0, maximum=14.0, step=0.1, value=-1.0, elem_id="video_guidance_true") with gr.Accordion(open=True, label="Decode", elem_id='video_decode_accordion'): with gr.Row(): vae_type = gr.Dropdown(label='VAE decode', choices=['Default', 'Tiny', 'Remote'], value='Default', elem_id="video_vae_type") @@ -121,7 +133,8 @@ def create_ui(): random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed]) # handle engine and model change engine.change(fn=engine_change, inputs=[engine], outputs=[model]) - model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, url]) + model.change(fn=model_change, inputs=[engine, model], outputs=[url]) + btn_load.click(fn=model_load, inputs=[engine, model], outputs=[html_log]) # setup extra networks ui_extra_networks.setup_ui(extra_networks_ui, gallery) @@ -154,6 +167,7 @@ def create_ui(): vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, + faster_cache, pyramid_attention, override_settings, ] # generate function diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 4395eb817..271318b69 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -3,6 +3,32 @@ import diffusers import transformers +""" +Hunyuan Video T2V: pass/pass/pass +Hunyuan Video I2V: pass/pass/pass, transformers incompatibility +SkyReels Hunyuan T2V: +SkyReels Hunyuan I2V: +Fast Hunyuan T2V: +LTXVideo 0.9.5 T2V: +LTXVideo 0.9.5 I2V: +LTXVideo 0.9.1 T2V: +LTXVideo 0.9.1 I2V: +LTXVideo 0.9.0 T2V: +LTXVideo 0.9.0 I2V: +WAN 2.1 1.3B T2V: pass/pass/pass +WAN 2.1 14B T2V: pass/fail/fail, error loading shard +WAN 2.1 14B I2V 480p: +WAN 2.1 14B I2V 720p: +Mochi 1 T2V: pass/pass/pass +Latte 1 T2V: pass/fail/fail, float vs bfloat during generate +Allegro T2V: pass/pass/fail, output is pure gray +CogVideoX 1.0 2B T2V: pass/pass/pass +CogVideoX 1.0 5B T2V: +CogVideoX 1.0 5B I2V: +CogVideoX 1.5 5B T2V: pass/pass/fail, output is pure black +CogVideoX 1.5 5B I2V: pass/pass/pass +""" + @dataclass class Model(): name: str @@ -19,6 +45,9 @@ class Model(): vae_hijack: bool = True vae_remote: bool = False + def __str__(self): + return f'name="{self.name}" url="{self.url}" repo="{self.repo}" repo_cls="{self.repo_cls}" dit="{self.dit}" dit_cls="{self.dit_cls}" dit_folder="{self.dit_folder}" te="{self.te}" te_cls="{self.te_cls}" te_folder="{self.te_folder}" te_hijack={self.te_hijack} vae_hijack={self.vae_hijack} vae_remote={self.vae_remote}' + models = { 'None': [], @@ -167,7 +196,7 @@ models = { dit_cls=diffusers.CogVideoXTransformer3DModel), Model(name='CogVideoX 1.0 5B T2V', url='https://huggingface.co/THUDM/CogVideoX-5b', - repo='THUDM/THUDM/CogVideoX-5b', + repo='THUDM/CogVideoX-5b', repo_cls=diffusers.CogVideoXPipeline, te_cls=transformers.T5EncoderModel, dit_cls=diffusers.CogVideoXTransformer3DModel), diff --git a/modules/video_models/run_allegro.py b/modules/video_models/run_allegro.py deleted file mode 100644 index 091be7333..000000000 --- a/modules/video_models/run_allegro.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import time -from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae - - -debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None - - -def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args - if engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__: - video_load.load_model(selected) - if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__: - return video_utils.queue_err('model not loaded') - debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') - - p = processing.StableDiffusionProcessingVideo( - sd_model=shared.sd_model, - prompt=prompt, - negative_prompt=negative, - styles=styles, - seed=int(seed), - sampler_name = processing.get_sampler_name(sampler_index), - sampler_shift=float(sampler_shift), - steps=int(steps), - width=8 * int(width // 8), - height=8 * int(height // 8), - frames=int(frames), - init_image=init_image, - cfg_scale=float(guidance_scale), - diffusers_guidance_rescale=float(guidance_true), - vae_type=vae_type, - vae_tile_frames=int(vae_tile_frames), - override_settings=override_settings, - ) - p.scripts = None - p.script_args = None - p.state = ui_state - p.do_not_save_grid = True - p.do_not_save_samples = not save_frames - if 'I2V' in model: - if init_image is None: - return video_utils.queue_err('init image not set') - p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - - # cleanup memory - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - # set args - processing.fix_seed(p) - video_vae.set_vae_params(p) - video_utils.set_prompt(p) - p.task_args['output_type'] = 'pil' - p.ops.append('video') - orig_dynamic_shift = shared.opts.schedulers_dynamic_shift - orig_sampler_shift = shared.opts.schedulers_shift - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - debug(f'Video: task_args={p.task_args}') - - # run processing - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') - err = None - t0 = time.time() - try: - processed = processing.process_images(p) - except Exception as e: - err = str(e) - errors.display(e, 'video') - t1 = time.time() - shared.state.disable_preview = False - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift - p.close() - - # done - if err: - return video_utils.queue_err(err) - if processed is None or len(processed.images) == 0: - return video_utils.queue_err('processing failed') - shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) - generation_info_js = processed.js() if processed is not None else '' - return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_cog.py b/modules/video_models/run_cog.py deleted file mode 100644 index 0aaee4b97..000000000 --- a/modules/video_models/run_cog.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import time -from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae - - -debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None - - -def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args - if engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__: - video_load.load_model(selected) - if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__: - return video_utils.queue_err('model not loaded') - debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') - - p = processing.StableDiffusionProcessingVideo( - sd_model=shared.sd_model, - prompt=prompt, - negative_prompt=negative, - styles=styles, - seed=int(seed), - sampler_name = processing.get_sampler_name(sampler_index), - sampler_shift=float(sampler_shift), - steps=int(steps), - width=8 * int(width // 8), - height=8 * int(height // 8), - frames=int(frames), - init_image=init_image, - cfg_scale=float(guidance_scale), - diffusers_guidance_rescale=float(guidance_true), - vae_type=vae_type, - vae_tile_frames=int(vae_tile_frames), - override_settings=override_settings, - ) - p.scripts = None - p.script_args = None - p.state = ui_state - p.do_not_save_grid = True - p.do_not_save_samples = not save_frames - if 'I2V' in model: - if init_image is None: - return video_utils.queue_err('init image not set') - p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - - # cleanup memory - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - # set args - processing.fix_seed(p) - video_vae.set_vae_params(p) - video_utils.set_prompt(p) - p.task_args['output_type'] = 'pil' - p.ops.append('video') - orig_dynamic_shift = shared.opts.schedulers_dynamic_shift - orig_sampler_shift = shared.opts.schedulers_shift - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - debug(f'Video: task_args={p.task_args}') - - # run processing - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') - err = None - t0 = time.time() - try: - processed = processing.process_images(p) - except Exception as e: - err = str(e) - errors.display(e, 'video') - t1 = time.time() - shared.state.disable_preview = False - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift - p.close() - - # done - if err: - return video_utils.queue_err(err) - if processed is None or len(processed.images) == 0: - return video_utils.queue_err('processing failed') - shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) - generation_info_js = processed.js() if processed is not None else '' - return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_hunyuan.py b/modules/video_models/run_hunyuan.py deleted file mode 100644 index 0f722b890..000000000 --- a/modules/video_models/run_hunyuan.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import time -from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae - - -debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None - - -def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args - if engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: - video_load.load_model(selected) - if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: - return video_utils.queue_err('model not loaded') - debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') - - p = processing.StableDiffusionProcessingVideo( - sd_model=shared.sd_model, - prompt=prompt, - negative_prompt=negative, - styles=styles, - seed=int(seed), - sampler_name = processing.get_sampler_name(sampler_index), - sampler_shift=float(sampler_shift), - steps=int(steps), - width=16 * int(width // 16), - height=16 * int(height // 16), - frames=int(frames), - init_image=init_image, - cfg_scale=float(guidance_scale), - diffusers_guidance_rescale=float(guidance_true), - vae_type=vae_type, - vae_tile_frames=int(vae_tile_frames), - override_settings=override_settings, - ) - if p.vae_type == 'Remote' and not selected.vae_remote: - shared.log.warning(f'Video: model={selected.name} remote vae not supported') - p.vae_type = 'Default' - p.scripts = None - p.script_args = None - p.state = ui_state - p.do_not_save_grid = True - p.do_not_save_samples = not save_frames - if 'I2V' in model: - if init_image is None: - return video_utils.queue_err('init image not set') - p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - - # cleanup memory - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - # set args - processing.fix_seed(p) - video_vae.set_vae_params(p) - video_utils.set_prompt(p) - p.task_args['output_type'] = 'latent' if (p.vae_type == 'Remote') else 'pil' - p.ops.append('video') - orig_dynamic_shift = shared.opts.schedulers_dynamic_shift - orig_sampler_shift = shared.opts.schedulers_shift - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - debug(f'Video: task_args={p.task_args}') - - # run processing - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') - err = None - t0 = time.time() - try: - processed = processing.process_images(p) - except Exception as e: - err = str(e) - errors.display(e, 'video') - t1 = time.time() - shared.state.disable_preview = False - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift - p.close() - - # done - if err: - return video_utils.queue_err(err) - if processed is None or len(processed.images) == 0: - return video_utils.queue_err('processing failed') - shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) - generation_info_js = processed.js() if processed is not None else '' - return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_ltx.py b/modules/video_models/run_ltx.py deleted file mode 100644 index ef3fa08b8..000000000 --- a/modules/video_models/run_ltx.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import time -from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae - - -debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None - - -def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args - if engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__: - video_load.load_model(selected) - if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__: - return video_utils.queue_err('model not loaded') - debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') - - p = processing.StableDiffusionProcessingVideo( - sd_model=shared.sd_model, - prompt=prompt, - negative_prompt=negative, - styles=styles, - seed=int(seed), - sampler_name = processing.get_sampler_name(sampler_index), - sampler_shift=float(sampler_shift), - steps=int(steps), - width=32 * int(width // 32), - height=32 * int(height // 32), - frames=int(frames), - init_image=init_image, - cfg_scale=float(guidance_scale), - diffusers_guidance_rescale=float(guidance_true), - vae_type=vae_type, - vae_tile_frames=int(vae_tile_frames), - override_settings=override_settings, - ) - p.scripts = None - p.script_args = None - p.state = ui_state - p.do_not_save_grid = True - p.do_not_save_samples = not save_frames - if 'I2V' in model: - if init_image is None: - return video_utils.queue_err('init image not set') - p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - - # cleanup memory - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - # set args - processing.fix_seed(p) - video_vae.set_vae_params(p) - video_utils.set_prompt(p) - p.task_args['output_type'] = 'pil' - p.ops.append('video') - orig_dynamic_shift = shared.opts.schedulers_dynamic_shift - orig_sampler_shift = shared.opts.schedulers_shift - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - debug(f'Video: task_args={p.task_args}') - - # run processing - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') - err = None - t0 = time.time() - try: - processed = processing.process_images(p) - except Exception as e: - err = str(e) - errors.display(e, 'video') - t1 = time.time() - shared.state.disable_preview = False - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift - p.close() - - # done - if err: - return video_utils.queue_err(err) - if processed is None or len(processed.images) == 0: - return video_utils.queue_err('processing failed') - shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) - generation_info_js = processed.js() if processed is not None else '' - return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_mochi.py b/modules/video_models/run_mochi.py deleted file mode 100644 index ef705880e..000000000 --- a/modules/video_models/run_mochi.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import time -from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae - - -debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None - - -def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args - if engine is None or model is None or engine == 'None' or model == 'None': - return video_utils.queue_err('model not selected') - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__: - video_load.load_model(selected) - if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__: - return video_utils.queue_err('model not loaded') - debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') - - p = processing.StableDiffusionProcessingVideo( - sd_model=shared.sd_model, - prompt=prompt, - negative_prompt=negative, - styles=styles, - seed=int(seed), - sampler_name = processing.get_sampler_name(sampler_index), - sampler_shift=float(sampler_shift), - steps=int(steps), - width=8 * int(width // 8), - height=8 * int(height // 8), - frames=int(frames), - init_image=init_image, - cfg_scale=float(guidance_scale), - diffusers_guidance_rescale=float(guidance_true), - vae_type=vae_type, - vae_tile_frames=int(vae_tile_frames), - override_settings=override_settings, - ) - p.scripts = None - p.script_args = None - p.state = ui_state - p.do_not_save_grid = True - p.do_not_save_samples = not save_frames - if 'I2V' in model: - if init_image is None: - return video_utils.queue_err('init image not set') - p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - - # cleanup memory - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - # set args - processing.fix_seed(p) - video_vae.set_vae_params(p) - video_utils.set_prompt(p) - p.task_args['output_type'] = 'pil' - p.ops.append('video') - orig_dynamic_shift = shared.opts.schedulers_dynamic_shift - orig_sampler_shift = shared.opts.schedulers_shift - shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift - shared.opts.data['schedulers_shift'] = sampler_shift - debug(f'Video: task_args={p.task_args}') - - # run processing - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') - err = None - t0 = time.time() - try: - processed = processing.process_images(p) - except Exception as e: - err = str(e) - errors.display(e, 'video') - t1 = time.time() - shared.state.disable_preview = False - shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift - shared.opts.data['schedulers_shift'] = orig_sampler_shift - p.close() - - # done - if err: - return video_utils.queue_err(err) - if processed is None or len(processed.images) == 0: - return video_utils.queue_err('processing failed') - shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) - generation_info_js = processed.js() if processed is not None else '' - return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/video_cache.py b/modules/video_models/video_cache.py new file mode 100644 index 000000000..be43068d1 --- /dev/null +++ b/modules/video_models/video_cache.py @@ -0,0 +1,45 @@ +import os +import diffusers +from modules import shared, errors + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def set_cache(faster_cache=False, pyramid_attention_broadcast=False): + if not shared.sd_loaded or not hasattr(shared.sd_model, 'transformer'): + return + if not hasattr(shared.sd_model.transformer, 'enable_cache'): + debug(f'Video cache: cls={shared.sd_model.transformer.__class__.__name__} not supported') + return + try: + if faster_cache: # https://github.com/huggingface/diffusers/pull/10163 + config = diffusers.FasterCacheConfig( + spatial_attention_block_skip_range=2, + spatial_attention_timestep_skip_range=(-1, 681), + current_timestep_callback=lambda: shared.sd_model.current_timestep, + attention_weight_callback=lambda _: 0.3, + unconditional_batch_skip_range=5, + unconditional_batch_timestep_skip_range=(-1, 781), + tensor_format="BFCHW", + ) + shared.sd_model.transformer.disable_cache() + shared.sd_model.transformer.enable_cache(config) + shared.log.debug(f'Video cache: type={config.__class__.__name__}') + debug(f'Video cache: {vars(config)}') + elif pyramid_attention_broadcast: # https://github.com/huggingface/diffusers/pull/9562 + config = diffusers.PyramidAttentionBroadcastConfig( + spatial_attention_block_skip_range=2, + spatial_attention_timestep_skip_range=(100, 800), + current_timestep_callback=lambda: shared.sd_model.current_timestep, + ) + shared.sd_model.transformer.disable_cache() + shared.sd_model.transformer.enable_cache(config) + shared.log.debug(f'Video cache: type={config.__class__.__name__}') + debug(f'Video cache: {vars(config)}') + else: + debug('Video cache: not enabled') + shared.sd_model.transformer.disable_cache() + except Exception as e: + shared.log.error(f'Video cache: error={e}') + errors.display(e, 'video cache') diff --git a/modules/video_models/run_wan.py b/modules/video_models/video_run.py similarity index 90% rename from modules/video_models/run_wan.py rename to modules/video_models/video_run.py index fd50c1fc5..09dee3826 100644 --- a/modules/video_models/run_wan.py +++ b/modules/video_models/video_run.py @@ -1,21 +1,21 @@ import os import time from modules import shared, errors, sd_models, processing, devices, images, ui_common -from modules.video_models import models_def, video_utils, video_load, video_vae +from modules.video_models import models_def, video_utils, video_load, video_vae, video_cache debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args +def generate(keyword, *args, **kwargs): + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, faster_cache, pyramid_attention, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': return video_utils.queue_err('model not selected') found = [model.name for model in models_def.models.get(engine, [])] selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None - if not shared.sd_loaded or 'Wan' not in shared.sd_model.__class__.__name__: + if not shared.sd_loaded or keyword not in shared.sd_model.__class__.__name__: video_load.load_model(selected) - if not shared.sd_loaded or 'Wan' not in shared.sd_model.__class__.__name__: + if not shared.sd_loaded or keyword not in shared.sd_model.__class__.__name__: return video_utils.queue_err('model not loaded') debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') @@ -58,6 +58,7 @@ def generate(*args, **kwargs): # set args processing.fix_seed(p) video_vae.set_vae_params(p) + video_cache.set_cache(faster_cache=faster_cache, pyramid_attention_broadcast=pyramid_attention) video_utils.set_prompt(p) p.task_args['output_type'] = 'latent' if (p.vae_type == 'Remote') else 'pil' p.ops.append('video') diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 0ed4791eb..8a69e19ec 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -18,7 +18,7 @@ def get_quant(args): def get_url(url): - return f'  {url}
' if url else '' + return f'  {url}

' if url else '

' def set_prompt(p):