diff --git a/CHANGELOG.md b/CHANGELOG.md index 67745eacf..4a6ba094e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Gemma3 requires `transformers==git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3` - Remote VAE encode for SD15 and Flux.1: - HunyuanVideo-I2V: + - LTXVideo condition input ### Highlights for 2025-03-17 diff --git a/installer.py b/installer.py index fb54f25e4..0411ef496 100644 --- a/installer.py +++ b/installer.py @@ -538,7 +538,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all or args.skip_git: return - sha = '82188cef0487837b8c70fc3f36ea63c05c85f341' # diffusers commit hash + sha = '56f740051dae2d410677292a5c9e5b66e60f87dc' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/ui_video.py b/modules/ui_video.py index 2c5c324a5..05a3bfc09 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -1,7 +1,7 @@ import gradio as gr from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste from modules.ui_components import ToolButton -from modules.video_models import models_def, video_utils, hunyuan, ltx +from modules.video_models import models_def, video_utils def engine_change(engine): @@ -12,6 +12,10 @@ def engine_change(engine): def model_change(engine, model): found = [model.name for model in models_def.models.get(engine, [])] selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + yield ['Video model loading', + gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False), + video_utils.get_url(selected.url if selected else None), + ] if selected: if 'None' in selected.name: sd_models.unload_model_weights() @@ -21,7 +25,10 @@ def model_change(engine, model): else: sd_models.unload_model_weights() msg = 'Video model unloaded' - return [msg, gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False)] + return [msg, + gr.update(visible='I2V' in selected.name) if selected else gr.update(visible=False), + video_utils.get_url(selected.url if selected else None), + ] def run_video(*args): @@ -29,10 +36,20 @@ def run_video(*args): found = [model.name for model in models_def.models.get(engine, [])] selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None if selected and 'Hunyuan' in selected.name: - return hunyuan.generate(*args) + from modules.video_models import run_hunyuan + return run_hunyuan.generate(*args) elif selected and 'LTX' in selected.name: - pass - # return ltx.generate(*args) + from modules.video_models import run_ltx + return run_ltx.generate(*args) + elif selected and 'Mochi' in selected.name: + from modules.video_models import run_mochi + return run_mochi.generate(*args) + elif selected and 'Cog' in selected.name: + from modules.video_models import run_cog + return run_cog.generate(*args) + elif selected and 'Allegro' in selected.name: + from modules.video_models import run_allegro + return run_allegro.generate(*args) shared.log.error(f'Video model not found: args={args}') return [], None, '', '', f'Video model not found: engine={engine} model={model}' @@ -55,6 +72,8 @@ def create_ui(): with gr.Row(): engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine") model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model") + with gr.Row(): + url = gr.HTML(label='Model URL', elem_id='video_model_url', value='') with gr.Row(): width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480) with gr.Row(): @@ -65,7 +84,7 @@ def create_ui(): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") with gr.Row(): sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift") - dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift") + dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift", interactive=False) with gr.Row(): guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale") guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true") @@ -95,7 +114,7 @@ def create_ui(): random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed]) # handle engine and model change engine.change(fn=engine_change, inputs=[engine], outputs=[model]) - model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group]) + model.change(fn=model_change, inputs=[engine, model], outputs=[html_log, image_group, url]) # setup extra networks ui_extra_networks.setup_ui(extra_networks_ui, gallery) diff --git a/modules/video_models/ltx.py b/modules/video_models/ltx.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index aba1b76b9..82e6f108a 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -6,6 +6,7 @@ import transformers @dataclass class Model(): name: str + url: str = '' repo: str = None repo_cls: classmethod = None dit: str = None @@ -23,16 +24,19 @@ models = { 'Hunyuan Video': [ Model(name='None'), Model(name='Hunyuan Video T2V', + url='https://huggingface.co/tencent/HunyuanVideo', repo='hunyuanvideo-community/HunyuanVideo', repo_cls=diffusers.HunyuanVideoPipeline, te_cls=transformers.LlamaModel, dit_cls=diffusers.HunyuanVideoTransformer3DModel), Model(name='Hunyuan Video I2V', # https://github.com/huggingface/diffusers/pull/10983 + url='https://huggingface.co/tencent/HunyuanVideo-I2V', repo='hunyuanvideo-community/HunyuanVideo-I2V', repo_cls=diffusers.HunyuanVideoImageToVideoPipeline, te_cls=transformers.LlavaForConditionalGeneration, dit_cls=diffusers.HunyuanVideoTransformer3DModel), Model(name='SkyReels Hunyuan T2V', # https://github.com/huggingface/diffusers/pull/10837 + url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-T2V', repo='hunyuanvideo-community/HunyuanVideo', repo_cls=diffusers.HunyuanVideoPipeline, te_cls=transformers.LlamaModel, @@ -40,28 +44,108 @@ models = { dit_folder=None, dit_cls=diffusers.HunyuanVideoTransformer3DModel), Model(name='SkyReels Hunyuan I2V', # https://github.com/huggingface/diffusers/pull/10837 + url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-I2V', repo='hunyuanvideo-community/HunyuanVideo', te_cls=transformers.LlamaModel, dit='Skywork/SkyReels-V1-Hunyuan-I2V', dit_folder=None, dit_cls=diffusers.HunyuanVideoTransformer3DModel), Model(name='Fast Hunyuan T2V', # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 + url='https://huggingface.co/FastVideo/FastHunyuan', repo='hunyuanvideo-community/HunyuanVideo', repo_cls=diffusers.HunyuanVideoPipeline, te_cls=transformers.LlamaModel, dit='FastVideo/FastHunyuan-diffusers', dit_cls=diffusers.HunyuanVideoTransformer3DModel), ], + 'LTX Video': [ + Model(name='None'), + Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968 + url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', + repo='YiYiXu/ltx-95', + repo_cls=diffusers.LTXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.5 I2V', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', + repo='YiYiXu/ltx-95', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.1 T2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo_cls=diffusers.LTXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.1 I2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo_cls=diffusers.LTXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.0 T2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers', + repo='a-r-r-o-w/LTX-Video-diffusers', + repo_cls=diffusers.LTXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.0 I2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers', + repo='a-r-r-o-w/LTX-Video-diffusers', + repo_cls=diffusers.LTXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + ], + 'Mochi Video': [ + Model(name='None'), + Model(name='Mochi 1 T2V', + url='https://huggingface.co/genmo/mochi-1-preview', + repo='genmo/mochi-1-preview', + repo_cls=diffusers.MochiPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.MochiTransformer3DModel), + ], + 'Allegro Video': [ + Model(name='None'), + Model(name='Allegro T2V', + url='https://huggingface.co/rhymes-ai/Allegro', + repo='rhymes-ai/Allegro', + repo_cls=diffusers.AllegroPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.AllegroTransformer3DModel), + ], + 'Cog Video': [ + Model(name='None'), + Model(name='CogVideoX 1.0 2B T2V', + url='https://huggingface.co/THUDM/CogVideoX-2b', + repo='THUDM/CogVideoX-2b', + repo_cls=diffusers.CogVideoXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.0 5B T2V', + url='https://huggingface.co/THUDM/CogVideoX-5b', + repo='THUDM/THUDM/CogVideoX-5b', + repo_cls=diffusers.CogVideoXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.0 5B I2V', + url='https://huggingface.co/THUDM/CogVideoX-5b-I2V', + repo='THUDM/CogVideoX-5b-I2V', + repo_cls=diffusers.CogVideoXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.5 5B T2V', + url='https://huggingface.co/THUDM/THUDM/CogVideoX1.5-5B', + repo='THUDM/CogVideoX1.5-5B', + repo_cls=diffusers.CogVideoXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.5 5B I2V', + url='https://huggingface.co/THUDM/CogVideoX1.5-5B-I2V', + repo='THUDM/CogVideoX1.5-5B-I2V', + repo_cls=diffusers.CogVideoXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + ], } - -""" -'LTX Video': [ - Model(name='None'), - Model(name='LTXVideo 0.9.0 T2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'), - Model(name='LTXVideo 0.9.1 T2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'), - Model(name='LTXVideo 0.9.5 T2V', repo='Lightricks/LTX-Video-0.9.5'), # https://github.com/huggingface/diffusers/pull/10968 - Model(name='LTXVideo 0.9.0 I2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'), - Model(name='LTXVideo 0.9.1 I2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'), - Model(name='LTXVideo 0.9.5 I2V', repo='Lightricks/LTX-Video-0.9.5', subfolder='transformer'), -]," -""" diff --git a/modules/video_models/run_allegro.py b/modules/video_models/run_allegro.py new file mode 100644 index 000000000..be1b0da8d --- /dev/null +++ b/modules/video_models/run_allegro.py @@ -0,0 +1,91 @@ +import os +import time +from modules import shared, errors, sd_models, processing, devices, images, ui_common +from modules.video_models import models_def, video_utils + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def generate(*args, **kwargs): + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + if engine is None or model is None or engine == 'None' or model == 'None': + return video_utils.queue_err('model not selected') + found = [model.name for model in models_def.models.get(engine, [])] + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__: + video_utils.load_model(selected) + if not shared.sd_loaded or 'Allegro' not in shared.sd_model.__class__.__name__: + return video_utils.queue_err('model not loaded') + debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + + p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, + prompt=prompt, + negative_prompt=negative, + styles=styles, + seed=int(seed), + sampler_name = processing.get_sampler_name(sampler_index), + sampler_shift=float(sampler_shift), + steps=int(steps), + width=8 * int(width // 8), + height=8 * int(height // 8), + frames=int(frames), + init_image=init_image, + cfg_scale=float(guidance_scale), + diffusers_guidance_rescale=float(guidance_true), + vae_type=vae_type, + vae_tile_frames=int(vae_tile_frames), + override_settings=override_settings, + ) + p.scripts = None + p.script_args = None + p.state = ui_state + p.do_not_save_grid = True + p.do_not_save_samples = not save_frames + if 'I2V' in model: + if init_image is None: + return video_utils.queue_err('init image not set') + p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + + # cleanup memory + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + # set args + processing.fix_seed(p) + video_utils.set_vae_params(p.frames, vae_tile_frames) + video_utils.set_prompt(p) + p.task_args['output_type'] = 'pil' + p.ops.append('video') + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift + shared.opts.data['schedulers_shift'] = sampler_shift + debug(f'Video: task_args={p.task_args}') + + # run processing + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') + err = None + t0 = time.time() + try: + processed = processing.process_images(p) + except Exception as e: + err = str(e) + errors.display(e, 'video') + t1 = time.time() + shared.state.disable_preview = False + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + p.close() + + # done + if err: + return video_utils.queue_err(err) + if processed is None or len(processed.images) == 0: + return video_utils.queue_err('processing failed') + shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') + video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + generation_info_js = processed.js() if processed is not None else '' + return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_cog.py b/modules/video_models/run_cog.py new file mode 100644 index 000000000..5fbe1d320 --- /dev/null +++ b/modules/video_models/run_cog.py @@ -0,0 +1,91 @@ +import os +import time +from modules import shared, errors, sd_models, processing, devices, images, ui_common +from modules.video_models import models_def, video_utils + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def generate(*args, **kwargs): + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + if engine is None or model is None or engine == 'None' or model == 'None': + return video_utils.queue_err('model not selected') + found = [model.name for model in models_def.models.get(engine, [])] + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__: + video_utils.load_model(selected) + if not shared.sd_loaded or 'Cog' not in shared.sd_model.__class__.__name__: + return video_utils.queue_err('model not loaded') + debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + + p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, + prompt=prompt, + negative_prompt=negative, + styles=styles, + seed=int(seed), + sampler_name = processing.get_sampler_name(sampler_index), + sampler_shift=float(sampler_shift), + steps=int(steps), + width=8 * int(width // 8), + height=8 * int(height // 8), + frames=int(frames), + init_image=init_image, + cfg_scale=float(guidance_scale), + diffusers_guidance_rescale=float(guidance_true), + vae_type=vae_type, + vae_tile_frames=int(vae_tile_frames), + override_settings=override_settings, + ) + p.scripts = None + p.script_args = None + p.state = ui_state + p.do_not_save_grid = True + p.do_not_save_samples = not save_frames + if 'I2V' in model: + if init_image is None: + return video_utils.queue_err('init image not set') + p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + + # cleanup memory + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + # set args + processing.fix_seed(p) + video_utils.set_vae_params(p.frames, vae_tile_frames) + video_utils.set_prompt(p) + p.task_args['output_type'] = 'pil' + p.ops.append('video') + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift + shared.opts.data['schedulers_shift'] = sampler_shift + debug(f'Video: task_args={p.task_args}') + + # run processing + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') + err = None + t0 = time.time() + try: + processed = processing.process_images(p) + except Exception as e: + err = str(e) + errors.display(e, 'video') + t1 = time.time() + shared.state.disable_preview = False + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + p.close() + + # done + if err: + return video_utils.queue_err(err) + if processed is None or len(processed.images) == 0: + return video_utils.queue_err('processing failed') + shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') + video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + generation_info_js = processed.js() if processed is not None else '' + return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/hunyuan.py b/modules/video_models/run_hunyuan.py similarity index 77% rename from modules/video_models/hunyuan.py rename to modules/video_models/run_hunyuan.py index e42139189..aa076f935 100644 --- a/modules/video_models/hunyuan.py +++ b/modules/video_models/run_hunyuan.py @@ -11,9 +11,9 @@ def generate(*args, **kwargs): task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': return video_utils.queue_err('model not selected') + found = [model.name for model in models_def.models.get(engine, [])] + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: - found = [model.name for model in models_def.models.get(engine, [])] - selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None video_utils.load_model(selected) if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: return video_utils.queue_err('model not loaded') @@ -21,6 +21,8 @@ def generate(*args, **kwargs): p = processing.StableDiffusionProcessingVideo( sd_model=shared.sd_model, + prompt=prompt, + negative_prompt=negative, styles=styles, seed=int(seed), sampler_name = processing.get_sampler_name(sampler_index), @@ -50,29 +52,16 @@ def generate(*args, **kwargs): shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) devices.torch_gc(force=True) - # handle sampler and seed + # set args + processing.fix_seed(p) + video_utils.set_vae_params(p.frames, vae_tile_frames) + video_utils.set_prompt(p) + p.task_args['output_type'] = 'pil' + p.ops.append('video') orig_dynamic_shift = shared.opts.schedulers_dynamic_shift orig_sampler_shift = shared.opts.schedulers_shift shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift shared.opts.data['schedulers_shift'] = sampler_shift - - # handle vae - if vae_tile_frames > p.frames: - shared.sd_model.vae.tile_sample_min_num_frames = vae_tile_frames - shared.sd_model.vae.use_framewise_decoding = True - shared.sd_model.vae.enable_tiling() - else: - shared.sd_model.vae.use_framewise_decoding = False - shared.sd_model.vae.disable_tiling() - - # set args - processing.fix_seed(p) - p.prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, p.styles) - p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(negative, p.styles) - p.task_args['prompt'] = p.prompt - p.task_args['negative_prompt'] = p.negative_prompt - p.task_args['output_type'] = 'pil' - p.ops.append('video') debug(f'Video: task_args={p.task_args}') # run processing @@ -90,15 +79,13 @@ def generate(*args, **kwargs): shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift shared.opts.data['schedulers_shift'] = orig_sampler_shift p.close() + + # done if err: return video_utils.queue_err(err) if processed is None or len(processed.images) == 0: return video_utils.queue_err('processing failed') shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') - if video_type != 'None': - video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) - else: - video_file = None - + video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) generation_info_js = processed.js() if processed is not None else '' return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_ltx.py b/modules/video_models/run_ltx.py new file mode 100644 index 000000000..ba01e45a4 --- /dev/null +++ b/modules/video_models/run_ltx.py @@ -0,0 +1,91 @@ +import os +import time +from modules import shared, errors, sd_models, processing, devices, images, ui_common +from modules.video_models import models_def, video_utils + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def generate(*args, **kwargs): + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + if engine is None or model is None or engine == 'None' or model == 'None': + return video_utils.queue_err('model not selected') + found = [model.name for model in models_def.models.get(engine, [])] + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__: + video_utils.load_model(selected) + if not shared.sd_loaded or 'LTX' not in shared.sd_model.__class__.__name__: + return video_utils.queue_err('model not loaded') + debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + + p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, + prompt=prompt, + negative_prompt=negative, + styles=styles, + seed=int(seed), + sampler_name = processing.get_sampler_name(sampler_index), + sampler_shift=float(sampler_shift), + steps=int(steps), + width=32 * int(width // 32), + height=32 * int(height // 32), + frames=int(frames), + init_image=init_image, + cfg_scale=float(guidance_scale), + diffusers_guidance_rescale=float(guidance_true), + vae_type=vae_type, + vae_tile_frames=int(vae_tile_frames), + override_settings=override_settings, + ) + p.scripts = None + p.script_args = None + p.state = ui_state + p.do_not_save_grid = True + p.do_not_save_samples = not save_frames + if 'I2V' in model: + if init_image is None: + return video_utils.queue_err('init image not set') + p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + + # cleanup memory + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + # set args + processing.fix_seed(p) + video_utils.set_vae_params(p.frames, vae_tile_frames) + video_utils.set_prompt(p) + p.task_args['output_type'] = 'pil' + p.ops.append('video') + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift + shared.opts.data['schedulers_shift'] = sampler_shift + debug(f'Video: task_args={p.task_args}') + + # run processing + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') + err = None + t0 = time.time() + try: + processed = processing.process_images(p) + except Exception as e: + err = str(e) + errors.display(e, 'video') + t1 = time.time() + shared.state.disable_preview = False + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + p.close() + + # done + if err: + return video_utils.queue_err(err) + if processed is None or len(processed.images) == 0: + return video_utils.queue_err('processing failed') + shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') + video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + generation_info_js = processed.js() if processed is not None else '' + return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/run_mochi.py b/modules/video_models/run_mochi.py new file mode 100644 index 000000000..fff3a6d04 --- /dev/null +++ b/modules/video_models/run_mochi.py @@ -0,0 +1,91 @@ +import os +import time +from modules import shared, errors, sd_models, processing, devices, images, ui_common +from modules.video_models import models_def, video_utils + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def generate(*args, **kwargs): + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + if engine is None or model is None or engine == 'None' or model == 'None': + return video_utils.queue_err('model not selected') + found = [model.name for model in models_def.models.get(engine, [])] + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__: + video_utils.load_model(selected) + if not shared.sd_loaded or 'Mochi' not in shared.sd_model.__class__.__name__: + return video_utils.queue_err('model not loaded') + debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') + + p = processing.StableDiffusionProcessingVideo( + sd_model=shared.sd_model, + prompt=prompt, + negative_prompt=negative, + styles=styles, + seed=int(seed), + sampler_name = processing.get_sampler_name(sampler_index), + sampler_shift=float(sampler_shift), + steps=int(steps), + width=8 * int(width // 8), + height=8 * int(height // 8), + frames=int(frames), + init_image=init_image, + cfg_scale=float(guidance_scale), + diffusers_guidance_rescale=float(guidance_true), + vae_type=vae_type, + vae_tile_frames=int(vae_tile_frames), + override_settings=override_settings, + ) + p.scripts = None + p.script_args = None + p.state = ui_state + p.do_not_save_grid = True + p.do_not_save_samples = not save_frames + if 'I2V' in model: + if init_image is None: + return video_utils.queue_err('init image not set') + p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + + # cleanup memory + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + # set args + processing.fix_seed(p) + video_utils.set_vae_params(p.frames, vae_tile_frames) + video_utils.set_prompt(p) + p.task_args['output_type'] = 'pil' + p.ops.append('video') + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift + shared.opts.data['schedulers_shift'] = sampler_shift + debug(f'Video: task_args={p.task_args}') + + # run processing + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') + err = None + t0 = time.time() + try: + processed = processing.process_images(p) + except Exception as e: + err = str(e) + errors.display(e, 'video') + t1 = time.time() + shared.state.disable_preview = False + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift + p.close() + + # done + if err: + return video_utils.queue_err(err) + if processed is None or len(processed.images) == 0: + return video_utils.queue_err('processing failed') + shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') + video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) + generation_info_js = processed.js() if processed is not None else '' + return processed.images, video_file, generation_info_js, processed.info, ui_common.plaintext_to_html(processed.comments) diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 3313cab69..c2b3878cf 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -1,6 +1,6 @@ import os import time -from modules import shared, timer, sd_models, sd_checkpoint, model_quant, devices +from modules import shared, errors, timer, sd_models, sd_checkpoint, model_quant, devices from modules.video_models import models_def @@ -18,6 +18,32 @@ def get_quant(args): return None +def get_url(url): + return f'  {url}
' if url else '' + + +def set_prompt(p): + p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + p.task_args['prompt'] = p.prompt + p.task_args['negative_prompt'] = p.negative_prompt + + +def set_vae_params(frames, tile_frames): + if tile_frames > frames: + if hasattr(shared.sd_model.vae, 'tile_sample_min_num_frames'): + shared.sd_model.vae.tile_sample_min_num_frames = tile_frames + if hasattr(shared.sd_model.vae, 'use_framewise_decoding'): + shared.sd_model.vae.use_framewise_decoding = True + if hasattr(shared.sd_model.vae, 'enable_tiling'): + shared.sd_model.vae.enable_tiling() + else: + if hasattr(shared.sd_model.vae, 'use_framewise_decoding'): + shared.sd_model.vae.use_framewise_decoding = False + if hasattr(shared.sd_model.vae, 'disable_tiling'): + shared.sd_model.vae.disable_tiling() + + def hijack_vae_decode(*args, **kwargs): t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) @@ -62,6 +88,7 @@ def load_model(selected: models_def.Model): ) except Exception as e: shared.log.error(f'video load: module=te cls={selected.te_cls.__name__} {e}') + errors.display(e, 'video') text_encoder = None # transformer @@ -77,6 +104,7 @@ def load_model(selected: models_def.Model): ) except Exception as e: shared.log.error(f'video load: module=transformer cls={selected.dit_cls.__name__} {e}') + errors.display(e, 'video') transformer = None # model @@ -91,6 +119,7 @@ def load_model(selected: models_def.Model): ) except Exception as e: shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__} {e}') + errors.display(e, 'video') t1 = time.time() sd_models.set_diffuser_options(shared.sd_model)