diff --git a/TODO.md b/TODO.md index 0575e97c7..f262d8b78 100644 --- a/TODO.md +++ b/TODO.md @@ -4,6 +4,14 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Current +- Video tab: + - remote vae + - tiny vae + - lora + - accelerators: teacache, pab, fastercache, paraattention, perflow + - modernui tab +- Detailer daemon: https://github.com/muerrilla/sd-webui-detail-daemon/blob/main/scripts/detail_daemon.py + ## Future Candidates - Redesign postprocessing diff --git a/modules/onnx_impl/execution_providers.py b/modules/onnx_impl/execution_providers.py index e38199d0f..25e0056b3 100644 --- a/modules/onnx_impl/execution_providers.py +++ b/modules/onnx_impl/execution_providers.py @@ -107,10 +107,14 @@ def install_execution_provider(ep: ExecutionProvider): elif ep == ExecutionProvider.OpenVINO: packages.append("openvino") packages.append("onnxruntime-openvino") + log.info(f'ONNX install: {packages}') for package in packages: res += install(package) res += '
' res += 'Server restart required' log.info("Server restart required") - importlib.reload(ort) + try: + importlib.reload(ort) + except Exception: + pass return res diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index a5f6b87c9..366a62995 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -71,8 +71,10 @@ def process_base(p: processing.StableDiffusionProcessing): eta=shared.opts.scheduler_eta, guidance_scale=p.cfg_scale, guidance_rescale=p.diffusers_guidance_rescale, + true_cfg_scale=p.diffusers_guidance_rescale, denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None, denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None, + num_frames=getattr(p, 'frames', None), output_type='latent', clip_skip=p.clip_skip, desc='Base', @@ -364,7 +366,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): else: width = getattr(p, 'width', 0) height = getattr(p, 'height', 0) - frames = p.task_args.get('num_frames', None) + frames = p.task_args.get('num_frames', None) or getattr(p, 'frames', None) if isinstance(output.images, list): results = [] for i in range(len(output.images)): diff --git a/modules/ui_caption.py b/modules/ui_caption.py index 474427e9d..ef8d2d6e2 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -45,9 +45,9 @@ def create_ui(): vlm_model = gr.Dropdown(list(vqa.vlm_models), value=list(vqa.vlm_models)[0], label='VLM Model', elem_id='vlm_model') with gr.Accordion(label='Advanced options', open=False, visible=True): with gr.Row(): - vlm_max_tokens = gr.Slider(label='Max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens') - vlm_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams') - vlm_temperature = gr.Slider(label='Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature') + vlm_max_tokens = gr.Slider(label='VLM max tokens', value=shared.opts.interrogate_vlm_max_length, minimum=16, maximum=4096, step=1, elem_id='vlm_max_tokens') + vlm_num_beams = gr.Slider(label='VLM num beams', value=shared.opts.interrogate_vlm_num_beams, minimum=1, maximum=16, step=1, elem_id='vlm_num_beams') + vlm_temperature = gr.Slider(label='VLM Temperature', value=shared.opts.interrogate_vlm_temperature, minimum=0.1, maximum=1.0, step=0.01, elem_id='vlm_temperature') with gr.Row(): vlm_top_k = gr.Slider(label='Top-K', value=shared.opts.interrogate_vlm_top_k, minimum=0, maximum=99, step=1, elem_id='vlm_top_k') vlm_top_p = gr.Slider(label='Top-P', value=shared.opts.interrogate_vlm_top_p, minimum=0.0, maximum=1.0, step=0.01, elem_id='vlm_top_p') @@ -90,7 +90,7 @@ def create_ui(): clip_max_flavors = gr.Slider(label='Max flavors', value=shared.opts.interrogate_clip_max_flavors, minimum=1, maximum=64, step=1, elem_id='clip_max_flavors') clip_flavor_count = gr.Slider(label='Intermediates', value=shared.opts.interrogate_clip_flavor_count, minimum=256, maximum=4096, step=8, elem_id='clip_flavor_intermediate_count') with gr.Row(): - clip_num_beams = gr.Slider(label='Num beams', value=shared.opts.interrogate_clip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams') + clip_num_beams = gr.Slider(label='CLiP num beams', value=shared.opts.interrogate_clip_num_beams, minimum=1, maximum=16, step=1, elem_id='clip_num_beams') clip_min_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) clip_max_length.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) clip_chunk_size.change(fn=update_clip_params, inputs=[clip_min_length, clip_max_length, clip_chunk_size, clip_min_flavors, clip_max_flavors, clip_flavor_count, clip_num_beams], outputs=[]) diff --git a/modules/ui_video.py b/modules/ui_video.py index 97b9d2ec7..2c5c324a5 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -1,30 +1,23 @@ -# TODO hunyuanvideo: prompt_template, lora -# TODO hunyuanvideo: teacache, pab, fastercache, paraattention, perflow -# TODO modernui video tab - import gradio as gr from modules import shared, sd_models, timer, images, ui_common, ui_sections, ui_symbols, call_queue, generation_parameters_copypaste from modules.ui_components import ToolButton -from modules.video_models import hunyuan +from modules.video_models import models_def, video_utils, hunyuan, ltx def engine_change(engine): - found = [model.name for model in hunyuan.models.get(engine, [])] + found = [model.name for model in models_def.models.get(engine, [])] return gr.update(choices=found, value=found[0] if len(found) > 0 else None) def model_change(engine, model): - found = [model.name for model in hunyuan.models.get(engine, [])] - selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None + found = [model.name for model in models_def.models.get(engine, [])] + selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None if selected: if 'None' in selected.name: sd_models.unload_model_weights() msg = 'Video model unloaded' - elif 'Hunyuan' in selected.name: - msg = hunyuan.load(selected) - elif model != 'None': - msg = f'Video model not found: engine={engine} model={model}' - shared.log.error(msg) + else: + msg = video_utils.load_model(selected) else: sd_models.unload_model_weights() msg = 'Video model unloaded' @@ -33,10 +26,13 @@ def model_change(engine, model): def run_video(*args): engine, model = args[2], args[3] - found = [model.name for model in hunyuan.models.get(engine, [])] - selected = [m for m in hunyuan.models[engine] if m.name == model][0] if len(found) > 0 else None + found = [model.name for model in models_def.models.get(engine, [])] + selected = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None if selected and 'Hunyuan' in selected.name: return hunyuan.generate(*args) + elif selected and 'LTX' in selected.name: + pass + # return ltx.generate(*args) shared.log.error(f'Video model not found: args={args}') return [], None, '', '', f'Video model not found: engine={engine} model={model}' @@ -57,7 +53,7 @@ def create_ui(): with gr.Column(variant='compact', elem_id="video_settings", scale=1): with gr.Row(): - engine = gr.Dropdown(label='Engine', choices=list(hunyuan.models), value='None', elem_id="video_engine") + engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine") model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model") with gr.Row(): width, height = ui_sections.create_resolution_inputs('video', default_width=720, default_height=480) @@ -69,6 +65,7 @@ def create_ui(): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") with gr.Row(): sampler_shift = gr.Slider(label='Sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0, elem_id="video_scheduler_shift") + dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift") with gr.Row(): guidance_scale = gr.Slider(label='Guidance scale', minimum=0.0, maximum=14.0, step=0.1, value=6.0, elem_id="video_guidance_scale") guidance_true = gr.Slider(label='True guidance', minimum=0.0, maximum=14.0, step=0.1, value=1.0, elem_id="video_guidance_true") @@ -86,7 +83,7 @@ def create_ui(): override_settings = ui_common.create_override_inputs('video') # output panel with gallery and video tabs - with gr.Column(elem_id='video-output-column', scale=3) as _column_output: + with gr.Column(elem_id='video-output-column', scale=2) as _column_output: with gr.Tabs(elem_classes=['video-output-tabs'], elem_id='video-output-tabs'): with gr.Tab('Frames', id='out-gallery'): gallery, gen_info, html_info, _html_info_formatted, html_log = ui_common.create_output_panel("video", prompt=prompt, preview=False, transfer=False, scale=2) @@ -124,7 +121,7 @@ def create_ui(): width, height, frames, steps, sampler_index, - sampler_shift, + sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, diff --git a/modules/video_models/hunyuan.py b/modules/video_models/hunyuan.py index 2b83e6b63..e42139189 100644 --- a/modules/video_models/hunyuan.py +++ b/modules/video_models/hunyuan.py @@ -1,187 +1,22 @@ -from dataclasses import dataclass import os import time -import torch -import transformers -import diffusers -from modules import shared, errors, sd_models, sd_checkpoint, sd_samplers, processing, model_quant, devices, images, timer, ui_common +from modules import shared, errors, sd_models, processing, devices, images, ui_common +from modules.video_models import models_def, video_utils -@dataclass -class Model(): - name: str - repo: str - dit: str - subfolder: str - -models = { - 'None': [], - 'Hunyuan Video': [ - Model(name='None', repo=None, dit=None, subfolder=None), - Model(name='Hunyuan Video T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='hunyuanvideo-community/HunyuanVideo', subfolder='transformer'), - Model(name='Hunyuan Video I2V', repo='hunyuanvideo-community/HunyuanVideo-I2V', dit='hunyuanvideo-community/HunyuanVideo-I2V', subfolder='transformer'), # https://github.com/huggingface/diffusers/pull/10983 - Model(name='SkyReels Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-T2V', subfolder=None), # https://github.com/huggingface/diffusers/pull/10837 - Model(name='SkyReels Hunyuan I2V', repo='hunyuanvideo-community/HunyuanVideo', dit='Skywork/SkyReels-V1-Hunyuan-I2V', subfolder=None), - Model(name='Fast Hunyuan T2V', repo='hunyuanvideo-community/HunyuanVideo', dit='FastVideo/FastHunyuan-diffusers', subfolder='transformer'), # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 - ] -} debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None -loaded_model = None -prompt_template = { - "template": ( - "<|start_header_id|>system<|end_header_id|>" - "\nDescribe the video by detailing the following aspects: \n" - "1. The main content and theme of the video.\n" - "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n" - "3. Actions, events, behaviors, temporal relationships, and physical movement changes of the objects.\n" - "4. Background environment, light, style and atmosphere.\n" - "5. Camera angles, movements, and transitions used in the video.\n" - "<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" - ), - "crop_start": 95, -} - - -def hijack_decode(*args, **kwargs): - t0 = time.time() - vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) - res = shared.sd_model.vae.orig_decode(*args, **kwargs) - t1 = time.time() - timer.process.add('vae', t1-t0) - debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}') - return res - - -def hijack_encode_prompt(*args, **kwargs): - t0 = time.time() - res = shared.sd_model.orig_encode_prompt(*args, **kwargs) - t1 = time.time() - timer.process.add('te', t1-t0) - debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - return res - - -def get_quant(args): - if args is not None and "quantization_config" in args: - return args['quantization_config'].__class__.__name__ - return None - - -def load(selected): - if selected is None: - return - global loaded_model # pylint: disable=global-statement - if loaded_model == selected.name: - return - sd_models.unload_model_weights() - t0 = time.time() - - quant_args = model_quant.create_config(module='Model') - cls = diffusers.HunyuanVideoTransformer3DModel - try: - debug(f'Video load: module=transofrmer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} quant={get_quant(quant_args)}') - transformer = cls.from_pretrained( - pretrained_model_name_or_path=selected.dit, - subfolder=selected.subfolder, - torch_dtype=devices.dtype, - cache_dir=shared.opts.hfcache_dir, - **quant_args - ) - except Exception as e: - shared.log.error(f'video load: module=transformer repo="{selected.dit}" subfolder="{selected.subfolder}" cls={cls.__name__} {e}') - - quant_args = model_quant.create_config(module='Text Encoder') - if 'I2V' in selected.repo: - cls = transformers.LlavaForConditionalGeneration - else: - cls = transformers.LlamaModel - try: - debug(f'Video load: module=te repo="{selected.repo}" cls={cls.__name__} quant={get_quant(quant_args)}') - text_encoder = cls.from_pretrained( - pretrained_model_name_or_path=selected.repo, - subfolder="text_encoder", - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - # torch_dtype='auto', # special case as text and vision nested models have different dtypes - # attn_implementation="flash_attention_2", # testing different attention types - **quant_args - ) - except Exception as e: - shared.log.error(f'video load: module=te repo="{selected.repo}" cls={cls.__name__} {e}') - - cls = transformers.CLIPTextModel - try: - debug(f'Video load: module=clip repo="{selected.repo}" cls={cls.__name__} quant=None') - text_encoder_2 = transformers.CLIPTextModel.from_pretrained( - pretrained_model_name_or_path=selected.repo, - subfolder="text_encoder_2", - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - ) - except Exception as e: - shared.log.error(f'video load: module=clip repo="{selected.repo}" cls={cls.__name__} {e}') - - cls = diffusers.AutoencoderKLHunyuanVideo - try: - debug(f'Video load: module=vae repo="{selected.repo}" cls={cls.__name__} quant=None') - vae = diffusers.AutoencoderKLHunyuanVideo.from_pretrained( - pretrained_model_name_or_path=selected.repo, - subfolder="vae", - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - ) - except Exception as e: - shared.log.error(f'video load: module=vae repo="{selected.repo}" cls={cls.__name__} {e}') - - if selected.name == 'Hunyuan Video I2V': - cls = diffusers.HunyuanVideoImageToVideoPipeline - elif selected.name == 'SkyReels Hunyuan I2V': - cls = diffusers.HunyuanSkyreelsImageToVideoPipeline - else: - cls = diffusers.HunyuanVideoPipeline - try: - debug(f'Video load: module=pipe repo="{selected.repo}" cls={cls.__name__} quant=None') - shared.sd_model = cls.from_pretrained( - pretrained_model_name_or_path=selected.repo, - transformer=transformer, - text_encoder=text_encoder, - text_encoder_2=text_encoder_2, - vae=vae, - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - ) - except Exception as e: - shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={cls.__name__} {e}') - - t1 = time.time() - sd_models.set_diffuser_options(shared.sd_model) - shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo) - shared.sd_model.sd_model_hash = None - shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode - shared.sd_model.vae.decode = hijack_decode - shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt - shared.sd_model.encode_prompt = hijack_encode_prompt - shared.sd_model.vae.enable_slicing() - loaded_model = selected.name - msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' - shared.log.info(msg) - return msg def generate(*args, **kwargs): - task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args + task_id, ui_state, engine, model, prompt, negative, styles, width, height, frames, steps, sampler_index, sampler_shift, dynamic_shift, seed, guidance_scale, guidance_true, init_image, vae_type, vae_tile_frames, save_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, override_settings = args if engine is None or model is None or engine == 'None' or model == 'None': - shared.log.error('Video: model not selected') - return [], None, '', '', 'Video model not selected' + return video_utils.queue_err('model not selected') if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: - found = [model.name for model in models.get(engine, [])] - selected = [m for m in models[engine] if m.name == model][0] if len(found) > 0 else None - load(selected) + found = [model.name for model in models_def.models.get(engine, [])] + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] if len(found) > 0 else None + video_utils.load_model(selected) if not shared.sd_loaded or 'Hunyuan' not in shared.sd_model.__class__.__name__: - shared.log.error('Video: model not loaded') - return [], None, '', '', 'Video model not loaded' + return video_utils.queue_err('model not loaded') debug(f'Video generate: task={task_id} args={args} kwargs={kwargs}') p = processing.StableDiffusionProcessingVideo( @@ -202,27 +37,24 @@ def generate(*args, **kwargs): override_settings=override_settings, ) p.scripts = None - p.script_args = args + p.script_args = None p.state = ui_state p.do_not_save_grid = True p.do_not_save_samples = not save_frames if 'I2V' in model: if init_image is None: - shared.log.error('Video: init image not set') - return [], None, '', '', 'Error: init image not set' - p.task_args['image'] = init_image - # from PIL import Image - # p.task_args['image'] = init_image.resize((336, 336), Image.Resampling.LANCZOS) + return video_utils.queue_err('init image not set') + p.task_args['image'] = images.resize_image(resize_mode=2, im=init_image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + # cleanup memory shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) devices.torch_gc(force=True) # handle sampler and seed - if p.sampler_name != 'Default': - shared.sd_model.scheduler = sd_samplers.create_sampler(p.sampler_name, shared.sd_model) - p.sampler_name = 'Default' # avoid double creation - if hasattr(shared.sd_model.scheduler, '_shift') and sampler_shift > 0: - shared.sd_model.scheduler._shift = sampler_shift # pylint: disable=protected-access + orig_dynamic_shift = shared.opts.schedulers_dynamic_shift + orig_sampler_shift = shared.opts.schedulers_shift + shared.opts.data['schedulers_dynamic_shift'] = dynamic_shift # todo video sampler dynamic shift + shared.opts.data['schedulers_shift'] = sampler_shift # handle vae if vae_tile_frames > p.frames: @@ -237,39 +69,32 @@ def generate(*args, **kwargs): processing.fix_seed(p) p.prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(negative, p.styles) - p.task_args['width'] = p.width - p.task_args['height'] = p.height - p.task_args['num_inference_steps'] = p.steps - p.task_args['num_frames'] = p.frames - p.task_args['generator'] = torch.manual_seed(p.seed) - p.task_args['guidance_scale'] = p.cfg_scale - p.task_args['true_cfg_scale'] = p.diffusers_guidance_rescale - # p.task_args['prompt_template'] = prompt_template # t2v and i2v have different templates - p.task_args['output_type'] = 'pil' p.task_args['prompt'] = p.prompt p.task_args['negative_prompt'] = p.negative_prompt + p.task_args['output_type'] = 'pil' p.ops.append('video') debug(f'Video: task_args={p.task_args}') # run processing shared.state.disable_preview = True shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={p.frames} steps={p.steps}') + err = None t0 = time.time() try: processed = processing.process_images(p) except Exception as e: - shared.log.error(f'Video: exception={e}') + err = str(e) errors.display(e, 'video') - processed = None - shared.state.disable_preview = False - return [], None, '', '', str(e) t1 = time.time() shared.state.disable_preview = False - + shared.opts.data['schedulers_dynamic_shift'] = orig_dynamic_shift + shared.opts.data['schedulers_shift'] = orig_sampler_shift p.close() + if err: + return video_utils.queue_err(err) if processed is None or len(processed.images) == 0: - return [], None, '', '', 'Video: processing failed' - shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') + return video_utils.queue_err('processing failed') + shared.log.info(f'Video: name="{selected.name}" cls={shared.sd_model.__class__.__name__} frames={len(processed.images)} time={t1-t0:.2f}') if video_type != 'None': video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) else: diff --git a/modules/video_models/ltx.py b/modules/video_models/ltx.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py new file mode 100644 index 000000000..aba1b76b9 --- /dev/null +++ b/modules/video_models/models_def.py @@ -0,0 +1,67 @@ +from dataclasses import dataclass +import diffusers +import transformers + + +@dataclass +class Model(): + name: str + repo: str = None + repo_cls: classmethod = None + dit: str = None + dit_cls: classmethod = None + dit_folder: str = 'transformer' + te: str = None + te_cls: classmethod = None + te_folder: str = 'text_encoder' + te_hijack: bool = True + vae_hijack: bool = True + + +models = { + 'None': [], + 'Hunyuan Video': [ + Model(name='None'), + Model(name='Hunyuan Video T2V', + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanVideoPipeline, + te_cls=transformers.LlamaModel, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='Hunyuan Video I2V', # https://github.com/huggingface/diffusers/pull/10983 + repo='hunyuanvideo-community/HunyuanVideo-I2V', + repo_cls=diffusers.HunyuanVideoImageToVideoPipeline, + te_cls=transformers.LlavaForConditionalGeneration, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='SkyReels Hunyuan T2V', # https://github.com/huggingface/diffusers/pull/10837 + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanVideoPipeline, + te_cls=transformers.LlamaModel, + dit='Skywork/SkyReels-V1-Hunyuan-T2V', + dit_folder=None, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='SkyReels Hunyuan I2V', # https://github.com/huggingface/diffusers/pull/10837 + repo='hunyuanvideo-community/HunyuanVideo', + te_cls=transformers.LlamaModel, + dit='Skywork/SkyReels-V1-Hunyuan-I2V', + dit_folder=None, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='Fast Hunyuan T2V', # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanVideoPipeline, + te_cls=transformers.LlamaModel, + dit='FastVideo/FastHunyuan-diffusers', + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + ], +} + +""" +'LTX Video': [ + Model(name='None'), + Model(name='LTXVideo 0.9.0 T2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'), + Model(name='LTXVideo 0.9.1 T2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'), + Model(name='LTXVideo 0.9.5 T2V', repo='Lightricks/LTX-Video-0.9.5'), # https://github.com/huggingface/diffusers/pull/10968 + Model(name='LTXVideo 0.9.0 I2V', repo='a-r-r-o-w/LTX-Video-diffusers', subfolder='transformer'), + Model(name='LTXVideo 0.9.1 I2V', repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', subfolder='transformer'), + Model(name='LTXVideo 0.9.5 I2V', repo='Lightricks/LTX-Video-0.9.5', subfolder='transformer'), +]," +""" diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py new file mode 100644 index 000000000..3313cab69 --- /dev/null +++ b/modules/video_models/video_utils.py @@ -0,0 +1,109 @@ +import os +import time +from modules import shared, timer, sd_models, sd_checkpoint, model_quant, devices +from modules.video_models import models_def + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def queue_err(msg): + shared.log.error(f'Video: {msg}') + return [], None, '', '', f'Error: {msg}' + + +def get_quant(args): + if args is not None and "quantization_config" in args: + return args['quantization_config'].__class__.__name__ + return None + + +def hijack_vae_decode(*args, **kwargs): + t0 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + t1 = time.time() + timer.process.add('vae', t1-t0) + debug(f'Video decode: vae={shared.sd_model.vae.__class__.__name__} time={t1-t0:.2f}') + return res + + +def hijack_encode_prompt(*args, **kwargs): + t0 = time.time() + res = shared.sd_model.orig_encode_prompt(*args, **kwargs) + t1 = time.time() + timer.process.add('te', t1-t0) + debug(f'Video encode: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return res + +loaded_model = None + + +def load_model(selected: models_def.Model): + if selected is None: + return + global loaded_model # pylint: disable=global-statement + if loaded_model == selected.name: + return + sd_models.unload_model_weights() + t0 = time.time() + + # text encoder + try: + quant_args = model_quant.create_config(module='Text Encoder') + debug(f'Video load: module=te repo="{selected.te or selected.repo}" folder="{selected.te_folder}" cls={selected.te_cls.__name__} quant={get_quant(quant_args)}') + text_encoder = selected.te_cls.from_pretrained( + pretrained_model_name_or_path=selected.te or selected.repo, + subfolder=selected.te_folder, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **quant_args + ) + except Exception as e: + shared.log.error(f'video load: module=te cls={selected.te_cls.__name__} {e}') + text_encoder = None + + # transformer + try: + quant_args = model_quant.create_config(module='Model') + debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{selected.dit_folder}" cls={selected.dit_cls.__name__} quant={get_quant(quant_args)}') + transformer = selected.dit_cls.from_pretrained( + pretrained_model_name_or_path=selected.dit or selected.repo, + subfolder=selected.dit_folder, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args + ) + except Exception as e: + shared.log.error(f'video load: module=transformer cls={selected.dit_cls.__name__} {e}') + transformer = None + + # model + try: + debug(f'Video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__}') + shared.sd_model = selected.repo_cls.from_pretrained( + pretrained_model_name_or_path=selected.repo, + transformer=transformer, + text_encoder=text_encoder, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + except Exception as e: + shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__} {e}') + + t1 = time.time() + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(selected.repo) + shared.sd_model.sd_model_hash = None + if selected.vae_hijack: + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.decode = hijack_vae_decode + if selected.te_hijack: + shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt + shared.sd_model.encode_prompt = hijack_encode_prompt + shared.sd_model.vae.enable_slicing() + loaded_model = selected.name + msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' + shared.log.info(msg) + return msg diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 135256ef2..a1933673e 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -11,6 +11,7 @@ from modules.teacache.teacache_ltx import teacache_forward repos = { '0.9.0': 'a-r-r-o-w/LTX-Video-diffusers', '0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers', + '0.9.5': 'Lightricks/LTX-Video-0.9.5', 'custom': None, } @@ -31,7 +32,6 @@ def load_quants(kwargs, repo_id): def hijack_decode(*args, **kwargs): t0 = time.time() - # vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) res = shared.sd_model.vae.orig_decode(*args, **kwargs) t1 = time.time()