diff --git a/CHANGELOG.md b/CHANGELOG.md index 9281b7557..da8b359ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2025-07-07 +## Update for 2025-07-09 -### Highlights for 2025-07-08 +### Highlights for 2025-07-09 In this release we finally break with legacy with the removal of the original [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui/) codebase which has not been maintained for a while now This plus major cleanup of codebase and external dependencies resulted in ~53k LoC (*lines-of-code*) reduction and spread over [~680 files](https://github.com/vladmandic/sdnext/pull/4017)! @@ -11,9 +11,9 @@ We also switched project license to [Apache-2.0](https://github.com/vladmandic/s Feature highlights include: - **ModernUI** layout redesign which should make it more user friendly and easier to navigate -- Redesigned **Video** interface with +- Redesigned **Video** interface with native **FramePack** support - New background replacement and relightning methods using **Latent Bridge Matching** and new **PixelArt** processing filter -- New LLM/VLM models available for captioning and prompt enhance +- New **LLM/VLM** models available for captioning and prompt enhance - Compute improvements And (as always) many bugfixes and improvements to existing features! @@ -23,7 +23,7 @@ Although upgrades and existing installations are tested and should work fine! [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-07-08 +### Details for 2025-07-09 - **License** - SD.Next [license](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) switched from **aGPL-v3.0** to **Apache-v2.0** diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 4c6761c02..44e982e42 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -505,7 +505,10 @@ function setupExtraNetworksForTab(tabname) { if (window.opts.extra_networks_card_cover === 'sidebar') en.style.width = 0; gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset'; } - if (tabname === 'video') gradioApp().getElementById('framepack_settings').parentNode.style.width = gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width; + if (tabname === 'video') { + gradioApp().getElementById('framepack_settings').parentNode.style.width = gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width; + gradioApp().getElementById('ltx_settings').parentNode.style.width = gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width; + } } }); intersectionObserver.observe(en); // monitor visibility diff --git a/javascript/ui.js b/javascript/ui.js index ef6445a92..4df527ceb 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -273,6 +273,15 @@ function submit_framepack(...args) { return args; } +function submit_ltx(...args) { + const id = randomId(); + log('submitFramepack', id); + requestProgress(id, null, null); + window.submit_state = ''; + args[0] = id; + return args; +} + function submit_video_wrapper(...args) { const modernEl = gradioApp().querySelector('.video_output.fade-in'); let id = modernEl ? modernEl.id : args[0]; @@ -280,6 +289,7 @@ function submit_video_wrapper(...args) { log('submitVideoWrapper', id); const btn = gradioApp().getElementById(`${id}_generate_btn`); if (btn) btn.click(); + else console.log('submit_video_wrapper: no button found', id); } function submit_postprocessing(...args) { diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py new file mode 100644 index 000000000..d5396ebc5 --- /dev/null +++ b/modules/ltx/ltx_process.py @@ -0,0 +1,214 @@ +""" +- condition upload image +- condition upload video +- condition video get frames +- custom sampler +- new way of generate video +- modernui +- lora loader +""" +import os +import time +import threading +import diffusers +from modules import shared, sd_models, errors, timer, memstats, progress +from modules.processing_callbacks import diffusers_callback +from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, get_conditions, get_generator, get_prompts, vae_decode + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None +engine, model = 'LTX Video', 'LTXVideo 0.9.7 13B' +upsample_repo_id = "a-r-r-o-w/LTX-Video-0.9.7-Latent-Spatial-Upsampler-diffusers" +upsample_pipe = None +queue_lock = threading.Lock() + + +def run_ltx(task_id, + _ui_state, + prompt:str, + negative:str, + styles:list[str], + width:int, + height:int, + frames:int, + steps:int, + sampler_index:int, + seed:int, + upsample_enable:bool, + upsample_ratio:float, + refine_enable:bool, + refine_strength:float, + condition_image_strength:float, + condition_video_strength:float, + condition_video_frames:int, + condition_image, + condition_video, + decode_timestep:float, + image_cond_noise_scale:float, + _overrides, + ): + + def abort(e, ok:bool=False): + if ok: + shared.log.info(e) + else: + shared.log.error(f'Video: cls={shared.sd_model.__class__.__name__} op=base {e}') + errors.display(e, 'LTX') + shared.state.end() + progress.finish_task(task_id) + yield None, f'LTX Error: {str(e)}' + + progress.add_task_to_queue(task_id) + with queue_lock: + progress.start_task(task_id) + memstats.reset_stats() + timer.process.reset() + yield None, 'LTX: Loading...' + load_model(engine, model) + + shared.state.begin('Video', task_id=task_id) + shared.state.job_count = 1 + + conditions = get_conditions( + condition_image, + condition_image_strength, + condition_video, + condition_video_strength, + condition_video_frames, + ) + + prompt, negative, networks = get_prompts(prompt, negative, styles) + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init prompt="{prompt}" negative="{negative}" styles={styles} networks={networks}') + + t0 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t1 = time.time() + base_args = { + "prompt": prompt, + "negative_prompt": negative, + "width": get_bucket(width), + "height": get_bucket(height), + "num_frames": get_frames(frames), + "num_inference_steps": steps, + "image_cond_noise_scale": image_cond_noise_scale, + "generator": get_generator(seed), + "callback_on_step_end": diffusers_callback, + "output_type": "latent", + } + if len(conditions) > 0: + base_args["conditions"] = conditions + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=base {base_args}') + yield None, 'LTX: Generate in progress...' + try: + latents = shared.sd_model(**base_args).frames[0] + except AssertionError as e: + yield from abort(e, ok=True) + return + except Exception as e: + yield from abort(e, ok=False) + return + t2 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t3 = time.time() + timer.process.add('offload', t1 - t0) + timer.process.add('base', t2 - t1) + timer.process.add('offload', t3 - t2) + # diffusers.LTXConditionPipeline + + if upsample_enable: + t4 = time.time() + shared.state.begin('Upsample') + global upsample_pipe + upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id) + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + upscale_args = { + "width": get_bucket(upsample_ratio * width), + "height": get_bucket(upsample_ratio * height), + "generator": get_generator(seed), + "output_type": "latent", + } + if latents.ndim == 4: + latents = latents.unsqueeze(0) # add batch dimension + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=upsample latents={latents.shape} {upscale_args}') + yield None, 'LTX: Upsample in progress...' + try: + upsampled_latents = upsample_pipe(latents=latents, **upscale_args).frames[0] + except AssertionError as e: + yield from abort(e, ok=True) + return + except Exception as e: + yield from abort(e, ok=False) + return + latents = upsampled_latents + t5 = time.time() + upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) + t6 = time.time() + timer.process.add('upsample', t5 - t4) + timer.process.add('offload', t6 - t5) + shared.state.end() + + if refine_enable: + t7 = time.time() + shared.state.begin('Refine') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + refine_args = { + "prompt": prompt, + "negative_prompt": negative, + "width": get_bucket(upsample_ratio * width), + "height": get_bucket(upsample_ratio * height), + "num_frames": get_frames(frames), + "denoise_strength": refine_strength, + "num_inference_steps": steps, + "image_cond_noise_scale": image_cond_noise_scale, + "generator": get_generator(seed), + "callback_on_step_end": diffusers_callback, + "output_type": "latent", + } + if len(conditions) > 0: + refine_args["conditions"] = conditions + if latents.ndim == 4: + latents = latents.unsqueeze(0) # add batch dimension + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=refine latents={latents.shape} {refine_args}') + yield None, 'LTX: Refine in progress...' + try: + refined_latents = shared.sd_model(latents=latents, **refine_args).frames[0] + except AssertionError as e: + yield from abort(e, ok=True) + return + except Exception as e: + yield from abort(e, ok=False) + return + latents = refined_latents + t8 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t9 = time.time() + timer.process.add('refine', t8 - t7) + timer.process.add('offload', t9 - t8) + shared.state.end() + + yield None, 'LTX: VAE decode in progress...' + try: + frames = vae_decode(latents, decode_timestep, seed) + except AssertionError as e: + yield from abort(e, ok=True) + return + except Exception as e: + yield from abort(e, ok=False) + return + t10 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t11 = time.time() + timer.process.add('offload', t11 - t10) + shared.state.end() + progress.finish_task(task_id) + + t_end = time.time() + num_frames = len(frames) + resolution = f'{frames[0].width}x{frames[0].height}' if num_frames > 0 else None + summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') + memory = shared.mem_mon.summary() + fps = f'{num_frames/(t_end-t0):.2f}' + its = f'{(steps)/(t_end-t0):.2f}' + + shared.log.info(f'Processed: frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') + yield frames, f'LTX: Generation completed | Frames {len(frames)} | Resolution {resolution} | f/s {fps} | it/s {its} '+ f"

{summary} {memory}

" diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py new file mode 100644 index 000000000..ec003f6de --- /dev/null +++ b/modules/ltx/ltx_ui.py @@ -0,0 +1,79 @@ +import os +import gradio as gr +from modules import shared, sd_models, ui_sections, ui_symbols +from modules.ui_components import ToolButton +from modules.ltx import ltx_process + + +debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def create_ui(prompt, negative, styles, overrides): + with gr.Row(): + with gr.Column(variant='compact', elem_id="ltx_settings", elem_classes=['settings-column'], scale=1): + with gr.Row(): + generate = gr.Button('Generate', elem_id="ltx_generate_btn", variant='primary', visible=False) + with gr.Accordion(open=True, label="Size", elem_id='ltx_generate_accordion'): + with gr.Row(): + width, height = ui_sections.create_resolution_inputs('ltx', default_width=704, default_height=512) + with gr.Row(): + frames = gr.Slider(label='Frames', minimum=1, maximum=513, step=1, value=17, elem_id="ltx_frames") + seed = gr.Number(label='Initial seed', value=-1, elem_id="ltx_seed", container=True) + random_seed = ToolButton(ui_symbols.random, elem_id="ltx_random_seed") + with gr.Accordion(open=False, label="Condition", elem_id='ltx_condition_accordion'): + with gr.Tabs(): + with gr.Tab('Image', id='ltx_condition_image_tab'): + condition_image_strength = gr.Slider(label='Condition strength', minimum=0.1, maximum=1.0, step=0.05, value=0.8, elem_id="ltx_condition_image_strength") + condition_image = gr.Image(label='Image', type='filepath', elem_id="ltx_condition_image", visible=False) + with gr.Tab('Video', id='ltx_condition_video_tab'): + condition_video_strength = gr.Slider(label='Condition strength', minimum=0.1, maximum=1.0, step=0.05, value=0.8, elem_id="ltx_condition_video_strength") + condition_video_frames = gr.Slider(label='Condition frames', minimum=1, maximum=1024, step=1, value=15, elem_id="ltx_condition_video_frames") + condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", visible=False) + with gr.Accordion(open=False, label="Upsample", elem_id='ltx_upsample_accordion'): + upsample_enable = gr.Checkbox(label='Enable upsampling', value=False, elem_id="ltx_upsample_enable") + upsample_ratio = gr.Slider(label='Upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio", interactive=False) + with gr.Accordion(open=False, label="Refine", elem_id='ltx_refine_accordion'): + refine_enable = gr.Checkbox(label='Enable refinement', value=False, elem_id="ltx_refine_enable") + refine_strength = gr.Slider(label='Refine strength', minimum=0.1, maximum=1.0, step=0.05, value=0.4, elem_id="ltx_refine_strength") + with gr.Accordion(open=False, label="Advanced", elem_id='ltx_parameters_accordion'): + steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "ltx", default_steps=50) + with gr.Row(): + decode_timestep = gr.Slider(label='Decode timestep', minimum=0.01, maximum=1.0, step=0.01, value=0.05, elem_id="ltx_decode_timestep") + image_cond_noise_scale = gr.Slider(label='Noise scale', minimum=0.01, maximum=1.0, step=0.01, value=0.025, elem_id="ltx_image_cond_noise_scale") + + with gr.Column(elem_id='ltx-output-column', scale=2) as _column_output: + with gr.Row(): + # video = gr.Video(label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512, autoplay=False) + video = gr.Gallery(value=[], label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512) + with gr.Row(): + text = gr.HTML('', elem_id='ltx_generation_info', show_label=False) + + random_seed.click(fn=lambda: -1, show_progress=False, inputs=[], outputs=[seed]) + task_id = gr.Textbox(visible=False, value='') + ui_state = gr.Textbox(visible=False, value='') + state_inputs = [task_id, ui_state] + + video_inputs = [ + prompt, negative, styles, + width, height, frames, + steps, sampler_index, seed, + upsample_enable, upsample_ratio, + refine_enable, refine_strength, + condition_image_strength, condition_video_strength, condition_video_frames, + condition_image, condition_video, + decode_timestep, image_cond_noise_scale, + overrides, + ] + video_outputs = [ + video, + text, + ] + + video_dict = dict( + fn=ltx_process.run_ltx, + _js="submit_ltx", + inputs=state_inputs + video_inputs, + outputs=video_outputs, + show_progress=False, + ) + generate.click(**video_dict) diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py new file mode 100644 index 000000000..2ee82e7d1 --- /dev/null +++ b/modules/ltx/ltx_util.py @@ -0,0 +1,100 @@ +import time +import torch +from modules import devices, shared, sd_models, timer, extra_networks + + +def get_bucket(size: int): + return int(size) - (int(size) % shared.sd_model.vae_temporal_compression_ratio) + + +def get_frames(frames: int): + return int(8 * (int(frames) // 8)) + 1 + + +def load_model(engine: str, model: str): + t0 = time.time() + from modules.video_models import models_def, video_load + selected: models_def.Model = [m for m in models_def.models[engine] if m.name == model][0] + shared.log.info(f'Video load: cls={selected.repo_cls.__name__} repo="{selected.repo}"') + video_load.load_model(selected) + t1 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + t2 = time.time() + timer.process.add('load', t1 - t0) + timer.process.add('offload', t2 - t1) + + +def load_upsample(upsample_pipe, upsample_repo_id): + if upsample_pipe is None: + t0 = time.time() + shared.state.begin('Load') + from diffusers.pipelines.ltx.pipeline_ltx_latent_upsample import LTXLatentUpsamplePipeline + shared.log.info(f'Video load: cls={LTXLatentUpsamplePipeline.__class__.__name__} repo="{upsample_repo_id}"') + upsample_pipe = LTXLatentUpsamplePipeline.from_pretrained( + upsample_repo_id, + vae=shared.sd_model.vae, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + shared.state.end() + t1 = time.time() + timer.process.add('load', t1 - t0) + return upsample_pipe + + +def get_conditions(condition_image, condition_image_strength, condition_video, condition_video_strength, condition_video_frames): + def get_video_frames(fn: str): + pass + + conditions = [] + if condition_image is not None: + from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXImageCondition + conditions.append(LTXImageCondition(image=condition_image, strength=condition_image_strength)) + if condition_video is not None: + from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition + condition_frames = get_video_frames(condition_video, num_frames=condition_video_frames) + conditions.append(LTXVideoCondition(video=condition_frames, frame_index=0, strength=condition_video_strength)) + return conditions + + +def get_prompts(prompt, negative, styles): + prompt = shared.prompt_styles.apply_styles_to_prompt(prompt, styles) + negative = shared.prompt_styles.apply_negative_styles_to_prompt(negative, styles) + prompts, networks = extra_networks.parse_prompts([prompt]) + prompt = prompts[0] if len(prompts) > 0 else prompt + return prompt, negative, networks + + +def get_generator(seed): + import random + if seed is None or seed < 0: + random.seed() + seed = int(random.randrange(4294967294)) + return torch.Generator().manual_seed(seed) + + +def vae_decode(latents, decode_timestep, seed): + t0 = time.time() + shared.state.begin('VAE') + shared.log.debug(f'Video: cls={shared.sd_model.vae.__class__.__name__} op=vae latents={latents.shape} timestep={decode_timestep}') + from diffusers.utils.torch_utils import randn_tensor + latents = shared.sd_model._denormalize_latents( + latents, + shared.sd_model.vae.latents_mean, + shared.sd_model.vae.latents_std, + shared.sd_model.vae.config.scaling_factor + ) + latents = latents.to(device=devices.device, dtype=devices.dtype) + if not shared.sd_model.vae.config.timestep_conditioning: + timestep = None + else: + noise = randn_tensor(latents.shape, generator=get_generator(seed), device=devices.device, dtype=devices.dtype) + timestep = torch.tensor([decode_timestep], device=devices.device, dtype=latents.dtype) + noise_scale = torch.tensor([decode_timestep], device=devices.device, dtype=devices.dtype)[:, None, None, None, None] + latents = (1 - noise_scale) * latents + noise_scale * noise + frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] + frames = shared.sd_model.video_processor.postprocess_video(frames, output_type='pil') + shared.state.end() + t1 = time.time() + timer.process.add('vae', t1 - t0) + return frames[0] diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 409a18f1f..72429cf55 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -55,11 +55,11 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} t0 = time.time() if devices.backend == "zluda": torch.cuda.synchronize(devices.device) - if p is None: - return kwargs latents = kwargs.get('latents', None) if debug: debug_callback(f'Callback: step={step} timestep={timestep} latents={latents.shape if latents is not None else None} kwargs={list(kwargs)}') + if shared.state.sampling_steps == 0 and getattr(pipe, 'num_timesteps', 0) > 0: + shared.state.sampling_steps = pipe.num_timesteps shared.state.step() if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') @@ -73,6 +73,8 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} return kwargs elif shared.opts.nan_skip: assert not torch.isnan(latents[..., 0, 0]).all(), f'NaN detected at step {step}: Skipping...' + if p is None: + return kwargs if len(getattr(p, 'ip_adapter_names', [])) > 0 and p.ip_adapter_names[0] != 'None': ip_adapter_scales = list(p.ip_adapter_scales) ip_adapter_starts = list(p.ip_adapter_starts) diff --git a/modules/ui_sections.py b/modules/ui_sections.py index ab436d71b..100fea3f7 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -200,12 +200,12 @@ def create_correction_inputs(tab): return hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio -def create_sampler_and_steps_selection(choices, tabname): +def create_sampler_and_steps_selection(choices, tabname, default_steps:int=20): if choices is None: sd_samplers.set_samplers() choices = [x for x in sd_samplers.samplers if not x.name == 'Same as primary'] with gr.Row(elem_classes=['flex-break']): - steps = gr.Slider(minimum=1, maximum=100, step=1, label="Steps", elem_id=f"{tabname}_steps", value=20) + steps = gr.Slider(minimum=1, maximum=100, step=1, label="Steps", elem_id=f"{tabname}_steps", value=default_steps) sampler_index = gr.Dropdown(label='Sampling method', elem_id=f"{tabname}_sampling", choices=[x.name for x in choices], value='Default', type="index") return steps, sampler_index diff --git a/modules/ui_video.py b/modules/ui_video.py index cf48f591d..e12f08bc9 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -27,12 +27,15 @@ def create_ui(): with gr.Row(elem_id="video_interface", equal_height=False): with gr.Tabs(elem_classes=['video-tabs'], elem_id='video-tabs'): overrides = ui_common.create_override_inputs('video') - with gr.Tab('Video', id='video-tab') as video_tab: + with gr.Tab('Generic', id='video-tab') as video_tab: from modules.video_models import video_ui video_ui.create_ui(prompt, negative, styles, overrides) with gr.Tab('FramePack', id='framepack-tab') as framepack_tab: from modules.framepack import framepack_ui framepack_ui.create_ui(prompt, negative, styles, overrides) + with gr.Tab('LTX', id='ltx-tab') as ltx_tab: + from modules.ltx import ltx_ui + ltx_ui.create_ui(prompt, negative, styles, overrides) paste_fields = [ (prompt, "Prompt"), # cannot add more fields as they are not defined yet @@ -44,6 +47,8 @@ def create_ui(): current_tab = gr.Textbox(visible=False, value='video') video_tab.select(fn=lambda: 'video', inputs=[], outputs=[current_tab]) framepack_tab.select(fn=lambda: 'framepack', inputs=[], outputs=[current_tab]) + ltx_tab.select(fn=lambda: 'ltx', inputs=[], outputs=[current_tab]) + generate_btn.click(fn=None, _js='submit_video_wrapper', inputs=[current_tab], outputs=[]) # from framepack_api import create_api # pylint: disable=wrong-import-order diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index c26d3fd40..8122fa031 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -71,6 +71,12 @@ models = { ], 'LTX Video': [ Model(name='None'), + Model(name='LTXVideo 0.9.7 13B', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.7-dev', + repo='a-r-r-o-w/LTX-Video-0.9.7-diffusers', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), Model(name='LTXVideo 0.9.6 2B T2V', url='https://huggingface.co/Lightricks/LTX-Video', repo='Lightricks/LTX-Video', diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 2a959f0d3..a798d26b3 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -10,6 +10,7 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e def load_model(selected: models_def.Model): + shared.state.begin('Load') if selected is None: return '' global loaded_model # pylint: disable=global-statement @@ -83,14 +84,20 @@ def load_model(selected: models_def.Model): shared.sd_model.vae.orig_encode = shared.sd_model.vae.encode shared.sd_model.vae.encode = video_vae.hijack_vae_encode if selected.te_hijack and hasattr(shared.sd_model, 'encode_prompt'): - shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt + # shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt sd_hijack_te.init_hijack(shared.sd_model) if selected.image_hijack and hasattr(shared.sd_model, 'encode_image'): shared.sd_model.orig_encode_image = shared.sd_model.encode_image shared.sd_model.encode_image = video_utils.hijack_encode_image if hasattr(shared.sd_model.vae, 'enable_slicing'): shared.sd_model.vae.enable_slicing() + if hasattr(shared.sd_model.vae, 'enable_tiling'): + shared.sd_model.vae.enable_tiling() + if hasattr(shared.sd_model, "set_progress_bar_config"): + shared.sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba') + loaded_model = selected.name msg = f'Video load: cls={shared.sd_model.__class__.__name__} model="{selected.name}" time={t1-t0:.2f}' shared.log.info(msg) + shared.state.end() return msg