diff --git a/CHANGELOG.md b/CHANGELOG.md index da8b359ba..d65539336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2025-07-09 +## Update for 2025-07-10 -### Highlights for 2025-07-09 +### Highlights for 2025-07-10 In this release we finally break with legacy with the removal of the original [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui/) codebase which has not been maintained for a while now This plus major cleanup of codebase and external dependencies resulted in ~53k LoC (*lines-of-code*) reduction and spread over [~680 files](https://github.com/vladmandic/sdnext/pull/4017)! @@ -11,19 +11,19 @@ We also switched project license to [Apache-2.0](https://github.com/vladmandic/s Feature highlights include: - **ModernUI** layout redesign which should make it more user friendly and easier to navigate -- Redesigned **Video** interface with native **FramePack** support +- Redesigned [LTXVideo](https://vladmandic.github.io/sdnext-docs/Video) interface with support for general video models plus optimized [FramePack](https://vladmandic.github.io/sdnext-docs/FramePack) and [LTXVideo](https://vladmandic.github.io/sdnext-docs/LTX) support - New background replacement and relightning methods using **Latent Bridge Matching** and new **PixelArt** processing filter - New **LLM/VLM** models available for captioning and prompt enhance - Compute improvements -And (as always) many bugfixes and improvements to existing features! +And (*as always*) many bugfixes and improvements to existing features! *Note*: We recommend clean install for this release due to sheer size of changes Although upgrades and existing installations are tested and should work fine! [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-07-09 +### Details for 2025-07-10 - **License** - SD.Next [license](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) switched from **aGPL-v3.0** to **Apache-v2.0** @@ -35,14 +35,27 @@ Although upgrades and existing installations are tested and should work fine! *relighting*: automatic background replacement with reglighting so source image fits desired background with optional composite blending available in *img2img or control -> scripts* - - Add **FLUX.1-Kontext-Dev** inpaint workflow - - Support **FLUX.1** all-in-one safetensors - - Support **TAESD** preview and remote VAE for **HunyuanDit** - - Support for [Gemma 3n](https://huggingface.co/google/gemma-3n-E4B-it) E2B and E4B LLM/VLM models in **prompt enhance** and process **captioning** - - **FramePack** support is now fully integrated instead of being a separate extension + - add **FLUX.1-Kontext-Dev** inpaint workflow + - support **FLUX.1** all-in-one safetensors + - support **TAESD** preview and remote VAE for **HunyuanDit** + - support for [Gemma 3n](https://huggingface.co/google/gemma-3n-E4B-it) E2B and E4B LLM/VLM models in **prompt enhance** and process **captioning** +- **Video** + - redesigned **Video** interface + - support for **Generic** video models + includes support for many video models without specific per-model optimizations + included: *Hunyuan, LTX, WAN, Mochi, Latte, Allegro, Cog* + supports quantization, offloading, frame interpolation, etc. + - support for optimized [FramePack](https://vladmandic.github.io/sdnext-docs/FramePack) + with *t2i, i2i, flf2v* workflows + LoRA support, prompt enhance, etc. + now fully integrated instead of being a separate extension + - support for optmized [LTXVideo](https://vladmandic.github.io/sdnext-docs/LTX) + with *t2i, i2i, v2v* workflows + optional native upsampling and video refine workflows + LoRA support with different conditioning types such as Canny/Depth/Pose, etc. - **UI** - major update to modernui layout - - redesign of the Flat UI theme + - redesign of the *Flat* UI theme - **WIKI** - new [Parameters](https://vladmandic.github.io/sdnext-docs/Parameters/) page that lists and explains all generation parameters - **Compute** @@ -68,6 +81,7 @@ Although upgrades and existing installations are tested and should work fine! - fix diffusers models non-unique hash - fix loading of manually downloaded diffuser models - fix api `/sdapi/v1/embeddings` endpoint + - fix incorrect reporting of deleted and modified files - improve infotext param parsing - improve extensions ui search - improve model type autodetection diff --git a/TODO.md b/TODO.md index 22199b89c..b30865bf2 100644 --- a/TODO.md +++ b/TODO.md @@ -10,6 +10,12 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Feature: LoRA add OMI format support for SD35/FLUX.1 - Refactor: sampler options - Video: API support +- Remove: CodeFormer +- Remove: GFPGAN +- Video: LTX TeaCache and others +- Video: LTX API +- Video: LTX PromptEnhance +- Video: LTX Condition preprocess ### Blocked items @@ -40,7 +46,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma #### Stable - [Diffusers-0.34.0](https://github.com/huggingface/diffusers/releases/tag/v0.34.0) - [WanAI-2.1 VACE](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B)(https://github.com/huggingface/diffusers/pull/11582) -- [LTXVideo-0.9.7](https://github.com/Lightricks/LTX-Video?tab=readme-ov-file#diffusers-integration)(https://github.com/huggingface/diffusers/pull/11516) - [Cosmos-Predict2-Video](https://huggingface.co/nvidia/Cosmos-Predict2-2B-Video2World)(https://github.com/huggingface/diffusers/pull/11695) #### Pending - [Magi](https://github.com/SandAI-org/MAGI-1)(https://github.com/huggingface/diffusers/pull/11713) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 9a4ff1ead..54f844d29 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 9a4ff1ead28257cd399e10f202b98d8c38c33b1a +Subproject commit 54f844d291a003339fff5d96ed913a79acd4d810 diff --git a/installer.py b/installer.py index c2e4f5d39..0e9d0ca4f 100644 --- a/installer.py +++ b/installer.py @@ -949,10 +949,10 @@ def check_modified_files(): files = [x for x in files if len(x) > 0 and (not x.startswith('extensions')) and (not x.startswith('wiki')) and (not x.endswith('.json')) and ('.log' not in x)] deleted = [x for x in files if not os.path.exists(x)] if len(deleted) > 0: - log.warning(f'Deleted files: {files}') - files = [x for x in files if os.path.exists(x) and not os.path.isdir(x)] - if len(files) > 0: - log.warning(f'Modified files: {files}') + log.warning(f'Deleted files: {deleted}') + modified = [x for x in files if os.path.exists(x) and not os.path.isdir(x)] + if len(modified) > 0: + log.warning(f'Modified files: {modified}') except Exception: pass ts('files', t_start) diff --git a/modules/framepack/framepack_ui.py b/modules/framepack/framepack_ui.py index 48cc2da44..d2f6fd091 100644 --- a/modules/framepack/framepack_ui.py +++ b/modules/framepack/framepack_ui.py @@ -1,8 +1,9 @@ import gradio as gr from modules import ui_sections, ui_common, ui_video_vlm +from modules.video_models.video_utils import get_codecs from modules.framepack import framepack_load from modules.framepack.framepack_worker import get_latent_paddings -from modules.framepack.framepack_wrappers import get_codecs, load_model, unload_model +from modules.framepack.framepack_wrappers import load_model, unload_model from modules.framepack.framepack_wrappers import run_framepack # pylint: disable=wrong-import-order diff --git a/modules/framepack/framepack_worker.py b/modules/framepack/framepack_worker.py index 081345a91..8ce562b8e 100644 --- a/modules/framepack/framepack_worker.py +++ b/modules/framepack/framepack_worker.py @@ -4,7 +4,7 @@ import rich.progress as rp from modules import shared, errors ,devices, sd_models, timer, memstats from modules.framepack import framepack_vae # pylint: disable=wrong-import-order from modules.framepack import framepack_hijack # pylint: disable=wrong-import-order -from modules.framepack import framepack_video # pylint: disable=wrong-import-order +from modules.video_models.video_save import save_video # pylint: disable=wrong-import-order stream = None # AsyncStream @@ -302,12 +302,12 @@ def worker( if is_last_section: break - total_generated_frames = framepack_video.save_video(history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate, pbar=pbar, stream=stream, metadata=metadata) + total_generated_frames, _video_filename = save_video(history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate, pbar=pbar, stream=stream, metadata=metadata) except AssertionError: shared.log.info('FramePack: interrupted') if shared.opts.keep_incomplete: - framepack_video.save_video(history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate=0, stream=stream, metadata=metadata) + save_video(history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate=0, stream=stream, metadata=metadata) except Exception as e: shared.log.error(f'FramePack: {e}') errors.display(e, 'FramePack') diff --git a/modules/framepack/framepack_wrappers.py b/modules/framepack/framepack_wrappers.py index 0c3b0d345..959bc99e5 100644 --- a/modules/framepack/framepack_wrappers.py +++ b/modules/framepack/framepack_wrappers.py @@ -6,6 +6,7 @@ import numpy as np import torch import gradio as gr from modules import shared, processing, timer, paths, extra_networks, progress, ui_video_vlm +from modules.video_models.video_utils import check_av from modules.framepack import framepack_install # pylint: disable=wrong-import-order from modules.framepack import framepack_load # pylint: disable=wrong-import-order from modules.framepack import framepack_worker # pylint: disable=wrong-import-order @@ -20,38 +21,6 @@ queue_lock = threading.Lock() loaded_variant = None -def check_av(): - try: - import av - except Exception as e: - shared.log.error(f'av package: {e}') - return False - return av - - -def get_codecs(): - av = check_av() - if av is None: - return [] - codecs = [] - for codec in av.codecs_available: - try: - c = av.Codec(codec, mode='w') - if c.type == 'video' and c.is_encoder and len(c.video_formats) > 0: - if not any(c.name == ca.name for ca in codecs): - codecs.append(c) - except Exception: - pass - hw_codecs = [c for c in codecs if (c.capabilities & 0x40000 > 0) or (c.capabilities & 0x80000 > 0)] - sw_codecs = [c for c in codecs if c not in hw_codecs] - shared.log.debug(f'Video codecs: hardware={len(hw_codecs)} software={len(sw_codecs)}') - # for c in hw_codecs: - # shared.log.trace(f'codec={c.name} cname="{c.canonical_name}" decs="{c.long_name}" intra={c.intra_only} lossy={c.lossy} lossless={c.lossless} capabilities={c.capabilities} hw=True') - # for c in sw_codecs: - # shared.log.trace(f'codec={c.name} cname="{c.canonical_name}" decs="{c.long_name}" intra={c.intra_only} lossy={c.lossy} lossless={c.lossless} capabilities={c.capabilities} hw=False') - return ['none'] + [c.name for c in hw_codecs + sw_codecs] - - def prepare_image(image, resolution): from modules.framepack.pipeline.utils import resize_and_center_crop buckets = [ diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 8f1787501..4a857561c 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -1,17 +1,13 @@ """ -- condition upload image -- condition upload video -- condition video get frames -- custom sampler -- new way of generate video - modernui -- lora loader +- teacache and others """ -# import diffusers.LTXConditionPipeline import os import time import threading -from modules import shared, sd_models, errors, timer, memstats, progress +from modules import shared, errors, timer, memstats, progress, processing, sd_models, sd_samplers, extra_networks +from modules.video_models.video_save import save_video +from modules.video_models.video_utils import check_av from modules.processing_callbacks import diffusers_callback from modules.ltx.ltx_util import get_bucket, get_frames, load_model, load_upsample, get_conditions, get_generator, get_prompts, vae_decode @@ -38,26 +34,39 @@ def run_ltx(task_id, upsample_ratio:float, refine_enable:bool, refine_strength:float, - condition_image_strength:float, - condition_video_strength:float, - condition_video_frames:int, + condition_strength: float, condition_image, + condition_files, condition_video, + condition_video_frames:int, + condition_video_skip:int, decode_timestep:float, image_cond_noise_scale:float, + mp4_fps:int, + mp4_interpolate:int, + mp4_codec:str, + mp4_ext:str, + mp4_opt:str, + mp4_video:bool, + mp4_frames:bool, + mp4_sf:bool, _overrides, ): - def abort(e, ok:bool=False): + def abort(e, ok:bool=False, p=None): if ok: shared.log.info(e) else: shared.log.error(f'Video: cls={shared.sd_model.__class__.__name__} op=base {e}') errors.display(e, 'LTX') + if p is not None: + extra_networks.deactivate(p) shared.state.end() progress.finish_task(task_id) yield None, f'LTX Error: {str(e)}' + from diffusers import LTXConditionPipeline # pylint: disable=unused-import + check_av() progress.add_task_to_queue(task_id) with queue_lock: progress.start_task(task_id) @@ -69,16 +78,34 @@ def run_ltx(task_id, shared.state.begin('Video', task_id=task_id) shared.state.job_count = 1 + p = processing.StableDiffusionProcessingVideo( + prompt=prompt, + negative_prompt=negative, + styles=styles, + width=width, + height=height, + frames=frames, + steps=steps, + sampler_index=sampler_index, + seed=seed, + ) + conditions = get_conditions( + width, + height, + condition_strength, condition_image, - condition_image_strength, + condition_files, condition_video, - condition_video_strength, condition_video_frames, + condition_video_skip, ) prompt, negative, networks = get_prompts(prompt, negative, styles) - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init prompt="{prompt}" negative="{negative}" styles={styles} networks={networks}') + sampler_name = processing.get_sampler_name(sampler_index) + sd_samplers.create_sampler(sampler_name, shared.sd_model) + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=init prompt="{prompt}" negative="{negative}" styles={styles} networks={networks} sampler={shared.sd_model.scheduler.__class__.__name__}') + extra_networks.activate(p, networks) t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) @@ -95,17 +122,17 @@ def run_ltx(task_id, "callback_on_step_end": diffusers_callback, "output_type": "latent", } + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=base {base_args}') if len(conditions) > 0: base_args["conditions"] = conditions - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=base {base_args}') yield None, 'LTX: Generate in progress...' try: latents = shared.sd_model(**base_args).frames[0] except AssertionError as e: - yield from abort(e, ok=True) + yield from abort(e, ok=True, p=p) return except Exception as e: - yield from abort(e, ok=False) + yield from abort(e, ok=False, p=p) return t2 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) @@ -117,7 +144,7 @@ def run_ltx(task_id, if upsample_enable: t4 = time.time() shared.state.begin('Upsample') - global upsample_pipe # python-lint: disable=global-statement + global upsample_pipe # pylint: disable=global-statement upsample_pipe = load_upsample(upsample_pipe, upsample_repo_id) upsample_pipe = sd_models.apply_balanced_offload(upsample_pipe) upscale_args = { @@ -133,10 +160,10 @@ def run_ltx(task_id, try: upsampled_latents = upsample_pipe(latents=latents, **upscale_args).frames[0] except AssertionError as e: - yield from abort(e, ok=True) + yield from abort(e, ok=True, p=p) return except Exception as e: - yield from abort(e, ok=False) + yield from abort(e, ok=False, p=p) return latents = upsampled_latents t5 = time.time() @@ -163,19 +190,19 @@ def run_ltx(task_id, "callback_on_step_end": diffusers_callback, "output_type": "latent", } - if len(conditions) > 0: - refine_args["conditions"] = conditions if latents.ndim == 4: latents = latents.unsqueeze(0) # add batch dimension shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=refine latents={latents.shape} {refine_args}') + if len(conditions) > 0: + refine_args["conditions"] = conditions yield None, 'LTX: Refine in progress...' try: refined_latents = shared.sd_model(latents=latents, **refine_args).frames[0] except AssertionError as e: - yield from abort(e, ok=True) + yield from abort(e, ok=True, p=p) return except Exception as e: - yield from abort(e, ok=False) + yield from abort(e, ok=False, p=p) return latents = refined_latents t8 = time.time() @@ -185,29 +212,45 @@ def run_ltx(task_id, timer.process.add('offload', t9 - t8) shared.state.end() + extra_networks.deactivate(p) + yield None, 'LTX: VAE decode in progress...' try: frames = vae_decode(latents, decode_timestep, seed) except AssertionError as e: - yield from abort(e, ok=True) + yield from abort(e, ok=True, p=p) return except Exception as e: - yield from abort(e, ok=False) + yield from abort(e, ok=False, p=p) return t10 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) t11 = time.time() timer.process.add('offload', t11 - t10) - shared.state.end() - progress.finish_task(task_id) + + num_frames, video_file = save_video( + pixels=frames, + mp4_fps=mp4_fps, + mp4_codec=mp4_codec, + mp4_opt=mp4_opt, + mp4_ext=mp4_ext, + mp4_sf=mp4_sf, + mp4_video=mp4_video, + mp4_frames=mp4_frames, + mp4_interpolate=mp4_interpolate, + metadata={}, + ) t_end = time.time() - num_frames = len(frames) - resolution = f'{frames[0].width}x{frames[0].height}' if num_frames > 0 else None + _n, _c, _t, h, w = frames.shape + resolution = f'{w}x{h}' if num_frames > 0 else None summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') memory = shared.mem_mon.summary() fps = f'{num_frames/(t_end-t0):.2f}' its = f'{(steps)/(t_end-t0):.2f}' - shared.log.info(f'Processed: frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') - yield frames, f'LTX: Generation completed | Frames {len(frames)} | Resolution {resolution} | f/s {fps} | it/s {its} '+ f"

{summary} {memory}

" + shared.state.end() + progress.finish_task(task_id) + + shared.log.info(f'Processed: fn="{video_file}" frames={num_frames} fps={fps} its={its} resolution={resolution} time={t_end-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') + yield video_file, f'LTX: Generation completed | File {video_file} | Frames {len(frames)} | Resolution {resolution} | f/s {fps} | it/s {its} '+ f"

{summary} {memory}

" diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index d8a50f625..fe5699d0b 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -1,7 +1,8 @@ import os import gradio as gr -from modules import shared, ui_sections, ui_symbols +from modules import shared, ui_sections, ui_symbols, ui_common from modules.ui_components import ToolButton +from modules.video_models.video_utils import get_codecs from modules.ltx import ltx_process @@ -15,26 +16,44 @@ def create_ui(prompt, negative, styles, overrides): generate = gr.Button('Generate', elem_id="ltx_generate_btn", variant='primary', visible=False) with gr.Accordion(open=True, label="Size", elem_id='ltx_generate_accordion'): with gr.Row(): - width, height = ui_sections.create_resolution_inputs('ltx', default_width=704, default_height=512) + width, height = ui_sections.create_resolution_inputs('ltx', default_width=832, default_height=480) with gr.Row(): frames = gr.Slider(label='Frames', minimum=1, maximum=513, step=1, value=17, elem_id="ltx_frames") seed = gr.Number(label='Initial seed', value=-1, elem_id="ltx_seed", container=True) random_seed = ToolButton(ui_symbols.random, elem_id="ltx_random_seed") with gr.Accordion(open=False, label="Condition", elem_id='ltx_condition_accordion'): + condition_strength = gr.Slider(label='Condition strength', minimum=0.1, maximum=1.0, step=0.05, value=0.8, elem_id="ltx_condition_image_strength") with gr.Tabs(): with gr.Tab('Image', id='ltx_condition_image_tab'): - condition_image_strength = gr.Slider(label='Condition strength', minimum=0.1, maximum=1.0, step=0.05, value=0.8, elem_id="ltx_condition_image_strength") - condition_image = gr.Image(label='Image', type='filepath', elem_id="ltx_condition_image", visible=False) + condition_image = gr.Image(sources='upload', type="pil", label="Image", width=256, height=256, interactive=True, tool="editor", image_mode='RGB', elem_id="ltx_condition_image") with gr.Tab('Video', id='ltx_condition_video_tab'): - condition_video_strength = gr.Slider(label='Condition strength', minimum=0.1, maximum=1.0, step=0.05, value=0.8, elem_id="ltx_condition_video_strength") - condition_video_frames = gr.Slider(label='Condition frames', minimum=1, maximum=1024, step=1, value=15, elem_id="ltx_condition_video_frames") - condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", visible=False) + condition_video = gr.Video(label='Video', type='filepath', elem_id="ltx_condition_video", width=256, height=256, source='upload') + with gr.Row(): + condition_video_frames = gr.Slider(label='Frames number', minimum=-1, maximum=1024, step=1, value=-1, elem_id="ltx_condition_video_frames") + condition_video_skip = gr.Slider(label='Frames skip', minimum=0, maximum=1024, step=1, value=0, elem_id="ltx_condition_video_sip") + with gr.Tab('Gallery', id='ltx_condition_batch_tab'): + condition_files = gr.Files(label="Image Batch", interactive=True, elem_id="ltx_condition_batch") with gr.Accordion(open=False, label="Upsample", elem_id='ltx_upsample_accordion'): - upsample_enable = gr.Checkbox(label='Enable upsampling', value=False, elem_id="ltx_upsample_enable") - upsample_ratio = gr.Slider(label='Upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio", interactive=False) + with gr.Row(): + upsample_enable = gr.Checkbox(label='Enable upsampling', value=False, elem_id="ltx_upsample_enable") + upsample_ratio = gr.Slider(label='Upsample ratio', minimum=1.0, maximum=4.0, step=0.1, value=2.0, elem_id="ltx_upsample_ratio", interactive=False) with gr.Accordion(open=False, label="Refine", elem_id='ltx_refine_accordion'): - refine_enable = gr.Checkbox(label='Enable refinement', value=False, elem_id="ltx_refine_enable") - refine_strength = gr.Slider(label='Refine strength', minimum=0.1, maximum=1.0, step=0.05, value=0.4, elem_id="ltx_refine_strength") + with gr.Row(): + refine_enable = gr.Checkbox(label='Enable refinement', value=False, elem_id="ltx_refine_enable") + refine_strength = gr.Slider(label='Refine strength', minimum=0.1, maximum=1.0, step=0.05, value=0.4, elem_id="ltx_refine_strength") + with gr.Accordion(label="Video", open=False): + with gr.Row(): + mp4_fps = gr.Slider(label="FPS", minimum=1, maximum=60, value=24, step=1) + mp4_interpolate = gr.Slider(label="Interpolation", minimum=0, maximum=10, value=0, step=1) + with gr.Row(): + mp4_codec = gr.Dropdown(label="Codec", choices=['none', 'libx264'], value='libx264', type='value') + ui_common.create_refresh_button(mp4_codec, get_codecs) + mp4_ext = gr.Textbox(label="Format", value='mp4', elem_id="framepack_mp4_ext") + mp4_opt = gr.Textbox(label="Options", value='crf:16', elem_id="framepack_mp4_ext") + with gr.Row(): + mp4_video = gr.Checkbox(label='Save Video', value=True, elem_id="framepack_mp4_video") + mp4_frames = gr.Checkbox(label='Save Frames', value=False, elem_id="framepack_mp4_frames") + mp4_sf = gr.Checkbox(label='Save SafeTensors', value=False, elem_id="framepack_mp4_sf") with gr.Accordion(open=False, label="Advanced", elem_id='ltx_parameters_accordion'): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "ltx", default_steps=50) with gr.Row(): @@ -43,8 +62,8 @@ def create_ui(prompt, negative, styles, overrides): with gr.Column(elem_id='ltx-output-column', scale=2) as _column_output: with gr.Row(): - # video = gr.Video(label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512, autoplay=False) - video = gr.Gallery(value=[], label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512) + video = gr.Video(label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512, autoplay=False) + # video = gr.Gallery(value=[], label="Output", show_label=False, elem_id='ltx_output_video', elem_classes=['control-image'], height=512) with gr.Row(): text = gr.HTML('', elem_id='ltx_generation_info', show_label=False) @@ -59,9 +78,9 @@ def create_ui(prompt, negative, styles, overrides): steps, sampler_index, seed, upsample_enable, upsample_ratio, refine_enable, refine_strength, - condition_image_strength, condition_video_strength, condition_video_frames, - condition_image, condition_video, + condition_strength, condition_image, condition_files, condition_video, condition_video_frames, condition_video_skip, decode_timestep, image_cond_noise_scale, + mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, overrides, ] video_outputs = [ diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index 1bf4f0aac..9906094d6 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -1,5 +1,6 @@ import time import torch +from PIL import Image from modules import devices, shared, sd_models, timer, extra_networks @@ -42,17 +43,43 @@ def load_upsample(upsample_pipe, upsample_repo_id): return upsample_pipe -def get_conditions(condition_image, condition_image_strength, condition_video, condition_video_strength, condition_video_frames): - def get_video_frames(fn: str): - pass - +def get_conditions(width, height, condition_strength, condition_image, condition_files, condition_video, condition_video_frames, condition_video_skip): from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition conditions = [] if condition_image is not None: - conditions.append(LTXVideoCondition(image=condition_image, strength=condition_image_strength)) + try: + if isinstance(condition_image, str): + from modules.api.api import decode_base64_to_image + condition_image = decode_base64_to_image(condition_image) + condition_image = condition_image.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) + conditions.append(LTXVideoCondition(image=condition_image, frame_index=0, strength=condition_strength)) + shared.log.debug(f'Video condition: image={condition_image.size} strength={condition_strength}') + except Exception as e: + shared.log.error(f'LTX condition image: {e}') + if condition_files is not None: + condition_images = [] + for fn in condition_files: + try: + if hasattr(fn, 'name'): + condition_image = Image.open(fn.name).convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) + else: + condition_image = fn.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) + condition_images.append(condition_image) + except Exception as e: + shared.log.error(f'LTX condition files: {e}') + if len(condition_images) > 0: + conditions.append(LTXVideoCondition(video=condition_images, frame_index=0, strength=condition_strength)) + shared.log.debug(f'Video condition: files={len(condition_images)} size={condition_images[0].size} strength={condition_strength}') if condition_video is not None: - condition_frames = get_video_frames(condition_video, num_frames=condition_video_frames) - conditions.append(LTXVideoCondition(video=condition_frames, frame_index=0, strength=condition_video_strength)) + from modules.video_models.video_utils import get_video_frames + try: + condition_frames = get_video_frames(condition_video, num_frames=condition_video_frames, skip_frames=condition_video_skip) + condition_frames = [f.convert('RGB').resize((width, height), resample=Image.Resampling.LANCZOS) for f in condition_frames] + if len(condition_frames) > 0: + conditions.append(LTXVideoCondition(video=condition_frames, frame_index=0, strength=condition_strength)) + shared.log.debug(f'Video condition: frames={len(condition_frames)} size={condition_frames[0].size} strength={condition_strength}') + except Exception as e: + shared.log.error(f'LTX condition video: {e}') return conditions @@ -91,9 +118,11 @@ def vae_decode(latents, decode_timestep, seed): timestep = torch.tensor([decode_timestep], device=devices.device, dtype=latents.dtype) noise_scale = torch.tensor([decode_timestep], device=devices.device, dtype=devices.dtype)[:, None, None, None, None] latents = (1 - noise_scale) * latents + noise_scale * noise - frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] - frames = shared.sd_model.video_processor.postprocess_video(frames, output_type='pil') + frames = shared.sd_model.vae.decode(latents, timestep, return_dict=False)[0] # n, c, f, h, w + # frames = frames.squeeze(0) if frames.ndim == 5 else frames + # frames = frames.permute(1, 2, 3, 0) + # frames = shared.sd_model.video_processor.postprocess_video(frames, output_type='pil') shared.state.end() t1 = time.time() timer.process.add('vae', t1 - t0) - return frames[0] + return frames diff --git a/modules/framepack/framepack_video.py b/modules/video_models/video_save.py similarity index 73% rename from modules/framepack/framepack_video.py rename to modules/video_models/video_save.py index 4de9515c9..274ff69a1 100644 --- a/modules/framepack/framepack_video.py +++ b/modules/video_models/video_save.py @@ -7,12 +7,18 @@ import einops from modules import shared, errors ,timer, rife +def get_video_filename(frames:int, codec:str): + timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + output_filename = os.path.join(shared.opts.outdir_video, f'{timestamp}-{codec}-f{frames}') + return output_filename + + def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='libx264', pix_fmt:str='yuv420p', options:str='', metadata:dict={}, pbar=None): try: import av av.logging.set_level(av.logging.ERROR) # pylint: disable=c-extension-no-member except Exception as e: - shared.log.error(f'FramePack video: {e}') + shared.log.error(f'Video: {e}') return frames, height, width, _channels = tensor.shape @@ -27,7 +33,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li else: continue options[key.strip()] = value.strip() - shared.log.info(f'FramePack video: file="{filename}" codec={codec} frames={frames} width={width} height={height} fps={rate} options={options}') + shared.log.info(f'Video: file="{filename}" codec={codec} frames={frames} width={width} height={height} fps={rate} options={options}') video_array = torch.as_tensor(tensor, dtype=torch.uint8).numpy(force=True) task = pbar.add_task('encoding', total=frames) if pbar is not None else None if task is not None: @@ -64,13 +70,14 @@ def save_video( metadata:dict={}, # metadata for video pbar=None, # progress bar for video ): + output_video = None if pixels is None: - return 0 + return 0, output_video t_save = time.time() n, _c, t, h, w = pixels.shape size = pixels.element_size() * pixels.numel() - shared.log.debug(f'FramePack video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}') - shared.log.debug(f'FramePack video: encode={t} raw={size} latent={pixels.shape} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"') + shared.log.debug(f'Video: video={mp4_video} export={mp4_frames} safetensors={mp4_sf} interpolate={mp4_interpolate}') + shared.log.debug(f'Video: encode={t} raw={size} latent={pixels.shape} fps={mp4_fps} codec={mp4_codec} ext={mp4_ext} options="{mp4_opt}"') try: if stream is not None: stream.output_queue.push(('progress', (None, 'Saving video...'))) @@ -86,18 +93,17 @@ def save_video( x = einops.rearrange(x, '(m n) c t h w -> t (m h) (n w) c', n=n) x = x.contiguous() - timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') - output_filename = os.path.join(shared.opts.outdir_video, f'{timestamp}-{mp4_codec}-f{t}') + output_filename = get_video_filename(t, mp4_codec) if mp4_sf: fn = f'{output_filename}.safetensors' - shared.log.info(f'FramePack export: file="{fn}" type=savetensors shape={x.shape}') + shared.log.info(f'Video export: file="{fn}" type=savetensors shape={x.shape}') from safetensors.torch import save_file shared.state.outputs(fn) save_file({ 'frames': x }, fn, metadata={'format': 'video', 'frames': str(t), 'width': str(w), 'height': str(h), 'fps': str(mp4_fps), 'codec': mp4_codec, 'options': mp4_opt, 'ext': mp4_ext, 'interpolate': str(mp4_interpolate)}) if mp4_frames: - shared.log.info(f'FramePack frames: files="{output_filename}-00000.jpg" frames={t} width={w} height={h}') + shared.log.info(f'Video frames: files="{output_filename}-00000.jpg" frames={t} width={w} height={h}') for i in range(t): image = cv2.cvtColor(x[i].numpy(), cv2.COLOR_RGB2BGR) fn = f'{output_filename}-{i:05d}.jpg' @@ -105,17 +111,17 @@ def save_video( cv2.imwrite(fn, image) if mp4_video and (mp4_codec != 'none'): - fn = f'{output_filename}.{mp4_ext}' - atomic_save_video(fn, tensor=x, fps=mp4_fps, codec=mp4_codec, options=mp4_opt, metadata=metadata, pbar=pbar) + output_video = f'{output_filename}.{mp4_ext}' + atomic_save_video(output_video, tensor=x, fps=mp4_fps, codec=mp4_codec, options=mp4_opt, metadata=metadata, pbar=pbar) if stream is not None: - stream.output_queue.push(('progress', (None, f'Video {os.path.basename(fn)} | Codec {mp4_codec} | Size {w}x{h}x{t} | FPS {mp4_fps}'))) - stream.output_queue.push(('file', fn)) + stream.output_queue.push(('progress', (None, f'Video {os.path.basename(output_video)} | Codec {mp4_codec} | Size {w}x{h}x{t} | FPS {mp4_fps}'))) + stream.output_queue.push(('file', output_video)) else: if stream is not None: stream.output_queue.push(('progress', (None, ''))) except Exception as e: - shared.log.error(f'FramePack video: raw={size} {e}') - errors.display(e, 'FramePack video') + shared.log.error(f'Video save: raw={size} {e}') + errors.display(e, 'video') timer.process.add('save', time.time()-t_save) - return t + return t, output_video diff --git a/modules/video_models/video_ui.py b/modules/video_models/video_ui.py index 81ac06b96..46b48a2d6 100644 --- a/modules/video_models/video_ui.py +++ b/modules/video_models/video_ui.py @@ -90,12 +90,12 @@ def create_ui(prompt, negative, styles, overrides): with gr.Row(): width, height = ui_sections.create_resolution_inputs('video', default_width=832, default_height=480) with gr.Row(): - frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=15, elem_id="video_frames") + frames = gr.Slider(label='Frames', minimum=1, maximum=1024, step=1, value=17, elem_id="video_frames") seed = gr.Number(label='Initial seed', value=-1, elem_id="video_seed", container=True) random_seed = ToolButton(ui_symbols.random, elem_id="video_random_seed") reuse_seed = ToolButton(ui_symbols.reuse, elem_id="video_reuse_seed") with gr.Accordion(open=False, label="Parameters", elem_id='video_parameters_accordion'): - steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video") + steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "video", default_steps=50) with gr.Row(): sampler_shift = gr.Slider(label='Sampler shift', minimum=-1.0, maximum=20.0, step=0.1, value=-1.0, elem_id="video_scheduler_shift") dynamic_shift = gr.Checkbox(label='Dynamic shift', value=False, elem_id="video_dynamic_shift") diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index b9510dd36..cbca9a3f5 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -1,5 +1,7 @@ import os +import sys import time +from PIL import Image from modules import shared, sd_models, timer, errors, devices @@ -15,6 +17,15 @@ def get_url(url): return f'  {url}

' if url else '

' +def check_av(): + try: + import av + except Exception as e: + shared.log.error(f'av package: {e}') + return False + return av + + def set_prompt(p): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) @@ -38,3 +49,68 @@ def hijack_encode_image(*args, **kwargs): debug(f'Video encode image: te={shared.sd_model.image_encoder.__class__.__name__} time={t1-t0:.2f}') shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) return res + + +def get_codecs(): + av = check_av() + if av is None: + return [] + codecs = [] + for codec in av.codecs_available: + try: + c = av.Codec(codec, mode='w') + if c.type == 'video' and c.is_encoder and len(c.video_formats) > 0: + if not any(c.name == ca.name for ca in codecs): + codecs.append(c) + except Exception: + pass + hw_codecs = [c for c in codecs if (c.capabilities & 0x40000 > 0) or (c.capabilities & 0x80000 > 0)] + sw_codecs = [c for c in codecs if c not in hw_codecs] + shared.log.debug(f'Video codecs: hardware={len(hw_codecs)} software={len(sw_codecs)}') + # for c in hw_codecs: + # shared.log.trace(f'codec={c.name} cname="{c.canonical_name}" decs="{c.long_name}" intra={c.intra_only} lossy={c.lossy} lossless={c.lossless} capabilities={c.capabilities} hw=True') + # for c in sw_codecs: + # shared.log.trace(f'codec={c.name} cname="{c.canonical_name}" decs="{c.long_name}" intra={c.intra_only} lossy={c.lossy} lossless={c.lossless} capabilities={c.capabilities} hw=False') + return ['none'] + [c.name for c in hw_codecs + sw_codecs] + + +def decode_fourcc(cc): + cc_bytes = int(cc).to_bytes(4, byteorder=sys.byteorder) # convert code to a bytearray + cc_str = cc_bytes.decode() # decode byteaarray to a string + return cc_str + + +def get_video_frames(fn: str, num_frames: int = -1, skip_frames: int = 0): + import cv2 + frames = [] + try: + video = cv2.VideoCapture(fn) + if not video.isOpened(): + return frames + frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = int(video.get(cv2.CAP_PROP_FPS)) + w, h = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)), int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + codec = decode_fourcc(video.get(cv2.CAP_PROP_FOURCC)) + skip = 0 + while True: + status, frame = video.read() + if skip_frames > 0: + if skip < skip_frames: + skip += 1 + _status, _frame = video.read() + continue + skip = 0 + if status: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frame = Image.fromarray(frame) + frames.append(frame) + else: + break + if len(frames) >= num_frames > 0: + break + video.release() + shared.log.debug(f'Video open: file="{fn}" frames={len(frames)} total={frame_count} skip={skip} fps={fps} size={w}x{h} codec={codec}') + except Exception as e: + shared.log.error(f'Video open: file="{fn}" {e}') + return frames + return frames diff --git a/wiki b/wiki index ae25f6f6c..8d7346a40 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit ae25f6f6c669285031361ec322c83b7ecfc1dc82 +Subproject commit 8d7346a4010529b72a0b3ade8711cb0fc6c45600