diff --git a/CHANGELOG.md b/CHANGELOG.md index 1931bd2b5..13b831797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ ## Update for 2024-12-18 +### Highlights + +*What's new?* + +While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: +- New memory management engine: list of changes that went into this one is too long for here, + but main goal is enabling modern large models to run on standard consumer GPUs + without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks +- New [documentation website](https://vladmandic.github.io/sdnext-docs/) + with full search and tons of new documentation +- New settings panel with simplified and streamlined configuration + +We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list): +- [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + +And a lot of Control goodies and related goodies +- for SDXL there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* +- for FLUX.1 there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model +- for SD 3.5 there are official *Canny*, *Blur* and *Depth* in addition to existing 3rd party models + +Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) + +[README](https://github.com/vladmandic/automatic/blob/master/README.md) | [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) + ### New models and integrations - [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) @@ -13,6 +38,13 @@ *reference values*: sampler: default (or any flow-match variant), width/height: 1024, guidance scale: 4.5 *note* like other LLM-based text-encoders, sana prefers long and descriptive prompts any short prompt below 300 characters will be auto-expanded using built in Gemma LLM before encoding while long prompts will be passed as-is +- **ControlNet** + - improved support for **Union** controlnets with granular control mode type + - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet + - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) + *note*: when selecting tiles in control settings, you can also specify non-square ratios + in which case it will use context-aware image resize to maintain overall composition + *note*: available tiling options can be set in settings -> control - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model @@ -36,6 +68,11 @@ both **Depth** and **Canny** LoRAs are available in standard control menus - [StabilityAI SD35 ControlNets](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets) - In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + basic support for LTX-Video for text-to-video and image-to-video + to use, select in *scripts -> ltx-video* + *note* you may need to enable sequential offload for maximum gpu memory savings + *note* ltx-video requires very long and descriptive prompt, see original link for examples - [Style Aligned Image Generation](https://style-aligned-gen.github.io/) enable in scripts, compatible with sd-xl enter multiple prompts in prompt field separated by new line @@ -47,13 +84,6 @@ run iterative generation of images at different scales to achieve better results can render 4k sdxl images *note*: disable live preview to avoid memory issues when generating large images -- **ControlNet** - - improved support for **Union** controlnets with granular control mode type - - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet - - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) - *note*: when selecting tiles in control settings, you can also specify non-square ratios - in which case it will use context-aware image resize to maintain overall composition - *note*: available tiling options can be set in settings -> control ### UI and workflow improvements diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 581589262..d43660ca8 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -355,6 +355,8 @@ def process_decode(p: processing.StableDiffusionProcessing, output): if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] + if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image): + return output.images model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner if not hasattr(model, 'vae'): if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'): diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py new file mode 100644 index 000000000..54e2685a8 --- /dev/null +++ b/scripts/ltxvideo.py @@ -0,0 +1,130 @@ +import time +import torch +import gradio as gr +import diffusers +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint + + +repo_id = 'a-r-r-o-w/LTX-Video-diffusers' +presets = [ + {"label": "custom", "width": 0, "height": 0, "num_frames": 0}, + {"label": "1216x704, 41 frames", "width": 1216, "height": 704, "num_frames": 41}, + {"label": "1088x704, 49 frames", "width": 1088, "height": 704, "num_frames": 49}, + {"label": "1056x640, 57 frames", "width": 1056, "height": 640, "num_frames": 57}, + {"label": "992x608, 65 frames", "width": 992, "height": 608, "num_frames": 65}, + {"label": "896x608, 73 frames", "width": 896, "height": 608, "num_frames": 73}, + {"label": "896x544, 81 frames", "width": 896, "height": 544, "num_frames": 81}, + {"label": "832x544, 89 frames", "width": 832, "height": 544, "num_frames": 89}, + {"label": "800x512, 97 frames", "width": 800, "height": 512, "num_frames": 97}, + {"label": "768x512, 97 frames", "width": 768, "height": 512, "num_frames": 97}, + {"label": "800x480, 105 frames", "width": 800, "height": 480, "num_frames": 105}, + {"label": "736x480, 113 frames", "width": 736, "height": 480, "num_frames": 113}, + {"label": "704x480, 121 frames", "width": 704, "height": 480, "num_frames": 121}, + {"label": "704x448, 129 frames", "width": 704, "height": 448, "num_frames": 129}, + {"label": "672x448, 137 frames", "width": 672, "height": 448, "num_frames": 137}, + {"label": "640x416, 153 frames", "width": 640, "height": 416, "num_frames": 153}, + {"label": "672x384, 161 frames", "width": 672, "height": 384, "num_frames": 161}, + {"label": "640x384, 169 frames", "width": 640, "height": 384, "num_frames": 169}, + {"label": "608x384, 177 frames", "width": 608, "height": 384, "num_frames": 177}, + {"label": "576x384, 185 frames", "width": 576, "height": 384, "num_frames": 185}, + {"label": "608x352, 193 frames", "width": 608, "height": 352, "num_frames": 193}, + {"label": "576x352, 201 frames", "width": 576, "height": 352, "num_frames": 201}, + {"label": "544x352, 209 frames", "width": 544, "height": 352, "num_frames": 209}, + {"label": "512x352, 225 frames", "width": 512, "height": 352, "num_frames": 225}, + {"label": "512x352, 233 frames", "width": 512, "height": 352, "num_frames": 233}, + {"label": "544x320, 241 frames", "width": 544, "height": 320, "num_frames": 241}, + {"label": "512x320, 249 frames", "width": 512, "height": 320, "num_frames": 249}, + {"label": "512x320, 257 frames", "width": 512, "height": 320, "num_frames": 257}, +] + + +class Script(scripts.Script): + def title(self): + return 'Video: LTX Video' + + def show(self, is_img2img): + return shared.native + + # return signature is array of gradio components + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + def preset_change(preset): + return gr.update(visible=preset == 'custom') + + with gr.Row(): + gr.HTML('  LTX Video
') + with gr.Row(): + preset_name = gr.Dropdown(label='Preset', choices=[p['label'] for p in presets], value='custom') + num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=9) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + preset_name.change(fn=preset_change, inputs=[preset_name], outputs=num_frames) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [preset_name, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, preset_name, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + # set params + preset = [p for p in presets if p['label'] == preset_name][0] + image = getattr(p, 'init_images', None) + image = None if image is None or len(image) == 0 else image[0] + if p.width == 0 or p.height == 0 and image is not None: + p.width = image.width + p.height = image.height + if preset['label'] != 'custom': + num_frames = preset['num_frames'] + p.width = preset['width'] + p.height = preset['height'] + else: + num_frames = 8 * int(num_frames // 8) + 1 + p.width = 32 * int(p.width // 32) + p.height = 32 * int(p.height // 32) + if image: + image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + p.task_args['image'] = image + p.task_args['output_type'] = 'pil' + p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['num_frames'] = num_frames + p.sampler_name = 'Default' + p.do_not_save_grid = True + p.ops.append('ltx') + + # load model + cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline + diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel + diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo + if shared.sd_model.__class__ != cls: + sd_models.unload_model_weights() + shared.sd_model = cls.from_pretrained( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) + shared.sd_model.sd_model_hash = None + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model.vae.enable_slicing() + shared.sd_model.vae.enable_tiling() + devices.torch_gc(force=True) + shared.log.debug(f'LTX: cls={shared.sd_model.__class__.__name__} preset={preset_name} args={p.task_args}') + + # run processing + t0 = time.time() + processed = processing.process_images(p) + t1 = time.time() + if processed is not None and len(processed.images) > 0: + shared.log.info(f'LTX: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed diff --git a/wiki b/wiki index 470e75f0c..34ba1df45 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 470e75f0c70a22ed3d65187c70f04c131400b35d +Subproject commit 34ba1df45d17da4ee09a2e5278e384bc1929dd8b