diff --git a/CHANGELOG.md b/CHANGELOG.md
index 703804ab3..0dac17552 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,8 @@
# Change Log for SD.Next
-## Update for 2024-12-22
+## Update for 2024-12-23
-### Highlights for 2024-12-22
+### Highlights for 2024-12-23
### SD.Next Xmass edition: *What's new?*
@@ -32,7 +32,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full
[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867)
-## Details for 2024-12-22
+## Details for 2024-12-23
### New models and integrations
@@ -93,17 +93,20 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full
### Video models
-- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video)
- model size: 27.75gb
+- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video)
+ model size: 27.75gb
+ support for 0.9.0, 0.9.1 and custom safetensor-based models with full quantization and offloading support
support for text-to-video and image-to-video, to use, select in *scripts -> ltx-video*
- *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0
+ *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0
- [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo)
- model size: 40.92gb
+ model size: 40.92gb
support for text-to-video, to use, select in *scripts -> hunyuan video*
- *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0
-- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview)
+ basic support only
+ *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0
+- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview)
support for text-to-video, to use, select in *scripts -> mochi.1 video*
- *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5
+ basic support only
+ *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5
*Notes*:
- all video models are very large and resource intensive!
diff --git a/installer.py b/installer.py
index 26d9a3a11..46ccce92c 100644
--- a/installer.py
+++ b/installer.py
@@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None):
def check_diffusers():
if args.skip_all or args.skip_git:
return
- sha = '233dffdc3f56b26abaaba8363a5dd30dab7f0e40' # diffusers commit hash
+ sha = '4b557132ce955d58fd84572c03e79f43bdc91450' # diffusers commit hash
pkg = pkg_resources.working_set.by_key.get('diffusers', None)
minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
cur = opts.get('diffusers_version', '') if minor > 0 else ''
diff --git a/modules/processing.py b/modules/processing.py
index 6d9b64c17..c85938268 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -357,7 +357,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
for i, sample in enumerate(samples):
debug(f'Processing result: index={i+1}/{len(samples)} iteration={n+1}/{p.n_iter}')
p.batch_index = i
- if type(sample) == Image.Image:
+ if isinstance(sample, Image.Image) or (isinstance(sample, list) and isinstance(sample[0], Image.Image)):
image = sample
sample = np.array(sample)
else:
@@ -399,11 +399,20 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i, all_negative_prompts=p.negative_prompts)
infotexts.append(info)
- image.info["parameters"] = info
- output_images.append(image)
+ if isinstance(image, list):
+ for img in image:
+ img.info["parameters"] = info
+ output_images = image
+ else:
+ image.info["parameters"] = info
+ output_images.append(image)
if shared.opts.samples_save and not p.do_not_save_samples and p.outpath_samples is not None:
info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i)
- images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
+ if isinstance(image, list):
+ for img in image:
+ images.save_image(img, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
+ else:
+ images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image
if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]):
image_mask = p.mask_for_overlay.convert('RGB')
image1 = image.convert('RGBA').convert('RGBa')
diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index aa820aa34..33d875e95 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -361,6 +361,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
else:
width = getattr(p, 'width', 0)
height = getattr(p, 'height', 0)
+ frames = p.task_args.get('num_frames', None)
if isinstance(output.images, list):
results = []
for i in range(len(output.images)):
@@ -370,6 +371,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
full_quality = p.full_quality,
width = width,
height = height,
+ frames = frames,
)
for result in list(result_batch):
results.append(result)
@@ -380,6 +382,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
full_quality = p.full_quality,
width = width,
height = height,
+ frames = frames,
)
elif hasattr(output, 'images'):
results = output.images
diff --git a/modules/processing_vae.py b/modules/processing_vae.py
index 5d8f9fc84..1710bd992 100644
--- a/modules/processing_vae.py
+++ b/modules/processing_vae.py
@@ -203,7 +203,7 @@ def taesd_vae_encode(image):
return encoded
-def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None):
+def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None, frames=None):
t0 = time.time()
model = model or shared.sd_model
if not hasattr(model, 'vae') and hasattr(model, 'pipe'):
@@ -221,7 +221,11 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None,
shared.log.error('VAE not found in model')
return []
- if hasattr(model, "_unpack_latents") and hasattr(model, "vae_scale_factor") and width is not None and height is not None: # FLUX
+ if hasattr(model, '_unpack_latents') and hasattr(model, 'transformer_spatial_patch_size') and frames is not None: # LTX
+ latent_num_frames = (frames - 1) // model.vae_temporal_compression_ratio + 1
+ latents = model._unpack_latents(latents.unsqueeze(0), latent_num_frames, height // 32, width // 32, model.transformer_spatial_patch_size, model.transformer_temporal_patch_size) # pylint: disable=protected-access
+ latents = model._denormalize_latents(latents, model.vae.latents_mean, model.vae.latents_std, model.vae.config.scaling_factor) # pylint: disable=protected-access
+ if hasattr(model, '_unpack_latents') and hasattr(model, "vae_scale_factor") and width is not None and height is not None: # FLUX
latents = model._unpack_latents(latents, height, width, model.vae_scale_factor) # pylint: disable=protected-access
if len(latents.shape) == 3: # lost a batch dim in hires
latents = latents.unsqueeze(0)
@@ -238,7 +242,9 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None,
decoded = taesd_vae_decode(latents=latents)
if torch.is_tensor(decoded):
- if hasattr(model, 'image_processor'):
+ if hasattr(model, 'video_processor'):
+ imgs = model.video_processor.postprocess_video(decoded, output_type='pil')
+ elif hasattr(model, 'image_processor'):
imgs = model.image_processor.postprocess(decoded, output_type=output_type)
elif hasattr(model, "vqgan"):
imgs = decoded.permute(0, 2, 3, 1).cpu().float().numpy()
diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py
index 148fd4481..007c4f4cc 100644
--- a/scripts/ltxvideo.py
+++ b/scripts/ltxvideo.py
@@ -1,11 +1,40 @@
+import os
import time
import torch
import gradio as gr
import diffusers
+import transformers
from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant
-repo_id = 'a-r-r-o-w/LTX-Video-diffusers'
+repos = {
+ '0.9.0': 'a-r-r-o-w/LTX-Video-diffusers',
+ '0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers',
+ 'custom': None,
+}
+
+
+def load_quants(kwargs, repo_id):
+ if len(shared.opts.bnb_quantization) > 0:
+ quant_args = {}
+ quant_args = model_quant.create_bnb_config(quant_args)
+ quant_args = model_quant.create_ao_config(quant_args)
+ if not quant_args:
+ return kwargs
+ model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}')
+ if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs:
+ kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args)
+ shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}')
+ if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs:
+ kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args)
+ shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}')
+ return kwargs
+
+
+def hijack_decode(*args, **kwargs):
+ shared.log.debug('Video: decode')
+ shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
+ return shared.sd_model.vae.orig_decode(*args, **kwargs)
class Script(scripts.Script):
@@ -24,11 +53,19 @@ class Script(scripts.Script):
gr.update(visible=video_type == 'MP4'),
gr.update(visible=video_type == 'MP4'),
]
+ def model_change(model):
+ return gr.update(visible=model == 'custom')
with gr.Row():
gr.HTML('  LTX Video
')
+ with gr.Row():
+ model = gr.Dropdown(label='LTX Model', choices=list(repos), value='0.9.1')
+ decode = gr.Dropdown(label='Decode', choices=['diffusers', 'native'], value='diffusers', visible=False)
with gr.Row():
num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41)
+ sampler = gr.Checkbox(label='Override sampler', value=True)
+ with gr.Row():
+ model_custom = gr.Textbox(value='', label='Path to model file', visible=False)
with gr.Row():
video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None')
duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False)
@@ -37,9 +74,10 @@ class Script(scripts.Script):
mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False)
mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate])
- return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
+ model.change(fn=model_change, inputs=[model], outputs=[model_custom])
+ return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
- def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
+ def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
# set params
image = getattr(p, 'init_images', None)
image = None if image is None or len(image) == 0 else image[0]
@@ -49,32 +87,48 @@ class Script(scripts.Script):
num_frames = 8 * int(num_frames // 8) + 1
p.width = 32 * int(p.width // 32)
p.height = 32 * int(p.height // 32)
+ processing.fix_seed(p)
if image:
image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
p.task_args['image'] = image
- p.task_args['output_type'] = 'pil'
- p.task_args['generator'] = torch.manual_seed(p.seed)
+ p.task_args['output_type'] = 'latent' if decode == 'native' else 'pil'
+ p.task_args['generator'] = torch.Generator(devices.device).manual_seed(p.seed)
p.task_args['num_frames'] = num_frames
- p.sampler_name = 'Default'
p.do_not_save_grid = True
+ if sampler:
+ p.sampler_name = 'Default'
p.ops.append('video')
# load model
cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline
diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel
diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo
+ repo_id = repos[model]
+ if repo_id is None:
+ repo_id = model_custom
if shared.sd_model.__class__ != cls:
sd_models.unload_model_weights()
kwargs = {}
kwargs = model_quant.create_bnb_config(kwargs)
kwargs = model_quant.create_ao_config(kwargs)
- shared.sd_model = cls.from_pretrained(
- repo_id,
- cache_dir = shared.opts.hfcache_dir,
- torch_dtype=devices.dtype,
- **kwargs
- )
+ if os.path.isfile(repo_id):
+ shared.sd_model = cls.from_single_file(
+ repo_id,
+ cache_dir = shared.opts.hfcache_dir,
+ torch_dtype=devices.dtype,
+ **kwargs
+ )
+ else:
+ kwargs = load_quants(kwargs, repo_id)
+ shared.sd_model = cls.from_pretrained(
+ repo_id,
+ cache_dir = shared.opts.hfcache_dir,
+ torch_dtype=devices.dtype,
+ **kwargs
+ )
sd_models.set_diffuser_options(shared.sd_model)
+ shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
+ shared.sd_model.vae.decode = hijack_decode
shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id)
shared.sd_model.sd_model_hash = None
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)