diff --git a/CHANGELOG.md b/CHANGELOG.md index 703804ab3..0dac17552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2024-12-22 +## Update for 2024-12-23 -### Highlights for 2024-12-22 +### Highlights for 2024-12-23 ### SD.Next Xmass edition: *What's new?* @@ -32,7 +32,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -## Details for 2024-12-22 +## Details for 2024-12-23 ### New models and integrations @@ -93,17 +93,20 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full ### Video models -- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) - model size: 27.75gb +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + model size: 27.75gb + support for 0.9.0, 0.9.1 and custom safetensor-based models with full quantization and offloading support support for text-to-video and image-to-video, to use, select in *scripts -> ltx-video* - *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0 + *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0 - [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) - model size: 40.92gb + model size: 40.92gb support for text-to-video, to use, select in *scripts -> hunyuan video* - *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0 -- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) + basic support only + *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0 +- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) support for text-to-video, to use, select in *scripts -> mochi.1 video* - *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5 + basic support only + *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5 *Notes*: - all video models are very large and resource intensive! diff --git a/installer.py b/installer.py index 26d9a3a11..46ccce92c 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = '233dffdc3f56b26abaaba8363a5dd30dab7f0e40' # diffusers commit hash + sha = '4b557132ce955d58fd84572c03e79f43bdc91450' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/processing.py b/modules/processing.py index 6d9b64c17..c85938268 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -357,7 +357,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for i, sample in enumerate(samples): debug(f'Processing result: index={i+1}/{len(samples)} iteration={n+1}/{p.n_iter}') p.batch_index = i - if type(sample) == Image.Image: + if isinstance(sample, Image.Image) or (isinstance(sample, list) and isinstance(sample[0], Image.Image)): image = sample sample = np.array(sample) else: @@ -399,11 +399,20 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i, all_negative_prompts=p.negative_prompts) infotexts.append(info) - image.info["parameters"] = info - output_images.append(image) + if isinstance(image, list): + for img in image: + img.info["parameters"] = info + output_images = image + else: + image.info["parameters"] = info + output_images.append(image) if shared.opts.samples_save and not p.do_not_save_samples and p.outpath_samples is not None: info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i) - images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image + if isinstance(image, list): + for img in image: + images.save_image(img, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image + else: + images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]): image_mask = p.mask_for_overlay.convert('RGB') image1 = image.convert('RGBA').convert('RGBa') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index aa820aa34..33d875e95 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -361,6 +361,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): else: width = getattr(p, 'width', 0) height = getattr(p, 'height', 0) + frames = p.task_args.get('num_frames', None) if isinstance(output.images, list): results = [] for i in range(len(output.images)): @@ -370,6 +371,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): full_quality = p.full_quality, width = width, height = height, + frames = frames, ) for result in list(result_batch): results.append(result) @@ -380,6 +382,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): full_quality = p.full_quality, width = width, height = height, + frames = frames, ) elif hasattr(output, 'images'): results = output.images diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 5d8f9fc84..1710bd992 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -203,7 +203,7 @@ def taesd_vae_encode(image): return encoded -def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None): +def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None, frames=None): t0 = time.time() model = model or shared.sd_model if not hasattr(model, 'vae') and hasattr(model, 'pipe'): @@ -221,7 +221,11 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, shared.log.error('VAE not found in model') return [] - if hasattr(model, "_unpack_latents") and hasattr(model, "vae_scale_factor") and width is not None and height is not None: # FLUX + if hasattr(model, '_unpack_latents') and hasattr(model, 'transformer_spatial_patch_size') and frames is not None: # LTX + latent_num_frames = (frames - 1) // model.vae_temporal_compression_ratio + 1 + latents = model._unpack_latents(latents.unsqueeze(0), latent_num_frames, height // 32, width // 32, model.transformer_spatial_patch_size, model.transformer_temporal_patch_size) # pylint: disable=protected-access + latents = model._denormalize_latents(latents, model.vae.latents_mean, model.vae.latents_std, model.vae.config.scaling_factor) # pylint: disable=protected-access + if hasattr(model, '_unpack_latents') and hasattr(model, "vae_scale_factor") and width is not None and height is not None: # FLUX latents = model._unpack_latents(latents, height, width, model.vae_scale_factor) # pylint: disable=protected-access if len(latents.shape) == 3: # lost a batch dim in hires latents = latents.unsqueeze(0) @@ -238,7 +242,9 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, decoded = taesd_vae_decode(latents=latents) if torch.is_tensor(decoded): - if hasattr(model, 'image_processor'): + if hasattr(model, 'video_processor'): + imgs = model.video_processor.postprocess_video(decoded, output_type='pil') + elif hasattr(model, 'image_processor'): imgs = model.image_processor.postprocess(decoded, output_type=output_type) elif hasattr(model, "vqgan"): imgs = decoded.permute(0, 2, 3, 1).cpu().float().numpy() diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 148fd4481..007c4f4cc 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -1,11 +1,40 @@ +import os import time import torch import gradio as gr import diffusers +import transformers from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant -repo_id = 'a-r-r-o-w/LTX-Video-diffusers' +repos = { + '0.9.0': 'a-r-r-o-w/LTX-Video-diffusers', + '0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers', + 'custom': None, +} + + +def load_quants(kwargs, repo_id): + if len(shared.opts.bnb_quantization) > 0: + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + quant_args = model_quant.create_ao_config(quant_args) + if not quant_args: + return kwargs + model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + return kwargs + + +def hijack_decode(*args, **kwargs): + shared.log.debug('Video: decode') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return shared.sd_model.vae.orig_decode(*args, **kwargs) class Script(scripts.Script): @@ -24,11 +53,19 @@ class Script(scripts.Script): gr.update(visible=video_type == 'MP4'), gr.update(visible=video_type == 'MP4'), ] + def model_change(model): + return gr.update(visible=model == 'custom') with gr.Row(): gr.HTML('  LTX Video
') + with gr.Row(): + model = gr.Dropdown(label='LTX Model', choices=list(repos), value='0.9.1') + decode = gr.Dropdown(label='Decode', choices=['diffusers', 'native'], value='diffusers', visible=False) with gr.Row(): num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41) + sampler = gr.Checkbox(label='Override sampler', value=True) + with gr.Row(): + model_custom = gr.Textbox(value='', label='Path to model file', visible=False) with gr.Row(): video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) @@ -37,9 +74,10 @@ class Script(scripts.Script): mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) - return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + model.change(fn=model_change, inputs=[model], outputs=[model_custom]) + return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument # set params image = getattr(p, 'init_images', None) image = None if image is None or len(image) == 0 else image[0] @@ -49,32 +87,48 @@ class Script(scripts.Script): num_frames = 8 * int(num_frames // 8) + 1 p.width = 32 * int(p.width // 32) p.height = 32 * int(p.height // 32) + processing.fix_seed(p) if image: image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['image'] = image - p.task_args['output_type'] = 'pil' - p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['output_type'] = 'latent' if decode == 'native' else 'pil' + p.task_args['generator'] = torch.Generator(devices.device).manual_seed(p.seed) p.task_args['num_frames'] = num_frames - p.sampler_name = 'Default' p.do_not_save_grid = True + if sampler: + p.sampler_name = 'Default' p.ops.append('video') # load model cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo + repo_id = repos[model] + if repo_id is None: + repo_id = model_custom if shared.sd_model.__class__ != cls: sd_models.unload_model_weights() kwargs = {} kwargs = model_quant.create_bnb_config(kwargs) kwargs = model_quant.create_ao_config(kwargs) - shared.sd_model = cls.from_pretrained( - repo_id, - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **kwargs - ) + if os.path.isfile(repo_id): + shared.sd_model = cls.from_single_file( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) + else: + kwargs = load_quants(kwargs, repo_id) + shared.sd_model = cls.from_pretrained( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.decode = hijack_decode shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) shared.sd_model.sd_model_hash = None shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)