From 506f5e1c2942f5e0f0e0c1338446d1ee9fd22ef7 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 2 May 2026 01:46:09 +0100 Subject: [PATCH 1/6] fix(ltx): audio toggle ux and audio mux stream registration relabel toggle to 'LTX save audio' (default true) since audio always generates on 2.x audio-capable models; the toggle gates mux only. hint added to locale_en.json. split add_audio_stream from write_audio. avformat_write_header runs on first container.mux() and freezes the stream set, so audio added after video packets has time_base=0/0 and raises 'Cannot rebase to zero time.' atomic_save_video registers the audio stream before the encode loop. --- html/locale_en.json | 2 +- modules/ltx/ltx_ui.py | 2 +- modules/video_models/video_save.py | 29 +++++++++++++++-------------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/html/locale_en.json b/html/locale_en.json index d387c860a..049cd9067 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -830,7 +830,7 @@ {"id":"","label":"LTX enable refine","localized":"","hint":"","ui":"video"}, {"id":"","label":"LTX refine strength","localized":"","hint":"","ui":"video"}, {"id":"","label":"LTX decode timestep","localized":"","hint":"","ui":"video"}, - {"id":"","label":"LTX enable audio","localized":"","hint":"","ui":"video"}, + {"id":"","label":"LTX save audio","localized":"","hint":"LTX-2 audio-capable models always generate audio from the same prompt as video; this toggle controls whether the audio track is included in the saved video file","ui":"video"}, {"id":"","label":"Loop","localized":"","hint":"","ui":"extras"}, {"id":"","label":"Local directory name","localized":"","hint":"Directory where to install extension, leave blank for default","ui":"component-8746"}, {"id":"","label":"Libs","localized":"","hint":"","ui":"component-8779"}, diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 896a89cf7..7385d5351 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -109,7 +109,7 @@ def create_ui(prompt, negative, styles, overrides, mp4_fps, mp4_interpolate, mp4 audio_accordion = gr.Accordion(open=False, label="Audio", elem_id='ltx_audio_accordion', visible=False) with audio_accordion: with gr.Row(): - audio_enable = gr.Checkbox(label='LTX enable audio', value=False, elem_id="ltx_audio_enable") + audio_enable = gr.Checkbox(label='LTX save audio', value=True, elem_id="ltx_audio_enable") with gr.Column(elem_id='ltx-output-column', scale=2) as _column_output: with gr.Row(): diff --git a/modules/video_models/video_save.py b/modules/video_models/video_save.py index db01c605b..f63ae5d3b 100644 --- a/modules/video_models/video_save.py +++ b/modules/video_models/video_save.py @@ -80,22 +80,25 @@ def numpy_to_tensor(images): return tensor +def add_audio_stream(container, audio_sample_rate: int): + # Must be registered before the first container.mux(); avformat_write_header runs there + # and freezes the stream set, after which new streams have time_base=0/0. + audio_stream = container.add_stream("aac", rate=audio_sample_rate) + audio_stream.codec_context.sample_rate = audio_sample_rate + audio_stream.codec_context.layout = "stereo" + audio_stream.codec_context.time_base = Fraction(1, audio_sample_rate) + log.debug(f'Audio: codec={audio_stream.codec_context.name} rate={audio_stream.codec_context.sample_rate} layout={audio_stream.codec_context.layout} base={audio_stream.codec_context.time_base}') + return audio_stream + + def write_audio( container, + audio_stream, samples: torch.Tensor, audio_sample_rate: int, ) -> None: av = check_av() - # create stream - audio_options = { 'time_base': f'1/{audio_sample_rate}' } - audio_stream = container.add_stream("aac", rate=audio_sample_rate, options=audio_options) - audio_stream.codec_context.sample_rate = audio_sample_rate - audio_stream.codec_context.layout = "stereo" audio_stream.codec_context.format = "fltp" - audio_stream.codec_context.time_base = Fraction(1, audio_sample_rate) - # audio_stream.time_base = audio_stream.codec_context.time_base # TODO audio set time-base - log.debug(f'Audio: codec={audio_stream.codec_context.name} rate={audio_stream.codec_context.sample_rate} layout={audio_stream.codec_context.layout} format={audio_stream.codec_context.format} base={audio_stream.codec_context.time_base}') - # init input samples if samples.ndim == 1: samples = samples[:, None] if samples.shape[1] != 2 and samples.shape[0] == 2: @@ -111,13 +114,11 @@ def write_audio( layout="stereo", ) audio_frames.sample_rate = audio_sample_rate - # init resampler audio_resampler = av.audio.resampler.AudioResampler( format=audio_stream.codec_context.format, layout=audio_stream.codec_context.layout, rate=audio_stream.codec_context.sample_rate, ) - # resample pts = 0 for resampled in audio_resampler.resample(audio_frames): resampled.pts = resampled.pts or 0 @@ -126,7 +127,6 @@ def write_audio( for packet in packets: container.mux(packet) pts += resampled.samples - # flush audio encoder for packet in audio_stream.encode(): container.mux(packet) @@ -177,6 +177,7 @@ def atomic_save_video( stream.width = video_array.shape[2] stream.height = video_array.shape[1] stream.pix_fmt = pix_fmt + audio_stream = add_audio_stream(container, aac) if audio is not None else None for img in video_array: frame = av.VideoFrame.from_ndarray(img, format="rgb24") for packet in stream.encode_lazy(frame): @@ -185,9 +186,9 @@ def atomic_save_video( pbar.update(task, advance=1) for packet in stream.encode(): # flush container.mux(packet) - if audio is not None: + if audio_stream is not None: try: - write_audio(container, audio, aac) + write_audio(container, audio_stream, audio, aac) except Exception as e: log.error(f'Video audio encoding: {e}') errors.display(e, 'Audio') From a6870f7d279937eaf1919c0e3e7d3baf1558f66b Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 2 May 2026 02:08:03 +0100 Subject: [PATCH 2/6] fix(ltx): decode stage 1 audio directly, discard stage 2 audio output stop threading stage 1 audio_latents into stage 2 refine. Lightricks/LTX-2#126 reports the two-stage pipeline degrades audio quality, confirmed locally as clean speech with tinny ambient/foley/music on the threaded path. root cause: stage 2 prepare_audio_latents calls _create_noised_state at noise_scale=0.909 (pipeline_ltx2.py:704-714, 598-603), keeping ~9% of stage 1 signal. 3 refine steps recover speech via video<->audio cross-attention but not broadband content. new path: _latent_pass decodes audio_latents to waveform via audio_vae + vocoder mirroring pipeline_ltx2.py:1471-1473 exactly (input cast to audio_vae.dtype, no module dtype mutation). stage 2 result.audio is discarded; cross-attention still runs each block for video conditioning. --- modules/ltx/ltx_process.py | 42 +++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 28c58b2dd..2fc79922c 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -93,13 +93,25 @@ def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_ base_args['use_cross_timestep'] = True log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}') result = shared.sd_model(**base_args) - # video latents strip the batch dim; audio latents keep it so LTX2Pipeline.prepare_audio_latents - # can rewrap them when re-entered as ndim==4 at Stage 2. latents = result.frames[0] if hasattr(result, 'frames') else None - audio_latents = None + # output_type='latent' already returns denormalized + unpacked audio_latents. Threading + # them into Stage 2 re-noises at sigma=0.909 (prepare_audio_latents) and 3 refine steps + # cannot recover broadband content. Decode here mirroring the pipeline's non-latent path; + # cast only the input tensor since mutating module dtypes shifts BWE activations. + audio_waveform = None if hasattr(result, 'audio') and result.audio is not None: - audio_latents = result.audio - return latents, audio_latents + pipe = shared.sd_model + if hasattr(pipe, 'audio_vae') and hasattr(pipe, 'vocoder'): + try: + audio_latents = result.audio.to(device=devices.device, dtype=pipe.audio_vae.dtype) + with torch.no_grad(): + mel = pipe.audio_vae.decode(audio_latents, return_dict=False)[0] + waveform = pipe.vocoder(mel) + audio_waveform = waveform[0].float().cpu() + except Exception as e: + log.warning(f'LTX: Stage 1 audio decode failed: {e}') + audio_waveform = None + return latents, audio_waveform def run_ltx(task_id, @@ -318,7 +330,7 @@ def run_ltx(task_id, yield None, 'LTX: Generate in progress...' audio = None - stage1_audio_latents = None + stage1_audio = None pixels = None frames_out = None needs_latent_path = upsample_enable or refine_enable @@ -327,7 +339,7 @@ def run_ltx(task_id, if needs_latent_path: prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) extra_networks.activate(p, networks) - latents, stage1_audio_latents = _latent_pass( + latents, stage1_audio = _latent_pass( caps=caps, prompt=prompt_final, negative=negative_final, @@ -457,14 +469,10 @@ def run_ltx(task_id, # Thread Stage-1 I2V init image through Stage 2 so first-frame identity survives refine. if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and p.task_args.get('image') is not None: refine_args['image'] = p.task_args['image'] - # Thread Stage-1 audio latents into Stage 2 on 2.x. The video branch cross-attends - # audio every layer; letting prepare_audio_latents fall back to fresh noise biases - # the video branch off-distribution (desaturated output on distilled 2.x). - if caps.family == '2.x': - if stage1_audio_latents is not None: - refine_args['audio_latents'] = stage1_audio_latents.to(device=devices.device) - if caps.use_cross_timestep: - refine_args['use_cross_timestep'] = True + # Audio cross-attention still runs in Stage 2 for video conditioning, but the + # vocoder output is discarded; Stage 1 decode (see _latent_pass) is authoritative. + if caps.family == '2.x' and caps.use_cross_timestep: + refine_args['use_cross_timestep'] = True saved_scheduler_stage2 = None try: @@ -503,8 +511,6 @@ def run_ltx(task_id, try: result = shared.sd_model(latents=latents, **refine_args) pixels = result.frames[0] if hasattr(result, 'frames') else None - if hasattr(result, 'audio') and result.audio is not None: - audio = result.audio[0].float().cpu() latents = None except AssertionError as e: yield from abort(e, ok=True, p=p) @@ -555,6 +561,8 @@ def run_ltx(task_id, t11 = time.time() timer.process.add('offload', t11 - t10) + if stage1_audio is not None: + audio = stage1_audio if not audio_enable: audio = None From 207e7d8b3bffcd19a8ab940f51367809808dcf6c Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 2 May 2026 04:08:55 +0100 Subject: [PATCH 3/6] fix(processing): preserve pipeline audio across process_decode process_decode strips video pipeline output to a flat list of frames at the PIL early-return (processing_diffusers.py:461-465), so any output.audio is lost before processing.process_images returns. video pipelines that produce synchronized audio (LTX-2 audio-capable models) were getting silent mp4s on the non-latent path. stash output.audio on p.audio_capture before process_decode runs and let processing read it back as a fallback when samples is a flat list. ltx_process non-latent branch strips the (B, 2, N) batch dim with [0] so write_audio's .T+contiguous() path produces interleaved bytes for AAC s16. --- modules/ltx/ltx_process.py | 7 +++++-- modules/processing.py | 2 +- modules/processing_diffusers.py | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 2fc79922c..5cbe5cb0c 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -360,8 +360,11 @@ def run_ltx(task_id, yield from abort('Video: process_images returned no frames', ok=True, p=p) return pixels = processed.images - if getattr(processed, 'audio', None) is not None: - audio = processed.audio + raw_audio = getattr(processed, 'audio', None) + if raw_audio is not None: + # Strip batch dim from (B, 2, N); write_audio expects (2, N) for the + # transpose-to-interleaved path used by AAC s16. + audio = raw_audio[0].float().cpu() if raw_audio.ndim == 3 else raw_audio.float().cpu() latents = None except AssertionError as e: yield from abort(e, ok=True, p=p) diff --git a/modules/processing.py b/modules/processing.py index 0306b20a8..36c600ebe 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -528,7 +528,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: output_images.append(batch_image) infotexts.append(batch_infotext) - audio = getattr(samples, 'audio', None) + audio = getattr(samples, 'audio', None) or getattr(p, 'audio_capture', None) if shared.cmd_opts.lowvram: devices.torch_gc(force=True, reason='lowvram') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index b7b1030ef..6b701a966 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -628,6 +628,10 @@ def process_diffusers(p: processing.StableDiffusionProcessing): timer.process.add('lora', lora_common.timer.total) lora_common.timer.clear(complete=True) + # process_decode flattens video output to a frame list and drops the audio attribute; + # stash it on `p` so video pipelines can recover it after process_images returns. + if output is not None and getattr(output, 'audio', None) is not None: + p.audio_capture = output.audio results = process_decode(p, output) timer.process.record('decode') From 41ffa7843824ba1b7446aeb0dc033445ebde3d67 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sat, 2 May 2026 19:56:30 +0100 Subject: [PATCH 4/6] fix(ltx): align stage 2 audio to canonical pr #13217 drop the project-specific stage 1 direct audio decode (a6870f7d2). smoke testing showed broadband tinniness on distilled is BWE-bound, not stage 2 corruption, so the deviation didn't fix the underlying issue. revert to canonical: - _latent_pass returns video latents only; result.audio is unused. - stage 2 receives audio_latents=None (default), prepare_audio_latents generates fresh gaussian noise, audio scheduler runs the 3 stage-2 sigmas under identity guidance, video<->audio cross-attention conditions the audio branch. - capture stage 2 result.audio[0].float().cpu() for save. upstream evidence: pipeline_ltx2.py:937-940 documents audio_latents as pre-generated noisy latents (initial gaussian, not stage 1 output); no caller in diffusers threads stage outputs into the kwarg. non-latent path is unchanged and continues to work via p.audio_capture. --- modules/ltx/ltx_process.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 5cbe5cb0c..ad0df0ef1 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -94,24 +94,7 @@ def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_ log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}') result = shared.sd_model(**base_args) latents = result.frames[0] if hasattr(result, 'frames') else None - # output_type='latent' already returns denormalized + unpacked audio_latents. Threading - # them into Stage 2 re-noises at sigma=0.909 (prepare_audio_latents) and 3 refine steps - # cannot recover broadband content. Decode here mirroring the pipeline's non-latent path; - # cast only the input tensor since mutating module dtypes shifts BWE activations. - audio_waveform = None - if hasattr(result, 'audio') and result.audio is not None: - pipe = shared.sd_model - if hasattr(pipe, 'audio_vae') and hasattr(pipe, 'vocoder'): - try: - audio_latents = result.audio.to(device=devices.device, dtype=pipe.audio_vae.dtype) - with torch.no_grad(): - mel = pipe.audio_vae.decode(audio_latents, return_dict=False)[0] - waveform = pipe.vocoder(mel) - audio_waveform = waveform[0].float().cpu() - except Exception as e: - log.warning(f'LTX: Stage 1 audio decode failed: {e}') - audio_waveform = None - return latents, audio_waveform + return latents def run_ltx(task_id, @@ -330,7 +313,6 @@ def run_ltx(task_id, yield None, 'LTX: Generate in progress...' audio = None - stage1_audio = None pixels = None frames_out = None needs_latent_path = upsample_enable or refine_enable @@ -339,7 +321,7 @@ def run_ltx(task_id, if needs_latent_path: prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) extra_networks.activate(p, networks) - latents, stage1_audio = _latent_pass( + latents = _latent_pass( caps=caps, prompt=prompt_final, negative=negative_final, @@ -514,6 +496,8 @@ def run_ltx(task_id, try: result = shared.sd_model(latents=latents, **refine_args) pixels = result.frames[0] if hasattr(result, 'frames') else None + if hasattr(result, 'audio') and result.audio is not None: + audio = result.audio[0].float().cpu() latents = None except AssertionError as e: yield from abort(e, ok=True, p=p) @@ -564,8 +548,6 @@ def run_ltx(task_id, t11 = time.time() timer.process.add('offload', t11 - t10) - if stage1_audio is not None: - audio = stage1_audio if not audio_enable: audio = None From 80fde086f94e85a20dd8aac3429c6701fd48d89e Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Sun, 3 May 2026 06:18:56 +0100 Subject: [PATCH 5/6] fix(ltx): close three two-stage audit gaps - resolve seed=-1 once at top of run_ltx and thread the int through every stage (StableDiffusionProcessingVideo, _latent_pass, upsample 0.9/2.x, refine, post-refine vae_decode). get_generator(-1) reseeds globally per call so each stage was rolling an uncorrelated value; p.seed now carries the resolved seed so reruns reproduce. - pre-encode prompts via shared.sd_model.encode_prompt before the latent path; park the four tensors on CPU and pass them as prompt_embeds / *_attention_mask kwargs through _latent_pass and refine_args. Stage 2 reuses Stage 1's embeds instead of re-running the text encoder. The manual encode is outside pipe.__call__ so the post-forward offload hook never fires; apply_balanced_offload(force=True) re-anchors the device map so the text encoder doesn't stay pinned through Stage 1. - skip audio_vae.decode + vocoder on refine when audio_enable=False: output_type='latent' bypasses the internal audio + video decode and hands off to the post-refine vae_decode block. Per-step audio cross-attention still runs for video conditioning. --- modules/ltx/ltx_process.py | 80 ++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index ad0df0ef1..31930bb3c 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -62,10 +62,12 @@ def _canonical_stage2_kwargs() -> dict: } -def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_scale, mp4_fps, conditions, image_cond_noise_scale, seed, image=None): +def _latent_pass(caps, prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask, width, height, frames, steps, guidance_scale, mp4_fps, conditions, image_cond_noise_scale, seed, image=None): base_args = { - 'prompt': prompt, - 'negative_prompt': negative, + 'prompt_embeds': prompt_embeds, + 'prompt_attention_mask': prompt_attention_mask, + 'negative_prompt_embeds': negative_prompt_embeds, + 'negative_prompt_attention_mask': negative_prompt_attention_mask, 'width': get_bucket(width), 'height': get_bucket(height), 'num_frames': get_frames(frames), @@ -168,6 +170,15 @@ def run_ltx(task_id, yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True) return + # get_generator(-1) reseeds globally per call, so every stage would otherwise + # roll an uncorrelated seed. Resolve once and thread the int through. + import random + if seed is None or int(seed) < 0: + random.seed() + resolved_seed = int(random.randrange(4294967294)) + else: + resolved_seed = int(seed) + # Lightricks TI2VidTwoStagesPipeline: Stage 1 at half-res, 2x upsample, Stage 2 refine at target. # Auto-couple when the user picks Refine but not Upsample. Both Dev and Distilled refine paths # expect upsampled latents; same-res refine on Distilled produces oversaturation. Condition @@ -251,7 +262,7 @@ def run_ltx(task_id, prompt=prompt, negative_prompt=negative, styles=styles, - seed=int(seed) if seed is not None else -1, + seed=resolved_seed, sampler_name=sampler_name, sampler_shift=float(sampler_shift), steps=int(steps), @@ -321,10 +332,28 @@ def run_ltx(task_id, if needs_latent_path: prompt_final, negative_final, networks = get_prompts(prompt, negative, styles) extra_networks.activate(p, networks) + # Encode once and reuse across stages; encode_prompt short-circuits when + # embeds are passed to __call__. CPU park keeps them off GPU between stages. + prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask = shared.sd_model.encode_prompt( + prompt=prompt_final, + negative_prompt=negative_final, + do_classifier_free_guidance=True, + device=devices.device, + ) + prompt_embeds = prompt_embeds.cpu() + prompt_attention_mask = prompt_attention_mask.cpu() if prompt_attention_mask is not None else None + negative_prompt_embeds = negative_prompt_embeds.cpu() if negative_prompt_embeds is not None else None + negative_prompt_attention_mask = negative_prompt_attention_mask.cpu() if negative_prompt_attention_mask is not None else None + # encode_prompt outside pipe.__call__ bypasses the post-forward offload hook; + # re-anchor so the text encoder doesn't stay pinned through Stage 1 forward. + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, force=True, silent=True) + devices.torch_gc(force=True, reason='ltx:encode') latents = _latent_pass( caps=caps, - prompt=prompt_final, - negative=negative_final, + prompt_embeds=prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_attention_mask=negative_prompt_attention_mask, width=base_w, height=base_h, frames=frames, @@ -333,7 +362,7 @@ def run_ltx(task_id, mp4_fps=mp4_fps, conditions=conditions, image_cond_noise_scale=image_cond_noise_scale if caps.supports_image_cond_noise_scale else None, - seed=int(seed) if seed is not None else -1, + seed=resolved_seed, image=p.task_args.get('image'), ) else: @@ -385,7 +414,7 @@ def run_ltx(task_id, up_args = { 'width': final_w, 'height': final_h, - 'generator': get_generator(int(seed) if seed is not None else -1), + 'generator': get_generator(resolved_seed), 'output_type': 'latent', } if latents.ndim == 4: @@ -406,7 +435,7 @@ def run_ltx(task_id, 'height': final_h, 'num_frames': get_frames(frames), 'latents_normalized': False, - 'generator': get_generator(int(seed) if seed is not None else -1), + 'generator': get_generator(resolved_seed), 'output_type': 'latent', } if latents.ndim == 4: @@ -433,13 +462,15 @@ def run_ltx(task_id, # Refine is terminal: let the pipe decode internally so the final VAE pass runs inside # the same offload/cudnn context as a normal generation (matches Generic Video tab). refine_args = { - 'prompt': prompt_final, - 'negative_prompt': negative_final, + 'prompt_embeds': prompt_embeds, + 'prompt_attention_mask': prompt_attention_mask, + 'negative_prompt_embeds': negative_prompt_embeds, + 'negative_prompt_attention_mask': negative_prompt_attention_mask, 'width': final_w, 'height': final_h, 'num_frames': get_frames(frames), 'num_inference_steps': steps, - 'generator': get_generator(int(seed) if seed is not None else -1), + 'generator': get_generator(resolved_seed), 'callback_on_step_end': diffusers_callback, 'output_type': 'pil', } @@ -454,10 +485,14 @@ def run_ltx(task_id, # Thread Stage-1 I2V init image through Stage 2 so first-frame identity survives refine. if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and p.task_args.get('image') is not None: refine_args['image'] = p.task_args['image'] - # Audio cross-attention still runs in Stage 2 for video conditioning, but the - # vocoder output is discarded; Stage 1 decode (see _latent_pass) is authoritative. if caps.family == '2.x' and caps.use_cross_timestep: refine_args['use_cross_timestep'] = True + # output_type='latent' skips the post-loop audio_vae + vocoder pass when audio + # is unwanted; per-step audio cross-attention still runs for video conditioning. + # Internal video decode is also skipped; vae_decode below picks it up. + want_audio = caps.supports_audio and audio_enable + if not want_audio: + refine_args['output_type'] = 'latent' saved_scheduler_stage2 = None try: @@ -495,10 +530,14 @@ def run_ltx(task_id, yield None, 'LTX: Refine in progress...' try: result = shared.sd_model(latents=latents, **refine_args) - pixels = result.frames[0] if hasattr(result, 'frames') else None - if hasattr(result, 'audio') and result.audio is not None: - audio = result.audio[0].float().cpu() - latents = None + out = result.frames[0] if hasattr(result, 'frames') else None + if want_audio: + pixels = out + if hasattr(result, 'audio') and result.audio is not None: + audio = result.audio[0].float().cpu() + latents = None + else: + latents = out except AssertionError as e: yield from abort(e, ok=True, p=p) return @@ -526,14 +565,15 @@ def run_ltx(task_id, extra_networks.deactivate(p) if needs_latent_path and latents is not None: - # Only reached on upsample-without-refine; refine decodes through the pipe and nulls latents. + # Decode any path that leaves latents intact: upsample-without-refine, or + # refine with output_type='latent' (audio_enable=False). shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'], force=True, silent=True) devices.torch_gc(force=True, reason='ltx:vae') yield None, 'LTX: VAE decode in progress...' try: if torch.is_tensor(latents): # 0.9.x returns raw latents with output_type='latent'; 2.x pre-denormalizes. - frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, int(seed) if seed is not None else -1, denormalize=caps.family == '0.9') + frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, resolved_seed, denormalize=caps.family == '0.9') else: frames_out = latents except AssertionError as e: From 4523e7c3ac00d0667429cac6d4946ee274356f32 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Mon, 4 May 2026 02:47:33 +0100 Subject: [PATCH 6/6] fix(ltx,processing): address pr #4819 review review feedback from vladmandic on commit 80fde086f. - run_ltx: drop the inline random.seed() / randrange resolve and use processing.fix_seed(p) right after the StableDiffusionProcessingVideo construction. p.seed carries the resolved int across every stage (latent_pass, both upsample paths, refine, post-refine vae_decode). matches the existing fix_seed(p) call sites in img2img.py:30, video_run.py:101, xyz_grid.py:266 and 8 others. - process_decode: add AudioFrameList(list) subclass and attach_audio helper to carry output.audio onto the returned frame list. mirrors the existing output.bytes early-return contract: samples.audio survives downstream so process_images_inner can collect it via the Processed.audio kwarg. - drop the p.audio_capture transit in process_diffusers and the fallback read in process_images_inner. p is input params, not output state. --- modules/ltx/ltx_process.py | 22 +++++++--------------- modules/processing.py | 2 +- modules/processing_diffusers.py | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/modules/ltx/ltx_process.py b/modules/ltx/ltx_process.py index 31930bb3c..e29c5fd79 100644 --- a/modules/ltx/ltx_process.py +++ b/modules/ltx/ltx_process.py @@ -170,15 +170,6 @@ def run_ltx(task_id, yield from abort(f'Video: cls={shared.sd_model.__class__.__name__} selected model is not LTX', ok=True) return - # get_generator(-1) reseeds globally per call, so every stage would otherwise - # roll an uncorrelated seed. Resolve once and thread the int through. - import random - if seed is None or int(seed) < 0: - random.seed() - resolved_seed = int(random.randrange(4294967294)) - else: - resolved_seed = int(seed) - # Lightricks TI2VidTwoStagesPipeline: Stage 1 at half-res, 2x upsample, Stage 2 refine at target. # Auto-couple when the user picks Refine but not Upsample. Both Dev and Distilled refine paths # expect upsampled latents; same-res refine on Distilled produces oversaturation. Condition @@ -262,7 +253,7 @@ def run_ltx(task_id, prompt=prompt, negative_prompt=negative, styles=styles, - seed=resolved_seed, + seed=int(seed) if seed is not None else -1, sampler_name=sampler_name, sampler_shift=float(sampler_shift), steps=int(steps), @@ -275,6 +266,7 @@ def run_ltx(task_id, vae_type='Default', vae_tile_frames=16, ) + processing.fix_seed(p) p.scripts = None p.script_args = None p.do_not_save_grid = True @@ -362,7 +354,7 @@ def run_ltx(task_id, mp4_fps=mp4_fps, conditions=conditions, image_cond_noise_scale=image_cond_noise_scale if caps.supports_image_cond_noise_scale else None, - seed=resolved_seed, + seed=p.seed, image=p.task_args.get('image'), ) else: @@ -414,7 +406,7 @@ def run_ltx(task_id, up_args = { 'width': final_w, 'height': final_h, - 'generator': get_generator(resolved_seed), + 'generator': get_generator(p.seed), 'output_type': 'latent', } if latents.ndim == 4: @@ -435,7 +427,7 @@ def run_ltx(task_id, 'height': final_h, 'num_frames': get_frames(frames), 'latents_normalized': False, - 'generator': get_generator(resolved_seed), + 'generator': get_generator(p.seed), 'output_type': 'latent', } if latents.ndim == 4: @@ -470,7 +462,7 @@ def run_ltx(task_id, 'height': final_h, 'num_frames': get_frames(frames), 'num_inference_steps': steps, - 'generator': get_generator(resolved_seed), + 'generator': get_generator(p.seed), 'callback_on_step_end': diffusers_callback, 'output_type': 'pil', } @@ -573,7 +565,7 @@ def run_ltx(task_id, try: if torch.is_tensor(latents): # 0.9.x returns raw latents with output_type='latent'; 2.x pre-denormalizes. - frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, resolved_seed, denormalize=caps.family == '0.9') + frames_out = vae_decode(latents, decode_timestep if caps.supports_decode_timestep else 0.0, p.seed, denormalize=caps.family == '0.9') else: frames_out = latents except AssertionError as e: diff --git a/modules/processing.py b/modules/processing.py index 36c600ebe..0306b20a8 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -528,7 +528,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: output_images.append(batch_image) infotexts.append(batch_infotext) - audio = getattr(samples, 'audio', None) or getattr(p, 'audio_capture', None) + audio = getattr(samples, 'audio', None) if shared.cmd_opts.lowvram: devices.torch_gc(force=True, reason='lowvram') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 6b701a966..38144a033 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -452,17 +452,32 @@ def process_refine(p: processing.StableDiffusionProcessing, output): return output +class AudioFrameList(list): + """list subclass with an audio attribute. Plain lists can't hold attributes, + so process_decode uses this when the pipeline output includes audio.""" + audio = None + + +def attach_audio(results, audio): + if audio is None: + return results + wrapped = AudioFrameList(results if isinstance(results, list) else list(results)) + wrapped.audio = audio + return wrapped + + def process_decode(p: processing.StableDiffusionProcessing, output): shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) if output is not None: if hasattr(output, 'bytes') and output.bytes is not None: log.debug(f'Generated: bytes={len(output.bytes)}') return output + audio = getattr(output, 'audio', None) if not hasattr(output, 'images') and hasattr(output, 'frames'): log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image): - return output.images + return attach_audio(output.images, audio) model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner if not hasattr(model, 'vae'): if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'): @@ -506,8 +521,9 @@ def process_decode(p: processing.StableDiffusionProcessing, output): results = [] else: log.warning('Processing: no results') + audio = None results = [] - return results + return attach_audio(results, audio) def update_pipeline(sd_model, p: processing.StableDiffusionProcessing): @@ -628,10 +644,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): timer.process.add('lora', lora_common.timer.total) lora_common.timer.clear(complete=True) - # process_decode flattens video output to a frame list and drops the audio attribute; - # stash it on `p` so video pipelines can recover it after process_images returns. - if output is not None and getattr(output, 'audio', None) is not None: - p.audio_capture = output.audio results = process_decode(p, output) timer.process.record('decode')