mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 09:14:35 +02:00
fix(ltx): decode stage 1 audio directly, discard stage 2 audio output
stop threading stage 1 audio_latents into stage 2 refine. Lightricks/LTX-2#126 reports the two-stage pipeline degrades audio quality, confirmed locally as clean speech with tinny ambient/foley/music on the threaded path. root cause: stage 2 prepare_audio_latents calls _create_noised_state at noise_scale=0.909 (pipeline_ltx2.py:704-714, 598-603), keeping ~9% of stage 1 signal. 3 refine steps recover speech via video<->audio cross-attention but not broadband content. new path: _latent_pass decodes audio_latents to waveform via audio_vae + vocoder mirroring pipeline_ltx2.py:1471-1473 exactly (input cast to audio_vae.dtype, no module dtype mutation). stage 2 result.audio is discarded; cross-attention still runs each block for video conditioning.
This commit is contained in:
+25
-17
@@ -93,13 +93,25 @@ def _latent_pass(caps, prompt, negative, width, height, frames, steps, guidance_
|
|||||||
base_args['use_cross_timestep'] = True
|
base_args['use_cross_timestep'] = True
|
||||||
log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}')
|
log.debug(f'Video: cls={shared.sd_model.__class__.__name__} op=latent_pass args_keys={list(base_args.keys())}')
|
||||||
result = shared.sd_model(**base_args)
|
result = shared.sd_model(**base_args)
|
||||||
# video latents strip the batch dim; audio latents keep it so LTX2Pipeline.prepare_audio_latents
|
|
||||||
# can rewrap them when re-entered as ndim==4 at Stage 2.
|
|
||||||
latents = result.frames[0] if hasattr(result, 'frames') else None
|
latents = result.frames[0] if hasattr(result, 'frames') else None
|
||||||
audio_latents = None
|
# output_type='latent' already returns denormalized + unpacked audio_latents. Threading
|
||||||
|
# them into Stage 2 re-noises at sigma=0.909 (prepare_audio_latents) and 3 refine steps
|
||||||
|
# cannot recover broadband content. Decode here mirroring the pipeline's non-latent path;
|
||||||
|
# cast only the input tensor since mutating module dtypes shifts BWE activations.
|
||||||
|
audio_waveform = None
|
||||||
if hasattr(result, 'audio') and result.audio is not None:
|
if hasattr(result, 'audio') and result.audio is not None:
|
||||||
audio_latents = result.audio
|
pipe = shared.sd_model
|
||||||
return latents, audio_latents
|
if hasattr(pipe, 'audio_vae') and hasattr(pipe, 'vocoder'):
|
||||||
|
try:
|
||||||
|
audio_latents = result.audio.to(device=devices.device, dtype=pipe.audio_vae.dtype)
|
||||||
|
with torch.no_grad():
|
||||||
|
mel = pipe.audio_vae.decode(audio_latents, return_dict=False)[0]
|
||||||
|
waveform = pipe.vocoder(mel)
|
||||||
|
audio_waveform = waveform[0].float().cpu()
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f'LTX: Stage 1 audio decode failed: {e}')
|
||||||
|
audio_waveform = None
|
||||||
|
return latents, audio_waveform
|
||||||
|
|
||||||
|
|
||||||
def run_ltx(task_id,
|
def run_ltx(task_id,
|
||||||
@@ -318,7 +330,7 @@ def run_ltx(task_id,
|
|||||||
yield None, 'LTX: Generate in progress...'
|
yield None, 'LTX: Generate in progress...'
|
||||||
|
|
||||||
audio = None
|
audio = None
|
||||||
stage1_audio_latents = None
|
stage1_audio = None
|
||||||
pixels = None
|
pixels = None
|
||||||
frames_out = None
|
frames_out = None
|
||||||
needs_latent_path = upsample_enable or refine_enable
|
needs_latent_path = upsample_enable or refine_enable
|
||||||
@@ -327,7 +339,7 @@ def run_ltx(task_id,
|
|||||||
if needs_latent_path:
|
if needs_latent_path:
|
||||||
prompt_final, negative_final, networks = get_prompts(prompt, negative, styles)
|
prompt_final, negative_final, networks = get_prompts(prompt, negative, styles)
|
||||||
extra_networks.activate(p, networks)
|
extra_networks.activate(p, networks)
|
||||||
latents, stage1_audio_latents = _latent_pass(
|
latents, stage1_audio = _latent_pass(
|
||||||
caps=caps,
|
caps=caps,
|
||||||
prompt=prompt_final,
|
prompt=prompt_final,
|
||||||
negative=negative_final,
|
negative=negative_final,
|
||||||
@@ -457,14 +469,10 @@ def run_ltx(task_id,
|
|||||||
# Thread Stage-1 I2V init image through Stage 2 so first-frame identity survives refine.
|
# Thread Stage-1 I2V init image through Stage 2 so first-frame identity survives refine.
|
||||||
if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and p.task_args.get('image') is not None:
|
if caps.is_i2v and caps.repo_cls_name in ('LTXImageToVideoPipeline', 'LTX2ImageToVideoPipeline') and p.task_args.get('image') is not None:
|
||||||
refine_args['image'] = p.task_args['image']
|
refine_args['image'] = p.task_args['image']
|
||||||
# Thread Stage-1 audio latents into Stage 2 on 2.x. The video branch cross-attends
|
# Audio cross-attention still runs in Stage 2 for video conditioning, but the
|
||||||
# audio every layer; letting prepare_audio_latents fall back to fresh noise biases
|
# vocoder output is discarded; Stage 1 decode (see _latent_pass) is authoritative.
|
||||||
# the video branch off-distribution (desaturated output on distilled 2.x).
|
if caps.family == '2.x' and caps.use_cross_timestep:
|
||||||
if caps.family == '2.x':
|
refine_args['use_cross_timestep'] = True
|
||||||
if stage1_audio_latents is not None:
|
|
||||||
refine_args['audio_latents'] = stage1_audio_latents.to(device=devices.device)
|
|
||||||
if caps.use_cross_timestep:
|
|
||||||
refine_args['use_cross_timestep'] = True
|
|
||||||
|
|
||||||
saved_scheduler_stage2 = None
|
saved_scheduler_stage2 = None
|
||||||
try:
|
try:
|
||||||
@@ -503,8 +511,6 @@ def run_ltx(task_id,
|
|||||||
try:
|
try:
|
||||||
result = shared.sd_model(latents=latents, **refine_args)
|
result = shared.sd_model(latents=latents, **refine_args)
|
||||||
pixels = result.frames[0] if hasattr(result, 'frames') else None
|
pixels = result.frames[0] if hasattr(result, 'frames') else None
|
||||||
if hasattr(result, 'audio') and result.audio is not None:
|
|
||||||
audio = result.audio[0].float().cpu()
|
|
||||||
latents = None
|
latents = None
|
||||||
except AssertionError as e:
|
except AssertionError as e:
|
||||||
yield from abort(e, ok=True, p=p)
|
yield from abort(e, ok=True, p=p)
|
||||||
@@ -555,6 +561,8 @@ def run_ltx(task_id,
|
|||||||
t11 = time.time()
|
t11 = time.time()
|
||||||
timer.process.add('offload', t11 - t10)
|
timer.process.add('offload', t11 - t10)
|
||||||
|
|
||||||
|
if stage1_audio is not None:
|
||||||
|
audio = stage1_audio
|
||||||
if not audio_enable:
|
if not audio_enable:
|
||||||
audio = None
|
audio = None
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user