mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 09:14:35 +02:00
remove obsolete video scripts
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
+4
-1
@@ -5,9 +5,12 @@
|
||||
- **Features**
|
||||
- **wildcards**: allow recursive inline wildcards using curly braces syntax
|
||||
- **sdnq**: simplify pre-quantization saved config
|
||||
- **attention**: refactor settings and improve handling of attention mechanisms
|
||||
- **attention**: additional torch attention settings
|
||||
- **lora**: separate fuse setting for native-vs-diffuser implementations
|
||||
- **auth**: strong-enforce auth check on all api endpoints
|
||||
- **Internal**
|
||||
- refactor attention handling
|
||||
- remove obsolete video scripts
|
||||
- **Fixes**
|
||||
- hires: strength save/load in metadata, thanks @awsr
|
||||
- imgi2img: fix initial scale tab, thanks @awsr
|
||||
|
||||
+35
-36
@@ -1088,51 +1088,50 @@ def set_diffuser_pipe(pipe, new_pipe_type):
|
||||
if 'Onnx' in cls:
|
||||
return pipe
|
||||
|
||||
new_pipe = None
|
||||
# in some cases we want to reset the pipeline to parent as they dont have their own variants
|
||||
if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE or new_pipe_type == DiffusersTaskType.INPAINTING:
|
||||
if (new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE) or (new_pipe_type == DiffusersTaskType.INPAINTING):
|
||||
if cls == 'StableDiffusionPAGPipeline':
|
||||
pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe)
|
||||
if cls == 'StableDiffusionXLPAGPipeline':
|
||||
pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe)
|
||||
|
||||
new_pipe = None
|
||||
components_backup = backup_pipe_components(pipe)
|
||||
|
||||
if new_pipe is None:
|
||||
if hasattr(pipe, 'config'): # real pipeline which can be auto-switched
|
||||
try:
|
||||
if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
|
||||
new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe)
|
||||
elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE:
|
||||
new_pipe = diffusers.AutoPipelineForImage2Image.from_pipe(pipe)
|
||||
elif new_pipe_type == DiffusersTaskType.INPAINTING:
|
||||
new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe)
|
||||
else:
|
||||
shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}')
|
||||
return pipe
|
||||
except Exception as e: # pylint: disable=unused-variable
|
||||
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
|
||||
shared.log.trace(f"Pipeline class change requested: target={new_pipe_type} fn={fn}") # pylint: disable=protected-access
|
||||
shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}')
|
||||
has_errors = True
|
||||
if not hasattr(pipe, 'config') or has_errors:
|
||||
try: # maybe a wrapper pipeline so just change the class
|
||||
if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
|
||||
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
|
||||
new_pipe = pipe
|
||||
elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE:
|
||||
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
|
||||
new_pipe = pipe
|
||||
elif new_pipe_type == DiffusersTaskType.INPAINTING:
|
||||
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
|
||||
new_pipe = pipe
|
||||
else:
|
||||
shared.log.error(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls}')
|
||||
return pipe
|
||||
except Exception as e: # pylint: disable=unused-variable
|
||||
shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}')
|
||||
has_errors = True
|
||||
if hasattr(pipe, 'config'): # real pipeline which can be auto-switched
|
||||
try:
|
||||
if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
|
||||
new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe)
|
||||
elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE:
|
||||
new_pipe = diffusers.AutoPipelineForImage2Image.from_pipe(pipe)
|
||||
elif new_pipe_type == DiffusersTaskType.INPAINTING:
|
||||
new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe)
|
||||
else:
|
||||
shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}')
|
||||
return pipe
|
||||
except Exception as e: # pylint: disable=unused-variable
|
||||
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
|
||||
shared.log.trace(f"Pipeline class change requested: target={new_pipe_type} fn={fn}") # pylint: disable=protected-access
|
||||
shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}')
|
||||
has_errors = True
|
||||
if not hasattr(pipe, 'config') or has_errors:
|
||||
try: # maybe a wrapper pipeline so just change the class
|
||||
if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
|
||||
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
|
||||
new_pipe = pipe
|
||||
elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE:
|
||||
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
|
||||
new_pipe = pipe
|
||||
elif new_pipe_type == DiffusersTaskType.INPAINTING:
|
||||
pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access
|
||||
new_pipe = pipe
|
||||
else:
|
||||
shared.log.error(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls}')
|
||||
return pipe
|
||||
except Exception as e: # pylint: disable=unused-variable
|
||||
shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}')
|
||||
has_errors = True
|
||||
return pipe
|
||||
|
||||
if new_pipe is None:
|
||||
return pipe
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
import time
|
||||
import gradio as gr
|
||||
import transformers
|
||||
import diffusers
|
||||
from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer, sd_hijack_te
|
||||
|
||||
|
||||
repo_id = 'rhymes-ai/Allegro'
|
||||
|
||||
|
||||
def hijack_decode(*args, **kwargs):
|
||||
t0 = time.time()
|
||||
vae: diffusers.AutoencoderKLAllegro = shared.sd_model.vae
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
|
||||
res = shared.sd_model.vae.orig_decode(*args, **kwargs)
|
||||
t1 = time.time()
|
||||
timer.process.add('vae', t1-t0)
|
||||
shared.log.debug(f'Video: vae={vae.__class__.__name__} time={t1-t0:.2f}')
|
||||
return res
|
||||
|
||||
|
||||
class Script(scripts_manager.Script):
|
||||
def title(self):
|
||||
return 'Video: Allegro (Legacy)'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return not is_img2img
|
||||
|
||||
# return signature is array of gradio components
|
||||
def ui(self, is_img2img):
|
||||
with gr.Row():
|
||||
gr.HTML('<a href="https://huggingface.co/rhymes-ai/Allegro">  Allegro Video</a><br>')
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(label='Frames', minimum=4, maximum=88, step=1, value=22)
|
||||
with gr.Row():
|
||||
override_scheduler = gr.Checkbox(label='Override scheduler', value=True)
|
||||
with gr.Row():
|
||||
from modules.ui_sections import create_video_inputs
|
||||
video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img')
|
||||
return [num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
|
||||
def run(self, p: processing.StableDiffusionProcessing, num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
# set params
|
||||
num_frames = int(num_frames)
|
||||
p.width = 8 * int(p.width // 8)
|
||||
p.height = 8 * int(p.height // 8)
|
||||
p.do_not_save_grid = True
|
||||
p.ops.append('video')
|
||||
|
||||
# load model
|
||||
if shared.sd_model.__class__ != diffusers.AllegroPipeline:
|
||||
sd_models.unload_model_weights()
|
||||
t0 = time.time()
|
||||
quant_args = model_quant.create_config()
|
||||
transformer = diffusers.AllegroTransformer3DModel.from_pretrained(
|
||||
repo_id,
|
||||
subfolder="transformer",
|
||||
torch_dtype=devices.dtype,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**quant_args
|
||||
)
|
||||
shared.log.debug(f'Video: module={transformer.__class__.__name__}')
|
||||
text_encoder = transformers.T5EncoderModel.from_pretrained(
|
||||
repo_id,
|
||||
subfolder="text_encoder",
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**quant_args
|
||||
)
|
||||
shared.log.debug(f'Video: module={text_encoder.__class__.__name__}')
|
||||
shared.sd_model = diffusers.AllegroPipeline.from_pretrained(
|
||||
repo_id,
|
||||
# transformer=transformer,
|
||||
# text_encoder=text_encoder,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**quant_args
|
||||
)
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}')
|
||||
sd_models.set_diffuser_options(shared.sd_model)
|
||||
shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id)
|
||||
shared.sd_model.sd_model_hash = None
|
||||
shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
|
||||
shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
|
||||
shared.sd_model.vae.decode = hijack_decode
|
||||
shared.sd_model.vae.enable_tiling()
|
||||
# shared.sd_model.vae.enable_slicing()
|
||||
sd_hijack_te.init_hijack(shared.sd_model)
|
||||
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
||||
devices.torch_gc(force=True)
|
||||
|
||||
processing.fix_seed(p)
|
||||
if override_scheduler:
|
||||
p.sampler_name = 'Default'
|
||||
p.steps = 100
|
||||
p.task_args['num_frames'] = num_frames
|
||||
p.task_args['output_type'] = 'pil'
|
||||
p.task_args['clean_caption'] = False
|
||||
|
||||
p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts([p.prompt], [p.negative_prompt], p.styles, [p.seed])
|
||||
p.task_args['prompt'] = p.all_prompts[0]
|
||||
p.task_args['negative_prompt'] = p.all_negative_prompts[0]
|
||||
|
||||
# w = shared.sd_model.transformer.config.sample_width * shared.sd_model.vae_scale_factor_spatial
|
||||
# h = shared.sd_model.transformer.config.sample_height * shared.sd_model.vae_scale_factor_spatial
|
||||
# n = shared.sd_model.transformer.config.sample_frames * shared.sd_model.vae_scale_factor_temporal
|
||||
|
||||
# run processing
|
||||
t0 = time.time()
|
||||
shared.state.disable_preview = True
|
||||
shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}')
|
||||
processed = processing.process_images(p)
|
||||
shared.state.disable_preview = False
|
||||
t1 = time.time()
|
||||
if processed is not None and len(processed.images) > 0:
|
||||
shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
|
||||
if video_type != 'None':
|
||||
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
return processed
|
||||
@@ -1,215 +0,0 @@
|
||||
"""
|
||||
models: https://huggingface.co/THUDM/CogVideoX-2b https://huggingface.co/THUDM/CogVideoX-5b
|
||||
source: https://github.com/THUDM/CogVideo
|
||||
quanto: https://gist.github.com/a-r-r-o-w/31be62828b00a9292821b85c1017effa
|
||||
torchao: https://gist.github.com/a-r-r-o-w/4d9732d17412888c885480c6521a9897
|
||||
venhancer: https://github.com/THUDM/CogVideo/blob/dcb82ae30b454ab898aeced0633172d75dbd55b8/tools/venhancer/README.md
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import cv2
|
||||
import gradio as gr
|
||||
import torch
|
||||
from torchvision import transforms
|
||||
import diffusers
|
||||
import numpy as np
|
||||
from modules import scripts_manager, shared, devices, errors, sd_models, processing
|
||||
from modules.processing_callbacks import diffusers_callback, set_callbacks_p
|
||||
|
||||
|
||||
debug = (os.environ.get('SD_LOAD_DEBUG', None) is not None) or (os.environ.get('SD_PROCESS_DEBUG', None) is not None)
|
||||
|
||||
|
||||
class Script(scripts_manager.Script):
|
||||
def title(self):
|
||||
return 'Video: CogVideoX (Legacy)'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return True
|
||||
|
||||
|
||||
def ui(self, is_img2img):
|
||||
with gr.Row():
|
||||
gr.HTML("<span>  CogVideoX</span><br>")
|
||||
with gr.Row():
|
||||
model = gr.Dropdown(label='Model', choices=['None', 'THUDM/CogVideoX-2b', 'THUDM/CogVideoX-5b', 'THUDM/CogVideoX-5b-I2V'], value='THUDM/CogVideoX-2b')
|
||||
sampler = gr.Dropdown(label='Sampler', choices=['DDIM', 'DPM'], value='DDIM')
|
||||
with gr.Row():
|
||||
frames = gr.Slider(label='Frames', minimum=1, maximum=100, step=1, value=49)
|
||||
guidance = gr.Slider(label='Guidance', minimum=0.0, maximum=14.0, step=0.5, value=6.0)
|
||||
with gr.Row():
|
||||
offload = gr.Dropdown(label='Offload', choices=['none', 'balanced', 'model', 'sequential'], value='balanced')
|
||||
override = gr.Checkbox(label='Override resolution', value=True)
|
||||
with gr.Accordion('Optional init image or video', open=False):
|
||||
with gr.Row():
|
||||
image = gr.Image(value=None, label='Image', type='pil', width=256, height=256)
|
||||
video = gr.Video(value=None, label='Video', width=256, height=256)
|
||||
with gr.Row():
|
||||
from modules.ui_sections import create_video_inputs
|
||||
video_type, duration, loop, pad, interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img')
|
||||
return [model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video]
|
||||
|
||||
def load(self, model):
|
||||
if (shared.sd_model_type != 'cogvideo' or shared.sd_model.sd_model_checkpoint != model) and model != 'None':
|
||||
sd_models.unload_model_weights('model')
|
||||
shared.log.info(f'CogVideoX load: model="{model}"')
|
||||
try:
|
||||
shared.sd_model = None
|
||||
cls = diffusers.CogVideoXImageToVideoPipeline if 'I2V' in model else diffusers.CogVideoXPipeline
|
||||
shared.sd_model = cls.from_pretrained(model, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir)
|
||||
shared.sd_model.sd_checkpoint_info = sd_models.CheckpointInfo(model)
|
||||
shared.sd_model.sd_model_hash = ''
|
||||
shared.sd_model.sd_model_checkpoint = model
|
||||
except Exception as e:
|
||||
shared.log.error(f'Load CogVideoX: {e}')
|
||||
if debug:
|
||||
errors.display(e, 'CogVideoX')
|
||||
if shared.sd_model_type == 'cogvideo' and model != 'None':
|
||||
shared.sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba')
|
||||
shared.log.debug(f'CogVideoX load: class="{shared.sd_model.__class__.__name__}"')
|
||||
if shared.sd_model is not None and model == 'None':
|
||||
shared.log.info(f'CogVideoX unload: model={model}')
|
||||
shared.sd_model = None
|
||||
devices.torch_gc(force=True)
|
||||
devices.torch_gc()
|
||||
|
||||
def offload(self, offload):
|
||||
if shared.sd_model_type != 'cogvideo':
|
||||
return
|
||||
if offload == 'none':
|
||||
sd_models.move_model(shared.sd_model, devices.device)
|
||||
shared.log.debug(f'CogVideoX: offload={offload}')
|
||||
if offload == 'balanced':
|
||||
sd_models.apply_balanced_offload(shared.sd_model)
|
||||
if offload == 'model':
|
||||
shared.sd_model.enable_model_cpu_offload()
|
||||
if offload == 'sequential':
|
||||
shared.sd_model.enable_model_cpu_offload()
|
||||
shared.sd_model.enable_sequential_cpu_offload()
|
||||
shared.sd_model.vae.enable_slicing()
|
||||
shared.sd_model.vae.enable_tiling()
|
||||
|
||||
def video(self, p, fn):
|
||||
frames = []
|
||||
try:
|
||||
from modules.control.util import decode_fourcc
|
||||
video = cv2.VideoCapture(fn)
|
||||
if not video.isOpened():
|
||||
shared.log.error(f'Video: file="{fn}" open failed')
|
||||
return frames
|
||||
frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = int(video.get(cv2.CAP_PROP_FPS))
|
||||
w, h = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)), int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
codec = decode_fourcc(video.get(cv2.CAP_PROP_FOURCC))
|
||||
shared.log.debug(f'CogVideoX input: video="{fn}" fps={fps} width={w} height={h} codec={codec} frames={frame_count} target={len(frames)}')
|
||||
frames = []
|
||||
while True:
|
||||
ok, frame = video.read()
|
||||
if not ok:
|
||||
break
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
frame = cv2.resize(frame, (p.width, p.height))
|
||||
frames.append(frame)
|
||||
video.release()
|
||||
if len(frames) > p.frames:
|
||||
frames = np.asarray(frames)
|
||||
indices = np.linspace(0, len(frames) - 1, p.frames).astype(int) # reduce array from n_frames to p_frames
|
||||
frames = frames[indices]
|
||||
shared.log.debug(f'CogVideoX input reduce: source={len(frames)} target={p.frames}')
|
||||
frames = [transforms.ToTensor()(frame) for frame in frames]
|
||||
except Exception as e:
|
||||
shared.log.error(f'Video: file="{fn}" {e}')
|
||||
if debug:
|
||||
errors.display(e, 'CogVideoX')
|
||||
return frames
|
||||
|
||||
def image(self, p, img):
|
||||
img = img.resize((p.width, p.height))
|
||||
shared.log.debug(f'CogVideoX input: image={img}')
|
||||
# frames = [np.array(img)]
|
||||
# frames = [transforms.ToTensor()(frame) for frame in frames]
|
||||
return img
|
||||
|
||||
def generate(self, p: processing.StableDiffusionProcessing, model: str):
|
||||
if shared.sd_model_type != 'cogvideo':
|
||||
return []
|
||||
shared.log.info(f'CogVideoX: sampler={p.sampler} steps={p.steps} frames={p.frames} width={p.width} height={p.height} seed={p.seed} guidance={p.guidance}')
|
||||
if p.sampler == 'DDIM':
|
||||
shared.sd_model.scheduler = diffusers.CogVideoXDDIMScheduler.from_config(shared.sd_model.scheduler.config, timestep_spacing="trailing")
|
||||
if p.sampler == 'DPM':
|
||||
shared.sd_model.scheduler = diffusers.CogVideoXDPMScheduler.from_config(shared.sd_model.scheduler.config, timestep_spacing="trailing")
|
||||
t0 = time.time()
|
||||
frames = []
|
||||
set_callbacks_p(p)
|
||||
shared.state.job_count = 1
|
||||
shared.state.sampling_steps = p.steps - 1
|
||||
try:
|
||||
args = dict(
|
||||
prompt=p.prompt,
|
||||
negative_prompt=p.negative_prompt,
|
||||
height=p.height,
|
||||
width=p.width,
|
||||
num_videos_per_prompt=1,
|
||||
num_inference_steps=p.steps,
|
||||
guidance_scale=p.guidance,
|
||||
generator=torch.Generator(device=devices.device).manual_seed(p.seed),
|
||||
callback_on_step_end=diffusers_callback,
|
||||
callback_on_step_end_tensor_inputs=['latents'],
|
||||
)
|
||||
if 'I2V' in model:
|
||||
if hasattr(p, 'video') and p.video is not None:
|
||||
args['video'] = self.video(p, p.video)
|
||||
shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXVideoToVideoPipeline, shared.sd_model)
|
||||
elif (hasattr(p, 'image') and p.image is not None) or (hasattr(p, 'init_images') and len(p.init_images) > 0):
|
||||
p.init_images = [p.image] if hasattr(p, 'image') and p.image is not None else p.init_images
|
||||
args['image'] = self.image(p, p.init_images[0])
|
||||
shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXImageToVideoPipeline, shared.sd_model)
|
||||
else:
|
||||
shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXPipeline, shared.sd_model)
|
||||
args['num_frames'] = p.frames # only txt2vid has num_frames
|
||||
shared.log.info(f"CogVideoX: class={shared.sd_model.__class__.__name__} frames={p.frames} input={args.get('video', None) or args.get('image', None)}")
|
||||
if debug:
|
||||
shared.log.debug(f'CogVideoX args: {args}')
|
||||
frames = shared.sd_model(**args).frames[0]
|
||||
except AssertionError as e:
|
||||
shared.log.info(f'CogVideoX: {e}')
|
||||
except Exception as e:
|
||||
shared.log.error(f'CogVideoX: {e}')
|
||||
if debug:
|
||||
errors.display(e, 'CogVideoX')
|
||||
t1 = time.time()
|
||||
its = (len(frames) * p.steps) / (t1 - t0)
|
||||
shared.log.info(f'CogVideoX: frame={frames[0] if len(frames) > 0 else None} frames={len(frames)} its={its:.2f} time={t1 - t0:.2f}')
|
||||
return frames
|
||||
|
||||
# auto-executed by the script-callback
|
||||
def run(self, p: processing.StableDiffusionProcessing, model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video): # pylint: disable=arguments-differ, unused-argument
|
||||
processing.fix_seed(p)
|
||||
p.extra_generation_params['CogVideoX'] = model
|
||||
p.do_not_save_grid = True
|
||||
if 'animatediff' not in p.ops:
|
||||
p.ops.append('video')
|
||||
if override:
|
||||
p.width = 720
|
||||
p.height = 480
|
||||
p.sampler = sampler
|
||||
p.guidance = guidance
|
||||
p.frames = frames
|
||||
p.use_dynamic_cfg = sampler == 'DPM'
|
||||
p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles)
|
||||
p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles)
|
||||
p.image = image
|
||||
p.video = video
|
||||
self.load(model)
|
||||
self.offload(offload)
|
||||
frames = self.generate(p, model)
|
||||
devices.torch_gc()
|
||||
processed = processing.get_processed(p, images_list=frames)
|
||||
return processed
|
||||
|
||||
# auto-executed by the script-callback
|
||||
def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video): # pylint: disable=arguments-differ, unused-argument
|
||||
if video_type != 'None' and processed is not None and len(processed.images) > 0:
|
||||
from modules.images import save_video
|
||||
shared.log.info(f'CogVideoX video: type={video_type} duration={duration} loop={loop} pad={pad} interpolate={interpolate}')
|
||||
save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=loop, pad=pad, interpolate=interpolate)
|
||||
@@ -1,176 +0,0 @@
|
||||
import time
|
||||
import torch
|
||||
import gradio as gr
|
||||
import transformers
|
||||
import diffusers
|
||||
from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, sd_samplers, model_quant, timer, sd_hijack_te
|
||||
|
||||
|
||||
default_template = """Describe the video by detailing the following aspects:
|
||||
1. The main content and theme of the video.
|
||||
2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.
|
||||
3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.
|
||||
4. Background environment, light, style and atmosphere.
|
||||
5. Camera angles, movements, and transitions used in the video.
|
||||
6. Thematic and aesthetic concepts associated with the scene, i.e. realistic, futuristic, fairy tale, etc.
|
||||
"""
|
||||
|
||||
models = {
|
||||
'HunyuanVideo': { 'repo': 'tencent/HunyuanVideo', 'revision': 'refs/pr/18' },
|
||||
'FastHunyuan': { 'repo': 'FastVideo/FastHunyuan', 'revision': None },
|
||||
}
|
||||
loaded_model = None
|
||||
|
||||
|
||||
def get_template(template: str = None):
|
||||
# diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.DEFAULT_PROMPT_TEMPLATE
|
||||
base_template_pre = "<|start_header_id|>system<|end_header_id|>\n\n"
|
||||
base_template_post = "<|eot_id|>\n"
|
||||
base_template_end = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
|
||||
if template is None or len(template) == 0:
|
||||
template = default_template
|
||||
template_lines = '\n'.join([line for line in template.split('\n') if len(line) > 0])
|
||||
prompt_template = {
|
||||
"crop_start": 95,
|
||||
"template": base_template_pre + template_lines + base_template_post + base_template_end
|
||||
}
|
||||
return prompt_template
|
||||
|
||||
|
||||
def hijack_decode(*args, **kwargs):
|
||||
t0 = time.time()
|
||||
vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
|
||||
res = shared.sd_model.vae.orig_decode(*args, **kwargs)
|
||||
t1 = time.time()
|
||||
timer.process.add('vae', t1-t0)
|
||||
shared.log.debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}')
|
||||
return res
|
||||
|
||||
|
||||
class Script(scripts_manager.Script):
|
||||
def title(self):
|
||||
return 'Video: Hunyuan Video (Legacy)'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return not is_img2img
|
||||
|
||||
# return signature is array of gradio components
|
||||
def ui(self, is_img2img):
|
||||
with gr.Row():
|
||||
gr.HTML('<a href="https://huggingface.co/tencent/HunyuanVideo">  Hunyuan Video</a><br>')
|
||||
with gr.Row():
|
||||
model = gr.Dropdown(label='Model', choices=list(models.keys()), value=list(models.keys())[0])
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45)
|
||||
tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16)
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
override_scheduler = gr.Checkbox(label='HV override sampler', value=True)
|
||||
with gr.Column():
|
||||
scheduler_shift = gr.Slider(label='HV sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0)
|
||||
with gr.Row():
|
||||
template = gr.TextArea(label='HV prompt processor', lines=3, value=default_template, visible=False)
|
||||
with gr.Row():
|
||||
from modules.ui_sections import create_video_inputs
|
||||
video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img')
|
||||
return [model, num_frames, tile_frames, override_scheduler, scheduler_shift, template, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
|
||||
def load(self, model:str):
|
||||
global loaded_model # pylint: disable=global-statement
|
||||
if shared.sd_model.__class__ != diffusers.HunyuanVideoPipeline or model != loaded_model:
|
||||
sd_models.unload_model_weights()
|
||||
t0 = time.time()
|
||||
quant_args = model_quant.create_config()
|
||||
transformer = diffusers.HunyuanVideoTransformer3DModel.from_pretrained(
|
||||
pretrained_model_name_or_path='tencent/HunyuanVideo',
|
||||
subfolder="transformer",
|
||||
torch_dtype=devices.dtype,
|
||||
revision='refs/pr/18',
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**quant_args
|
||||
)
|
||||
shared.log.debug(f'Video: module={transformer.__class__.__name__}')
|
||||
text_encoder = transformers.LlamaModel.from_pretrained(
|
||||
pretrained_model_name_or_path=models.get(model)['repo'],
|
||||
subfolder="text_encoder",
|
||||
revision=models.get(model)['revision'],
|
||||
cache_dir = shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**quant_args
|
||||
)
|
||||
text_encoder_2 = transformers.CLIPTextModel.from_pretrained(
|
||||
pretrained_model_name_or_path=models.get(model)['repo'],
|
||||
subfolder="text_encoder_2",
|
||||
revision=models.get(model)['revision'],
|
||||
cache_dir = shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
)
|
||||
shared.log.debug(f'Video: module={text_encoder.__class__.__name__}')
|
||||
shared.sd_model = diffusers.HunyuanVideoPipeline.from_pretrained(
|
||||
pretrained_model_name_or_path='tencent/HunyuanVideo',
|
||||
transformer=transformer,
|
||||
text_encoder=text_encoder,
|
||||
text_encoder_2=text_encoder_2,
|
||||
revision='refs/pr/18',
|
||||
cache_dir = shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**quant_args
|
||||
)
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} model="{model}" repo={models.get(model)["repo"]} dtype={devices.dtype} time={t1-t0:.2f}')
|
||||
sd_models.set_diffuser_options(shared.sd_model)
|
||||
shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(models.get(model)['repo'])
|
||||
shared.sd_model.sd_model_hash = None
|
||||
shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
|
||||
shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
|
||||
shared.sd_model.vae.decode = hijack_decode
|
||||
shared.sd_model.vae.enable_slicing()
|
||||
shared.sd_model.vae.enable_tiling()
|
||||
shared.sd_model.vae.use_framewise_decoding = True
|
||||
sd_hijack_te.init_hijack(shared.sd_model)
|
||||
loaded_model = model
|
||||
|
||||
def run(self, p: processing.StableDiffusionProcessing, model, num_frames, tile_frames, override_scheduler, scheduler_shift, template, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
# set params
|
||||
num_frames = int(num_frames)
|
||||
p.width = 16 * int(p.width // 16)
|
||||
p.height = 16 * int(p.height // 16)
|
||||
p.do_not_save_grid = True
|
||||
p.ops.append('video')
|
||||
|
||||
# load model
|
||||
self.load(model)
|
||||
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
||||
devices.torch_gc(force=True)
|
||||
|
||||
if override_scheduler:
|
||||
p.sampler_name = 'Default'
|
||||
else:
|
||||
shared.sd_model.scheduler = sd_samplers.create_sampler(p.sampler_name, shared.sd_model)
|
||||
p.sampler_name = 'Default' # avoid double creation
|
||||
if hasattr(shared.sd_model.scheduler, '_shift'):
|
||||
shared.sd_model.scheduler._shift = scheduler_shift # pylint: disable=protected-access
|
||||
|
||||
# encode prompt
|
||||
processing.fix_seed(p)
|
||||
p.task_args['num_frames'] = num_frames
|
||||
p.task_args['output_type'] = 'pil'
|
||||
p.task_args['generator'] = torch.manual_seed(p.seed)
|
||||
# p.task_args['prompt'] = None
|
||||
# p.task_args['prompt_embeds'], p.task_args['pooled_prompt_embeds'], p.task_args['prompt_attention_mask'] = shared.sd_model.encode_prompt(prompt=p.prompt, prompt_template=get_template(template), device=devices.device)
|
||||
|
||||
# run processing
|
||||
t0 = time.time()
|
||||
shared.sd_model.vae.tile_sample_min_num_frames = tile_frames
|
||||
shared.state.disable_preview = True
|
||||
shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}')
|
||||
processed = processing.process_images(p)
|
||||
shared.state.disable_preview = False
|
||||
t1 = time.time()
|
||||
if processed is not None and len(processed.images) > 0:
|
||||
shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
|
||||
if video_type != 'None':
|
||||
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
return processed
|
||||
@@ -1,121 +0,0 @@
|
||||
import time
|
||||
import gradio as gr
|
||||
import transformers
|
||||
import diffusers
|
||||
from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer, sd_hijack_te
|
||||
|
||||
|
||||
repo_id = 'rhymes-ai/Allegro'
|
||||
|
||||
|
||||
def hijack_decode(*args, **kwargs):
|
||||
t0 = time.time()
|
||||
vae: diffusers.AutoencoderKLAllegro = shared.sd_model.vae
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
|
||||
res = shared.sd_model.vae.orig_decode(*args, **kwargs)
|
||||
t1 = time.time()
|
||||
timer.process.add('vae', t1-t0)
|
||||
shared.log.debug(f'Video: vae={vae.__class__.__name__} time={t1-t0:.2f}')
|
||||
return res
|
||||
|
||||
|
||||
class Script(scripts_manager.Script):
|
||||
def title(self):
|
||||
return 'Video: Allegro (Legacy)'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return not is_img2img
|
||||
|
||||
# return signature is array of gradio components
|
||||
def ui(self, is_img2img):
|
||||
with gr.Row():
|
||||
gr.HTML('<a href="https://huggingface.co/rhymes-ai/Allegro">  Allegro Video</a><br>')
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(label='Frames', minimum=4, maximum=88, step=1, value=22)
|
||||
with gr.Row():
|
||||
override_scheduler = gr.Checkbox(label='Override scheduler', value=True)
|
||||
with gr.Row():
|
||||
from modules.ui_sections import create_video_inputs
|
||||
video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img')
|
||||
return [num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
|
||||
def run(self, p: processing.StableDiffusionProcessing, num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
# set params
|
||||
num_frames = int(num_frames)
|
||||
p.width = 8 * int(p.width // 8)
|
||||
p.height = 8 * int(p.height // 8)
|
||||
p.do_not_save_grid = True
|
||||
p.ops.append('video')
|
||||
|
||||
# load model
|
||||
if shared.sd_model.__class__ != diffusers.AllegroPipeline:
|
||||
sd_models.unload_model_weights()
|
||||
t0 = time.time()
|
||||
quant_args = model_quant.create_config()
|
||||
transformer = diffusers.AllegroTransformer3DModel.from_pretrained(
|
||||
repo_id,
|
||||
subfolder="transformer",
|
||||
torch_dtype=devices.dtype,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
**quant_args
|
||||
)
|
||||
shared.log.debug(f'Video: module={transformer.__class__.__name__}')
|
||||
text_encoder = transformers.T5EncoderModel.from_pretrained(
|
||||
repo_id,
|
||||
subfolder="text_encoder",
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**quant_args
|
||||
)
|
||||
shared.log.debug(f'Video: module={text_encoder.__class__.__name__}')
|
||||
shared.sd_model = diffusers.AllegroPipeline.from_pretrained(
|
||||
repo_id,
|
||||
# transformer=transformer,
|
||||
# text_encoder=text_encoder,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**quant_args
|
||||
)
|
||||
t1 = time.time()
|
||||
shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}')
|
||||
sd_models.set_diffuser_options(shared.sd_model)
|
||||
shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id)
|
||||
shared.sd_model.sd_model_hash = None
|
||||
shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
|
||||
shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
|
||||
shared.sd_model.vae.decode = hijack_decode
|
||||
shared.sd_model.vae.enable_tiling()
|
||||
sd_hijack_te.init_hijack(shared.sd_model)
|
||||
# shared.sd_model.vae.enable_slicing()
|
||||
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
||||
devices.torch_gc(force=True)
|
||||
|
||||
processing.fix_seed(p)
|
||||
if override_scheduler:
|
||||
p.sampler_name = 'Default'
|
||||
p.steps = 100
|
||||
p.task_args['num_frames'] = num_frames
|
||||
p.task_args['output_type'] = 'pil'
|
||||
p.task_args['clean_caption'] = False
|
||||
|
||||
p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts([p.prompt], [p.negative_prompt], p.styles, [p.seed])
|
||||
p.task_args['prompt'] = p.all_prompts[0]
|
||||
p.task_args['negative_prompt'] = p.all_negative_prompts[0]
|
||||
|
||||
# w = shared.sd_model.transformer.config.sample_width * shared.sd_model.vae_scale_factor_spatial
|
||||
# h = shared.sd_model.transformer.config.sample_height * shared.sd_model.vae_scale_factor_spatial
|
||||
# n = shared.sd_model.transformer.config.sample_frames * shared.sd_model.vae_scale_factor_temporal
|
||||
|
||||
# run processing
|
||||
t0 = time.time()
|
||||
shared.state.disable_preview = True
|
||||
shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}')
|
||||
processed = processing.process_images(p)
|
||||
shared.state.disable_preview = False
|
||||
t1 = time.time()
|
||||
if processed is not None and len(processed.images) > 0:
|
||||
shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
|
||||
if video_type != 'None':
|
||||
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
return processed
|
||||
@@ -1,152 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import torch
|
||||
import gradio as gr
|
||||
import diffusers
|
||||
import transformers
|
||||
from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer, sd_hijack_te
|
||||
|
||||
|
||||
repos = {
|
||||
'0.9.0': 'a-r-r-o-w/LTX-Video-diffusers',
|
||||
'0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers',
|
||||
'0.9.5': 'Lightricks/LTX-Video-0.9.5',
|
||||
'custom': None,
|
||||
}
|
||||
|
||||
|
||||
def load_quants(kwargs, repo_id):
|
||||
quant_args = model_quant.create_config()
|
||||
if not quant_args:
|
||||
return kwargs
|
||||
model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}')
|
||||
if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization):
|
||||
kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args)
|
||||
shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}')
|
||||
if 'text_encoder' not in kwargs and ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization):
|
||||
kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args)
|
||||
shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}')
|
||||
return kwargs
|
||||
|
||||
|
||||
def hijack_decode(*args, **kwargs):
|
||||
t0 = time.time()
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
|
||||
res = shared.sd_model.vae.orig_decode(*args, **kwargs)
|
||||
t1 = time.time()
|
||||
timer.process.add('vae', t1-t0)
|
||||
shared.log.debug(f'Video: vae={shared.sd_model.vae.__class__.__name__} time={t1-t0:.2f}')
|
||||
return res
|
||||
|
||||
|
||||
class Script(scripts_manager.Script):
|
||||
def title(self):
|
||||
return 'Video: LTX Video (Legacy)'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return True
|
||||
|
||||
# return signature is array of gradio components
|
||||
def ui(self, is_img2img):
|
||||
def model_change(model):
|
||||
return gr.update(visible=model == 'custom')
|
||||
|
||||
with gr.Row():
|
||||
gr.HTML('<a href="https://www.ltxvideo.org/">  LTX Video</a><br>')
|
||||
with gr.Row():
|
||||
model = gr.Dropdown(label='LTX Model', choices=list(repos), value='0.9.1')
|
||||
decode = gr.Dropdown(label='Decode', choices=['diffusers', 'native'], value='diffusers', visible=False)
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41)
|
||||
sampler = gr.Checkbox(label='Override sampler', value=True)
|
||||
with gr.Row():
|
||||
teacache_enable = gr.Checkbox(label='Enable TeaCache', value=False)
|
||||
teacache_threshold = gr.Slider(label='Threshold', minimum=0.01, maximum=0.1, step=0.01, value=0.03)
|
||||
with gr.Row():
|
||||
model_custom = gr.Textbox(value='', label='Path to model file', visible=False)
|
||||
with gr.Row():
|
||||
from modules.ui_sections import create_video_inputs
|
||||
video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img')
|
||||
model.change(fn=model_change, inputs=[model], outputs=[model_custom])
|
||||
return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, teacache_enable, teacache_threshold]
|
||||
|
||||
def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, teacache_enable, teacache_threshold): # pylint: disable=arguments-differ, unused-argument
|
||||
# set params
|
||||
image = getattr(p, 'init_images', None)
|
||||
image = None if image is None or len(image) == 0 else image[0]
|
||||
if (p.width == 0 or p.height == 0) and image is not None:
|
||||
p.width = image.width
|
||||
p.height = image.height
|
||||
num_frames = 8 * int(num_frames // 8) + 1
|
||||
p.width = 32 * int(p.width // 32)
|
||||
p.height = 32 * int(p.height // 32)
|
||||
processing.fix_seed(p)
|
||||
if image:
|
||||
image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil')
|
||||
p.task_args['image'] = image
|
||||
p.task_args['output_type'] = 'latent' if decode == 'native' else 'pil'
|
||||
p.task_args['generator'] = torch.Generator(devices.device).manual_seed(p.seed)
|
||||
p.task_args['num_frames'] = num_frames
|
||||
p.do_not_save_grid = True
|
||||
if sampler:
|
||||
p.sampler_name = 'Default'
|
||||
p.ops.append('video')
|
||||
|
||||
# load model
|
||||
cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline
|
||||
diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel
|
||||
diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo
|
||||
repo_id = repos[model]
|
||||
if repo_id is None:
|
||||
repo_id = model_custom
|
||||
if shared.sd_model.__class__ != cls:
|
||||
sd_models.unload_model_weights()
|
||||
kwargs = model_quant.create_config()
|
||||
if os.path.isfile(repo_id):
|
||||
shared.sd_model = cls.from_single_file(
|
||||
repo_id,
|
||||
cache_dir = shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**kwargs
|
||||
)
|
||||
else:
|
||||
kwargs = load_quants(kwargs, repo_id)
|
||||
shared.sd_model = cls.from_pretrained(
|
||||
repo_id,
|
||||
cache_dir = shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**kwargs
|
||||
)
|
||||
sd_models.set_diffuser_options(shared.sd_model)
|
||||
shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode
|
||||
shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt
|
||||
shared.sd_model.vae.decode = hijack_decode
|
||||
shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id)
|
||||
shared.sd_model.sd_model_hash = None
|
||||
sd_hijack_te.init_hijack(shared.sd_model)
|
||||
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
||||
shared.sd_model.vae.enable_slicing()
|
||||
shared.sd_model.vae.enable_tiling()
|
||||
shared.sd_model.vae.use_framewise_decoding = True
|
||||
devices.torch_gc(force=True)
|
||||
|
||||
shared.sd_model.transformer.cnt = 0
|
||||
shared.sd_model.transformer.accumulated_rel_l1_distance = 0
|
||||
shared.sd_model.transformer.previous_modulated_input = None
|
||||
shared.sd_model.transformer.previous_residual = None
|
||||
shared.sd_model.transformer.enable_teacache = teacache_enable
|
||||
shared.sd_model.transformer.rel_l1_thresh = teacache_threshold
|
||||
shared.sd_model.transformer.num_steps = p.steps
|
||||
|
||||
shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args} steps={p.steps} teacache={teacache_enable} threshold={teacache_threshold}')
|
||||
|
||||
# run processing
|
||||
t0 = time.time()
|
||||
processed = processing.process_images(p)
|
||||
t1 = time.time()
|
||||
if processed is not None and len(processed.images) > 0:
|
||||
shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
|
||||
if video_type != 'None':
|
||||
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
return processed
|
||||
@@ -1,70 +0,0 @@
|
||||
import time
|
||||
import torch
|
||||
import gradio as gr
|
||||
import diffusers
|
||||
from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant
|
||||
|
||||
|
||||
repo_id = 'genmo/mochi-1-preview'
|
||||
|
||||
|
||||
class Script(scripts_manager.Script):
|
||||
def title(self):
|
||||
return 'Video: Mochi.1 Video (Legacy)'
|
||||
|
||||
def show(self, is_img2img):
|
||||
return not is_img2img
|
||||
|
||||
# return signature is array of gradio components
|
||||
def ui(self, is_img2img):
|
||||
with gr.Row():
|
||||
gr.HTML('<a href="https://huggingface.co/genmo/mochi-1-preview">  Mochi.1 Video</a><br>')
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45)
|
||||
with gr.Row():
|
||||
from modules.ui_sections import create_video_inputs
|
||||
video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img')
|
||||
return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
|
||||
|
||||
def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
|
||||
# set params
|
||||
num_frames = int(num_frames)
|
||||
p.width = 32 * int(p.width // 32)
|
||||
p.height = 32 * int(p.height // 32)
|
||||
p.task_args['output_type'] = 'pil'
|
||||
p.task_args['generator'] = torch.manual_seed(p.seed)
|
||||
p.task_args['num_frames'] = num_frames
|
||||
p.sampler_name = 'Default'
|
||||
p.do_not_save_grid = True
|
||||
p.ops.append('video')
|
||||
|
||||
# load model
|
||||
cls = diffusers.MochiPipeline
|
||||
if shared.sd_model.__class__ != cls:
|
||||
sd_models.unload_model_weights()
|
||||
kwargs = model_quant.create_config()
|
||||
shared.sd_model = cls.from_pretrained(
|
||||
repo_id,
|
||||
cache_dir = shared.opts.hfcache_dir,
|
||||
torch_dtype=devices.dtype,
|
||||
**kwargs
|
||||
)
|
||||
shared.sd_model.scheduler._shift = 7.0 # pylint: disable=protected-access
|
||||
sd_models.set_diffuser_options(shared.sd_model)
|
||||
shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id)
|
||||
shared.sd_model.sd_model_hash = None
|
||||
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
||||
shared.sd_model.vae.enable_slicing()
|
||||
shared.sd_model.vae.enable_tiling()
|
||||
devices.torch_gc(force=True)
|
||||
shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}')
|
||||
|
||||
# run processing
|
||||
t0 = time.time()
|
||||
processed = processing.process_images(p)
|
||||
t1 = time.time()
|
||||
if processed is not None and len(processed.images) > 0:
|
||||
shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}')
|
||||
if video_type != 'None':
|
||||
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
|
||||
return processed
|
||||
Reference in New Issue
Block a user