diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d9486ed4..51a8c8cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,12 @@ - **Features** - **wildcards**: allow recursive inline wildcards using curly braces syntax - **sdnq**: simplify pre-quantization saved config - - **attention**: refactor settings and improve handling of attention mechanisms + - **attention**: additional torch attention settings - **lora**: separate fuse setting for native-vs-diffuser implementations - **auth**: strong-enforce auth check on all api endpoints +- **Internal** + - refactor attention handling + - remove obsolete video scripts - **Fixes** - hires: strength save/load in metadata, thanks @awsr - imgi2img: fix initial scale tab, thanks @awsr diff --git a/modules/sd_models.py b/modules/sd_models.py index 2c3e9f0f0..4f49a97e6 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1088,51 +1088,50 @@ def set_diffuser_pipe(pipe, new_pipe_type): if 'Onnx' in cls: return pipe - new_pipe = None # in some cases we want to reset the pipeline to parent as they dont have their own variants - if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE or new_pipe_type == DiffusersTaskType.INPAINTING: + if (new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE) or (new_pipe_type == DiffusersTaskType.INPAINTING): if cls == 'StableDiffusionPAGPipeline': pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) if cls == 'StableDiffusionXLPAGPipeline': pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) + new_pipe = None components_backup = backup_pipe_components(pipe) - if new_pipe is None: - if hasattr(pipe, 'config'): # real pipeline which can be auto-switched - try: - if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: - new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe) - elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: - new_pipe = diffusers.AutoPipelineForImage2Image.from_pipe(pipe) - elif new_pipe_type == DiffusersTaskType.INPAINTING: - new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) - else: - shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}') - return pipe - except Exception as e: # pylint: disable=unused-variable - fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - shared.log.trace(f"Pipeline class change requested: target={new_pipe_type} fn={fn}") # pylint: disable=protected-access - shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}') - has_errors = True - if not hasattr(pipe, 'config') or has_errors: - try: # maybe a wrapper pipeline so just change the class - if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: - pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access - new_pipe = pipe - elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: - pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access - new_pipe = pipe - elif new_pipe_type == DiffusersTaskType.INPAINTING: - pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access - new_pipe = pipe - else: - shared.log.error(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls}') - return pipe - except Exception as e: # pylint: disable=unused-variable - shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}') - has_errors = True + if hasattr(pipe, 'config'): # real pipeline which can be auto-switched + try: + if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: + new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe) + elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + new_pipe = diffusers.AutoPipelineForImage2Image.from_pipe(pipe) + elif new_pipe_type == DiffusersTaskType.INPAINTING: + new_pipe = diffusers.AutoPipelineForInpainting.from_pipe(pipe) + else: + shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}') return pipe + except Exception as e: # pylint: disable=unused-variable + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + shared.log.trace(f"Pipeline class change requested: target={new_pipe_type} fn={fn}") # pylint: disable=protected-access + shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}') + has_errors = True + if not hasattr(pipe, 'config') or has_errors: + try: # maybe a wrapper pipeline so just change the class + if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access + new_pipe = pipe + elif new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE: + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access + new_pipe = pipe + elif new_pipe_type == DiffusersTaskType.INPAINTING: + pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access + new_pipe = pipe + else: + shared.log.error(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls}') + return pipe + except Exception as e: # pylint: disable=unused-variable + shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}') + has_errors = True + return pipe if new_pipe is None: return pipe diff --git a/scripts/allegrovideo.py b/scripts/allegrovideo.py deleted file mode 100644 index 4ff136762..000000000 --- a/scripts/allegrovideo.py +++ /dev/null @@ -1,121 +0,0 @@ -import time -import gradio as gr -import transformers -import diffusers -from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer, sd_hijack_te - - -repo_id = 'rhymes-ai/Allegro' - - -def hijack_decode(*args, **kwargs): - t0 = time.time() - vae: diffusers.AutoencoderKLAllegro = shared.sd_model.vae - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) - res = shared.sd_model.vae.orig_decode(*args, **kwargs) - t1 = time.time() - timer.process.add('vae', t1-t0) - shared.log.debug(f'Video: vae={vae.__class__.__name__} time={t1-t0:.2f}') - return res - - -class Script(scripts_manager.Script): - def title(self): - return 'Video: Allegro (Legacy)' - - def show(self, is_img2img): - return not is_img2img - - # return signature is array of gradio components - def ui(self, is_img2img): - with gr.Row(): - gr.HTML('  Allegro Video
') - with gr.Row(): - num_frames = gr.Slider(label='Frames', minimum=4, maximum=88, step=1, value=22) - with gr.Row(): - override_scheduler = gr.Checkbox(label='Override scheduler', value=True) - with gr.Row(): - from modules.ui_sections import create_video_inputs - video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') - return [num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - - def run(self, p: processing.StableDiffusionProcessing, num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - # set params - num_frames = int(num_frames) - p.width = 8 * int(p.width // 8) - p.height = 8 * int(p.height // 8) - p.do_not_save_grid = True - p.ops.append('video') - - # load model - if shared.sd_model.__class__ != diffusers.AllegroPipeline: - sd_models.unload_model_weights() - t0 = time.time() - quant_args = model_quant.create_config() - transformer = diffusers.AllegroTransformer3DModel.from_pretrained( - repo_id, - subfolder="transformer", - torch_dtype=devices.dtype, - cache_dir=shared.opts.hfcache_dir, - **quant_args - ) - shared.log.debug(f'Video: module={transformer.__class__.__name__}') - text_encoder = transformers.T5EncoderModel.from_pretrained( - repo_id, - subfolder="text_encoder", - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **quant_args - ) - shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') - shared.sd_model = diffusers.AllegroPipeline.from_pretrained( - repo_id, - # transformer=transformer, - # text_encoder=text_encoder, - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **quant_args - ) - t1 = time.time() - shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}') - sd_models.set_diffuser_options(shared.sd_model) - shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) - shared.sd_model.sd_model_hash = None - shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode - shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt - shared.sd_model.vae.decode = hijack_decode - shared.sd_model.vae.enable_tiling() - # shared.sd_model.vae.enable_slicing() - sd_hijack_te.init_hijack(shared.sd_model) - - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - processing.fix_seed(p) - if override_scheduler: - p.sampler_name = 'Default' - p.steps = 100 - p.task_args['num_frames'] = num_frames - p.task_args['output_type'] = 'pil' - p.task_args['clean_caption'] = False - - p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts([p.prompt], [p.negative_prompt], p.styles, [p.seed]) - p.task_args['prompt'] = p.all_prompts[0] - p.task_args['negative_prompt'] = p.all_negative_prompts[0] - - # w = shared.sd_model.transformer.config.sample_width * shared.sd_model.vae_scale_factor_spatial - # h = shared.sd_model.transformer.config.sample_height * shared.sd_model.vae_scale_factor_spatial - # n = shared.sd_model.transformer.config.sample_frames * shared.sd_model.vae_scale_factor_temporal - - # run processing - t0 = time.time() - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}') - processed = processing.process_images(p) - shared.state.disable_preview = False - t1 = time.time() - if processed is not None and len(processed.images) > 0: - shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') - if video_type != 'None': - images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) - return processed diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py deleted file mode 100644 index 1e7fbd076..000000000 --- a/scripts/cogvideo.py +++ /dev/null @@ -1,215 +0,0 @@ -""" -models: https://huggingface.co/THUDM/CogVideoX-2b https://huggingface.co/THUDM/CogVideoX-5b -source: https://github.com/THUDM/CogVideo -quanto: https://gist.github.com/a-r-r-o-w/31be62828b00a9292821b85c1017effa -torchao: https://gist.github.com/a-r-r-o-w/4d9732d17412888c885480c6521a9897 -venhancer: https://github.com/THUDM/CogVideo/blob/dcb82ae30b454ab898aeced0633172d75dbd55b8/tools/venhancer/README.md -""" -import os -import time -import cv2 -import gradio as gr -import torch -from torchvision import transforms -import diffusers -import numpy as np -from modules import scripts_manager, shared, devices, errors, sd_models, processing -from modules.processing_callbacks import diffusers_callback, set_callbacks_p - - -debug = (os.environ.get('SD_LOAD_DEBUG', None) is not None) or (os.environ.get('SD_PROCESS_DEBUG', None) is not None) - - -class Script(scripts_manager.Script): - def title(self): - return 'Video: CogVideoX (Legacy)' - - def show(self, is_img2img): - return True - - - def ui(self, is_img2img): - with gr.Row(): - gr.HTML("  CogVideoX
") - with gr.Row(): - model = gr.Dropdown(label='Model', choices=['None', 'THUDM/CogVideoX-2b', 'THUDM/CogVideoX-5b', 'THUDM/CogVideoX-5b-I2V'], value='THUDM/CogVideoX-2b') - sampler = gr.Dropdown(label='Sampler', choices=['DDIM', 'DPM'], value='DDIM') - with gr.Row(): - frames = gr.Slider(label='Frames', minimum=1, maximum=100, step=1, value=49) - guidance = gr.Slider(label='Guidance', minimum=0.0, maximum=14.0, step=0.5, value=6.0) - with gr.Row(): - offload = gr.Dropdown(label='Offload', choices=['none', 'balanced', 'model', 'sequential'], value='balanced') - override = gr.Checkbox(label='Override resolution', value=True) - with gr.Accordion('Optional init image or video', open=False): - with gr.Row(): - image = gr.Image(value=None, label='Image', type='pil', width=256, height=256) - video = gr.Video(value=None, label='Video', width=256, height=256) - with gr.Row(): - from modules.ui_sections import create_video_inputs - video_type, duration, loop, pad, interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') - return [model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video] - - def load(self, model): - if (shared.sd_model_type != 'cogvideo' or shared.sd_model.sd_model_checkpoint != model) and model != 'None': - sd_models.unload_model_weights('model') - shared.log.info(f'CogVideoX load: model="{model}"') - try: - shared.sd_model = None - cls = diffusers.CogVideoXImageToVideoPipeline if 'I2V' in model else diffusers.CogVideoXPipeline - shared.sd_model = cls.from_pretrained(model, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir) - shared.sd_model.sd_checkpoint_info = sd_models.CheckpointInfo(model) - shared.sd_model.sd_model_hash = '' - shared.sd_model.sd_model_checkpoint = model - except Exception as e: - shared.log.error(f'Load CogVideoX: {e}') - if debug: - errors.display(e, 'CogVideoX') - if shared.sd_model_type == 'cogvideo' and model != 'None': - shared.sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m', ncols=80, colour='#327fba') - shared.log.debug(f'CogVideoX load: class="{shared.sd_model.__class__.__name__}"') - if shared.sd_model is not None and model == 'None': - shared.log.info(f'CogVideoX unload: model={model}') - shared.sd_model = None - devices.torch_gc(force=True) - devices.torch_gc() - - def offload(self, offload): - if shared.sd_model_type != 'cogvideo': - return - if offload == 'none': - sd_models.move_model(shared.sd_model, devices.device) - shared.log.debug(f'CogVideoX: offload={offload}') - if offload == 'balanced': - sd_models.apply_balanced_offload(shared.sd_model) - if offload == 'model': - shared.sd_model.enable_model_cpu_offload() - if offload == 'sequential': - shared.sd_model.enable_model_cpu_offload() - shared.sd_model.enable_sequential_cpu_offload() - shared.sd_model.vae.enable_slicing() - shared.sd_model.vae.enable_tiling() - - def video(self, p, fn): - frames = [] - try: - from modules.control.util import decode_fourcc - video = cv2.VideoCapture(fn) - if not video.isOpened(): - shared.log.error(f'Video: file="{fn}" open failed') - return frames - frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) - fps = int(video.get(cv2.CAP_PROP_FPS)) - w, h = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)), int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) - codec = decode_fourcc(video.get(cv2.CAP_PROP_FOURCC)) - shared.log.debug(f'CogVideoX input: video="{fn}" fps={fps} width={w} height={h} codec={codec} frames={frame_count} target={len(frames)}') - frames = [] - while True: - ok, frame = video.read() - if not ok: - break - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - frame = cv2.resize(frame, (p.width, p.height)) - frames.append(frame) - video.release() - if len(frames) > p.frames: - frames = np.asarray(frames) - indices = np.linspace(0, len(frames) - 1, p.frames).astype(int) # reduce array from n_frames to p_frames - frames = frames[indices] - shared.log.debug(f'CogVideoX input reduce: source={len(frames)} target={p.frames}') - frames = [transforms.ToTensor()(frame) for frame in frames] - except Exception as e: - shared.log.error(f'Video: file="{fn}" {e}') - if debug: - errors.display(e, 'CogVideoX') - return frames - - def image(self, p, img): - img = img.resize((p.width, p.height)) - shared.log.debug(f'CogVideoX input: image={img}') - # frames = [np.array(img)] - # frames = [transforms.ToTensor()(frame) for frame in frames] - return img - - def generate(self, p: processing.StableDiffusionProcessing, model: str): - if shared.sd_model_type != 'cogvideo': - return [] - shared.log.info(f'CogVideoX: sampler={p.sampler} steps={p.steps} frames={p.frames} width={p.width} height={p.height} seed={p.seed} guidance={p.guidance}') - if p.sampler == 'DDIM': - shared.sd_model.scheduler = diffusers.CogVideoXDDIMScheduler.from_config(shared.sd_model.scheduler.config, timestep_spacing="trailing") - if p.sampler == 'DPM': - shared.sd_model.scheduler = diffusers.CogVideoXDPMScheduler.from_config(shared.sd_model.scheduler.config, timestep_spacing="trailing") - t0 = time.time() - frames = [] - set_callbacks_p(p) - shared.state.job_count = 1 - shared.state.sampling_steps = p.steps - 1 - try: - args = dict( - prompt=p.prompt, - negative_prompt=p.negative_prompt, - height=p.height, - width=p.width, - num_videos_per_prompt=1, - num_inference_steps=p.steps, - guidance_scale=p.guidance, - generator=torch.Generator(device=devices.device).manual_seed(p.seed), - callback_on_step_end=diffusers_callback, - callback_on_step_end_tensor_inputs=['latents'], - ) - if 'I2V' in model: - if hasattr(p, 'video') and p.video is not None: - args['video'] = self.video(p, p.video) - shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXVideoToVideoPipeline, shared.sd_model) - elif (hasattr(p, 'image') and p.image is not None) or (hasattr(p, 'init_images') and len(p.init_images) > 0): - p.init_images = [p.image] if hasattr(p, 'image') and p.image is not None else p.init_images - args['image'] = self.image(p, p.init_images[0]) - shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXImageToVideoPipeline, shared.sd_model) - else: - shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXPipeline, shared.sd_model) - args['num_frames'] = p.frames # only txt2vid has num_frames - shared.log.info(f"CogVideoX: class={shared.sd_model.__class__.__name__} frames={p.frames} input={args.get('video', None) or args.get('image', None)}") - if debug: - shared.log.debug(f'CogVideoX args: {args}') - frames = shared.sd_model(**args).frames[0] - except AssertionError as e: - shared.log.info(f'CogVideoX: {e}') - except Exception as e: - shared.log.error(f'CogVideoX: {e}') - if debug: - errors.display(e, 'CogVideoX') - t1 = time.time() - its = (len(frames) * p.steps) / (t1 - t0) - shared.log.info(f'CogVideoX: frame={frames[0] if len(frames) > 0 else None} frames={len(frames)} its={its:.2f} time={t1 - t0:.2f}') - return frames - - # auto-executed by the script-callback - def run(self, p: processing.StableDiffusionProcessing, model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video): # pylint: disable=arguments-differ, unused-argument - processing.fix_seed(p) - p.extra_generation_params['CogVideoX'] = model - p.do_not_save_grid = True - if 'animatediff' not in p.ops: - p.ops.append('video') - if override: - p.width = 720 - p.height = 480 - p.sampler = sampler - p.guidance = guidance - p.frames = frames - p.use_dynamic_cfg = sampler == 'DPM' - p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) - p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) - p.image = image - p.video = video - self.load(model) - self.offload(offload) - frames = self.generate(p, model) - devices.torch_gc() - processed = processing.get_processed(p, images_list=frames) - return processed - - # auto-executed by the script-callback - def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, model, sampler, frames, guidance, offload, override, video_type, duration, loop, pad, interpolate, image, video): # pylint: disable=arguments-differ, unused-argument - if video_type != 'None' and processed is not None and len(processed.images) > 0: - from modules.images import save_video - shared.log.info(f'CogVideoX video: type={video_type} duration={duration} loop={loop} pad={pad} interpolate={interpolate}') - save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=loop, pad=pad, interpolate=interpolate) diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py deleted file mode 100644 index b43c90b79..000000000 --- a/scripts/hunyuanvideo.py +++ /dev/null @@ -1,176 +0,0 @@ -import time -import torch -import gradio as gr -import transformers -import diffusers -from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, sd_samplers, model_quant, timer, sd_hijack_te - - -default_template = """Describe the video by detailing the following aspects: -1. The main content and theme of the video. -2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. -3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. -4. Background environment, light, style and atmosphere. -5. Camera angles, movements, and transitions used in the video. -6. Thematic and aesthetic concepts associated with the scene, i.e. realistic, futuristic, fairy tale, etc. -""" - -models = { - 'HunyuanVideo': { 'repo': 'tencent/HunyuanVideo', 'revision': 'refs/pr/18' }, - 'FastHunyuan': { 'repo': 'FastVideo/FastHunyuan', 'revision': None }, -} -loaded_model = None - - -def get_template(template: str = None): - # diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.DEFAULT_PROMPT_TEMPLATE - base_template_pre = "<|start_header_id|>system<|end_header_id|>\n\n" - base_template_post = "<|eot_id|>\n" - base_template_end = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" - if template is None or len(template) == 0: - template = default_template - template_lines = '\n'.join([line for line in template.split('\n') if len(line) > 0]) - prompt_template = { - "crop_start": 95, - "template": base_template_pre + template_lines + base_template_post + base_template_end - } - return prompt_template - - -def hijack_decode(*args, **kwargs): - t0 = time.time() - vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) - res = shared.sd_model.vae.orig_decode(*args, **kwargs) - t1 = time.time() - timer.process.add('vae', t1-t0) - shared.log.debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}') - return res - - -class Script(scripts_manager.Script): - def title(self): - return 'Video: Hunyuan Video (Legacy)' - - def show(self, is_img2img): - return not is_img2img - - # return signature is array of gradio components - def ui(self, is_img2img): - with gr.Row(): - gr.HTML('  Hunyuan Video
') - with gr.Row(): - model = gr.Dropdown(label='Model', choices=list(models.keys()), value=list(models.keys())[0]) - with gr.Row(): - num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45) - tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16) - with gr.Row(): - with gr.Column(): - override_scheduler = gr.Checkbox(label='HV override sampler', value=True) - with gr.Column(): - scheduler_shift = gr.Slider(label='HV sampler shift', minimum=0.0, maximum=20.0, step=0.1, value=7.0) - with gr.Row(): - template = gr.TextArea(label='HV prompt processor', lines=3, value=default_template, visible=False) - with gr.Row(): - from modules.ui_sections import create_video_inputs - video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') - return [model, num_frames, tile_frames, override_scheduler, scheduler_shift, template, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - - def load(self, model:str): - global loaded_model # pylint: disable=global-statement - if shared.sd_model.__class__ != diffusers.HunyuanVideoPipeline or model != loaded_model: - sd_models.unload_model_weights() - t0 = time.time() - quant_args = model_quant.create_config() - transformer = diffusers.HunyuanVideoTransformer3DModel.from_pretrained( - pretrained_model_name_or_path='tencent/HunyuanVideo', - subfolder="transformer", - torch_dtype=devices.dtype, - revision='refs/pr/18', - cache_dir=shared.opts.hfcache_dir, - **quant_args - ) - shared.log.debug(f'Video: module={transformer.__class__.__name__}') - text_encoder = transformers.LlamaModel.from_pretrained( - pretrained_model_name_or_path=models.get(model)['repo'], - subfolder="text_encoder", - revision=models.get(model)['revision'], - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **quant_args - ) - text_encoder_2 = transformers.CLIPTextModel.from_pretrained( - pretrained_model_name_or_path=models.get(model)['repo'], - subfolder="text_encoder_2", - revision=models.get(model)['revision'], - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - ) - shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') - shared.sd_model = diffusers.HunyuanVideoPipeline.from_pretrained( - pretrained_model_name_or_path='tencent/HunyuanVideo', - transformer=transformer, - text_encoder=text_encoder, - text_encoder_2=text_encoder_2, - revision='refs/pr/18', - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **quant_args - ) - t1 = time.time() - shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} model="{model}" repo={models.get(model)["repo"]} dtype={devices.dtype} time={t1-t0:.2f}') - sd_models.set_diffuser_options(shared.sd_model) - shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(models.get(model)['repo']) - shared.sd_model.sd_model_hash = None - shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode - shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt - shared.sd_model.vae.decode = hijack_decode - shared.sd_model.vae.enable_slicing() - shared.sd_model.vae.enable_tiling() - shared.sd_model.vae.use_framewise_decoding = True - sd_hijack_te.init_hijack(shared.sd_model) - loaded_model = model - - def run(self, p: processing.StableDiffusionProcessing, model, num_frames, tile_frames, override_scheduler, scheduler_shift, template, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - # set params - num_frames = int(num_frames) - p.width = 16 * int(p.width // 16) - p.height = 16 * int(p.height // 16) - p.do_not_save_grid = True - p.ops.append('video') - - # load model - self.load(model) - - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - if override_scheduler: - p.sampler_name = 'Default' - else: - shared.sd_model.scheduler = sd_samplers.create_sampler(p.sampler_name, shared.sd_model) - p.sampler_name = 'Default' # avoid double creation - if hasattr(shared.sd_model.scheduler, '_shift'): - shared.sd_model.scheduler._shift = scheduler_shift # pylint: disable=protected-access - - # encode prompt - processing.fix_seed(p) - p.task_args['num_frames'] = num_frames - p.task_args['output_type'] = 'pil' - p.task_args['generator'] = torch.manual_seed(p.seed) - # p.task_args['prompt'] = None - # p.task_args['prompt_embeds'], p.task_args['pooled_prompt_embeds'], p.task_args['prompt_attention_mask'] = shared.sd_model.encode_prompt(prompt=p.prompt, prompt_template=get_template(template), device=devices.device) - - # run processing - t0 = time.time() - shared.sd_model.vae.tile_sample_min_num_frames = tile_frames - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}') - processed = processing.process_images(p) - shared.state.disable_preview = False - t1 = time.time() - if processed is not None and len(processed.images) > 0: - shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') - if video_type != 'None': - images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) - return processed diff --git a/scripts/legacy_allegrovideo.py b/scripts/legacy_allegrovideo.py deleted file mode 100644 index 84c357589..000000000 --- a/scripts/legacy_allegrovideo.py +++ /dev/null @@ -1,121 +0,0 @@ -import time -import gradio as gr -import transformers -import diffusers -from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer, sd_hijack_te - - -repo_id = 'rhymes-ai/Allegro' - - -def hijack_decode(*args, **kwargs): - t0 = time.time() - vae: diffusers.AutoencoderKLAllegro = shared.sd_model.vae - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) - res = shared.sd_model.vae.orig_decode(*args, **kwargs) - t1 = time.time() - timer.process.add('vae', t1-t0) - shared.log.debug(f'Video: vae={vae.__class__.__name__} time={t1-t0:.2f}') - return res - - -class Script(scripts_manager.Script): - def title(self): - return 'Video: Allegro (Legacy)' - - def show(self, is_img2img): - return not is_img2img - - # return signature is array of gradio components - def ui(self, is_img2img): - with gr.Row(): - gr.HTML('  Allegro Video
') - with gr.Row(): - num_frames = gr.Slider(label='Frames', minimum=4, maximum=88, step=1, value=22) - with gr.Row(): - override_scheduler = gr.Checkbox(label='Override scheduler', value=True) - with gr.Row(): - from modules.ui_sections import create_video_inputs - video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') - return [num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - - def run(self, p: processing.StableDiffusionProcessing, num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - # set params - num_frames = int(num_frames) - p.width = 8 * int(p.width // 8) - p.height = 8 * int(p.height // 8) - p.do_not_save_grid = True - p.ops.append('video') - - # load model - if shared.sd_model.__class__ != diffusers.AllegroPipeline: - sd_models.unload_model_weights() - t0 = time.time() - quant_args = model_quant.create_config() - transformer = diffusers.AllegroTransformer3DModel.from_pretrained( - repo_id, - subfolder="transformer", - torch_dtype=devices.dtype, - cache_dir=shared.opts.hfcache_dir, - **quant_args - ) - shared.log.debug(f'Video: module={transformer.__class__.__name__}') - text_encoder = transformers.T5EncoderModel.from_pretrained( - repo_id, - subfolder="text_encoder", - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **quant_args - ) - shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') - shared.sd_model = diffusers.AllegroPipeline.from_pretrained( - repo_id, - # transformer=transformer, - # text_encoder=text_encoder, - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **quant_args - ) - t1 = time.time() - shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}') - sd_models.set_diffuser_options(shared.sd_model) - shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) - shared.sd_model.sd_model_hash = None - shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode - shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt - shared.sd_model.vae.decode = hijack_decode - shared.sd_model.vae.enable_tiling() - sd_hijack_te.init_hijack(shared.sd_model) - # shared.sd_model.vae.enable_slicing() - - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - devices.torch_gc(force=True) - - processing.fix_seed(p) - if override_scheduler: - p.sampler_name = 'Default' - p.steps = 100 - p.task_args['num_frames'] = num_frames - p.task_args['output_type'] = 'pil' - p.task_args['clean_caption'] = False - - p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts([p.prompt], [p.negative_prompt], p.styles, [p.seed]) - p.task_args['prompt'] = p.all_prompts[0] - p.task_args['negative_prompt'] = p.all_negative_prompts[0] - - # w = shared.sd_model.transformer.config.sample_width * shared.sd_model.vae_scale_factor_spatial - # h = shared.sd_model.transformer.config.sample_height * shared.sd_model.vae_scale_factor_spatial - # n = shared.sd_model.transformer.config.sample_frames * shared.sd_model.vae_scale_factor_temporal - - # run processing - t0 = time.time() - shared.state.disable_preview = True - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}') - processed = processing.process_images(p) - shared.state.disable_preview = False - t1 = time.time() - if processed is not None and len(processed.images) > 0: - shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') - if video_type != 'None': - images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) - return processed diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py deleted file mode 100644 index baaa150aa..000000000 --- a/scripts/ltxvideo.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import time -import torch -import gradio as gr -import diffusers -import transformers -from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer, sd_hijack_te - - -repos = { - '0.9.0': 'a-r-r-o-w/LTX-Video-diffusers', - '0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers', - '0.9.5': 'Lightricks/LTX-Video-0.9.5', - 'custom': None, -} - - -def load_quants(kwargs, repo_id): - quant_args = model_quant.create_config() - if not quant_args: - return kwargs - model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') - if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): - kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'text_encoder' not in kwargs and ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization): - kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - return kwargs - - -def hijack_decode(*args, **kwargs): - t0 = time.time() - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) - res = shared.sd_model.vae.orig_decode(*args, **kwargs) - t1 = time.time() - timer.process.add('vae', t1-t0) - shared.log.debug(f'Video: vae={shared.sd_model.vae.__class__.__name__} time={t1-t0:.2f}') - return res - - -class Script(scripts_manager.Script): - def title(self): - return 'Video: LTX Video (Legacy)' - - def show(self, is_img2img): - return True - - # return signature is array of gradio components - def ui(self, is_img2img): - def model_change(model): - return gr.update(visible=model == 'custom') - - with gr.Row(): - gr.HTML('  LTX Video
') - with gr.Row(): - model = gr.Dropdown(label='LTX Model', choices=list(repos), value='0.9.1') - decode = gr.Dropdown(label='Decode', choices=['diffusers', 'native'], value='diffusers', visible=False) - with gr.Row(): - num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41) - sampler = gr.Checkbox(label='Override sampler', value=True) - with gr.Row(): - teacache_enable = gr.Checkbox(label='Enable TeaCache', value=False) - teacache_threshold = gr.Slider(label='Threshold', minimum=0.01, maximum=0.1, step=0.01, value=0.03) - with gr.Row(): - model_custom = gr.Textbox(value='', label='Path to model file', visible=False) - with gr.Row(): - from modules.ui_sections import create_video_inputs - video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') - model.change(fn=model_change, inputs=[model], outputs=[model_custom]) - return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, teacache_enable, teacache_threshold] - - def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, teacache_enable, teacache_threshold): # pylint: disable=arguments-differ, unused-argument - # set params - image = getattr(p, 'init_images', None) - image = None if image is None or len(image) == 0 else image[0] - if (p.width == 0 or p.height == 0) and image is not None: - p.width = image.width - p.height = image.height - num_frames = 8 * int(num_frames // 8) + 1 - p.width = 32 * int(p.width // 32) - p.height = 32 * int(p.height // 32) - processing.fix_seed(p) - if image: - image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - p.task_args['image'] = image - p.task_args['output_type'] = 'latent' if decode == 'native' else 'pil' - p.task_args['generator'] = torch.Generator(devices.device).manual_seed(p.seed) - p.task_args['num_frames'] = num_frames - p.do_not_save_grid = True - if sampler: - p.sampler_name = 'Default' - p.ops.append('video') - - # load model - cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline - diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel - diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo - repo_id = repos[model] - if repo_id is None: - repo_id = model_custom - if shared.sd_model.__class__ != cls: - sd_models.unload_model_weights() - kwargs = model_quant.create_config() - if os.path.isfile(repo_id): - shared.sd_model = cls.from_single_file( - repo_id, - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **kwargs - ) - else: - kwargs = load_quants(kwargs, repo_id) - shared.sd_model = cls.from_pretrained( - repo_id, - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **kwargs - ) - sd_models.set_diffuser_options(shared.sd_model) - shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode - shared.sd_model.orig_encode_prompt = shared.sd_model.encode_prompt - shared.sd_model.vae.decode = hijack_decode - shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) - shared.sd_model.sd_model_hash = None - sd_hijack_te.init_hijack(shared.sd_model) - - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - shared.sd_model.vae.enable_slicing() - shared.sd_model.vae.enable_tiling() - shared.sd_model.vae.use_framewise_decoding = True - devices.torch_gc(force=True) - - shared.sd_model.transformer.cnt = 0 - shared.sd_model.transformer.accumulated_rel_l1_distance = 0 - shared.sd_model.transformer.previous_modulated_input = None - shared.sd_model.transformer.previous_residual = None - shared.sd_model.transformer.enable_teacache = teacache_enable - shared.sd_model.transformer.rel_l1_thresh = teacache_threshold - shared.sd_model.transformer.num_steps = p.steps - - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args} steps={p.steps} teacache={teacache_enable} threshold={teacache_threshold}') - - # run processing - t0 = time.time() - processed = processing.process_images(p) - t1 = time.time() - if processed is not None and len(processed.images) > 0: - shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') - if video_type != 'None': - images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) - return processed diff --git a/scripts/mochivideo.py b/scripts/mochivideo.py deleted file mode 100644 index 9a3d1f2fa..000000000 --- a/scripts/mochivideo.py +++ /dev/null @@ -1,70 +0,0 @@ -import time -import torch -import gradio as gr -import diffusers -from modules import scripts_manager, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant - - -repo_id = 'genmo/mochi-1-preview' - - -class Script(scripts_manager.Script): - def title(self): - return 'Video: Mochi.1 Video (Legacy)' - - def show(self, is_img2img): - return not is_img2img - - # return signature is array of gradio components - def ui(self, is_img2img): - with gr.Row(): - gr.HTML('  Mochi.1 Video
') - with gr.Row(): - num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45) - with gr.Row(): - from modules.ui_sections import create_video_inputs - video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') - return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - - def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - # set params - num_frames = int(num_frames) - p.width = 32 * int(p.width // 32) - p.height = 32 * int(p.height // 32) - p.task_args['output_type'] = 'pil' - p.task_args['generator'] = torch.manual_seed(p.seed) - p.task_args['num_frames'] = num_frames - p.sampler_name = 'Default' - p.do_not_save_grid = True - p.ops.append('video') - - # load model - cls = diffusers.MochiPipeline - if shared.sd_model.__class__ != cls: - sd_models.unload_model_weights() - kwargs = model_quant.create_config() - shared.sd_model = cls.from_pretrained( - repo_id, - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **kwargs - ) - shared.sd_model.scheduler._shift = 7.0 # pylint: disable=protected-access - sd_models.set_diffuser_options(shared.sd_model) - shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) - shared.sd_model.sd_model_hash = None - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - shared.sd_model.vae.enable_slicing() - shared.sd_model.vae.enable_tiling() - devices.torch_gc(force=True) - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}') - - # run processing - t0 = time.time() - processed = processing.process_images(p) - t1 = time.time() - if processed is not None and len(processed.images) > 0: - shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') - if video_type != 'None': - images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) - return processed