From 4457ea2e51f40f2b707cd6c2137f81ff46215aff Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 2 Apr 2025 17:22:45 -0400 Subject: [PATCH] mark legacy scripts Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + TODO.md | 33 ++++----- scripts/allegrovideo.py | 2 +- scripts/cogvideo.py | 2 +- scripts/flux_prompt_enhance.py | 2 +- scripts/hunyuanvideo.py | 2 +- scripts/legacy_allegrovideo.py | 131 +++++++++++++++++++++++++++++++++ scripts/ltxvideo.py | 2 +- scripts/mochivideo.py | 2 +- 9 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 scripts/legacy_allegrovideo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b73256a5d..5e6f1fb3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Models...And support for new models: **CogView-4**, **SANA 1.5**, *Plus...* - New **Prompt Enhance** using LLM, +- New pipelines such as **InfiniteYou** - New **CLiP** models, improvements to **remote VAE**, additional wiki/docs/guides - More quantization options and granular control - Pretty big performance updates to a) Any model using DiT based architecture due to new caching methods, b) ZLUDA with new attention methods, c) LoRA with much lower memory usage diff --git a/TODO.md b/TODO.md index 1a9ea9094..c6cef03da 100644 --- a/TODO.md +++ b/TODO.md @@ -6,10 +6,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ### Issues/Limitations -- Video: Hunyuan Video I2V: requires `transformers==4.47.1` -- Video: CogVideoX 1.5 5B T2V/I2V: all-gray output -- Video: Allegro T2V: all-gray output -- Video: Latte1 T2V: garbage output +N/A ## Future Candidates @@ -25,17 +22,19 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Code TODO -- control: support scripts via api -- enable ROCm for windows when available -- fc: autodetect distilled based on model -- fc: autodetect tensor format based on model -- hypertile: vae breaks when using non-standard sizes -- infotext: handle using regex instead -- lora: add other quantization types -- lora: force-reloading entire model as loading transformers only leads to massive memory usage -- lora: required for flux to reapply offload after lora has been applied, but fails with oom -- lora: support pre-quantized flux -- model loader: implement model in-memory caching -- modernui: monkey-patch for missing tabs.select event -- processing: remove duplicate mask params +> pnpm lint | grep W0511 | awk -F'TODO ' '{print "- "$NF}' | sed 's/ (fixme)//g' + +- install: enable ROCm for windows when available - resize image: enable full VAE mode for resize-latent +- infotext: handle using regex instead +- fc: autodetect tensor format based on model +- fc: autodetect distilled based on model +- processing: remove duplicate mask params +- model loader: implement model in-memory caching +- hypertile: vae breaks when using non-standard sizes +- model load: force-reloading entire model as loading transformers only leads to massive memory usage +- lora: add other quantization types +- lora: maybe force imediate quantization +- modules/lora/lora_extract.py:185:9: W0511: TODO: lora support pre-quantized flux +- control: support scripts via api +- modernui: monkey-patch for missing tabs.select event diff --git a/scripts/allegrovideo.py b/scripts/allegrovideo.py index fdcd52adf..cf35500fb 100644 --- a/scripts/allegrovideo.py +++ b/scripts/allegrovideo.py @@ -31,7 +31,7 @@ def hijack_encode_prompt(*args, **kwargs): class Script(scripts.Script): def title(self): - return 'Video: Allegro' + return 'Video: Allegro (Legacy)' def show(self, is_img2img): return not is_img2img if shared.native else False diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index c18b4eb2f..de3c7736c 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -22,7 +22,7 @@ debug = (os.environ.get('SD_LOAD_DEBUG', None) is not None) or (os.environ.get(' class Script(scripts.Script): def title(self): - return 'Video: CogVideoX' + return 'Video: CogVideoX (Legacy)' def show(self, is_img2img): return shared.native diff --git a/scripts/flux_prompt_enhance.py b/scripts/flux_prompt_enhance.py index abfbeae6d..0ab087e1b 100644 --- a/scripts/flux_prompt_enhance.py +++ b/scripts/flux_prompt_enhance.py @@ -27,7 +27,7 @@ class Script(scripts.Script): prompt: gr.Textbox = None def title(self): - return 'Prompt enhance' + return 'Flux Prompt enhance (Legacy)' def show(self, is_img2img): return shared.native diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py index dfd33e8ca..c39cec688 100644 --- a/scripts/hunyuanvideo.py +++ b/scripts/hunyuanvideo.py @@ -60,7 +60,7 @@ def hijack_encode_prompt(*args, **kwargs): class Script(scripts.Script): def title(self): - return 'Video: Hunyuan Video' + return 'Video: Hunyuan Video (Legacy)' def show(self, is_img2img): return not is_img2img if shared.native else False diff --git a/scripts/legacy_allegrovideo.py b/scripts/legacy_allegrovideo.py new file mode 100644 index 000000000..cf35500fb --- /dev/null +++ b/scripts/legacy_allegrovideo.py @@ -0,0 +1,131 @@ +import time +import gradio as gr +import transformers +import diffusers +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer + + +repo_id = 'rhymes-ai/Allegro' + + +def hijack_decode(*args, **kwargs): + t0 = time.time() + vae: diffusers.AutoencoderKLAllegro = shared.sd_model.vae + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + t1 = time.time() + timer.process.add('vae', t1-t0) + shared.log.debug(f'Video: vae={vae.__class__.__name__} time={t1-t0:.2f}') + return res + + +def hijack_encode_prompt(*args, **kwargs): + t0 = time.time() + res = shared.sd_model.vae.orig_encode_prompt(*args, **kwargs) + t1 = time.time() + timer.process.add('te', t1-t0) + shared.log.debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return res + + +class Script(scripts.Script): + def title(self): + return 'Video: Allegro (Legacy)' + + def show(self, is_img2img): + return not is_img2img if shared.native else False + + # return signature is array of gradio components + def ui(self, is_img2img): + with gr.Row(): + gr.HTML('  Allegro Video
') + with gr.Row(): + num_frames = gr.Slider(label='Frames', minimum=4, maximum=88, step=1, value=22) + with gr.Row(): + override_scheduler = gr.Checkbox(label='Override scheduler', value=True) + with gr.Row(): + from modules.ui_sections import create_video_inputs + video_type, duration, gif_loop, mp4_pad, mp4_interpolate = create_video_inputs(tab='img2img' if is_img2img else 'txt2img') + return [num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + # set params + num_frames = int(num_frames) + p.width = 8 * int(p.width // 8) + p.height = 8 * int(p.height // 8) + p.do_not_save_grid = True + p.ops.append('video') + + # load model + if shared.sd_model.__class__ != diffusers.AllegroPipeline: + sd_models.unload_model_weights() + t0 = time.time() + quant_args = model_quant.create_config() + transformer = diffusers.AllegroTransformer3DModel.from_pretrained( + repo_id, + subfolder="transformer", + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args + ) + shared.log.debug(f'Video: module={transformer.__class__.__name__}') + text_encoder = transformers.T5EncoderModel.from_pretrained( + repo_id, + subfolder="text_encoder", + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **quant_args + ) + shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') + shared.sd_model = diffusers.AllegroPipeline.from_pretrained( + repo_id, + # transformer=transformer, + # text_encoder=text_encoder, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **quant_args + ) + t1 = time.time() + shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}') + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) + shared.sd_model.sd_model_hash = None + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.orig_encode_prompt = shared.sd_model.encode_prompt + shared.sd_model.vae.decode = hijack_decode + shared.sd_model.encode_prompt = hijack_encode_prompt + shared.sd_model.vae.enable_tiling() + # shared.sd_model.vae.enable_slicing() + + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + processing.fix_seed(p) + if override_scheduler: + p.sampler_name = 'Default' + p.steps = 100 + p.task_args['num_frames'] = num_frames + p.task_args['output_type'] = 'pil' + p.task_args['clean_caption'] = False + + p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts([p.prompt], [p.negative_prompt], p.styles, [p.seed]) + p.task_args['prompt'] = p.all_prompts[0] + p.task_args['negative_prompt'] = p.all_negative_prompts[0] + + # w = shared.sd_model.transformer.config.sample_width * shared.sd_model.vae_scale_factor_spatial + # h = shared.sd_model.transformer.config.sample_height * shared.sd_model.vae_scale_factor_spatial + # n = shared.sd_model.transformer.config.sample_frames * shared.sd_model.vae_scale_factor_temporal + + # run processing + t0 = time.time() + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}') + processed = processing.process_images(p) + shared.state.disable_preview = False + t1 = time.time() + if processed is not None and len(processed.images) > 0: + shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 09f30360e..697e64021 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -52,7 +52,7 @@ def hijack_encode_prompt(*args, **kwargs): class Script(scripts.Script): def title(self): - return 'Video: LTX Video' + return 'Video: LTX Video (Legacy)' def show(self, is_img2img): return shared.native diff --git a/scripts/mochivideo.py b/scripts/mochivideo.py index e2c193c24..1e7ba5541 100644 --- a/scripts/mochivideo.py +++ b/scripts/mochivideo.py @@ -10,7 +10,7 @@ repo_id = 'genmo/mochi-1-preview' class Script(scripts.Script): def title(self): - return 'Video: Mochi.1 Video' + return 'Video: Mochi.1 Video (Legacy)' def show(self, is_img2img): return not is_img2img if shared.native else False