From 287c3600d798909cbe1d7403c7fe2898ac9c3025 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 20 Jul 2025 12:07:28 -0400 Subject: [PATCH] torch compile for llm Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 ++++--- TODO.md | 1 - modules/interrogate/vqa.py | 8 +++++++- modules/sd_models_compile.py | 7 +++++-- modules/shared.py | 2 +- requirements.txt | 4 ++-- scripts/prompt_enhance.py | 20 +++++++++++++++----- 7 files changed, 34 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa13aeacb..bd26c93d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2025-07-18 +## Update for 2025-07-20 -### Highlights for 2025-07-18 +### Highlights for 2025-07-20 Feature highlights include: - **ModernUI** layout redesign which should make it more user friendly and easier to navigate @@ -26,7 +26,7 @@ Although upgrades and existing installations are tested and should work fine! [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-07-18 +### Details for 2025-07-20 - **License** - SD.Next [license](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) switched from **aGPL-v3.0** to **Apache-v2.0** @@ -85,6 +85,7 @@ Although upgrades and existing installations are tested and should work fine! enable in *settings -> compute settings -> sdp options* *note*: SD.Next will use either SageAttention v1/v2/v2++, depending which one is installed until authors provide pre-build wheels for v2++, you need to install it manually or SD.Next will auto-install v1 + - support for `torch.compile` for LLM: captioning/prompt-enhannce - support for `torch.compile` with repeated-blocks reduces time-to-compile 5x without loss of performance! enable in *settings -> model compile -> repeated* diff --git a/TODO.md b/TODO.md index 9a760a8cc..0fddce9e6 100644 --- a/TODO.md +++ b/TODO.md @@ -8,7 +8,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915) - Refactor: Sampler options -- Refactor: [torch.compile](https://pytorch.org/blog/torch-compile-and-diffusers-a-hands-on-guide-to-peak-performance/) - Feature: Diffusers [group offloading](https://github.com/vladmandic/sdnext/issues/4049) - Feature: Common repo for `T5` and `CLiP` - Feature: LoRA add OMI format support for SD35/FLUX.1 diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index f75342703..0457af7ea 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -7,7 +7,7 @@ import torch import transformers import transformers.dynamic_module_utils from PIL import Image -from modules import shared, devices, errors, sd_models, model_quant +from modules import shared, devices, errors, model_quant, sd_models, sd_models_compile processor = None @@ -129,6 +129,8 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str **quant_args, ) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) + if 'LLM' in shared.opts.cuda_compile: + model = compile_torch(model) loaded = repo devices.torch_gc() sd_models.move_model(model, devices.device) @@ -177,6 +179,8 @@ def gemma(question: str, image: Image.Image, repo: str = None, system_prompt: st cache_dir=shared.opts.hfcache_dir, **quant_args, ) + if 'LLM' in shared.opts.cuda_compile: + model = compile_torch(model) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) loaded = repo devices.torch_gc() @@ -302,6 +306,8 @@ def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str **quant_args, ) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) + if 'LLM' in shared.opts.cuda_compile: + model = compile_torch(model) loaded = repo devices.torch_gc() sd_models.move_model(model, devices.device) diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index a396b8390..498df59c6 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -213,8 +213,11 @@ def compile_torch(sd_model): setup_logging() # compile messes with logging so reset is needed if 'precompile' in shared.opts.cuda_compile_options: - shared.log.debug("Model compile: task=torch precompile") - sd_model("dummy prompt") + try: + shared.log.debug("Model compile: task=torch precompile") + sd_model("dummy prompt") + except Exception: + pass t1 = time.time() shared.log.info(f"Model compile: task=torch time={t1-t0:.2f}") except Exception as e: diff --git a/modules/shared.py b/modules/shared.py index c0e01399e..32b8b180a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -419,7 +419,7 @@ options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { options_templates.update(options_section(('compile', "Model Compile"), { "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), - "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE", "Upscaler"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "TE", "VAE", "Upscaler"]}), + "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE", "Upscaler"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "TE", "VAE", "LLM", "Upscaler"]}), "cuda_compile_backend": OptionInfo("inductor" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'migraphx', 'ipex', 'onediff', 'stable-fast', 'deep-cache', 'olive-ai', 'openvino_fx']}), "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}), "cuda_compile_options": OptionInfo(["precompile", "repeated", "fullgraph", "dynamic"], "Model compile options", gr.CheckboxGroup, {"choices": ["precompile", "repeated", "fullgraph", "dynamic", "verbose"]}), diff --git a/requirements.txt b/requirements.txt index 4ec99eac7..30b00f06d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,10 +41,10 @@ torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.4 tqdm==4.67.1 -accelerate==1.8.1 +accelerate==1.9.0 opencv-contrib-python-headless==4.11.0.86 einops==0.8.1 -huggingface_hub==0.33.2 +huggingface_hub==0.33.4 numexpr==2.11.0 numpy==2.1.2 pandas==2.3.0 diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 7cc5ef064..3d27d95a4 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -3,6 +3,7 @@ import io import os import re import time +import random import base64 import torch import transformers @@ -68,7 +69,8 @@ class Options: 'file': 'Llama-3.2-1B-Instruct-Uncensored.i1-Q4_0.gguf', # gguf file inside repo }, } - default = list(models)[1] # gemma-3-4b-it + # default = list(models)[1] # gemma-3-4b-it + default = 'Qwen/Qwen3-0.6B-FP8' supported = list(transformers.integrations.ggml.GGUF_CONFIG_MAPPING) t2i_prompt: str = 'You are a helpful assistant. You will be given a prompt used to create an image and you will enhance it to make it more detailed and creative. ' i2i_prompt: str = 'You are a helpful assistant. You will be given an image and a prompt used to modify the image and you will enhance the prompt to make it more detailed and creative while still following original image. ' @@ -101,6 +103,12 @@ class Script(scripts_manager.Script): def show(self, _is_img2img): return scripts_manager.AlwaysVisible + def compile(self): + if self.llm is None or 'LLM' not in shared.opts.cuda_compile: + return + from modules.sd_models_compile import compile_torch + self.llm = compile_torch(self.llm) + def load(self, name:str=None, model_repo:str=None, model_gguf:str=None, model_type:str=None, model_file:str=None): name = name or self.options.default if self.busy: @@ -171,6 +179,7 @@ class Script(scripts_manager.Script): self.model = name t1 = time.time() shared.log.info(f'Prompt enhance: cls={self.llm.__class__.__name__} name="{name}" repo="{model_repo}" fn="{model_file}" time={t1-t0:.2f} loaded') + self.compile() except Exception as e: shared.log.error(f'Prompt enhance: load {e}') errors.display(e, 'Prompt enhance') @@ -253,8 +262,10 @@ class Script(scripts_manager.Script): while self.busy: time.sleep(0.1) self.load(model) - if seed is not None and seed >= 0: - torch.manual_seed(seed) + if seed is None or seed == -1: + random.seed() + seed = int(random.randrange(4294967294)) + torch.manual_seed(seed) if self.llm is None: shared.log.error('Prompt enhance: model not loaded') return prompt @@ -394,7 +405,6 @@ class Script(scripts_manager.Script): return prompt # Return original full prompt on censorship return response - # --- START OF CORRECTED METHOD --- def apply(self, prompt, image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, max_tokens, do_sample, temperature, repetition_penalty, thinking_mode, nsfw_mode): # Added nsfw_mode response = self.enhance( prompt=prompt, @@ -413,7 +423,6 @@ class Script(scripts_manager.Script): if apply_prompt: return [response, response] return [response, gr.update()] - # --- END OF CORRECTED METHOD --- def get_custom(self, name): model_repo = self.options.models.get(name, {}).get('repo', None) or name @@ -504,6 +513,7 @@ class Script(scripts_manager.Script): shared.state.begin('LLM') p.prompt = self.enhance( prompt=p.prompt, + seed=p.seed, image=self_image, prefix=prompt_prefix, suffix=prompt_suffix,