Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-20 01:31:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

torch compile for llm

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-07-20 12:07:28 -04:00
parent 3cb85e5cfc
commit 287c3600d7
7 changed files with 34 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/interrogate/vqa.py
+7 -1
View File
@@ -7,7 +7,7 @@ import torch
import transformers
import transformers.dynamic_module_utils
from PIL import Image
from modules import shared, devices, errors, sd_models, model_quant
from modules import shared, devices, errors, model_quant, sd_models, sd_models_compile
processor = None
@@ -129,6 +129,8 @@ def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str
**quant_args,
)
processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
if 'LLM' in shared.opts.cuda_compile:
model = compile_torch(model)
loaded = repo
devices.torch_gc()
sd_models.move_model(model, devices.device)
@@ -177,6 +179,8 @@ def gemma(question: str, image: Image.Image, repo: str = None, system_prompt: st
cache_dir=shared.opts.hfcache_dir,
**quant_args,
)
if 'LLM' in shared.opts.cuda_compile:
model = compile_torch(model)
processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
loaded = repo
devices.torch_gc()
@@ -302,6 +306,8 @@ def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str
**quant_args,
)
processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
if 'LLM' in shared.opts.cuda_compile:
model = compile_torch(model)
loaded = repo
devices.torch_gc()
sd_models.move_model(model, devices.device)