Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 01:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix T5 INT8 and add QINT8

Browse Source
This commit is contained in:
Disty0
2024-07-30 18:23:21 +03:00
parent 6c75bcca0a
commit b50a8601fe
3 changed files with 20 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/model_t5.py
+16 -2
View File
@@ -1,3 +1,4 @@
import torch
import transformers
@@ -37,12 +38,24 @@ def load_t5(t5=None, cache_dir=None):
cache_dir=cache_dir,
torch_dtype=devices.dtype,
)
elif 'qint8' in t5.lower():
modelloader.hf_login()
from installer import install
install('optimum-quanto', quiet=True)
from modules.sd_models_compile import optimum_quanto_model
t5 = transformers.T5EncoderModel.from_pretrained(
repo_id,
subfolder='text_encoder_3',
cache_dir=cache_dir,
torch_dtype=devices.dtype,
)
t5 = optimum_quanto_model(t5, weights="qint8")
elif 'int8' in t5.lower():
modelloader.hf_login()
from installer import install
install('nncf==2.7.0', quiet=True)
from modules.sd_models_compile import nncf_compress_model
from modules.sd_hijack import NNCF_T5DenseGatedActDense # T5DenseGatedActDense uses fp32
from modules.sd_hijack import NNCF_T5DenseGatedActDense
t5 = transformers.T5EncoderModel.from_pretrained(
repo_id,
subfolder='text_encoder_3',
@@ -51,7 +64,8 @@ def load_t5(t5=None, cache_dir=None):
)
for i in range(len(t5.encoder.block)):
t5.encoder.block[i].layer[1].DenseReluDense = NNCF_T5DenseGatedActDense(
t5.encoder.block[i].layer[1].DenseReluDense
t5.encoder.block[i].layer[1].DenseReluDense,
dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16
)
t5 = nncf_compress_model(t5)
else:
modules/sd_models_compile.py
+3 -2
View File
@@ -150,13 +150,14 @@ def nncf_compress_weights(sd_model):
shared.log.warning(f"NNCF Compress Weights: error: {e}")
return sd_model
def optimum_quanto_model(model):
def optimum_quanto_model(model, weights=None):
from optimum import quanto
weights = getattr(quanto, weights) if weights is not None else getattr(quanto, shared.opts.optimum_quanto_weights_type)
model.eval()
backup_embeddings = None
if hasattr(model, "get_input_embeddings"):
backup_embeddings = copy.deepcopy(model.get_input_embeddings())
quanto.quantize(model, weights=getattr(quanto, shared.opts.optimum_quanto_weights_type))
quanto.quantize(model, weights=weights)
quanto.freeze(model)
if hasattr(model, "set_input_embeddings") and backup_embeddings is not None:
model.set_input_embeddings(backup_embeddings)
modules/shared.py
+1 -1
View File
@@ -391,7 +391,7 @@ options_templates.update(options_section(('sd', "Execution & Models"), {
"sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
"sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list),
"sd_unet": OptionInfo("None", "UNET model", gr.Dropdown, lambda: {"choices": shared_items.sd_unet_items()}, refresh=shared_items.refresh_unet_list),
"sd_text_encoder": OptionInfo('None', "Text encoder model", gr.Dropdown, lambda: {"choices": ['None', 'T5 FP4', 'T5 FP8', 'T5 INT8', 'T5 FP16']}),
"sd_text_encoder": OptionInfo('None', "Text encoder model", gr.Dropdown, lambda: {"choices": ['None', 'T5 FP4', 'T5 FP8', 'T5 INT8', 'T5 QINT8', 'T5 FP16']}),
"sd_model_dict": OptionInfo('None', "Use separate base dict", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
"sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"),
"sd_textencoder_cache": OptionInfo(True, "Cache text encoder results"),