From a6ea26fb8d82dc59251ffe3e47b1efbeb7fbb133 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 16 Apr 2025 19:16:35 +0300 Subject: [PATCH] Use transformers QuantoConfig for TE and LLM --- modules/model_quant.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 99d5d6c9b..cfcd35627 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -88,9 +88,13 @@ def create_quanto_config(kwargs = None, allow_quanto: bool = True, module: str = load_quanto(silent=True) if optimum_quanto is None: return kwargs - quanto_config = diffusers.QuantoConfig(weights_dtype=shared.opts.quanto_quantization_type) - quanto_config.activations = None # patch so it works with transformers - quanto_config.weights = quanto_config.weights_dtype + if module in {'TE', 'LLM'}: + quanto_config = transformers.QuantoConfig(weights=shared.opts.quanto_quantization_type) + quanto_config.weights_dtype = quanto_config.weights + else: + quanto_config = diffusers.QuantoConfig(weights_dtype=shared.opts.quanto_quantization_type) + quanto_config.activations = None # patch so it works with transformers + quanto_config.weights = quanto_config.weights_dtype log.debug(f'Quantization: module="{module}" type=quanto dtype={shared.opts.quanto_quantization_type}') if kwargs is None: return quanto_config