ensure lora is not first to use quans

Signed-off-by: Vladimir Mandic <mandic00@live.com>

extensions-builtin/Lora/lora_patches.py

@@ -1,3 +1,4 @@
+import sys
 import torch
 import networks
 from modules import patches, shared, model_quant
@@ -21,16 +22,27 @@ class LoraPatches:
         self.QLinear_forward = None # optimum.quanto
         self.QConv2d_forward = None # optimum.quanto
 
+    def handle_quant(self, apply: bool):
+        if 'bitsandbytes' in sys.modules: # lora should not be first to initialize quantization
+            bnb = model_quant.load_bnb(silent=True)
+            if bnb is not None:
+                if apply:
+                    self.Linear4bit_forward = patches.patch(__name__, bnb.nn.Linear4bit, 'forward', networks.network_Linear4bit_forward)
+                else:
+                    self.Linear4bit_forward = patches.undo(__name__, bnb.nn.Linear4bit, 'forward') # pylint: disable=E1128
+        if 'optimum' in sys.modules:
+            quanto = model_quant.load_quanto(silent=True)
+            if quanto is not None:
+                if apply:
+                    self.QLinear_forward = patches.patch(__name__, quanto.nn.QLinear, 'forward', networks.network_QLinear_forward)
+                    self.QConv2d_forward = patches.patch(__name__, quanto.nn.QConv2d, 'forward', networks.network_QConv2d_forward)
+                else:
+                    self.QLinear_forward = patches.undo(__name__, quanto.nn.QLinear, 'forward') # pylint: disable=E1128
+                    self.QConv2d_forward = patches.undo(__name__, quanto.nn.QConv2d, 'forward') # pylint: disable=E1128
+
     def apply(self):
         if self.active or shared.opts.lora_force_diffusers:
             return
-        bnb = model_quant.load_bnb(silent=True)
-        if bnb is not None:
-            self.Linear4bit_forward = patches.patch(__name__, bnb.nn.Linear4bit, 'forward', networks.network_Linear4bit_forward)
-        quanto = model_quant.load_quanto(silent=True)
-        if quanto is not None:
-            self.QLinear_forward = patches.patch(__name__, quanto.nn.QLinear, 'forward', networks.network_QLinear_forward)
-            self.QConv2d_forward = patches.patch(__name__, quanto.nn.QConv2d, 'forward', networks.network_QConv2d_forward)
         self.Linear_forward = patches.patch(__name__, torch.nn.Linear, 'forward', networks.network_Linear_forward)
         self.Linear_load_state_dict = patches.patch(__name__, torch.nn.Linear, '_load_from_state_dict', networks.network_Linear_load_state_dict)
         self.Conv2d_forward = patches.patch(__name__, torch.nn.Conv2d, 'forward', networks.network_Conv2d_forward)
@@ -41,6 +53,7 @@ class LoraPatches:
         self.LayerNorm_load_state_dict = patches.patch(__name__, torch.nn.LayerNorm, '_load_from_state_dict', networks.network_LayerNorm_load_state_dict)
         self.MultiheadAttention_forward = patches.patch(__name__, torch.nn.MultiheadAttention, 'forward', networks.network_MultiheadAttention_forward)
         self.MultiheadAttention_load_state_dict = patches.patch(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict', networks.network_MultiheadAttention_load_state_dict)
+        self.handle_quant(apply=True)
         networks.timer['load'] = 0
         networks.timer['apply'] = 0
         networks.timer['restore'] = 0
@@ -49,13 +62,6 @@ class LoraPatches:
     def undo(self):
         if not self.active or shared.opts.lora_force_diffusers:
             return
-        bnb = model_quant.load_bnb(silent=True)
-        if bnb is not None:
-            self.Linear4bit_forward = patches.undo(__name__, bnb.nn.Linear4bit, 'forward') # pylint: disable=E1128
-        quanto = model_quant.load_quanto(silent=True)
-        if quanto is not None:
-            self.QLinear_forward = patches.undo(__name__, quanto.nn.QLinear, 'forward') # pylint: disable=E1128
-            self.QConv2d_forward = patches.undo(__name__, quanto.nn.QConv2d, 'forward') # pylint: disable=E1128
         self.Linear_forward = patches.undo(__name__, torch.nn.Linear, 'forward') # pylint: disable=E1128
         self.Linear_load_state_dict = patches.undo(__name__, torch.nn.Linear, '_load_from_state_dict') # pylint: disable=E1128
         self.Conv2d_forward = patches.undo(__name__, torch.nn.Conv2d, 'forward') # pylint: disable=E1128
@@ -66,5 +72,6 @@ class LoraPatches:
         self.LayerNorm_load_state_dict = patches.undo(__name__, torch.nn.LayerNorm, '_load_from_state_dict') # pylint: disable=E1128
         self.MultiheadAttention_forward = patches.undo(__name__, torch.nn.MultiheadAttention, 'forward') # pylint: disable=E1128
         self.MultiheadAttention_load_state_dict = patches.undo(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict') # pylint: disable=E1128
+        self.handle_quant(apply=False)
         patches.originals.pop(__name__, None)
         self.active = False

modules/shared.py

@@ -112,10 +112,10 @@ def readfile(filename, silent=False, lock=False):
             lock_file = fasteners.InterProcessReaderWriterLock(f"{filename}.lock")
             lock_file.logger.disabled = True
             locked = lock_file.acquire_read_lock(blocking=True, timeout=3)
-        except Exception as e:
+        except Exception as err:
             lock_file = None
             locking_available = False
-            log.error(f'File read lock: file="{filename}" {e}')
+            log.error(f'File read lock: file="{filename}" {err}')
             locked = False
     try:
         # if not os.path.exists(filename):
@@ -129,11 +129,11 @@ def readfile(filename, silent=False, lock=False):
         t1 = time.time()
         if not silent:
             log.debug(f'Read: file="{filename}" json={len(data)} bytes={os.path.getsize(filename)} time={t1-t0:.3f}')
-    except FileNotFoundError as e:
-        log.debug(f'Reading failed: {filename} {e}')
-    except Exception as e:
+    except FileNotFoundError as err:
+        log.debug(f'Reading failed: {filename} {err}')
+    except Exception as err:
         if not silent:
-            log.error(f'Reading failed: {filename} {e}')
+            log.error(f'Reading failed: {filename} {err}')
     try:
         if locking_available and lock_file is not None:
             lock_file.release_read_lock()
@@ -169,18 +169,18 @@ def writefile(data, filename, mode='w', silent=False, atomic=False):
             output = json.dumps(simple, indent=2, default=default)
         else:
             raise ValueError('not a valid object')
-    except Exception as e:
-        log.error(f'Save failed: file="{filename}" {e}')
+    except Exception as err:
+        log.error(f'Save failed: file="{filename}" {err}')
         return
     try:
         if locking_available:
             lock_file = fasteners.InterProcessReaderWriterLock(f"{filename}.lock") if locking_available else None
             lock_file.logger.disabled = True
             locked = lock_file.acquire_write_lock(blocking=True, timeout=3) if lock_file is not None else False
-    except Exception as e:
+    except Exception as err:
         locking_available = False
         lock_file = None
-        log.error(f'File write lock: file="{filename}" {e}')
+        log.error(f'File write lock: file="{filename}" {err}')
         locked = False
     try:
         if atomic:
@@ -195,8 +195,8 @@ def writefile(data, filename, mode='w', silent=False, atomic=False):
         t1 = time.time()
         if not silent:
             log.debug(f'Save: file="{filename}" json={len(data)} bytes={len(output)} time={t1-t0:.3f}')
-    except Exception as e:
-        log.error(f'Save failed: file="{filename}" {e}')
+    except Exception as err:
+        log.error(f'Save failed: file="{filename}" {err}')
     try:
         if locking_available and lock_file is not None:
             lock_file.release_write_lock()
@@ -283,6 +283,7 @@ def list_checkpoint_tiles():
     import modules.sd_models # pylint: disable=W0621
     return modules.sd_models.checkpoint_tiles()
 
+
 default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.safetensors"
 
 
@@ -420,6 +421,7 @@ def get_default_modes():
 
     return default_offload_mode, default_cross_attention, default_sdp_options
 
+
 startup_offload_mode, startup_cross_attention, startup_sdp_options = get_default_modes()
 
 options_templates.update(options_section(('sd', "Execution & Models"), {
@@ -499,7 +501,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
     "ipex_optimize": OptionInfo([], "IPEX Optimize for Intel GPUs", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}),
 
     "openvino_sep": OptionInfo("<h2>OpenVINO</h2>", "", gr.HTML, {"visible": cmd_opts.use_openvino}),
-    "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint disable:E0606
+    "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint: disable=E0606
     "nncf_quantize": OptionInfo([], "OpenVINO Quantize Models with NNCF", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": cmd_opts.use_openvino}),
     "nncf_quant_mode": OptionInfo("INT8", "OpenVINO quantization mode for NNCF", gr.Radio, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}),
     "nncf_compress_weights_mode": OptionInfo("INT8", "OpenVINO compress mode for NNCF", gr.Radio, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}),
@@ -955,8 +957,8 @@ class Options:
         if self.data_labels[key].onchange is not None:
             try:
                 self.data_labels[key].onchange()
-            except Exception as e:
-                log.error(f'Error in onchange callback: {key} {value} {e}')
+            except Exception as err:
+                log.error(f'Error in onchange callback: {key} {value} {err}')
                 errors.display(e, 'Error in onchange callback')
                 setattr(self, key, oldval)
                 return False
@@ -1000,8 +1002,8 @@ class Options:
             writefile(diff, filename, silent=silent)
             if len(unused_settings) > 0:
                 log.debug(f"Unused settings: {unused_settings}")
-        except Exception as e:
-            log.error(f'Save settings failed: {filename} {e}')
+        except Exception as err:
+            log.error(f'Save settings failed: {filename} {err}')
 
     def save(self, filename=None, silent=False):
         threading.Thread(target=self.save_atomic, args=(filename, silent)).start()
@@ -1155,8 +1157,8 @@ def restart_server(restart=True):
         time.sleep(1)
         sys.tracebacklimit = 100
         # os._exit(0)
-    except (Exception, BaseException) as e:
-        log.error(f'Server shutdown error: {e}')
+    except (Exception, BaseException) as err:
+        log.error(f'Server shutdown error: {err}')
     if restart:
         log.info('Server will restart')
 
@@ -1234,9 +1236,9 @@ def req(url_addr, headers = None, **kwargs):
         headers = { 'Content-type': 'application/json' }
     try:
         res = requests.get(url_addr, timeout=30, headers=headers, verify=False, allow_redirects=True, **kwargs)
-    except Exception as e:
+    except Exception as err:
         log.error(f'HTTP request error: url={url_addr} {e}')
-        res = { 'status_code': 500, 'text': f'HTTP request error: url={url_addr} {e}' }
+        res = { 'status_code': 500, 'text': f'HTTP request error: url={url_addr} {err}' }
         res = SimpleNamespace(**res)
     return res