diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py
index 5256ea94a..eb9a40b0d 100644
--- a/extensions-builtin/Lora/extra_networks_lora.py
+++ b/extensions-builtin/Lora/extra_networks_lora.py
@@ -65,7 +65,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
 
     def deactivate(self, p):
         if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"):
-            if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__:
+            if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__ and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"):
                 shared.sd_model.unload_lora_weights()
         if not self.active and getattr(networks, "originals", None ) is not None:
             networks.originals.undo() # remove patches
diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py
index b58e6d343..81c50db31 100644
--- a/extensions-builtin/Lora/networks.py
+++ b/extensions-builtin/Lora/networks.py
@@ -150,6 +150,10 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
                     recompile_model = True
                     shared.compiled_model_state.lora_model = []
                     break
+            if not recompile_model:
+                if len(loaded_networks) > 0 and debug:
+                    shared.log.debug(f'OpenVINO: Skipping LoRa loading')
+                return
         else:
             recompile_model = True
             shared.compiled_model_state.lora_model = []