diff --git a/modules/sd_models.py b/modules/sd_models.py
index 9cfa6b2a1..461351a75 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -1127,6 +1127,12 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
             sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining}', ncols=80, colour='#327fba')
 
         sd_unet.load_unet(sd_model)
+
+        from modules.textual_inversion import textual_inversion
+        sd_model.embedding_db = textual_inversion.EmbeddingDatabase()
+        sd_model.embedding_db.add_embedding_dir(shared.opts.embeddings_dir)
+        sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True)
+
         set_diffuser_options(sd_model, vae, op)
 
         if op == 'refiner' and shared.opts.diffusers_move_refiner:
@@ -1148,14 +1154,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
         shared.log.error("Failed to load diffusers model")
         errors.display(e, "loading Diffusers model")
 
-    from modules.textual_inversion import textual_inversion
-    sd_model.embedding_db = textual_inversion.EmbeddingDatabase()
     if op == 'refiner':
         model_data.sd_refiner = sd_model
     else:
         model_data.sd_model = sd_model
-    sd_model.embedding_db.add_embedding_dir(shared.opts.embeddings_dir)
-    sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True)
 
     timer.record("load")
     devices.torch_gc(force=True)