diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 9d6b18517..0453e30d0 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -654,11 +654,9 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
                         loss = shared.sd_model.forward(x, c)[0] / gradient_step
                     del x
                     del c
-                    if shared.cmd_opts.use_ipex and loss > (1 / gradient_step):
-                        loss = (loss - (1 / gradient_step)) * 10
                     _loss_step += loss.item()
-                scaler.scale(loss).backward()
 
+                scaler.scale(loss).backward()
                 # go back until we reach gradient accumulation steps
                 if (j + 1) % gradient_step != 0:
                     continue
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 7a859897e..2fdcb03d9 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -486,8 +486,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     else:
                         loss = shared.sd_model.forward(x, cond)[0] / gradient_step
                     del x
-                    if shared.cmd_opts.use_ipex and loss > (1 / gradient_step):
-                        loss = (loss - (1 / gradient_step)) * 10
                     _loss_step += loss.item()
 
                 scaler.scale(loss).backward()
@@ -496,6 +494,7 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st
                     continue
                 if clip_grad:
                     clip_grad(embedding.vec, clip_grad_sched.learn_rate)
+
                 scaler.step(optimizer)
                 scaler.update()
                 embedding.step += 1