From 3bef3e3eee9b380e223608a14de9103c62a09cea Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 7 Jun 2023 17:25:11 +0300 Subject: [PATCH] Train patches for IPEX --- modules/devices.py | 3 +++ modules/hypernetworks/hypernetwork.py | 10 ++++------ modules/textual_inversion/textual_inversion.py | 9 ++++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index b8d363cc5..1b7cb428b 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -169,6 +169,9 @@ if args.use_ipex: CondFunc('torch.nn.modules.GroupNorm.forward', lambda orig_func, *args, **kwargs: orig_func(args[0], args[1].to(args[0].weight.data.dtype)), lambda *args, **kwargs: args[2].dtype != args[1].weight.data.dtype) + CondFunc('torch.nn.modules.Linear.forward', + lambda orig_func, *args, **kwargs: orig_func(args[0], args[1].to(args[0].weight.data.dtype)), + lambda *args, **kwargs: args[2].dtype != args[1].weight.data.dtype) #Use XPU instead of CPU. %20 Perf improvement on weak CPUs. if args.device_id is not None: diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 49253c2c3..75d4b07f6 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -594,12 +594,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi print(e) if shared.cmd_opts.use_ipex: - scaler = ipex.cpu.autocast._grad_scaler.GradScaler() - #Remove these after Intel adds support for float16 training: - if shared.opts.cuda_dtype == 'BF16': - shared.sd_model = shared.sd_model.bfloat16() - elif shared.opts.cuda_dtype == 'FP32': - shared.sd_model = shared.sd_model.float32() + scaler = ipex.cpu.autocast._grad_scaler.GradScaler() #scaler.step(optimizer): PI_ERROR_INVALID_ARG_VALUE + shared.sd_model = shared.sd_model.to(dtype=torch.float32) + shared.sd_model.train() + shared.sd_model, optimizer = ipex.optimize(shared.sd_model, optimizer=optimizer, dtype=devices.dtype) else: scaler = torch.cuda.amp.GradScaler() diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 722593525..3025bd864 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -3,6 +3,10 @@ import html import csv from collections import namedtuple import torch +try: + import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import +except: + pass from tqdm import tqdm import safetensors.torch import numpy as np @@ -433,7 +437,10 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st shared.log.info("No saved optimizer exists in checkpoint") if shared.cmd_opts.use_ipex: - scaler = torch.xpu.amp.GradScaler() + scaler = ipex.cpu.autocast._grad_scaler.GradScaler() #scaler.step(optimizer): PI_ERROR_INVALID_ARG_VALUE + shared.sd_model = shared.sd_model.to(dtype=torch.float32) + shared.sd_model.train() + shared.sd_model, optimizer = ipex.optimize(shared.sd_model, optimizer=optimizer, dtype=devices.dtype) else: scaler = torch.cuda.amp.GradScaler()