From a49a8f8b462b96af9eb682a6bbcb6cd96f276868 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 01:43:19 +0900 Subject: [PATCH 01/26] First DirectML implementation. Unstable and not tested. --- TODO_DML.md | 22 +++++++++ modules/devices.py | 16 ++++++- modules/realesrgan_model.py | 71 +++++++++++++++++++++++++++++- modules/sd_hijack.py | 3 ++ modules/sd_hijack_inpainting.py | 1 + modules/sd_hijack_optimizations.py | 10 +++++ modules/shared.py | 2 +- setup.py | 38 +++++++++++----- 8 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 TODO_DML.md diff --git a/TODO_DML.md b/TODO_DML.md new file mode 100644 index 000000000..af6559ac1 --- /dev/null +++ b/TODO_DML.md @@ -0,0 +1,22 @@ +# TODO + +## Issues + +Stuff to be fixed... + +- `mat1 and mat2 must have the same dtype` error (half mode) +- Some samplers won't work (test later) + +## Something needs discussion + +- About memory optimization. + +Basically, we cannot get detailed vram information from `torch-directml`. + +It has `gpu_memory` method which returns an array contains used memory size, but it is almostly useless without any other information. + +What should we do? + +1. Use any fixed value as the available memory capacity. +2. Use `atiadlxx`(AMD/ATI GPU driver library) to infer vram information as similar as possible to the actual value. (works for AMDGPUs) +3. or another better way. diff --git a/modules/devices.py b/modules/devices.py index 7c869d60a..5f5e18768 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -27,12 +27,26 @@ def get_cuda_device_string(): return "cuda" +def get_dml_device_string(): + from modules import shared + if shared.cmd_opts.device_id is not None: + return f"privateuseone:{shared.cmd_opts.device_id}" + return "privateuseone" + + def get_optimal_device_name(): if torch.cuda.is_available(): return get_cuda_device_string() if has_mps(): return "mps" - return "cpu" + try: + import torch_directml + if torch_directml.is_available(): + return get_dml_device_string() + else: + return "cpu" + except: + return "cpu" def get_optimal_device(): diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py index 86007f906..ac1102e38 100644 --- a/modules/realesrgan_model.py +++ b/modules/realesrgan_model.py @@ -6,10 +6,76 @@ from PIL import Image from basicsr.utils.download_util import load_file_from_url from modules.upscaler import Upscaler, UpscalerData -from modules.shared import cmd_opts, opts +from modules.shared import cmd_opts, opts, device import modules.errors as errors +# DML ISSUE: Some tensors turn 0 after Extended Slices. +def realesrgan_tile_process_dml_fix(self): + import math + import torch + batch, channel, height, width = self.img.shape + output_height = height * self.scale + output_width = width * self.scale + output_shape = (batch, channel, output_height, output_width) + + # start with black image + self.output = self.img.new_zeros(output_shape, device='cpu' if self.device.type == 'privateuseone' else self.device) + tiles_x = math.ceil(width / self.tile_size) + tiles_y = math.ceil(height / self.tile_size) + + # loop over all tiles + for y in range(tiles_y): + for x in range(tiles_x): + # extract tile from input image + ofs_x = x * self.tile_size + ofs_y = y * self.tile_size + # input tile area on total image + input_start_x = ofs_x + input_end_x = min(ofs_x + self.tile_size, width) + input_start_y = ofs_y + input_end_y = min(ofs_y + self.tile_size, height) + + # input tile area on total image with padding + input_start_x_pad = max(input_start_x - self.tile_pad, 0) + input_end_x_pad = min(input_end_x + self.tile_pad, width) + input_start_y_pad = max(input_start_y - self.tile_pad, 0) + input_end_y_pad = min(input_end_y + self.tile_pad, height) + + # input tile dimensions + input_tile_width = input_end_x - input_start_x + input_tile_height = input_end_y - input_start_y + tile_idx = y * tiles_x + x + 1 + input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad] + + # upscale tile + try: + with torch.no_grad(): + output_tile = self.model(input_tile) + output_tile = output_tile.cpu() + except RuntimeError as error: + print('Error', error) + print(f'\tTile {tile_idx}/{tiles_x * tiles_y}') + + # output tile area on total image + output_start_x = input_start_x * self.scale + output_end_x = input_end_x * self.scale + output_start_y = input_start_y * self.scale + output_end_y = input_end_y * self.scale + + # output tile area without padding + output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale + output_end_x_tile = output_start_x_tile + input_tile_width * self.scale + output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale + output_end_y_tile = output_start_y_tile + input_tile_height * self.scale + + # put tile into output image + self.output[:, :, output_start_y:output_end_y, + output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile, + output_start_x_tile:output_end_x_tile] + self.output = self.output.to(self.device) + + class UpscalerRealESRGAN(Upscaler): def __init__(self, path): self.name = "RealESRGAN" @@ -37,6 +103,8 @@ class UpscalerRealESRGAN(Upscaler): try: from realesrgan import RealESRGANer + if device.type == 'privateuseone': + RealESRGANer.tile_process = realesrgan_tile_process_dml_fix except: print("Error importing Real-ESRGAN:", file=sys.stderr) return img @@ -53,6 +121,7 @@ class UpscalerRealESRGAN(Upscaler): half=not cmd_opts.no_half and not opts.upcast_sampling, tile=opts.ESRGAN_tile, tile_pad=opts.ESRGAN_tile_overlap, + device=device, ) upsampled = upsampler.enhance(np.array(img), outscale=info.scale)[0] diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index f817b7afd..cd859a6c3 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -257,6 +257,9 @@ class EmbeddingsWithFixes(torch.nn.Module): for offset, embedding in fixes: emb = devices.cond_cast_unet(embedding.vec) emb_len = min(tensor.shape[0] - offset - 1, emb.shape[0]) + # DML ISSUE: type mismatch on half mode + if tensor.dtype == torch.float16 and emb.dtype == torch.float32 and not shared.cmd_opts.no_half: + emb = emb.half() tensor = torch.cat([tensor[0:offset + 1], emb[0:emb_len], tensor[offset + 1 + emb_len:]]) vecs.append(tensor) diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py index 4b23c132d..0c37fef89 100644 --- a/modules/sd_hijack_inpainting.py +++ b/modules/sd_hijack_inpainting.py @@ -53,6 +53,7 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F def get_x_prev_and_pred_x0(e_t, index): # select parameters corresponding to the currently considered timestep + print(alphas[index]) # DML ISSUE: PLMS Sampling does not work without this print. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 776d91d6c..0f1f2a757 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -30,6 +30,9 @@ def get_available_vram(): mem_free_torch = mem_reserved - mem_active mem_free_total = mem_free_cuda + mem_free_torch return mem_free_total + elif shared.device.type == 'privateuseone': + # DML ISSUE: There's no way to get any memory info. + return 1048576 else: return psutil.virtual_memory().available @@ -195,6 +198,10 @@ def einsum_op_cuda(q, k, v): # Divide factor of safety as there's copying and fragmentation return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) +def einsum_op_dml(q, k, v): + # DML ISSUE: There's no way to get any memory info. + return einsum_op_tensor_mem(q, k, v, 1024) + def einsum_op(q, k, v): if q.device.type == 'cuda': return einsum_op_cuda(q, k, v) @@ -204,6 +211,9 @@ def einsum_op(q, k, v): return einsum_op_mps_v1(q, k, v) return einsum_op_mps_v2(q, k, v) + if q.device.type == 'privateuseone': + return einsum_op_dml(q, k, v) + # Smaller slices are faster due to L2/L3/SLC caches. # Tested on i7 with 8MB L3 cache. return einsum_op_tensor_mem(q, k, v, 32) diff --git a/modules/shared.py b/modules/shared.py index a2e202735..f147317cd 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -422,7 +422,7 @@ options_templates.update(options_section(('ui', "Live previews"), { "live_previews_enable": OptionInfo(True, "Show live previews of the created image"), "show_progress_grid": OptionInfo(True, "Show previews of all images generated in a batch as a grid"), "show_progress_every_n_steps": OptionInfo(1, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), - "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), + "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML ISSUE: Approx NN does not work well on DirectML device. "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}), "live_preview_refresh_period": OptionInfo(250, "Progressbar/preview update period, in milliseconds") })) diff --git a/setup.py b/setup.py index 73269b0df..aa01a3de5 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import json import time import shutil import logging +import platform import subprocess try: @@ -20,7 +21,7 @@ class Dot(dict): # dot notation access to dictionary attributes log = logging.getLogger("sd") -args = Dot({ 'debug': False, 'upgrade': False, 'noupdate': False, 'skip-extensions': False, 'skip-requirements': False, 'reset': False }) +args = Dot({ 'debug': False, 'upgrade': False, 'noupdate': False, 'nodirectml': False, 'skip-extensions': False, 'skip-requirements': False, 'reset': False }) quick_allowed = True errors = 0 opts = {} @@ -188,17 +189,21 @@ def check_torch(): torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: - log.info('Using CPU-only Torch') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') + machine = platform.machine() + if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64 + log.info('Using DirectML Backend') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.1 torchvision==0.14.1 torch-directml') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') + else: + log.info('Using CPU-only Torch') + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchaudio torchvision') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if 'torch' in torch_command: install(torch_command, 'torch torchvision torchaudio') try: import torch log.info(f'Torch {torch.__version__}') - if not torch.cuda.is_available(): - log.warning("Torch repoorts CUDA not available") - else: + if torch.cuda.is_available(): if torch.version.cuda: log.info(f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}') elif torch.version.hip: @@ -207,6 +212,16 @@ def check_torch(): log.warning('Unknown Torch backend') for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]: log.info(f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}') + else: + try: + import torch_directml + import pkg_resources + version = pkg_resources.get_distribution("torch-directml") + log.info(f'Torch backend: DirectML ({version})') + for i in range(0, torch_directml.device_count()): + log.info(f'Torch detected GPU: {torch_directml.device_name(i)}') + except: + log.warning("Torch repoorts CUDA not available") except Exception as e: log.error(f'Could not load torch: {e}') exit(1) @@ -239,14 +254,14 @@ def install_repositories(): return os.path.join(os.path.dirname(__file__), 'repositories', name) log.info('Installing repositories') os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True) - stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") - stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") + stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") # DML TODO: check samplers work well + stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341") clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git") taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318") clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit) - k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') - k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919") + k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') # DML TODO: check samplers work well + k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb") clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git') codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af") @@ -481,6 +496,7 @@ def parse_args(): parser.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s") parser.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s") parser.add_argument('--noupdate', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s") + parser.add_argument('--nodirectml', default = False, action='store_true', help = "Although nVidia and AMD toolkit aren't detected, use CPU not DirectML, default: %(default)s") parser.add_argument('--skip-requirements', default = False, action='store_true', help = "Skips checking and installing requirements, default: %(default)s") parser.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s") parser.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s") From 836324cd2c5ff362d5d15b09c395ae39134d07d4 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 01:55:27 +0900 Subject: [PATCH 02/26] Fix PLMS & DPM & DDIM. Unstable and not tested. --- modules/realesrgan_model.py | 2 +- modules/sd_hijack.py | 2 +- modules/sd_hijack_inpainting.py | 2 +- setup.py | 21 +++++++++++++++------ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py index ac1102e38..1d48ac42c 100644 --- a/modules/realesrgan_model.py +++ b/modules/realesrgan_model.py @@ -10,7 +10,7 @@ from modules.shared import cmd_opts, opts, device import modules.errors as errors -# DML ISSUE: Some tensors turn 0 after Extended Slices. +# DML Solution: Some tensors turn 0 after Extended Slices. Move output to cpu and get it back. def realesrgan_tile_process_dml_fix(self): import math import torch diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index cd859a6c3..4a68216df 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -257,7 +257,7 @@ class EmbeddingsWithFixes(torch.nn.Module): for offset, embedding in fixes: emb = devices.cond_cast_unet(embedding.vec) emb_len = min(tensor.shape[0] - offset - 1, emb.shape[0]) - # DML ISSUE: type mismatch on half mode + # DML Solution: type mismatch on half mode if tensor.dtype == torch.float16 and emb.dtype == torch.float32 and not shared.cmd_opts.no_half: emb = emb.half() tensor = torch.cat([tensor[0:offset + 1], emb[0:emb_len], tensor[offset + 1 + emb_len:]]) diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py index 0c37fef89..3405e8a40 100644 --- a/modules/sd_hijack_inpainting.py +++ b/modules/sd_hijack_inpainting.py @@ -53,7 +53,7 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F def get_x_prev_and_pred_x0(e_t, index): # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML ISSUE: PLMS Sampling does not work without this print. + print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) diff --git a/setup.py b/setup.py index aa01a3de5..237fd22d7 100644 --- a/setup.py +++ b/setup.py @@ -254,15 +254,24 @@ def install_repositories(): return os.path.join(os.path.dirname(__file__), 'repositories', name) log.info('Installing repositories') os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True) - stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") # DML TODO: check samplers work well - stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341") - clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) + try: + import torch_directml + stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/lshqqytiger/stablediffusion-directml.git") # DML Solution: DDIM sampler fix + stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341") + clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) + k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/lshqqytiger/k-diffusion-directml.git') # DML Solution: DPM fix + k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb") + clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) + except: + stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") + stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") + clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) + k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') + k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919") + clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git") taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318") clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit) - k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') # DML TODO: check samplers work well - k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb") - clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git') codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af") clone(codeformer_repo, d('CodeFormer'), codeformer_commit) From cb664cf3328d80c36be89e74f43bdffa6f24de88 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 02:02:12 +0900 Subject: [PATCH 03/26] Update TODO_DML. --- TODO_DML.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/TODO_DML.md b/TODO_DML.md index af6559ac1..f359ba1b3 100644 --- a/TODO_DML.md +++ b/TODO_DML.md @@ -20,3 +20,14 @@ What should we do? 1. Use any fixed value as the available memory capacity. 2. Use `atiadlxx`(AMD/ATI GPU driver library) to infer vram information as similar as possible to the actual value. (works for AMDGPUs) 3. or another better way. + +- Half mode does not work well. + +In half precision, we get an error like `mat1 and mat2 must have the same dtype`. + +I solved this problem by overriding forward of GroupNorm / LayerNorm / Conv2d / Linear to convert input tensor to full precision (and convert to float16 before return). + +What should we do? + +1. Override forwards (same solution) +2. When using DirectML, set the default to full precision and notify the user. From 9dc8581ba04cd092885674ffe6681490f1238532 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 21:22:47 +0900 Subject: [PATCH 04/26] Add hijack for DirectML. Unstable & Under testing. --- modules/sd_hijack_directml.py | 170 ++++++++++++++++++++++++++++++++++ modules/shared.py | 4 + setup.py | 21 ++--- 3 files changed, 180 insertions(+), 15 deletions(-) create mode 100644 modules/sd_hijack_directml.py diff --git a/modules/sd_hijack_directml.py b/modules/sd_hijack_directml.py new file mode 100644 index 000000000..e2e6fa50f --- /dev/null +++ b/modules/sd_hijack_directml.py @@ -0,0 +1,170 @@ +import torch +from tqdm.auto import tqdm + +from shared import device + +# k-diffusion +from k_diffusion import sampling + +def dpm_solver_adaptive(self, x, t_start, t_end, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None): + noise_sampler = sampling.default_noise_sampler(x) if noise_sampler is None else noise_sampler + if order not in {2, 3}: + raise ValueError('order should be 2 or 3') + forward = t_end > t_start + if not forward and eta: + raise ValueError('eta must be 0 for reverse sampling') + h_init = abs(h_init) * (1 if forward else -1) + atol = torch.tensor(atol).to(device) + rtol = torch.tensor(rtol).to(device) + s = t_start + x_prev = x + accept = True + pid = sampling.PIDStepSizeController(h_init, pcoeff, icoeff, dcoeff, 1.5 if eta else order, accept_safety) + info = {'steps': 0, 'nfe': 0, 'n_accept': 0, 'n_reject': 0} + + while s < t_end - 1e-5 if forward else s > t_end + 1e-5: + eps_cache = {} + t = torch.minimum(t_end, s + pid.h) if forward else torch.maximum(t_end, s + pid.h) + if eta: + sd, su = sampling.get_ancestral_step(self.sigma(s), self.sigma(t), eta) + t_ = torch.minimum(t_end, self.t(sd)) + su = (self.sigma(t) ** 2 - self.sigma(t_) ** 2) ** 0.5 + else: + t_, su = t, 0. + + eps, eps_cache = self.eps(eps_cache, 'eps', x, s) + denoised = x - self.sigma(s) * eps + + if order == 2: + x_low, eps_cache = self.dpm_solver_1_step(x, s, t_, eps_cache=eps_cache) + x_high, eps_cache = self.dpm_solver_2_step(x, s, t_, eps_cache=eps_cache) + else: + x_low, eps_cache = self.dpm_solver_2_step(x, s, t_, r1=1 / 3, eps_cache=eps_cache) + x_high, eps_cache = self.dpm_solver_3_step(x, s, t_, eps_cache=eps_cache) + delta = torch.maximum(atol, rtol * torch.maximum(x_low.abs(), x_prev.abs())) + error = torch.linalg.norm((x_low - x_high) / delta) / x.numel() ** 0.5 + accept = pid.propose_step(error) + if accept: + x_prev = x_low + x = x_high + su * s_noise * noise_sampler(self.sigma(s), self.sigma(t)) + s = t + info['n_accept'] += 1 + else: + info['n_reject'] += 1 + info['nfe'] += order + info['steps'] += 1 + + if self.info_callback is not None: + self.info_callback({'x': x, 'i': info['steps'] - 1, 't': s, 't_up': s, 'denoised': denoised, 'error': error, 'h': pid.h, **info}) + + return x, info + + +@torch.no_grad() +def sample_dpm_fast(model, x, sigma_min, sigma_max, n, extra_args=None, callback=None, disable=None, eta=0., s_noise=1., noise_sampler=None): + """DPM-Solver-Fast (fixed step size). See https://arxiv.org/abs/2206.00927.""" + if sigma_min <= 0 or sigma_max <= 0: + raise ValueError('sigma_min and sigma_max must not be 0') + with tqdm(total=n, disable=disable) as pbar: + dpm_solver = sampling.DPMSolver(model, extra_args, eps_callback=pbar.update) + if callback is not None: + dpm_solver.info_callback = lambda info: callback({'sigma': dpm_solver.sigma(info['t']), 'sigma_hat': dpm_solver.sigma(info['t_up']), **info}) + return dpm_solver.dpm_solver_fast(x, dpm_solver.t(torch.tensor(sigma_max).to(device)), dpm_solver.t(torch.tensor(sigma_min).to(device)), n, eta, s_noise, noise_sampler) + + +@torch.no_grad() +def sample_dpm_adaptive(model, x, sigma_min, sigma_max, extra_args=None, callback=None, disable=None, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None, return_info=False): + """DPM-Solver-12 and 23 (adaptive step size). See https://arxiv.org/abs/2206.00927.""" + if sigma_min <= 0 or sigma_max <= 0: + raise ValueError('sigma_min and sigma_max must not be 0') + with tqdm(disable=disable) as pbar: + dpm_solver = sampling.DPMSolver(model, extra_args, eps_callback=pbar.update) + if callback is not None: + dpm_solver.info_callback = lambda info: callback({'sigma': dpm_solver.sigma(info['t']), 'sigma_hat': dpm_solver.sigma(info['t_up']), **info}) + x, info = dpm_solver.dpm_solver_adaptive(x, dpm_solver.t(torch.tensor(sigma_max).to(device)), dpm_solver.t(torch.tensor(sigma_min).to(device)), order, rtol, atol, h_init, pcoeff, icoeff, dcoeff, accept_safety, eta, s_noise, noise_sampler) + if return_info: + return x, info + return x + +sampling.DPMSolver.dpm_solver_adaptive = dpm_solver_adaptive +sampling.sample_dpm_fast = sample_dpm_fast +sampling.sample_dpm_adaptive = sample_dpm_adaptive + +# stablediffusion +from ldm.models.diffusion.ddim import DDIMSampler + +@torch.no_grad() +def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False, + temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None, + unconditional_guidance_scale=1., unconditional_conditioning=None, + dynamic_threshold=None): + b, *_, device = *x.shape, x.device + + if unconditional_conditioning is None or unconditional_guidance_scale == 1.: + model_output = self.model.apply_model(x, t, c) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t] * 2) + if isinstance(c, dict): + assert isinstance(unconditional_conditioning, dict) + c_in = dict() + for k in c: + if isinstance(c[k], list): + c_in[k] = [torch.cat([ + unconditional_conditioning[k][i], + c[k][i]]) for i in range(len(c[k]))] + else: + c_in[k] = torch.cat([ + unconditional_conditioning[k], + c[k]]) + elif isinstance(c, list): + c_in = list() + assert isinstance(unconditional_conditioning, list) + for i in range(len(c)): + c_in.append(torch.cat([unconditional_conditioning[i], c[i]])) + else: + c_in = torch.cat([unconditional_conditioning, c]) + model_uncond, model_t = self.model.apply_model(x_in, t_in, c_in).chunk(2) + model_output = model_uncond + unconditional_guidance_scale * (model_t - model_uncond) + + if self.model.parameterization == "v": + e_t = self.model.predict_eps_from_z_and_v(x, t, model_output) + else: + e_t = model_output + + if score_corrector is not None: + assert self.model.parameterization == "eps", 'not implemented' + e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs) + + alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas + alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev + sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas + sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas + # select parameters corresponding to the currently considered timestep + print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print. + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device) + + # current prediction for x_0 + if self.model.parameterization != "v": + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + else: + pred_x0 = self.model.predict_start_from_z_and_v(x, t, model_output) + + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + + if dynamic_threshold is not None: + raise NotImplementedError() + + # direction pointing to x_t + dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t + noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature + if noise_dropout > 0.: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0 + +DDIMSampler.p_sample_ddim = p_sample_ddim diff --git a/modules/shared.py b/modules/shared.py index f147317cd..6267a4a83 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -62,6 +62,10 @@ sd_model = None clip_model = None +#if device.type == 'privateuseone': +# import sd_hijack_directml + + def reload_hypernetworks(): from modules.hypernetworks import hypernetwork global hypernetworks # pylint: disable=W0603 diff --git a/setup.py b/setup.py index 237fd22d7..292529300 100644 --- a/setup.py +++ b/setup.py @@ -254,21 +254,12 @@ def install_repositories(): return os.path.join(os.path.dirname(__file__), 'repositories', name) log.info('Installing repositories') os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True) - try: - import torch_directml - stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/lshqqytiger/stablediffusion-directml.git") # DML Solution: DDIM sampler fix - stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "d4c168b2ad29d82e5fdfea4d598075f40a3b0341") - clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) - k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/lshqqytiger/k-diffusion-directml.git') # DML Solution: DPM fix - k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "47b6ef08bca986ff5e72815e74a419ef6616bdbb") - clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) - except: - stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") - stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") - clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) - k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') - k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919") - clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) + stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") + stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") + clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) + k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') + k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919") + clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git") taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318") clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit) From db56da075a2740bac6ba93c9eec621db1238ac62 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 23:04:52 +0900 Subject: [PATCH 05/26] need full precision for model & vae. Stable & tested. --- modules/sd_hijack_directml.py | 11 +++++++++++ modules/sd_hijack_optimizations.py | 4 ++-- modules/shared.py | 12 +++++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/modules/sd_hijack_directml.py b/modules/sd_hijack_directml.py index e2e6fa50f..b640fc848 100644 --- a/modules/sd_hijack_directml.py +++ b/modules/sd_hijack_directml.py @@ -92,6 +92,7 @@ sampling.sample_dpm_adaptive = sample_dpm_adaptive # stablediffusion from ldm.models.diffusion.ddim import DDIMSampler +from ldm.modules.diffusionmodules.util import noise_like @torch.no_grad() def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False, @@ -168,3 +169,13 @@ def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=F return x_prev, pred_x0 DDIMSampler.p_sample_ddim = p_sample_ddim + +# torch + +Generator_init = torch.Generator.__init__ +def Generator_init_fix(self, device = None, *args, **kwargs): + if device is not None and device.type == 'privateuseone': + return Generator_init(self, 'cpu', *args, **kwargs) # DML Solution: torch.Generator fallback to cpu. + else: + return Generator_init(self, device, *args, **kwargs) +torch.Generator.__init__ = Generator_init_fix diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 0f1f2a757..7e1ba79b3 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -32,7 +32,7 @@ def get_available_vram(): return mem_free_total elif shared.device.type == 'privateuseone': # DML ISSUE: There's no way to get any memory info. - return 1048576 + return 1073741824 else: return psutil.virtual_memory().available @@ -200,7 +200,7 @@ def einsum_op_cuda(q, k, v): def einsum_op_dml(q, k, v): # DML ISSUE: There's no way to get any memory info. - return einsum_op_tensor_mem(q, k, v, 1024) + return einsum_op_tensor_mem(q, k, v, 1073741824) def einsum_op(q, k, v): if q.device.type == 'cuda': diff --git a/modules/shared.py b/modules/shared.py index 6267a4a83..9aa4e8d64 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -57,13 +57,15 @@ devices.device, devices.device_interrogate, devices.device_gfpgan, devices.devic (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer']) device = devices.device +is_device_dml = False sd_upscalers = [] sd_model = None clip_model = None -#if device.type == 'privateuseone': -# import sd_hijack_directml +if device.type == 'privateuseone': + import sd_hijack_directml + is_device_dml = True def reload_hypernetworks(): @@ -251,7 +253,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), { "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }), "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1}), "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"), - "cross_attention_optimization": OptionInfo("Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }), + "cross_attention_optimization": OptionInfo("Sub-quadratic" if is_device_dml else "Scaled-Dot-Product", "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }), "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}), "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size for the layer optimization to use", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}), "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attentionkv chunk size for the sub-quadratic cross-attention layer optimization to use", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}), @@ -328,8 +330,8 @@ options_templates.update(options_section(('saving-paths', "Image Paths"), { options_templates.update(options_section(('cuda', "CUDA Settings"), { "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}), "cuda_dtype": OptionInfo("FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}), - "no_half": OptionInfo(False, "Use full precision for model (--no-half)"), - "no_half_vae": OptionInfo(False, "Use full precision for VAE (--no-half-vae)"), + "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)"), + "no_half_vae": OptionInfo(True if is_device_dml else False, "Use full precision for VAE (--no-half-vae)"), "disable_nan_check": OptionInfo(True, "Do not check if produced images/latent spaces have NaN values"), "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "), "cudnn_benchmark": OptionInfo(False, "Enable cuDNN benchmark feature"), From 53736ea7cbb7c08dd976cc8deff61197319fe492 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 23:07:25 +0900 Subject: [PATCH 06/26] fix --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 9aa4e8d64..27018c5a7 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -64,7 +64,7 @@ clip_model = None if device.type == 'privateuseone': - import sd_hijack_directml + import modules.sd_hijack_directml is_device_dml = True From 32634298d7ff047f53fb4d17d28068969c10e285 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 23:09:01 +0900 Subject: [PATCH 07/26] fix --- modules/sd_hijack_directml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_hijack_directml.py b/modules/sd_hijack_directml.py index b640fc848..c782921af 100644 --- a/modules/sd_hijack_directml.py +++ b/modules/sd_hijack_directml.py @@ -1,7 +1,7 @@ import torch from tqdm.auto import tqdm -from shared import device +from modules.shared import device # k-diffusion from k_diffusion import sampling From eb072db23c3e2144a991274f37b68006d2ce7781 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 25 Apr 2023 23:19:03 +0900 Subject: [PATCH 08/26] Add dml_specific. --- modules/{sd_hijack_directml.py => dml_specific.py} | 9 ++------- modules/shared.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) rename modules/{sd_hijack_directml.py => dml_specific.py} (95%) diff --git a/modules/sd_hijack_directml.py b/modules/dml_specific.py similarity index 95% rename from modules/sd_hijack_directml.py rename to modules/dml_specific.py index c782921af..e8503d745 100644 --- a/modules/sd_hijack_directml.py +++ b/modules/dml_specific.py @@ -2,6 +2,7 @@ import torch from tqdm.auto import tqdm from modules.shared import device +from modules.sd_hijack_utils import CondFunc # k-diffusion from k_diffusion import sampling @@ -172,10 +173,4 @@ DDIMSampler.p_sample_ddim = p_sample_ddim # torch -Generator_init = torch.Generator.__init__ -def Generator_init_fix(self, device = None, *args, **kwargs): - if device is not None and device.type == 'privateuseone': - return Generator_init(self, 'cpu', *args, **kwargs) # DML Solution: torch.Generator fallback to cpu. - else: - return Generator_init(self, device, *args, **kwargs) -torch.Generator.__init__ = Generator_init_fix +CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone') diff --git a/modules/shared.py b/modules/shared.py index 27018c5a7..99f4a986b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -64,7 +64,7 @@ clip_model = None if device.type == 'privateuseone': - import modules.sd_hijack_directml + import modules.dml_specific is_device_dml = True From 09ae33cdf741e927f18713ff0c9b1b1d7e0465c2 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 26 Apr 2023 12:21:44 +0900 Subject: [PATCH 09/26] Implement torch.dml. VERY UNSTABLE & NOT TESTED. --- extensions-builtin/sd-webui-controlnet | 2 +- extensions-builtin/seed_travel | 2 +- modules/dml/__init__.py | 32 +++++++ .../{dml_specific.py => dml/kdiffusion.py} | 89 +------------------ modules/dml/optimizer/amd/__init__.py | 7 ++ modules/dml/optimizer/amd/driver/atiadlxx.py | 43 +++++++++ .../dml/optimizer/amd/driver/atiadlxx_apis.py | 50 +++++++++++ .../optimizer/amd/driver/atiadlxx_defines.py | 1 + .../amd/driver/atiadlxx_structures.py | 87 ++++++++++++++++++ modules/dml/optimizer/amd/memory.py | 0 modules/dml/optimizer/intel/__init__.py | 7 ++ modules/dml/optimizer/nvidia/__init__.py | 7 ++ modules/dml/optimizer/optimizer.py | 8 ++ modules/dml/optimizer/unknown/__init__.py | 6 ++ modules/dml/stablediffusion.py | 80 +++++++++++++++++ modules/dml/torch.py | 5 ++ modules/sd_hijack_optimizations.py | 12 ++- modules/shared.py | 4 +- wiki | 2 +- 19 files changed, 347 insertions(+), 97 deletions(-) create mode 100644 modules/dml/__init__.py rename modules/{dml_specific.py => dml/kdiffusion.py} (52%) create mode 100644 modules/dml/optimizer/amd/__init__.py create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx.py create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx_apis.py create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx_defines.py create mode 100644 modules/dml/optimizer/amd/driver/atiadlxx_structures.py create mode 100644 modules/dml/optimizer/amd/memory.py create mode 100644 modules/dml/optimizer/intel/__init__.py create mode 100644 modules/dml/optimizer/nvidia/__init__.py create mode 100644 modules/dml/optimizer/optimizer.py create mode 100644 modules/dml/optimizer/unknown/__init__.py create mode 100644 modules/dml/stablediffusion.py create mode 100644 modules/dml/torch.py diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index f16c9e522..c5fbfc31d 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit f16c9e5221bcf9490f5fd93ff0d38027a14bc4d6 +Subproject commit c5fbfc31d002ff83bc692e1f24b7b1c9183dbe72 diff --git a/extensions-builtin/seed_travel b/extensions-builtin/seed_travel index 1a97ebb43..ffe0553c5 160000 --- a/extensions-builtin/seed_travel +++ b/extensions-builtin/seed_travel @@ -1 +1 @@ -Subproject commit 1a97ebb43428b43c41001cfe79e50b468126d99f +Subproject commit ffe0553c59e91067ebf1e4fc7ad85ca9c870bf57 diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py new file mode 100644 index 000000000..fbda39384 --- /dev/null +++ b/modules/dml/__init__.py @@ -0,0 +1,32 @@ +import torch +import torch_directml + +import modules.dml.kdiffusion +import modules.dml.stablediffusion +import modules.dml.torch + +from optimizer.unknown import UnknownOptimizer + +class DirectML(): + def get_optimizer(self, device: torch.device): + assert(device.type == 'privateuseone') + try: + device_name = torch_directml.device_name(device.index) + if 'NVIDIA' in device_name or 'GeForce' in device_name: + from optimizer.nvidia import nVidiaOptimizer as optimizer + elif 'AMD' in device_name or 'Radeon' in device_name: + from optimizer.amd import AMDOptimizer as optimizer + elif 'Intel' in device_name: + from optimizer.intel import IntelOptimizer as optimizer + else: + return UnknownOptimizer + return optimizer + except: + return UnknownOptimizer + + def memory_stats(self, device: torch.device): + optimizer = self.get_optimizer(device) + return optimizer.memory_stats(device.index) + +# Alternative of torch.cuda for DirectML. +torch.dml = DirectML diff --git a/modules/dml_specific.py b/modules/dml/kdiffusion.py similarity index 52% rename from modules/dml_specific.py rename to modules/dml/kdiffusion.py index e8503d745..2eced885f 100644 --- a/modules/dml_specific.py +++ b/modules/dml/kdiffusion.py @@ -2,9 +2,6 @@ import torch from tqdm.auto import tqdm from modules.shared import device -from modules.sd_hijack_utils import CondFunc - -# k-diffusion from k_diffusion import sampling def dpm_solver_adaptive(self, x, t_start, t_end, order=3, rtol=0.05, atol=0.0078, h_init=0.05, pcoeff=0., icoeff=1., dcoeff=0., accept_safety=0.81, eta=0., s_noise=1., noise_sampler=None): @@ -89,88 +86,4 @@ def sample_dpm_adaptive(model, x, sigma_min, sigma_max, extra_args=None, callbac sampling.DPMSolver.dpm_solver_adaptive = dpm_solver_adaptive sampling.sample_dpm_fast = sample_dpm_fast -sampling.sample_dpm_adaptive = sample_dpm_adaptive - -# stablediffusion -from ldm.models.diffusion.ddim import DDIMSampler -from ldm.modules.diffusionmodules.util import noise_like - -@torch.no_grad() -def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False, - temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None, - unconditional_guidance_scale=1., unconditional_conditioning=None, - dynamic_threshold=None): - b, *_, device = *x.shape, x.device - - if unconditional_conditioning is None or unconditional_guidance_scale == 1.: - model_output = self.model.apply_model(x, t, c) - else: - x_in = torch.cat([x] * 2) - t_in = torch.cat([t] * 2) - if isinstance(c, dict): - assert isinstance(unconditional_conditioning, dict) - c_in = dict() - for k in c: - if isinstance(c[k], list): - c_in[k] = [torch.cat([ - unconditional_conditioning[k][i], - c[k][i]]) for i in range(len(c[k]))] - else: - c_in[k] = torch.cat([ - unconditional_conditioning[k], - c[k]]) - elif isinstance(c, list): - c_in = list() - assert isinstance(unconditional_conditioning, list) - for i in range(len(c)): - c_in.append(torch.cat([unconditional_conditioning[i], c[i]])) - else: - c_in = torch.cat([unconditional_conditioning, c]) - model_uncond, model_t = self.model.apply_model(x_in, t_in, c_in).chunk(2) - model_output = model_uncond + unconditional_guidance_scale * (model_t - model_uncond) - - if self.model.parameterization == "v": - e_t = self.model.predict_eps_from_z_and_v(x, t, model_output) - else: - e_t = model_output - - if score_corrector is not None: - assert self.model.parameterization == "eps", 'not implemented' - e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs) - - alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas - alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev - sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas - sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas - # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print. - a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) - a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) - sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) - sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device) - - # current prediction for x_0 - if self.model.parameterization != "v": - pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() - else: - pred_x0 = self.model.predict_start_from_z_and_v(x, t, model_output) - - if quantize_denoised: - pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) - - if dynamic_threshold is not None: - raise NotImplementedError() - - # direction pointing to x_t - dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t - noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature - if noise_dropout > 0.: - noise = torch.nn.functional.dropout(noise, p=noise_dropout) - x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise - return x_prev, pred_x0 - -DDIMSampler.p_sample_ddim = p_sample_ddim - -# torch - -CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone') +sampling.sample_dpm_adaptive = sample_dpm_adaptive \ No newline at end of file diff --git a/modules/dml/optimizer/amd/__init__.py b/modules/dml/optimizer/amd/__init__.py new file mode 100644 index 000000000..37ea3ea91 --- /dev/null +++ b/modules/dml/optimizer/amd/__init__.py @@ -0,0 +1,7 @@ +from modules.dml.optimizer.optimizer import Optimizer +from driver.atiadlxx import ATIADLxx + +class AMDOptimizer(Optimizer): + driver: ATIADLxx = ATIADLxx() + def memory_stats(self, index): + return (AMDOptimizer.driver.iHyperMemorySize, AMDOptimizer.driver.get_dedicated_vram_usage(index)) diff --git a/modules/dml/optimizer/amd/driver/atiadlxx.py b/modules/dml/optimizer/amd/driver/atiadlxx.py new file mode 100644 index 000000000..abb5eca96 --- /dev/null +++ b/modules/dml/optimizer/amd/driver/atiadlxx.py @@ -0,0 +1,43 @@ +import ctypes as C +from .atiadlxx_apis import * +from .atiadlxx_structures import * +from .atiadlxx_defines import * + +class ATIADLxx(object): + iHyperMemorySize = 0 + + def __init__(self): + self.context = ADL_CONTEXT_HANDLE() + ADL2_Main_Control_Create(ADL_Main_Memory_Alloc, 1, C.byref(self.context)) + num_adapters = C.c_int(-1) + ADL2_Adapter_NumberOfAdapters_Get(self.context, C.byref(num_adapters)) + AdapterInfoArray = (AdapterInfo * num_adapters.value)() + ADL2_Adapter_AdapterInfo_Get(self.context, C.cast(AdapterInfoArray, LPAdapterInfo), C.sizeof(AdapterInfoArray)) + self.devices = [] + for adapter in AdapterInfoArray: + self.devices.append(adapter) + self.iHyperMemorySize = self.get_memory_info2(0).iHyperMemorySize + + def get_memory_info2(self, adapterIndex: int) -> ADLMemoryInfo2: + info = ADLMemoryInfo2() + + if ADL2_Adapter_MemoryInfo2_Get(self.context, adapterIndex, C.byref(info)) != ADL_OK: + raise RuntimeError("ADL2: Failed to get MemoryInfo2") + + return info + + def get_dedicated_vram_usage(self, index: int) -> int: + usage = C.c_int(-1) + + if ADL2_Adapter_DedicatedVRAMUsage_Get(self.context, self.devices[index].iAdapterIndex, C.byref(usage)) != ADL_OK: + raise RuntimeError("ADL2: Failed to get DedicatedVRAMUsage") + + return usage.value + + def get_vram_usage(self, index: int) -> int: + usage = C.c_int(-1) + + if ADL2_Adapter_VRAMUsage_Get(self.context, self.devices[index].iAdapterIndex, C.byref(usage)) != ADL_OK: + raise RuntimeError("ADL2: Failed to get VRAMUsage") + + return usage.value diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py new file mode 100644 index 000000000..23b7da3b1 --- /dev/null +++ b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py @@ -0,0 +1,50 @@ +import ctypes as C +import platform +from .atiadlxx_structures import * + +_platform = platform.system() + +try: + if _platform == "Windows": + atiadlxx = C.WinDLL("atiadlxx.dll") + + ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int) + ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p)) + + @ADL_MAIN_MALLOC_CALLBACK + def ADL_Main_Memory_Alloc(iSize): + return C._malloc(iSize) + + @ADL_MAIN_FREE_CALLBACK + def ADL_Main_Memory_Free(lpBuffer): + if lpBuffer[0] is not None: + C._free(lpBuffer[0]) + lpBuffer[0] = None + + ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create + ADL2_Main_Control_Create.restype = C.c_int + ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE] + + ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get + ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int + ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)] + + ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get + ADL2_Adapter_AdapterInfo_Get.restype = C.c_int + ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int] + + ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get + ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int + ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)] + + ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get + ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int + ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)] + + ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get + ADL2_Adapter_VRAMUsage_Get.restype = C.c_int + ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)] + else: + print("Warning: experimental graphic memory optimization for AMDGPU is disabled. Because this is not Windows platform.") +except FileNotFoundError: + print("Warning: memory optimization for AMDGPU is disabled. Because couldn't find 'atiadlxx.dll'. Please install GPU driver downloaded from AMD.com.") \ No newline at end of file diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_defines.py b/modules/dml/optimizer/amd/driver/atiadlxx_defines.py new file mode 100644 index 000000000..c242b9819 --- /dev/null +++ b/modules/dml/optimizer/amd/driver/atiadlxx_defines.py @@ -0,0 +1 @@ +ADL_OK = 0 \ No newline at end of file diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_structures.py b/modules/dml/optimizer/amd/driver/atiadlxx_structures.py new file mode 100644 index 000000000..a68392ec4 --- /dev/null +++ b/modules/dml/optimizer/amd/driver/atiadlxx_structures.py @@ -0,0 +1,87 @@ +import ctypes as C + +class _ADLPMActivity(C.Structure): + __slot__ = [ + 'iActivityPercent', + 'iCurrentBusLanes', + 'iCurrentBusSpeed', + 'iCurrentPerformanceLevel', + 'iEngineClock', + 'iMaximumBusLanes', + 'iMemoryClock', + 'iReserved', + 'iSize', + 'iVddc', + ] +_ADLPMActivity._fields_ = [ + ('iActivityPercent', C.c_int), + ('iCurrentBusLanes', C.c_int), + ('iCurrentBusSpeed', C.c_int), + ('iCurrentPerformanceLevel', C.c_int), + ('iEngineClock', C.c_int), + ('iMaximumBusLanes', C.c_int), + ('iMemoryClock', C.c_int), + ('iReserved', C.c_int), + ('iSize', C.c_int), + ('iVddc', C.c_int), +] +ADLPMActivity = _ADLPMActivity + +class _ADLMemoryInfo2(C.Structure): + __slot__ = [ + 'iHyperMemorySize', + 'iInvisibleMemorySize', + 'iMemoryBandwidth', + 'iMemorySize', + 'iVisibleMemorySize', + 'strMemoryType' + ] +_ADLMemoryInfo2._fields_ = [ + ('iHyperMemorySize', C.c_longlong), + ('iInvisibleMemorySize', C.c_longlong), + ('iMemoryBandwidth', C.c_longlong), + ('iMemorySize', C.c_longlong), + ('iVisibleMemorySize', C.c_longlong), + ('strMemoryType', C.c_char * 256) +] +ADLMemoryInfo2 = _ADLMemoryInfo2 + +class _AdapterInfo(C.Structure): + __slot__ = [ + 'iSize', + 'iAdapterIndex', + 'strUDID', + 'iBusNumber', + 'iDeviceNumber', + 'iFunctionNumber', + 'iVendorID', + 'strAdapterName', + 'strDisplayName', + 'iPresent', + 'iExist', + 'strDriverPath', + 'strDriverPathExt', + 'strPNPString', + 'iOSDisplayIndex', + ] +_AdapterInfo._fields_ = [ + ('iSize', C.c_int), + ('iAdapterIndex', C.c_int), + ('strUDID', C.c_char * 256), + ('iBusNumber', C.c_int), + ('iDeviceNumber', C.c_int), + ('iFunctionNumber', C.c_int), + ('iVendorID', C.c_int), + ('strAdapterName', C.c_char * 256), + ('strDisplayName', C.c_char * 256), + ('iPresent', C.c_int), + ('iExist', C.c_int), + ('strDriverPath', C.c_char * 256), + ('strDriverPathExt', C.c_char * 256), + ('strPNPString', C.c_char * 256), + ('iOSDisplayIndex', C.c_int) +] +AdapterInfo = _AdapterInfo +LPAdapterInfo = C.POINTER(_AdapterInfo) + +ADL_CONTEXT_HANDLE = C.c_void_p \ No newline at end of file diff --git a/modules/dml/optimizer/amd/memory.py b/modules/dml/optimizer/amd/memory.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/dml/optimizer/intel/__init__.py b/modules/dml/optimizer/intel/__init__.py new file mode 100644 index 000000000..d17cd59d2 --- /dev/null +++ b/modules/dml/optimizer/intel/__init__.py @@ -0,0 +1,7 @@ +from modules.dml.optimizer.optimizer import Optimizer + +class IntelOptimizer(Optimizer): + def memory_stats(): + raise NotImplementedError() + # DML TODO: Implement + return diff --git a/modules/dml/optimizer/nvidia/__init__.py b/modules/dml/optimizer/nvidia/__init__.py new file mode 100644 index 000000000..d467e8ae5 --- /dev/null +++ b/modules/dml/optimizer/nvidia/__init__.py @@ -0,0 +1,7 @@ +from modules.dml.optimizer.optimizer import Optimizer + +class nVidiaOptimizer(Optimizer): + def memory_stats(): + raise NotImplementedError() + # DML TODO: Implement + return diff --git a/modules/dml/optimizer/optimizer.py b/modules/dml/optimizer/optimizer.py new file mode 100644 index 000000000..2b2d9ed64 --- /dev/null +++ b/modules/dml/optimizer/optimizer.py @@ -0,0 +1,8 @@ +from abc import * +from typing import * + +class Optimizer(metaclass=ABCMeta): + driver: Any = None + @abstractmethod + def memory_stats(self, index: int) -> Tuple[int, int]: + pass diff --git a/modules/dml/optimizer/unknown/__init__.py b/modules/dml/optimizer/unknown/__init__.py new file mode 100644 index 000000000..f2fbc2ae2 --- /dev/null +++ b/modules/dml/optimizer/unknown/__init__.py @@ -0,0 +1,6 @@ +from modules.dml.optimizer.optimizer import Optimizer + +class UnknownOptimizer(Optimizer): + def memory_stats(): + # DML TODO: Implement + return (1073741824, 0) diff --git a/modules/dml/stablediffusion.py b/modules/dml/stablediffusion.py new file mode 100644 index 000000000..cbc4b85fb --- /dev/null +++ b/modules/dml/stablediffusion.py @@ -0,0 +1,80 @@ +import torch + +from ldm.models.diffusion.ddim import DDIMSampler +from ldm.modules.diffusionmodules.util import noise_like + +@torch.no_grad() +def p_sample_ddim(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False, + temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None, + unconditional_guidance_scale=1., unconditional_conditioning=None, + dynamic_threshold=None): + b, *_, device = *x.shape, x.device + + if unconditional_conditioning is None or unconditional_guidance_scale == 1.: + model_output = self.model.apply_model(x, t, c) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t] * 2) + if isinstance(c, dict): + assert isinstance(unconditional_conditioning, dict) + c_in = dict() + for k in c: + if isinstance(c[k], list): + c_in[k] = [torch.cat([ + unconditional_conditioning[k][i], + c[k][i]]) for i in range(len(c[k]))] + else: + c_in[k] = torch.cat([ + unconditional_conditioning[k], + c[k]]) + elif isinstance(c, list): + c_in = list() + assert isinstance(unconditional_conditioning, list) + for i in range(len(c)): + c_in.append(torch.cat([unconditional_conditioning[i], c[i]])) + else: + c_in = torch.cat([unconditional_conditioning, c]) + model_uncond, model_t = self.model.apply_model(x_in, t_in, c_in).chunk(2) + model_output = model_uncond + unconditional_guidance_scale * (model_t - model_uncond) + + if self.model.parameterization == "v": + e_t = self.model.predict_eps_from_z_and_v(x, t, model_output) + else: + e_t = model_output + + if score_corrector is not None: + assert self.model.parameterization == "eps", 'not implemented' + e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs) + + alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas + alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev + sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas + sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas + # select parameters corresponding to the currently considered timestep + print(alphas[index]) # DML Solution: DDIM Sampling does not work without this print. + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device) + + # current prediction for x_0 + if self.model.parameterization != "v": + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + else: + pred_x0 = self.model.predict_start_from_z_and_v(x, t, model_output) + + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + + if dynamic_threshold is not None: + raise NotImplementedError() + + # direction pointing to x_t + dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t + noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature + if noise_dropout > 0.: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0 + +DDIMSampler.p_sample_ddim = p_sample_ddim diff --git a/modules/dml/torch.py b/modules/dml/torch.py new file mode 100644 index 000000000..04e777258 --- /dev/null +++ b/modules/dml/torch.py @@ -0,0 +1,5 @@ +import torch + +from modules.sd_hijack_utils import CondFunc + +CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone') diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 7e1ba79b3..79b8e3b47 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -20,6 +20,9 @@ if shared.opts.cross_attention_optimization == "xFormers": except Exception: pass +if shared.device.type == 'privateuseone': + import dml + def get_available_vram(): if shared.device.type == 'cuda': @@ -31,8 +34,8 @@ def get_available_vram(): mem_free_total = mem_free_cuda + mem_free_torch return mem_free_total elif shared.device.type == 'privateuseone': - # DML ISSUE: There's no way to get any memory info. - return 1073741824 + mem_total, mem_active = torch.dml.memory_stats(shared.device) + return mem_total - mem_active * (1 << 20) else: return psutil.virtual_memory().available @@ -199,8 +202,9 @@ def einsum_op_cuda(q, k, v): return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) def einsum_op_dml(q, k, v): - # DML ISSUE: There's no way to get any memory info. - return einsum_op_tensor_mem(q, k, v, 1073741824) + mem_total, mem_active = devices.adl.memory_stats() + mem_reserved = mem_total / (1 << 20) * 0.7 + return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1) def einsum_op(q, k, v): if q.device.type == 'cuda': diff --git a/modules/shared.py b/modules/shared.py index 34748c0b9..08ff22cfa 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -64,7 +64,7 @@ clip_model = None if device.type == 'privateuseone': - import modules.dml_specific + import modules.dml is_device_dml = True @@ -429,7 +429,7 @@ options_templates.update(options_section(('ui', "Live previews"), { "live_previews_enable": OptionInfo(True, "Show live previews of the created image"), "show_progress_grid": OptionInfo(True, "Show previews of all images generated in a batch as a grid"), "show_progress_every_n_steps": OptionInfo(1, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), - "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML ISSUE: Approx NN does not work well on DirectML device. + "show_progress_type": OptionInfo("Approx cheap" if is_device_dml else "Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML Solution: Use Approx cheap instead of Approx NN as a default progress type. "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}), "live_preview_refresh_period": OptionInfo(250, "Progressbar/preview update period, in milliseconds") })) diff --git a/wiki b/wiki index 066ea609f..12603bcde 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 066ea609f6a2630bceb07c679142d83a41a6db0f +Subproject commit 12603bcdec55df780b18612d58b6d0dcd4c27f96 From 8b75033a111e41d185cbef782b60520559a20e20 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 26 Apr 2023 12:34:27 +0900 Subject: [PATCH 10/26] fix --- modules/sd_hijack_optimizations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 79b8e3b47..e9d065a32 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -202,7 +202,7 @@ def einsum_op_cuda(q, k, v): return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) def einsum_op_dml(q, k, v): - mem_total, mem_active = devices.adl.memory_stats() + mem_total, mem_active = torch.dml.memory_stats(q.device) mem_reserved = mem_total / (1 << 20) * 0.7 return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1) From df0e89be480f67e63a27061c174de835e98f0910 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 26 Apr 2023 12:45:44 +0900 Subject: [PATCH 11/26] fix. Unstable & need more test. --- modules/dml/__init__.py | 12 +++++------- modules/dml/hijack/__init__.py | 3 +++ modules/dml/{ => hijack}/kdiffusion.py | 0 modules/dml/{ => hijack}/stablediffusion.py | 0 modules/dml/{ => hijack}/torch.py | 0 modules/sd_hijack_optimizations.py | 3 --- 6 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 modules/dml/hijack/__init__.py rename modules/dml/{ => hijack}/kdiffusion.py (100%) rename modules/dml/{ => hijack}/stablediffusion.py (100%) rename modules/dml/{ => hijack}/torch.py (100%) diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py index fbda39384..90beb96d7 100644 --- a/modules/dml/__init__.py +++ b/modules/dml/__init__.py @@ -1,11 +1,9 @@ import torch import torch_directml -import modules.dml.kdiffusion -import modules.dml.stablediffusion -import modules.dml.torch +import modules.dml.hijack -from optimizer.unknown import UnknownOptimizer +from modules.dml.optimizer.unknown import UnknownOptimizer class DirectML(): def get_optimizer(self, device: torch.device): @@ -13,11 +11,11 @@ class DirectML(): try: device_name = torch_directml.device_name(device.index) if 'NVIDIA' in device_name or 'GeForce' in device_name: - from optimizer.nvidia import nVidiaOptimizer as optimizer + from modules.dml.optimizer.nvidia import nVidiaOptimizer as optimizer elif 'AMD' in device_name or 'Radeon' in device_name: - from optimizer.amd import AMDOptimizer as optimizer + from modules.dml.optimizer.amd import AMDOptimizer as optimizer elif 'Intel' in device_name: - from optimizer.intel import IntelOptimizer as optimizer + from modules.dml.optimizer.intel import IntelOptimizer as optimizer else: return UnknownOptimizer return optimizer diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py new file mode 100644 index 000000000..5cf59a704 --- /dev/null +++ b/modules/dml/hijack/__init__.py @@ -0,0 +1,3 @@ +import modules.dml.hijack.kdiffusion +import modules.dml.hijack.stablediffusion +import modules.dml.hijack.torch diff --git a/modules/dml/kdiffusion.py b/modules/dml/hijack/kdiffusion.py similarity index 100% rename from modules/dml/kdiffusion.py rename to modules/dml/hijack/kdiffusion.py diff --git a/modules/dml/stablediffusion.py b/modules/dml/hijack/stablediffusion.py similarity index 100% rename from modules/dml/stablediffusion.py rename to modules/dml/hijack/stablediffusion.py diff --git a/modules/dml/torch.py b/modules/dml/hijack/torch.py similarity index 100% rename from modules/dml/torch.py rename to modules/dml/hijack/torch.py diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index e9d065a32..12ee9f956 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -20,9 +20,6 @@ if shared.opts.cross_attention_optimization == "xFormers": except Exception: pass -if shared.device.type == 'privateuseone': - import dml - def get_available_vram(): if shared.device.type == 'cuda': From d2d5011bd3cf7c982b92ef9019c5bb30d89f2bf2 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 26 Apr 2023 17:44:32 +0900 Subject: [PATCH 12/26] Implement memory estimation for AMDGPUs. Stable. --- modules/devices.py | 2 +- modules/dml/__init__.py | 14 ++-- modules/dml/optimizer/amd/__init__.py | 4 +- modules/dml/optimizer/amd/driver/atiadlxx.py | 5 +- .../dml/optimizer/amd/driver/atiadlxx_apis.py | 67 ++++++++----------- modules/dml/optimizer/amd/memory.py | 0 modules/dml/optimizer/intel/__init__.py | 2 +- modules/dml/optimizer/nvidia/__init__.py | 2 +- modules/dml/optimizer/optimizer.py | 2 +- modules/dml/optimizer/unknown/__init__.py | 2 +- 10 files changed, 47 insertions(+), 53 deletions(-) delete mode 100644 modules/dml/optimizer/amd/memory.py diff --git a/modules/devices.py b/modules/devices.py index 5f5e18768..e317d91f4 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -31,7 +31,7 @@ def get_dml_device_string(): from modules import shared if shared.cmd_opts.device_id is not None: return f"privateuseone:{shared.cmd_opts.device_id}" - return "privateuseone" + return "privateuseone:0" def get_optimal_device_name(): diff --git a/modules/dml/__init__.py b/modules/dml/__init__.py index 90beb96d7..b77db18d0 100644 --- a/modules/dml/__init__.py +++ b/modules/dml/__init__.py @@ -3,27 +3,27 @@ import torch_directml import modules.dml.hijack -from modules.dml.optimizer.unknown import UnknownOptimizer +from .optimizer.unknown import UnknownOptimizer class DirectML(): - def get_optimizer(self, device: torch.device): + def get_optimizer(device: torch.device): assert(device.type == 'privateuseone') try: device_name = torch_directml.device_name(device.index) if 'NVIDIA' in device_name or 'GeForce' in device_name: - from modules.dml.optimizer.nvidia import nVidiaOptimizer as optimizer + from .optimizer.nvidia import nVidiaOptimizer as optimizer elif 'AMD' in device_name or 'Radeon' in device_name: - from modules.dml.optimizer.amd import AMDOptimizer as optimizer + from .optimizer.amd import AMDOptimizer as optimizer elif 'Intel' in device_name: - from modules.dml.optimizer.intel import IntelOptimizer as optimizer + from .optimizer.intel import IntelOptimizer as optimizer else: return UnknownOptimizer return optimizer except: return UnknownOptimizer - def memory_stats(self, device: torch.device): - optimizer = self.get_optimizer(device) + def memory_stats(device: torch.device): + optimizer = DirectML.get_optimizer(device) return optimizer.memory_stats(device.index) # Alternative of torch.cuda for DirectML. diff --git a/modules/dml/optimizer/amd/__init__.py b/modules/dml/optimizer/amd/__init__.py index 37ea3ea91..23c6e57e3 100644 --- a/modules/dml/optimizer/amd/__init__.py +++ b/modules/dml/optimizer/amd/__init__.py @@ -1,7 +1,7 @@ from modules.dml.optimizer.optimizer import Optimizer -from driver.atiadlxx import ATIADLxx +from .driver.atiadlxx import ATIADLxx class AMDOptimizer(Optimizer): driver: ATIADLxx = ATIADLxx() - def memory_stats(self, index): + def memory_stats(index): return (AMDOptimizer.driver.iHyperMemorySize, AMDOptimizer.driver.get_dedicated_vram_usage(index)) diff --git a/modules/dml/optimizer/amd/driver/atiadlxx.py b/modules/dml/optimizer/amd/driver/atiadlxx.py index abb5eca96..81a578a5c 100644 --- a/modules/dml/optimizer/amd/driver/atiadlxx.py +++ b/modules/dml/optimizer/amd/driver/atiadlxx.py @@ -14,8 +14,11 @@ class ATIADLxx(object): AdapterInfoArray = (AdapterInfo * num_adapters.value)() ADL2_Adapter_AdapterInfo_Get(self.context, C.cast(AdapterInfoArray, LPAdapterInfo), C.sizeof(AdapterInfoArray)) self.devices = [] + busNumbers = [] for adapter in AdapterInfoArray: - self.devices.append(adapter) + if adapter.iBusNumber not in busNumbers: # filter duplicate device + self.devices.append(adapter) + busNumbers.append(adapter.iBusNumber) self.iHyperMemorySize = self.get_memory_info2(0).iHyperMemorySize def get_memory_info2(self, adapterIndex: int) -> ADLMemoryInfo2: diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py index 23b7da3b1..fef70b543 100644 --- a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py +++ b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py @@ -1,50 +1,41 @@ import ctypes as C -import platform from .atiadlxx_structures import * -_platform = platform.system() +atiadlxx = C.WinDLL("atiadlxx.dll") -try: - if _platform == "Windows": - atiadlxx = C.WinDLL("atiadlxx.dll") +ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int) +ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p)) - ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int) - ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p)) +@ADL_MAIN_MALLOC_CALLBACK +def ADL_Main_Memory_Alloc(iSize): + return C._malloc(iSize) - @ADL_MAIN_MALLOC_CALLBACK - def ADL_Main_Memory_Alloc(iSize): - return C._malloc(iSize) +@ADL_MAIN_FREE_CALLBACK +def ADL_Main_Memory_Free(lpBuffer): + if lpBuffer[0] is not None: + C._free(lpBuffer[0]) + lpBuffer[0] = None - @ADL_MAIN_FREE_CALLBACK - def ADL_Main_Memory_Free(lpBuffer): - if lpBuffer[0] is not None: - C._free(lpBuffer[0]) - lpBuffer[0] = None +ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create +ADL2_Main_Control_Create.restype = C.c_int +ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE] - ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create - ADL2_Main_Control_Create.restype = C.c_int - ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE] +ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get +ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int +ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)] - ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get - ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int - ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)] +ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get +ADL2_Adapter_AdapterInfo_Get.restype = C.c_int +ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int] - ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get - ADL2_Adapter_AdapterInfo_Get.restype = C.c_int - ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int] +ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get +ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int +ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)] - ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get - ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int - ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)] +ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get +ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int +ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)] - ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get - ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int - ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)] - - ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get - ADL2_Adapter_VRAMUsage_Get.restype = C.c_int - ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)] - else: - print("Warning: experimental graphic memory optimization for AMDGPU is disabled. Because this is not Windows platform.") -except FileNotFoundError: - print("Warning: memory optimization for AMDGPU is disabled. Because couldn't find 'atiadlxx.dll'. Please install GPU driver downloaded from AMD.com.") \ No newline at end of file +ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get +ADL2_Adapter_VRAMUsage_Get.restype = C.c_int +ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)] diff --git a/modules/dml/optimizer/amd/memory.py b/modules/dml/optimizer/amd/memory.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/modules/dml/optimizer/intel/__init__.py b/modules/dml/optimizer/intel/__init__.py index d17cd59d2..bffda69f1 100644 --- a/modules/dml/optimizer/intel/__init__.py +++ b/modules/dml/optimizer/intel/__init__.py @@ -1,7 +1,7 @@ from modules.dml.optimizer.optimizer import Optimizer class IntelOptimizer(Optimizer): - def memory_stats(): + def memory_stats(index): raise NotImplementedError() # DML TODO: Implement return diff --git a/modules/dml/optimizer/nvidia/__init__.py b/modules/dml/optimizer/nvidia/__init__.py index d467e8ae5..e5fda97b8 100644 --- a/modules/dml/optimizer/nvidia/__init__.py +++ b/modules/dml/optimizer/nvidia/__init__.py @@ -1,7 +1,7 @@ from modules.dml.optimizer.optimizer import Optimizer class nVidiaOptimizer(Optimizer): - def memory_stats(): + def memory_stats(index): raise NotImplementedError() # DML TODO: Implement return diff --git a/modules/dml/optimizer/optimizer.py b/modules/dml/optimizer/optimizer.py index 2b2d9ed64..db88c6579 100644 --- a/modules/dml/optimizer/optimizer.py +++ b/modules/dml/optimizer/optimizer.py @@ -4,5 +4,5 @@ from typing import * class Optimizer(metaclass=ABCMeta): driver: Any = None @abstractmethod - def memory_stats(self, index: int) -> Tuple[int, int]: + def memory_stats(index: int) -> Tuple[int, int]: pass diff --git a/modules/dml/optimizer/unknown/__init__.py b/modules/dml/optimizer/unknown/__init__.py index f2fbc2ae2..40ff476cd 100644 --- a/modules/dml/optimizer/unknown/__init__.py +++ b/modules/dml/optimizer/unknown/__init__.py @@ -1,6 +1,6 @@ from modules.dml.optimizer.optimizer import Optimizer class UnknownOptimizer(Optimizer): - def memory_stats(): + def memory_stats(index): # DML TODO: Implement return (1073741824, 0) From c9dde03fc54d0882679b90c6e717cf11b5969433 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 26 Apr 2023 18:54:24 +0900 Subject: [PATCH 13/26] Move realesrgan fallback to dml/hijack. --- TODO_DML.md | 33 ------------ modules/dml/hijack/__init__.py | 1 + modules/dml/hijack/realesrgan_model.py | 69 ++++++++++++++++++++++++++ modules/realesrgan_model.py | 68 ------------------------- setup.py | 6 +-- 5 files changed, 73 insertions(+), 104 deletions(-) delete mode 100644 TODO_DML.md create mode 100644 modules/dml/hijack/realesrgan_model.py diff --git a/TODO_DML.md b/TODO_DML.md deleted file mode 100644 index f359ba1b3..000000000 --- a/TODO_DML.md +++ /dev/null @@ -1,33 +0,0 @@ -# TODO - -## Issues - -Stuff to be fixed... - -- `mat1 and mat2 must have the same dtype` error (half mode) -- Some samplers won't work (test later) - -## Something needs discussion - -- About memory optimization. - -Basically, we cannot get detailed vram information from `torch-directml`. - -It has `gpu_memory` method which returns an array contains used memory size, but it is almostly useless without any other information. - -What should we do? - -1. Use any fixed value as the available memory capacity. -2. Use `atiadlxx`(AMD/ATI GPU driver library) to infer vram information as similar as possible to the actual value. (works for AMDGPUs) -3. or another better way. - -- Half mode does not work well. - -In half precision, we get an error like `mat1 and mat2 must have the same dtype`. - -I solved this problem by overriding forward of GroupNorm / LayerNorm / Conv2d / Linear to convert input tensor to full precision (and convert to float16 before return). - -What should we do? - -1. Override forwards (same solution) -2. When using DirectML, set the default to full precision and notify the user. diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py index 5cf59a704..dd71784c0 100644 --- a/modules/dml/hijack/__init__.py +++ b/modules/dml/hijack/__init__.py @@ -1,3 +1,4 @@ import modules.dml.hijack.kdiffusion import modules.dml.hijack.stablediffusion import modules.dml.hijack.torch +import modules.dml.hijack.realesrgan_model diff --git a/modules/dml/hijack/realesrgan_model.py b/modules/dml/hijack/realesrgan_model.py new file mode 100644 index 000000000..c82429fda --- /dev/null +++ b/modules/dml/hijack/realesrgan_model.py @@ -0,0 +1,69 @@ +import math +import torch + +from realesrgan import RealESRGANer + + +# DML Solution: Some tensors turn 0 after Extended Slices. Move output to cpu and get it back. +def tile_process(self): + batch, channel, height, width = self.img.shape + output_height = height * self.scale + output_width = width * self.scale + output_shape = (batch, channel, output_height, output_width) + + # start with black image + self.output = self.img.new_zeros(output_shape, device='cpu') + tiles_x = math.ceil(width / self.tile_size) + tiles_y = math.ceil(height / self.tile_size) + + # loop over all tiles + for y in range(tiles_y): + for x in range(tiles_x): + # extract tile from input image + ofs_x = x * self.tile_size + ofs_y = y * self.tile_size + # input tile area on total image + input_start_x = ofs_x + input_end_x = min(ofs_x + self.tile_size, width) + input_start_y = ofs_y + input_end_y = min(ofs_y + self.tile_size, height) + + # input tile area on total image with padding + input_start_x_pad = max(input_start_x - self.tile_pad, 0) + input_end_x_pad = min(input_end_x + self.tile_pad, width) + input_start_y_pad = max(input_start_y - self.tile_pad, 0) + input_end_y_pad = min(input_end_y + self.tile_pad, height) + + # input tile dimensions + input_tile_width = input_end_x - input_start_x + input_tile_height = input_end_y - input_start_y + tile_idx = y * tiles_x + x + 1 + input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad] + + # upscale tile + try: + with torch.no_grad(): + output_tile = self.model(input_tile) + output_tile = output_tile.cpu() + except RuntimeError as error: + print('Error', error) + print(f'\tTile {tile_idx}/{tiles_x * tiles_y}') + + # output tile area on total image + output_start_x = input_start_x * self.scale + output_end_x = input_end_x * self.scale + output_start_y = input_start_y * self.scale + output_end_y = input_end_y * self.scale + + # output tile area without padding + output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale + output_end_x_tile = output_start_x_tile + input_tile_width * self.scale + output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale + output_end_y_tile = output_start_y_tile + input_tile_height * self.scale + + # put tile into output image + self.output[:, :, output_start_y:output_end_y, + output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile, + output_start_x_tile:output_end_x_tile] + self.output = self.output.to(self.device) +RealESRGANer.tile_process = tile_process diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py index 1d48ac42c..398f2526b 100644 --- a/modules/realesrgan_model.py +++ b/modules/realesrgan_model.py @@ -10,72 +10,6 @@ from modules.shared import cmd_opts, opts, device import modules.errors as errors -# DML Solution: Some tensors turn 0 after Extended Slices. Move output to cpu and get it back. -def realesrgan_tile_process_dml_fix(self): - import math - import torch - batch, channel, height, width = self.img.shape - output_height = height * self.scale - output_width = width * self.scale - output_shape = (batch, channel, output_height, output_width) - - # start with black image - self.output = self.img.new_zeros(output_shape, device='cpu' if self.device.type == 'privateuseone' else self.device) - tiles_x = math.ceil(width / self.tile_size) - tiles_y = math.ceil(height / self.tile_size) - - # loop over all tiles - for y in range(tiles_y): - for x in range(tiles_x): - # extract tile from input image - ofs_x = x * self.tile_size - ofs_y = y * self.tile_size - # input tile area on total image - input_start_x = ofs_x - input_end_x = min(ofs_x + self.tile_size, width) - input_start_y = ofs_y - input_end_y = min(ofs_y + self.tile_size, height) - - # input tile area on total image with padding - input_start_x_pad = max(input_start_x - self.tile_pad, 0) - input_end_x_pad = min(input_end_x + self.tile_pad, width) - input_start_y_pad = max(input_start_y - self.tile_pad, 0) - input_end_y_pad = min(input_end_y + self.tile_pad, height) - - # input tile dimensions - input_tile_width = input_end_x - input_start_x - input_tile_height = input_end_y - input_start_y - tile_idx = y * tiles_x + x + 1 - input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad] - - # upscale tile - try: - with torch.no_grad(): - output_tile = self.model(input_tile) - output_tile = output_tile.cpu() - except RuntimeError as error: - print('Error', error) - print(f'\tTile {tile_idx}/{tiles_x * tiles_y}') - - # output tile area on total image - output_start_x = input_start_x * self.scale - output_end_x = input_end_x * self.scale - output_start_y = input_start_y * self.scale - output_end_y = input_end_y * self.scale - - # output tile area without padding - output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale - output_end_x_tile = output_start_x_tile + input_tile_width * self.scale - output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale - output_end_y_tile = output_start_y_tile + input_tile_height * self.scale - - # put tile into output image - self.output[:, :, output_start_y:output_end_y, - output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile, - output_start_x_tile:output_end_x_tile] - self.output = self.output.to(self.device) - - class UpscalerRealESRGAN(Upscaler): def __init__(self, path): self.name = "RealESRGAN" @@ -103,8 +37,6 @@ class UpscalerRealESRGAN(Upscaler): try: from realesrgan import RealESRGANer - if device.type == 'privateuseone': - RealESRGANer.tile_process = realesrgan_tile_process_dml_fix except: print("Error importing Real-ESRGAN:", file=sys.stderr) return img diff --git a/setup.py b/setup.py index 69b86012d..52da5d8e3 100644 --- a/setup.py +++ b/setup.py @@ -266,12 +266,12 @@ def install_repositories(): stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) - k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') - k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919") - clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git") taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318") clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit) + k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') + k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919") + clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git') codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af") clone(codeformer_repo, d('CodeFormer'), codeformer_commit) From 576ea88618d4f752e83483b2c30f541c6ba8747d Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Fri, 28 Apr 2023 10:56:00 +0900 Subject: [PATCH 14/26] torch 2.0.0 support. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 99d079463..dda7a4423 100644 --- a/setup.py +++ b/setup.py @@ -200,7 +200,7 @@ def check_torch(): machine = platform.machine() if 'arm' not in machine and 'aarch' not in machine and not args.nodirectml: # torch-directml is available on AMD64 log.info('Using DirectML Backend') - torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.1 torchvision==0.14.1 torch-directml') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0 torchvision torch-directml') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: log.info('Using CPU-only Torch') From c128c0770b31ccb0cf8bc788793f0a4b4f1f6244 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sun, 30 Apr 2023 01:15:47 +0900 Subject: [PATCH 15/26] Load libatiadlxx.so on Linux systems. --- modules/dml/optimizer/amd/driver/atiadlxx_apis.py | 6 +++++- setup.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py index fef70b543..25e6390ef 100644 --- a/modules/dml/optimizer/amd/driver/atiadlxx_apis.py +++ b/modules/dml/optimizer/amd/driver/atiadlxx_apis.py @@ -1,7 +1,11 @@ import ctypes as C +from platform import platform from .atiadlxx_structures import * -atiadlxx = C.WinDLL("atiadlxx.dll") +if 'Windows' in platform(): + atiadlxx = C.WinDLL("atiadlxx.dll") +else: + atiadlxx = C.CDLL("libatiadlxx.so") # Not tested on Linux system. But will be supported. ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int) ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p)) diff --git a/setup.py b/setup.py index 615ac49d4..c143924a7 100644 --- a/setup.py +++ b/setup.py @@ -229,6 +229,7 @@ def check_torch(): log.info(f'Torch backend: DirectML ({version})') for i in range(0, torch_directml.device_count()): log.info(f'Torch detected GPU: {torch_directml.device_name(i)}') + log.info(f'DirectML default device: {torch_directml.device_name(torch_directml.default_device())}') except: log.warning("Torch repoorts CUDA not available") except Exception as e: From 9d8d57a51a415aecacd4bff6f2d8ade3d1ecec01 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sun, 30 Apr 2023 01:34:40 +0900 Subject: [PATCH 16/26] Sync submodules. --- extensions-builtin/multidiffusion-upscaler-for-automatic1111 | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- extensions-builtin/seed_travel | 2 +- extensions-builtin/stable-diffusion-webui-images-browser | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 b/extensions-builtin/multidiffusion-upscaler-for-automatic1111 index f3d79a474..6931b89cb 160000 --- a/extensions-builtin/multidiffusion-upscaler-for-automatic1111 +++ b/extensions-builtin/multidiffusion-upscaler-for-automatic1111 @@ -1 +1 @@ -Subproject commit f3d79a474b9795f07143eaf8104737a403b5fb52 +Subproject commit 6931b89cb4507c7dc8fa81ac36c2c19d0691c44e diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index c5fbfc31d..940d4edfb 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit c5fbfc31d002ff83bc692e1f24b7b1c9183dbe72 +Subproject commit 940d4edfbab1525615b1827a9cb7b7ea21af8a6c diff --git a/extensions-builtin/seed_travel b/extensions-builtin/seed_travel index ffe0553c5..4bc8b2f10 160000 --- a/extensions-builtin/seed_travel +++ b/extensions-builtin/seed_travel @@ -1 +1 @@ -Subproject commit ffe0553c59e91067ebf1e4fc7ad85ca9c870bf57 +Subproject commit 4bc8b2f10d5c12958f48b67ad23fb445aff074df diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser index 704e42c10..2c988c08c 160000 --- a/extensions-builtin/stable-diffusion-webui-images-browser +++ b/extensions-builtin/stable-diffusion-webui-images-browser @@ -1 +1 @@ -Subproject commit 704e42c10d01e6c6965493ec956a82bb8fc2da51 +Subproject commit 2c988c08c7fc2f1c0f572bc4209f0baa1fac4fee From a41cf2bd84c4de166e0f6ea4871f72e78a1cee58 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sun, 30 Apr 2023 01:36:55 +0900 Subject: [PATCH 17/26] Sync submodules. --- modules/lora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lora b/modules/lora index ac4935bf7..d52c524fc 160000 --- a/modules/lora +++ b/modules/lora @@ -1 +1 @@ -Subproject commit ac4935bf79942f966d7b6578e8fbb9ee5f12d4ad +Subproject commit d52c524fc2942c053cf37c648188502a3a26df1b From 1469f8c69c8964be07fec0c67b554d9d22a11371 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 18:18:08 -0400 Subject: [PATCH 18/26] force lightbox --- javascript/imageviewer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javascript/imageviewer.js b/javascript/imageviewer.js index c928d9c5f..149177430 100644 --- a/javascript/imageviewer.js +++ b/javascript/imageviewer.js @@ -105,8 +105,8 @@ function setupImageForLightbox(e) { var isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1 var event = isFirefox ? 'mousedown' : 'click' e.addEventListener(event, function (evt) { - if(!opts.js_modal_lightbox || evt.button != 0) return; - modalZoomSet(gradioApp().getElementById('modalImage'), opts.js_modal_lightbox_initially_zoomed) + if (evt.button != 0) return; + modalZoomSet(gradioApp().getElementById('modalImage'), true) evt.preventDefault() showModal(evt) }, true); From 1f5d5a17e39a901c1f81a814026e17100dcf0d0c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 18:27:21 -0400 Subject: [PATCH 19/26] update lint rules --- .github/workflows/on_pull_request.yaml | 4 ---- .pylintrc | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 011fe9b2d..8693fd003 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -28,11 +28,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint - # This lets PyLint check to see if it can resolve imports - name: Install dependencies - run: | - export COMMANDLINE_ARGS="--skip-torch-cuda-test --exit" - python launch.py - name: Analysing the code with pylint run: | pylint $(git ls-files '*.py') diff --git a/.pylintrc b/.pylintrc index 72e1224e8..515dce01d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -11,6 +11,7 @@ fail-under=10 ignore=CVS ignore-paths=^repositories/.*$, ^extensions/.*$, + ^extensions-builtin/.*$, /usr/lib/.*$, ignore-patterns= ignored-modules= @@ -141,6 +142,8 @@ disable=raw-checker-failed, consider-using-dict-items, dangerous-default-value, unnecessary-dunder-call, + invalid-name, + R0801, enable=c-extension-no-member [METHOD_ARGS] From f813e6022b1e8471044fc38edbf5602a5635520d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 18:38:13 -0400 Subject: [PATCH 20/26] add test mode --- .github/workflows/on_pull_request.yaml | 10 +++++----- launch.py | 8 ++++++-- setup.py | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 8693fd003..57f46ab6e 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -24,11 +24,11 @@ jobs: python-version: 3.10.6 cache: pip cache-dependency-path: requirements.txt - - name: Install PyLint - run: | - python -m pip install --upgrade pip - pip install pylint - name: Install dependencies + run: | + python launch.py --test - name: Analysing the code with pylint run: | - pylint $(git ls-files '*.py') + python -m pip install --upgrade pip + pip install pylint + pylint $(git ls-files '*.py') diff --git a/launch.py b/launch.py index bf34ee507..ad55ca2e3 100644 --- a/launch.py +++ b/launch.py @@ -97,5 +97,9 @@ if __name__ == "__main__": setup.log.info(f"Server arguments: {sys.argv[1:]}") setup.log.debug('Starting WebUI') logging.disable(logging.INFO) - import webui - webui.webui() + if args.test: + setup.log.info(f"Test only") + import webui + exit(0) + else: + webui.webui() diff --git a/setup.py b/setup.py index 804ee35b0..649718bd0 100644 --- a/setup.py +++ b/setup.py @@ -496,6 +496,7 @@ def add_args(): parser.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s") parser.add_argument('--skip-git', default = False, action='store_true', help = "Skips running all GIT operations, default: %(default)s") parser.add_argument('--experimental', default = False, action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s") + parser.add_argument('--test', default = False, action='store_true', help = "Run test only, default: %(default)s") def parse_args(): From ddd6125e56940eb4b2d4580dce65ec423d8c7f77 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 18:44:03 -0400 Subject: [PATCH 21/26] set git triggers --- .github/workflows/on_pull_request.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 57f46ab6e..b9e0eb7db 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -29,6 +29,6 @@ jobs: python launch.py --test - name: Analysing the code with pylint run: | - python -m pip install --upgrade pip - pip install pylint - pylint $(git ls-files '*.py') + python -m pip install --upgrade pip + pip install pylint + pylint $(git ls-files '*.py') From 21b6e11b01433a4cc86fbe008d8dc06892b9d169 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 18:53:00 -0400 Subject: [PATCH 22/26] critical --- launch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/launch.py b/launch.py index ad55ca2e3..f80e9ae72 100644 --- a/launch.py +++ b/launch.py @@ -102,4 +102,5 @@ if __name__ == "__main__": import webui exit(0) else: + import webui webui.webui() From d4fd25a7fdc350e7edde601443279786396c1abf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 18:58:10 -0400 Subject: [PATCH 23/26] add debug mode --- launch.py | 2 +- modules/img2img.py | 4 ++-- modules/txt2img.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/launch.py b/launch.py index f80e9ae72..995f61894 100644 --- a/launch.py +++ b/launch.py @@ -98,7 +98,7 @@ if __name__ == "__main__": setup.log.debug('Starting WebUI') logging.disable(logging.INFO) if args.test: - setup.log.info(f"Test only") + setup.log.info("Test only") import webui exit(0) else: diff --git a/modules/img2img.py b/modules/img2img.py index de889d4b7..fc7165751 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -60,7 +60,7 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args): if processed_image.mode == 'RGBA': processed_image = processed_image.convert("RGB") processed_image.save(os.path.join(output_dir, filename)) - if cmd_opts.debug: + if cmd_opts.get('debug', False): log.info(f'Processed: {len(images)} Memory: {memory_stats()} batch') @@ -146,6 +146,6 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s processed = process_images(p) p.close() generation_info_js = processed.js() - if cmd_opts.debug: + if cmd_opts.get('debug', False): log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} img') return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments) diff --git a/modules/txt2img.py b/modules/txt2img.py index 17e5ce909..45541749b 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -46,6 +46,6 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step processed = process_images(p) p.close() generation_info_js = processed.js() - if cmd_opts.debug: + if cmd_opts.get('debug', False): log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} txt') return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments) From 9ac7f3771a2b8b488609709ffbf559d8c88ffd01 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 29 Apr 2023 19:30:26 -0400 Subject: [PATCH 24/26] update cmdflags --- .github/workflows/on_pull_request.yaml | 7 ++++--- TODO.md | 5 ++--- launch.py | 5 ++--- modules/cmd_args.py | 1 + modules/img2img.py | 4 ++-- modules/txt2img.py | 2 +- setup.py | 3 +-- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index b9e0eb7db..6cb1005b0 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -24,10 +24,11 @@ jobs: python-version: 3.10.6 cache: pip cache-dependency-path: requirements.txt - - name: Install dependencies + - name: Test Startup run: | - python launch.py --test - - name: Analysing the code with pylint + export COMMANDLINE_ARGS="--debug --test" + python launch.py + - name: Linting run: | python -m pip install --upgrade pip pip install pylint diff --git a/TODO.md b/TODO.md index 167a823ec..0a5b0e99d 100644 --- a/TODO.md +++ b/TODO.md @@ -4,10 +4,7 @@ Stuff to be fixed... -- Run VAE with hires at 1280 -- Transformers version - Move Restart Server from WebUI to Launch and reload modules -- Follow-up on `p.script_args` - Mdularize `cli` scripts ## Features @@ -16,9 +13,11 @@ Stuff to be added... - Update README - Add Gradio theme maker +- Transformers version - Create new GitHub hooks/actions for CI/CD - Redo Extensions tab: see - Stream-load models as option for slow storage +- Auto-test `torch.layer_norm` for FP16 ## Investigate diff --git a/launch.py b/launch.py index 995f61894..8df3f787c 100644 --- a/launch.py +++ b/launch.py @@ -101,6 +101,5 @@ if __name__ == "__main__": setup.log.info("Test only") import webui exit(0) - else: - import webui - webui.webui() + import webui + webui.webui() diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 3bca9b357..9b8438800 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -74,6 +74,7 @@ def compatibility_args(opts, args): parser.add_argument("--sub-quad-q-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_q_chunk_size) parser.add_argument("--sub-quad-kv-chunk-size", help=argparse.SUPPRESS, default=opts.sub_quad_kv_chunk_size) parser.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold) + parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s") opts.use_old_emphasis_implementation = False opts.use_old_karras_scheduler_sigmas = False diff --git a/modules/img2img.py b/modules/img2img.py index fc7165751..de889d4b7 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -60,7 +60,7 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args): if processed_image.mode == 'RGBA': processed_image = processed_image.convert("RGB") processed_image.save(os.path.join(output_dir, filename)) - if cmd_opts.get('debug', False): + if cmd_opts.debug: log.info(f'Processed: {len(images)} Memory: {memory_stats()} batch') @@ -146,6 +146,6 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s processed = process_images(p) p.close() generation_info_js = processed.js() - if cmd_opts.get('debug', False): + if cmd_opts.debug: log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} img') return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments) diff --git a/modules/txt2img.py b/modules/txt2img.py index 45541749b..17e5ce909 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -46,6 +46,6 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step processed = process_images(p) p.close() generation_info_js = processed.js() - if cmd_opts.get('debug', False): + if cmd_opts.debug: log.info(f'Processed: {len(processed.images)} Memory: {memory_stats()} txt') return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments) diff --git a/setup.py b/setup.py index 649718bd0..906eaf136 100644 --- a/setup.py +++ b/setup.py @@ -487,8 +487,7 @@ def check_timestamp(): def add_args(): - if vars(parser)['_option_string_actions'].get('--debug', None) is None: - parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s") + parser.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s") parser.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s") parser.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s") parser.add_argument('--noupdate', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s") From e65d74100fe1990affd1048dc26f5eed92729a0e Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sun, 30 Apr 2023 11:12:47 +0900 Subject: [PATCH 25/26] Move/Remove hijacks. Add warning. --- modules/dml/hijack/__init__.py | 1 + modules/sd_hijack.py | 3 --- modules/sd_hijack_inpainting.py | 1 - modules/shared.py | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/dml/hijack/__init__.py b/modules/dml/hijack/__init__.py index dd71784c0..d8cc0913a 100644 --- a/modules/dml/hijack/__init__.py +++ b/modules/dml/hijack/__init__.py @@ -2,3 +2,4 @@ import modules.dml.hijack.kdiffusion import modules.dml.hijack.stablediffusion import modules.dml.hijack.torch import modules.dml.hijack.realesrgan_model +import modules.dml.hijack.plms diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 4a68216df..f817b7afd 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -257,9 +257,6 @@ class EmbeddingsWithFixes(torch.nn.Module): for offset, embedding in fixes: emb = devices.cond_cast_unet(embedding.vec) emb_len = min(tensor.shape[0] - offset - 1, emb.shape[0]) - # DML Solution: type mismatch on half mode - if tensor.dtype == torch.float16 and emb.dtype == torch.float32 and not shared.cmd_opts.no_half: - emb = emb.half() tensor = torch.cat([tensor[0:offset + 1], emb[0:emb_len], tensor[offset + 1 + emb_len:]]) vecs.append(tensor) diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py index 3405e8a40..4b23c132d 100644 --- a/modules/sd_hijack_inpainting.py +++ b/modules/sd_hijack_inpainting.py @@ -53,7 +53,6 @@ def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=F def get_x_prev_and_pred_x0(e_t, index): # select parameters corresponding to the currently considered timestep - print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print. a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) diff --git a/modules/shared.py b/modules/shared.py index d8a3f9d64..7c5916cfd 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -322,7 +322,7 @@ options_templates.update(options_section(('cuda', "CUDA Settings"), { "memmon_poll_rate": OptionInfo(2, "VRAM usage polls per second during generation. Set to 0 to disable.", gr.Slider, {"minimum": 0, "maximum": 40, "step": 1}), "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}), "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}), - "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)"), + "no_half": OptionInfo(True if is_device_dml else False, "Use full precision for model (--no-half)", None, None, lambda: print("Warning: Most of DirectML devices do not fully support half mode. Recommend to use full precision to model.") if is_device_dml else None), "no_half_vae": OptionInfo(True if is_device_dml else False, "Use full precision for VAE (--no-half-vae)"), "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling. Usually produces similar results to --no-half with better performance while using less memory"), "disable_nan_check": OptionInfo(True, "Do not check if produced images/latent spaces have NaN values"), From 93d638d8b8b7d5226a8fa56e6d9b96c7710da994 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sun, 30 Apr 2023 11:15:09 +0900 Subject: [PATCH 26/26] Approx NN works fine. --- modules/dml/hijack/plms.py | 91 ++++++++++++++++++++++++++++++++++++++ modules/shared.py | 2 +- 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 modules/dml/hijack/plms.py diff --git a/modules/dml/hijack/plms.py b/modules/dml/hijack/plms.py new file mode 100644 index 000000000..49979db28 --- /dev/null +++ b/modules/dml/hijack/plms.py @@ -0,0 +1,91 @@ +import torch + +from ldm.models.diffusion.ddim import noise_like + +import modules.sd_hijack_inpainting as plms_hijack + + +@torch.no_grad() +def p_sample_plms(self, x, c, t, index, repeat_noise=False, use_original_steps=False, quantize_denoised=False, + temperature=1., noise_dropout=0., score_corrector=None, corrector_kwargs=None, + unconditional_guidance_scale=1., unconditional_conditioning=None, old_eps=None, t_next=None, dynamic_threshold=None): + b, *_, device = *x.shape, x.device + + def get_model_output(x, t): + if unconditional_conditioning is None or unconditional_guidance_scale == 1.: + e_t = self.model.apply_model(x, t, c) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t] * 2) + + if isinstance(c, dict): + assert isinstance(unconditional_conditioning, dict) + c_in = dict() + for k in c: + if isinstance(c[k], list): + c_in[k] = [ + torch.cat([unconditional_conditioning[k][i], c[k][i]]) + for i in range(len(c[k])) + ] + else: + c_in[k] = torch.cat([unconditional_conditioning[k], c[k]]) + else: + c_in = torch.cat([unconditional_conditioning, c]) + + e_t_uncond, e_t = self.model.apply_model(x_in, t_in, c_in).chunk(2) + e_t = e_t_uncond + unconditional_guidance_scale * (e_t - e_t_uncond) + + if score_corrector is not None: + assert self.model.parameterization == "eps" + e_t = score_corrector.modify_score(self.model, e_t, x, t, c, **corrector_kwargs) + + return e_t + + alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas + alphas_prev = self.model.alphas_cumprod_prev if use_original_steps else self.ddim_alphas_prev + sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod if use_original_steps else self.ddim_sqrt_one_minus_alphas + sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas + + def get_x_prev_and_pred_x0(e_t, index): + # select parameters corresponding to the currently considered timestep + print(alphas[index]) # DML Solution: PLMS Sampling does not work without this print. + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full((b, 1, 1, 1), sqrt_one_minus_alphas[index],device=device) + + # current prediction for x_0 + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + if dynamic_threshold is not None: + from ldm.models.diffusion.sampling_util import norm_thresholding + pred_x0 = norm_thresholding(pred_x0, dynamic_threshold) + # direction pointing to x_t + dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t + noise = sigma_t * noise_like(x.shape, device, repeat_noise) * temperature + if noise_dropout > 0.: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0 + + e_t = get_model_output(x, t) + if len(old_eps) == 0: + # Pseudo Improved Euler (2nd order) + x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t, index) + e_t_next = get_model_output(x_prev, t_next) + e_t_prime = (e_t + e_t_next) / 2 + elif len(old_eps) == 1: + # 2nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (3 * e_t - old_eps[-1]) / 2 + elif len(old_eps) == 2: + # 3nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (23 * e_t - 16 * old_eps[-1] + 5 * old_eps[-2]) / 12 + elif len(old_eps) >= 3: + # 4nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (55 * e_t - 59 * old_eps[-1] + 37 * old_eps[-2] - 9 * old_eps[-3]) / 24 + + x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t_prime, index) + + return x_prev, pred_x0, e_t +plms_hijack.p_sample_plms = p_sample_plms diff --git a/modules/shared.py b/modules/shared.py index 7c5916cfd..ee8ca7f0a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -416,7 +416,7 @@ options_templates.update(options_section(('ui', "Live previews"), { "live_previews_enable": OptionInfo(True, "Show live previews of the created image"), "show_progress_grid": OptionInfo(True, "Show previews of all images generated in a batch as a grid"), "show_progress_every_n_steps": OptionInfo(1, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), - "show_progress_type": OptionInfo("Approx cheap" if is_device_dml else "Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), # DML Solution: Use Approx cheap instead of Approx NN as a default progress type. + "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}), "live_preview_refresh_period": OptionInfo(250, "Progressbar/preview update period, in milliseconds") }))