From deb0546b4695defd3851beae2b679ad4fcc1841a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 1 May 2023 18:54:50 -0400 Subject: [PATCH 1/4] update requirements --- TODO.md | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- .../stable-diffusion-webui-images-browser | 2 +- modules/memmon.py | 73 ++++++++++--------- modules/sd_hijack_optimizations.py | 56 ++++++-------- requirements.txt | 2 +- 6 files changed, 63 insertions(+), 74 deletions(-) diff --git a/TODO.md b/TODO.md index a18ac0d0c..309c274fb 100644 --- a/TODO.md +++ b/TODO.md @@ -34,7 +34,7 @@ Stuff to be investigated... Pick & merge PRs from main repo... -- Merge backlog: +- Merge backlog: ## Models diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 070d08f75..6315bec9a 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 070d08f7524ae6d16973ee57b1e0bc6369d5388f +Subproject commit 6315bec9abbeaaff7ef5f59dc9bf7f8d4bf67c8c diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser index 84cb61749..a396a9f90 160000 --- a/extensions-builtin/stable-diffusion-webui-images-browser +++ b/extensions-builtin/stable-diffusion-webui-images-browser @@ -1 +1 @@ -Subproject commit 84cb6174983812da2dff242fb484431b4ae3b8f8 +Subproject commit a396a9f90c6cd2fbd16fd2dc5aef3d210491bbc6 diff --git a/modules/memmon.py b/modules/memmon.py index 77e5eb752..97199445d 100644 --- a/modules/memmon.py +++ b/modules/memmon.py @@ -3,7 +3,7 @@ import time from collections import defaultdict import torch try: - import intel_extension_for_pytorch as ipex + import intel_extension_for_pytorch as ipex # pylint: disable=unused-import,import-error except: pass @@ -28,20 +28,15 @@ class MemUsageMonitor(threading.Thread): if not torch.cuda.is_available(): self.disabled = True else: - if shared.cmd_opts.use_ipex: - try: + try: + if shared.cmd_opts.use_ipex: self.cuda_mem_get_info() torch.cuda.memory_stats("xpu") - except Exception as e: # AMD or whatever - print(f"Torch exception: {e}") - self.disabled = True - else: - try: + else: self.cuda_mem_get_info() torch.cuda.memory_stats(self.device) - except Exception as e: # AMD or whatever - print(f"Torch exception: {e}") - self.disabled = True + except Exception: + self.disabled = True def cuda_mem_get_info(self): if shared.cmd_opts.use_ipex: @@ -70,22 +65,25 @@ class MemUsageMonitor(threading.Thread): time.sleep(1 / self.opts.memmon_poll_rate) def dump_debug(self): - print(self, 'recorded data:') - for k, v in self.read().items(): - print(k, -(v // -(1024 ** 2))) - print(self, 'raw torch memory stats:') - if shared.cmd_opts.use_ipex: - tm = torch.xpu.memory_stats("xpu") - else: - tm = torch.cuda.memory_stats(self.device) - for k, v in tm.items(): - if 'bytes' not in k: - continue - print('\t' if 'peak' in k else '', k, -(v // -(1024 ** 2))) - if shared.cmd_opts.use_ipex: - print(torch.xpu.memory_summary()) - else: - print(torch.cuda.memory_summary()) + try: + print(self, 'recorded data:') + for k, v in self.read().items(): + print(k, -(v // -(1024 ** 2))) + print(self, 'raw torch memory stats:') + if shared.cmd_opts.use_ipex: + tm = torch.xpu.memory_stats("xpu") + else: + tm = torch.cuda.memory_stats(self.device) + for k, v in tm.items(): + if 'bytes' not in k: + continue + print('\t' if 'peak' in k else '', k, -(v // -(1024 ** 2))) + if shared.cmd_opts.use_ipex: + print(torch.xpu.memory_summary()) + else: + print(torch.cuda.memory_summary()) + except: + self.disabled = True def monitor(self): self.run_flag.set() @@ -95,15 +93,18 @@ class MemUsageMonitor(threading.Thread): free, total = self.cuda_mem_get_info() self.data["free"] = free self.data["total"] = total - if shared.cmd_opts.use_ipex: - torch_stats = torch.xpu.memory_stats("xpu") - else: - torch_stats = torch.cuda.memory_stats(self.device) - self.data["active"] = torch_stats["active.all.current"] - self.data["active_peak"] = torch_stats["active_bytes.all.peak"] - self.data["reserved"] = torch_stats["reserved_bytes.all.current"] - self.data["reserved_peak"] = torch_stats["reserved_bytes.all.peak"] - self.data["system_peak"] = total - self.data["min_free"] + try: + if shared.cmd_opts.use_ipex: + torch_stats = torch.xpu.memory_stats("xpu") + else: + torch_stats = torch.cuda.memory_stats(self.device) + self.data["active"] = torch_stats["active.all.current"] + self.data["active_peak"] = torch_stats["active_bytes.all.peak"] + self.data["reserved"] = torch_stats["reserved_bytes.all.current"] + self.data["reserved_peak"] = torch_stats["reserved_bytes.all.peak"] + self.data["system_peak"] = total - self.data["min_free"] + except: + self.disabled = True return self.data def stop(self): diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 3887e238d..6c6824175 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -3,7 +3,7 @@ import psutil import torch try: - import intel_extension_for_pytorch as ipex + import intel_extension_for_pytorch as ipex # pylint: disable=unused-import,import-error except: pass from torch import einsum @@ -14,12 +14,12 @@ from einops import rearrange from modules import shared, errors, devices from modules.hypernetworks import hypernetwork -from .sub_quadratic_attention import efficient_dot_product_attention +from .sub_quadratic_attention import efficient_dot_product_attention # pylint: disable=relative-beyond-top-level if shared.opts.cross_attention_optimization == "xFormers": try: - import xformers.ops + import xformers.ops # pylint: disable=import-error shared.xformers_available = True except Exception: pass @@ -35,12 +35,16 @@ def get_available_vram(): mem_free_total = mem_free_xpu + mem_free_torch return mem_free_total elif shared.device.type == 'cuda': - stats = torch.cuda.memory_stats(shared.device) - mem_active = stats['active_bytes.all.current'] - mem_reserved = stats['reserved_bytes.all.current'] - mem_free_cuda, _ = torch.cuda.mem_get_info(torch.cuda.current_device()) - mem_free_torch = mem_reserved - mem_active - mem_free_total = mem_free_cuda + mem_free_torch + try: + stats = torch.cuda.memory_stats(shared.device) + mem_active = stats['active_bytes.all.current'] + mem_reserved = stats['reserved_bytes.all.current'] + mem_free_cuda, _ = torch.cuda.mem_get_info(torch.cuda.current_device()) + mem_free_torch = mem_reserved - mem_active + mem_free_total = mem_free_cuda + mem_free_torch + except: + mem_free_total = 1024 * 1024 * 1024 + return mem_free_total elif shared.device.type == 'privateuseone': mem_total, mem_active = torch.dml.memory_stats(shared.device) @@ -74,10 +78,8 @@ def split_cross_attention_forward_v1(self, x, context=None, mask=None): end = i + 2 s1 = einsum('b i d, b j d -> b i j', q[i:end], k[i:end]) s1 *= self.scale - s2 = s1.softmax(dim=-1) del s1 - r1[i:end] = einsum('b i j, b j d -> b i d', s2, v[i:end]) del s2 del q, k, v @@ -93,7 +95,6 @@ def split_cross_attention_forward_v1(self, x, context=None, mask=None): # taken from https://github.com/Doggettx/stable-diffusion and modified def split_cross_attention_forward(self, x, context=None, mask=None): h = self.heads - q_in = self.to_q(x) context = default(context, x) @@ -107,47 +108,34 @@ def split_cross_attention_forward(self, x, context=None, mask=None): with devices.without_autocast(disable=not shared.opts.upcast_attn): k_in = k_in * self.scale - del context, x - q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q_in, k_in, v_in)) del q_in, k_in, v_in - r1 = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype) - mem_free_total = get_available_vram() - gb = 1024 ** 3 tensor_size = q.shape[0] * q.shape[1] * k.shape[1] * q.element_size() modifier = 3 if q.element_size() == 2 else 2.5 mem_required = tensor_size * modifier steps = 1 - if mem_required > mem_free_total: steps = 2 ** (math.ceil(math.log(mem_required / mem_free_total, 2))) # print(f"Expected tensor size:{tensor_size/gb:0.1f}GB, cuda free:{mem_free_cuda/gb:0.1f}GB " # f"torch free:{mem_free_torch/gb:0.1f} total:{mem_free_total/gb:0.1f} steps:{steps}") - if steps > 64: max_res = math.floor(math.sqrt(math.sqrt(mem_free_total / 2.5)) / 8) * 64 raise RuntimeError(f'Not enough memory, use lower resolution (max approx. {max_res}x{max_res}). ' f'Need: {mem_required / 64 / gb:0.1f}GB free, Have:{mem_free_total / gb:0.1f}GB free') - slice_size = q.shape[1] // steps if (q.shape[1] % steps) == 0 else q.shape[1] for i in range(0, q.shape[1], slice_size): end = i + slice_size s1 = einsum('b i d, b j d -> b i j', q[:, i:end], k) - s2 = s1.softmax(dim=-1, dtype=q.dtype) del s1 - r1[:, i:end] = einsum('b i j, b j d -> b i d', s2, v) del s2 - del q, k, v - r1 = r1.to(dtype) - r2 = rearrange(r1, '(b h) n d -> b n (h d)', h=h) del r1 @@ -211,12 +199,15 @@ def einsum_op_cuda(q, k, v): # Divide factor of safety as there's copying and fragmentation return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) else: - stats = torch.cuda.memory_stats(q.device) - mem_active = stats['active_bytes.all.current'] - mem_reserved = stats['reserved_bytes.all.current'] - mem_free_cuda, _ = torch.cuda.mem_get_info(q.device) - mem_free_torch = mem_reserved - mem_active - mem_free_total = mem_free_cuda + mem_free_torch + try: + stats = torch.cuda.memory_stats(q.device) + mem_active = stats['active_bytes.all.current'] + mem_reserved = stats['reserved_bytes.all.current'] + mem_free_cuda, _ = torch.cuda.mem_get_info(q.device) + mem_free_torch = mem_reserved - mem_active + mem_free_total = mem_free_cuda + mem_free_torch + except: + mem_free_total = 1024 * 1024 * 1024 # Divide factor of safety as there's copying and fragmentation return einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) @@ -261,7 +252,6 @@ def split_cross_attention_forward_invokeAI(self, x, context=None, mask=None): with devices.without_autocast(disable=not shared.opts.upcast_attn): k = k * self.scale - q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v)) r = einsum_op(q, k, v) r = r.to(dtype) @@ -400,9 +390,7 @@ def scaled_dot_product_attention_forward(self, x, context=None, mask=None): q = q_in.view(batch_size, -1, h, head_dim).transpose(1, 2) k = k_in.view(batch_size, -1, h, head_dim).transpose(1, 2) v = v_in.view(batch_size, -1, h, head_dim).transpose(1, 2) - del q_in, k_in, v_in - dtype = q.dtype if shared.opts.upcast_attn: q, k, v = q.float(), k.float(), v.float() diff --git a/requirements.txt b/requirements.txt index e87a63e85..15b606bc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,6 +58,6 @@ numexpr==2.8.4 pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 -transformers==4.28.1 +transformers==4.26.1 timm==0.6.13 tomesd==0.1.2 From d22103e992573051dcc40522d0aeb6e6dfa81d11 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 1 May 2023 19:12:10 -0400 Subject: [PATCH 2/4] update requirements --- requirements.txt | 2 +- wiki | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 15b606bc1..1409e2960 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,10 +45,10 @@ torch torchdiffeq torchsde torchvision -tqdm voluptuous yapf scikit-image +tqdm==4.65.0 accelerate==0.18.0 opencv-python==4.7.0.72 diffusers==0.16.1 diff --git a/wiki b/wiki index 4cbdffaa9..ab46c9f35 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 4cbdffaa95978d0a46758eac4a3fbe689eb4cdcd +Subproject commit ab46c9f3583bbd155623fd7d59a969a353aa96be From 10a9c2760a7dce566e5f37231cdcae24137d6744 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 1 May 2023 19:49:19 -0400 Subject: [PATCH 3/4] switch cmdargs --- installer.py | 8 +++++--- webui.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/installer.py b/installer.py index 56431ec20..010ee1980 100644 --- a/installer.py +++ b/installer.py @@ -20,7 +20,7 @@ class Dot(dict): # dot notation access to dictionary attributes log = logging.getLogger("sd") -args = Dot({ 'debug': False, 'upgrade': False, 'no_directml': False, 'skip_update': False, 'skip_extensions': False, 'skip_requirements': False, 'skip_git': False, 'reset': False, 'use_ipex': False, 'experimental': False, 'test': False }) +args = Dot({ 'debug': False, 'upgrade': False, 'skip_update': False, 'skip_extensions': False, 'skip_requirements': False, 'skip_git': False, 'reset': False, 'use_directml': False, 'use_ipex': False, 'experimental': False, 'test': False }) quick_allowed = True errors = 0 opts = {} @@ -203,7 +203,7 @@ def check_torch(): xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: machine = platform.machine() - if 'arm' not in machine and 'aarch' not in machine and not args.no_directml: # torch-directml is available on AMD64 + if 'arm' not in machine and 'aarch' not in machine and args.use_directml: # torch-directml is available on AMD64 log.info('Using DirectML Backend') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0 torchvision torch-directml') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') @@ -514,7 +514,7 @@ def add_args(): group.add_argument('--reset', default = False, action='store_true', help = "Reset main repository to latest version, default: %(default)s") group.add_argument('--upgrade', default = False, action='store_true', help = "Upgrade main repository to latest version, default: %(default)s") group.add_argument("--use-ipex", action='store_true', help="Use Intel OneAPI XPU backend, default: %(default)s", default=False) - group.add_argument('--no-directml', default = False, action='store_true', help = "Use CPU instead of DirectML if no compatible GPU is detected, default: %(default)s") + group.add_argument('--use-directml', default = False, action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s") group.add_argument('--skip-update', default = False, action='store_true', help = "Skip update of extensions and submodules, default: %(default)s") group.add_argument('--skip-requirements', default = False, action='store_true', help = "Skips checking and installing requirements, default: %(default)s") group.add_argument('--skip-extensions', default = False, action='store_true', help = "Skips running individual extension installers, default: %(default)s") @@ -594,5 +594,7 @@ def run_setup(): if __name__ == "__main__": + add_args() + ensure_base_requirements() parse_args() run_setup() diff --git a/webui.py b/webui.py index 81e196bac..8fae77f30 100644 --- a/webui.py +++ b/webui.py @@ -199,7 +199,7 @@ def start_ui(): if cmd_opts.disable_queue: print('Server queues disabled') else: - shared.demo.queue(16) + shared.demo.queue(concurrency_count=16) gradio_auth_creds = [] if cmd_opts.auth: @@ -220,6 +220,7 @@ def start_ui(): auth=[tuple(cred.split(':')) for cred in gradio_auth_creds] if gradio_auth_creds else None, inbrowser=cmd_opts.autolaunch, prevent_thread_lock=True, + show_api=True, favicon_path='automatic.ico', ) setup_middleware(app, cmd_opts) From 9e214b32d739a83d1c94205e0872cfadcf02d030 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Tue, 2 May 2023 09:04:35 +0900 Subject: [PATCH 4/4] remove NotImplementedError. --- modules/dml/optimizer/intel/__init__.py | 7 +++---- modules/dml/optimizer/nvidia/__init__.py | 7 +++---- modules/dml/optimizer/unknown/__init__.py | 3 +-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/modules/dml/optimizer/intel/__init__.py b/modules/dml/optimizer/intel/__init__.py index bffda69f1..aaa9e67b1 100644 --- a/modules/dml/optimizer/intel/__init__.py +++ b/modules/dml/optimizer/intel/__init__.py @@ -1,7 +1,6 @@ from modules.dml.optimizer.optimizer import Optimizer class IntelOptimizer(Optimizer): - def memory_stats(index): - raise NotImplementedError() - # DML TODO: Implement - return + def memory_stats(index: int): + # DML TODO: Implement or find a general (and also lightweight) way. + return (1073741824, 0) diff --git a/modules/dml/optimizer/nvidia/__init__.py b/modules/dml/optimizer/nvidia/__init__.py index e5fda97b8..5990c11a9 100644 --- a/modules/dml/optimizer/nvidia/__init__.py +++ b/modules/dml/optimizer/nvidia/__init__.py @@ -1,7 +1,6 @@ from modules.dml.optimizer.optimizer import Optimizer class nVidiaOptimizer(Optimizer): - def memory_stats(index): - raise NotImplementedError() - # DML TODO: Implement - return + def memory_stats(index: int): + # DML TODO: Implement or find a general (and also lightweight) way. + return (1073741824, 0) diff --git a/modules/dml/optimizer/unknown/__init__.py b/modules/dml/optimizer/unknown/__init__.py index 40ff476cd..79ed11ce5 100644 --- a/modules/dml/optimizer/unknown/__init__.py +++ b/modules/dml/optimizer/unknown/__init__.py @@ -1,6 +1,5 @@ from modules.dml.optimizer.optimizer import Optimizer class UnknownOptimizer(Optimizer): - def memory_stats(index): - # DML TODO: Implement + def memory_stats(index: int): return (1073741824, 0)