From 7e63b389c9706b0fb0d078bf693bd5f1d74582c4 Mon Sep 17 00:00:00 2001 From: iDeNoh Date: Thu, 13 Mar 2025 13:56:36 -0500 Subject: [PATCH 01/13] Added remote vae type to infotext.py and processing_info.py --- modules/infotext.py | 2 ++ modules/processing_info.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/modules/infotext.py b/modules/infotext.py index 78c1fd92e..4ee14dd01 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -109,6 +109,8 @@ def parse(infotext): params[key] = False elif key == 'VAE' and val == 'TAESD': params["VAE type"] = 'Tiny' + elif key == 'VAE' and val == 'Remote': + params["VAE type"] = 'Remote' elif size is not None: params[f"{key}-1"] = int(size.group(1)) params[f"{key}-2"] = int(size.group(2)) diff --git a/modules/processing_info.py b/modules/processing_info.py index 5944855b0..fa084a2fb 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -74,6 +74,8 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["VAE"] = (None if not shared.opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]) elif p.vae_type == 'Tiny': args["VAE"] = 'TAESD' + elif p.vae_type == 'Remote': + args["VAE"] = 'Remote' if shared.opts.add_model_name_to_info and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None: args["Model"] = shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', '') if shared.opts.add_model_hash_to_info and getattr(shared.sd_model, 'sd_model_hash', None) is not None: From 3c0ca2e815493603af75706e1663ca049c730e50 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 13 Mar 2025 22:31:47 +0300 Subject: [PATCH 02/13] Add --upgrade to nightly torch commands --- CHANGELOG.md | 4 +++- installer.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9c2c69ac..6ceb44399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Change Log for SD.Next -## Update for 2025-03-12 +## Update for 2025-03-13 - fix installer not starting when older version of rich is installed - fix circular imports when debug flags are enabled - fix cuda errors with directml - fix memory stats not displaying the ram usage +- fix remote vae not being stored in metadata, thanks @iDeNoh +- add --upgrade to torch_command when using --use-nightly for ipex and rocm - **ipex** - add xpu to profiler - fix untyped_storage, torch.eye and torch.cuda.device ops diff --git a/installer.py b/installer.py index 30e0cb377..153dae193 100644 --- a/installer.py +++ b/installer.py @@ -667,11 +667,11 @@ def install_rocm_zluda(): if args.use_nightly: if rocm.version is None or float(rocm.version) >= 6.3: # assume the latest if version check fails - torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.3') + torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.3') elif rocm.version == "6.2": # use rocm 6.2.4 instead of 6.2 as torch+rocm6.2 doesn't exists - torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.2.4') + torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.2.4') else: # oldest rocm version on nightly is 6.1 - torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.1') + torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.1') else: if rocm.version is None or float(rocm.version) >= 6.2: # assume the latest if version check fails # use rocm 6.2.4 instead of 6.2 as torch==2.6.0+rocm6.2 doesn't exists @@ -735,7 +735,7 @@ def install_ipex(torch_command): # os.environ.setdefault('IGC_EnableDPEmulation', '1') # FP64 Emulation if args.use_nightly: - torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') + torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') else: torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+xpu torchvision==0.21.0+xpu --index-url https://download.pytorch.org/whl/xpu') From aad6bfca3a9398905b655ef5068bb83a5de69b24 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 13 Mar 2025 23:18:20 +0300 Subject: [PATCH 03/13] Don't restore remove vae type --- modules/infotext.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/infotext.py b/modules/infotext.py index 4ee14dd01..78c1fd92e 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -109,8 +109,6 @@ def parse(infotext): params[key] = False elif key == 'VAE' and val == 'TAESD': params["VAE type"] = 'Tiny' - elif key == 'VAE' and val == 'Remote': - params["VAE type"] = 'Remote' elif size is not None: params[f"{key}-1"] = int(size.group(1)) params[f"{key}-2"] = int(size.group(2)) From 19741722962c5322fb2c17b555402899f9176ee0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 13 Mar 2025 23:53:04 +0300 Subject: [PATCH 04/13] IPEX fix triton --- modules/intel/ipex/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index cd272af70..6d3b2ab1d 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -18,7 +18,10 @@ def ipex_init(): # pylint: disable=too-many-statements if hasattr(torch, "cuda") and hasattr(torch.cuda, "is_xpu_hijacked") and torch.cuda.is_xpu_hijacked: return True, "Skipping IPEX hijack" else: - try: # force xpu device on torch compile and triton + try: + # force xpu device on torch compile and triton + # import inductor utils to get around lazy import + from torch._inductor import utils as torch_inductor_utils # pylint: disable=import-error, unused-import torch._inductor.utils.GPU_TYPES = ["xpu"] torch._inductor.utils.get_gpu_type = lambda *args, **kwargs: "xpu" from triton import backends as triton_backends # pylint: disable=import-error From cd9cb51a05841724d66bcf6c42772aee19f8af82 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 02:03:33 +0300 Subject: [PATCH 05/13] IPEX fix torch.compile on PyTorch 2.8 --- modules/intel/ipex/__init__.py | 3 +++ modules/intel/ipex/hijacks.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 6d3b2ab1d..b0240cb7a 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -190,11 +190,13 @@ def ipex_init(): # pylint: disable=too-many-statements ipex._C._DeviceProperties.multi_processor_count = ipex._C._DeviceProperties.gpu_subslice_count ipex._C._DeviceProperties.major = 12 ipex._C._DeviceProperties.minor = 1 + ipex._C._DeviceProperties.L2_cache_size = 16*1024*1024 # A770 and A750 else: torch._C._cuda_getCurrentRawStream = torch._C._xpu_getCurrentRawStream torch._C._XpuDeviceProperties.multi_processor_count = torch._C._XpuDeviceProperties.gpu_subslice_count torch._C._XpuDeviceProperties.major = 12 torch._C._XpuDeviceProperties.minor = 1 + torch._C._XpuDeviceProperties.L2_cache_size = 16*1024*1024 # A770 and A750 # Fix functions with ipex: # torch.xpu.mem_get_info always returns the total memory as free memory @@ -211,6 +213,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.get_device_capability = lambda *args, **kwargs: (12,1) torch.cuda.get_device_properties.major = 12 torch.cuda.get_device_properties.minor = 1 + torch.cuda.get_device_properties.L2_cache_size = 16*1024*1024 # A770 and A750 torch.cuda.ipc_collect = lambda *args, **kwargs: None torch.cuda.utilization = lambda *args, **kwargs: 0 diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 91a256aed..e47065a62 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -332,14 +332,6 @@ def torch_load(f, map_location=None, *args, **kwargs): else: return original_torch_load(f, *args, map_location=map_location, **kwargs) -original_torch_Generator = torch.Generator -@wraps(torch.Generator) -def torch_Generator(device=None): - if check_cuda(device): - return original_torch_Generator(return_xpu(device)) - else: - return original_torch_Generator(device) - @wraps(torch.cuda.synchronize) def torch_cuda_synchronize(device=None): if check_cuda(device): @@ -355,6 +347,17 @@ def torch_cuda_device(device): return torch.xpu.device(device) +# torch.Generator has to be a class for isinstance checks +original_torch_Generator = torch.Generator +class torch_Generator(original_torch_Generator): + def __new__(self, device=None): + # can't hijack __init__ because of C override so use return super().__new__ + if check_cuda(device): + return super().__new__(self, return_xpu(device)) + else: + return super().__new__(self, device) + + # Hijack Functions: def ipex_hijacks(): global device_supports_fp64, can_allocate_plus_4gb @@ -374,10 +377,12 @@ def ipex_hijacks(): torch.linspace = torch_linspace torch.eye = torch_eye torch.load = torch_load - torch.Generator = torch_Generator torch.cuda.synchronize = torch_cuda_synchronize torch.cuda.device = torch_cuda_device + torch.Generator = torch_Generator + torch._C.Generator = torch_Generator + torch.backends.cuda.sdp_kernel = return_null_context torch.nn.DataParallel = DummyDataParallel torch.UntypedStorage.is_cuda = is_cuda From 75ff932e792398cc4e9d1f90d6fcc49bac19c0b6 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 13:48:31 +0300 Subject: [PATCH 06/13] Add safety check for total ram available --- CHANGELOG.md | 1 + modules/call_queue.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ceb44399..ac6f45bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - fix untyped_storage, torch.eye and torch.cuda.device ops - fix torch 2.7 compatibility - fix performance with balanced offload + - fix triton and torch.compile ## Update for 2025-02-28 diff --git a/modules/call_queue.py b/modules/call_queue.py index 7e7ab16f5..367b5d5ee 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -88,7 +88,9 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None): gpu += f" | retries {retries} oom {ooms}" if retries > 0 or ooms > 0 else '' ram = shared.ram_stats() if ram['used'] > 0: - cpu += f"| RAM {ram['used']} GB {round(100.0 * ram['used'] / ram['total'])}%" + cpu += f"| RAM {ram['used']} GB" + if ram['total'] > 0: + cpu += f" {round(100.0 * ram['used'] / ram['total'])}%" if isinstance(res, list): res[-1] += f"

Time: {elapsed_text} | {summary} {gpu} {cpu}

" return tuple(res) From 463024c74968a038ba79f43d31313b6c3bf8b98b Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 14:36:13 +0300 Subject: [PATCH 07/13] fix control memstats --- modules/call_queue.py | 8 ++++---- modules/ui_control.py | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/call_queue.py b/modules/call_queue.py index 367b5d5ee..8368e5759 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -83,14 +83,14 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None): vram = {k: v//1048576 for k, v in mem_mon_read.items()} peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 - if used > 0: - gpu += f"| GPU {peak} MB {used}%" + if peak > 0: + gpu += f"| GPU {peak} MB" + gpu += f" {used}%" if used > 0 else '' gpu += f" | retries {retries} oom {ooms}" if retries > 0 or ooms > 0 else '' ram = shared.ram_stats() if ram['used'] > 0: cpu += f"| RAM {ram['used']} GB" - if ram['total'] > 0: - cpu += f" {round(100.0 * ram['used'] / ram['total'])}%" + cpu += f" {round(100.0 * ram['used'] / ram['total'])}%" if ram['total'] > 0 else '' if isinstance(res, list): res[-1] += f"

Time: {elapsed_text} | {summary} {gpu} {cpu}

" return tuple(res) diff --git a/modules/ui_control.py b/modules/ui_control.py index fba599d38..6aa6d0002 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -33,15 +33,20 @@ def return_stats(t: float = None): gpu = '' cpu = '' if not shared.mem_mon.disabled: - vram = {k: -(v//-(1024*1024)) for k, v in shared.mem_mon.read().items()} + mem_mon_read = shared.mem_mon.read() + ooms = mem_mon_read.pop("oom") + retries = mem_mon_read.pop("retries") + vram = {k: v//1048576 for k, v in mem_mon_read.items()} peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 - if used > 0: - gpu += f"| GPU {peak} MB {used}%" - gpu += f" | retries {vram['retries']} oom {vram['oom']}" if vram.get('retries', 0) > 0 or vram.get('oom', 0) > 0 else '' - ram = shared.ram_stats() - if ram['used'] > 0: - cpu += f"| RAM {ram['used']} GB {round(100.0 * ram['used'] / ram['total'])}%" + if peak > 0: + gpu += f"| GPU {peak} MB" + gpu += f" {used}%" if used > 0 else '' + gpu += f" | retries {retries} oom {ooms}" if retries > 0 or ooms > 0 else '' + ram = shared.ram_stats() + if ram['used'] > 0: + cpu += f"| RAM {ram['used']} GB" + cpu += f" {round(100.0 * ram['used'] / ram['total'])}%" if ram['total'] > 0 else '' return f"

{elapsed_text} {summary} {gpu} {cpu}

" From 851fb9d307e61def60b08e5944999c42d66383f8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 17:48:44 +0300 Subject: [PATCH 08/13] Fix runpod limit --- modules/memstats.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/memstats.py b/modules/memstats.py index f01cfc2e1..229deea26 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -31,7 +31,9 @@ def get_runpod_limit(): global runpod_limit # pylint: disable=global-statement if runpod_limit is not None: return runpod_limit - runpod_limit = float(os.environ.get('RUNPOD_MEM_GB', sys.float_info.max)) + runpod_limit = float(os.environ.get('RUNPOD_MEM_GB', 0)) + if runpod_limit == 0: # RUNPOD_MEM_GB can be set to 0 for no limit + runpod_limit = sys.float_info.max return runpod_limit From 75c1dcb8922005ec782a0e828b433bc74b221021 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 17:57:14 +0300 Subject: [PATCH 09/13] Update changelog --- CHANGELOG.md | 3 ++- modules/memstats.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac6f45bad..5e85ce44a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # Change Log for SD.Next -## Update for 2025-03-13 +## Update for 2025-03-14 - fix installer not starting when older version of rich is installed - fix circular imports when debug flags are enabled - fix cuda errors with directml - fix memory stats not displaying the ram usage +- fix runpod memory limit reporting when limit is set to 0 - fix remote vae not being stored in metadata, thanks @iDeNoh - add --upgrade to torch_command when using --use-nightly for ipex and rocm - **ipex** diff --git a/modules/memstats.py b/modules/memstats.py index 229deea26..c4f7f7f49 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -31,7 +31,7 @@ def get_runpod_limit(): global runpod_limit # pylint: disable=global-statement if runpod_limit is not None: return runpod_limit - runpod_limit = float(os.environ.get('RUNPOD_MEM_GB', 0)) + runpod_limit = float(os.environ.get('RUNPOD_MEM_GB', sys.float_info.max)) if runpod_limit == 0: # RUNPOD_MEM_GB can be set to 0 for no limit runpod_limit = sys.float_info.max return runpod_limit From f2409d38c468e3ed9f51b794b2f0039a8e1dfb20 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 18:52:13 +0300 Subject: [PATCH 10/13] IPEX use get_arch_list from torch.xpu --- modules/intel/ipex/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index b0240cb7a..9ae19565a 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -205,11 +205,11 @@ def ipex_init(): # pylint: disable=too-many-statements torch._utils._get_available_device_type = lambda: "xpu" torch.has_cuda = True torch.cuda.has_half = True - torch.cuda.is_bf16_supported = lambda *args, **kwargs: True + torch.cuda.is_bf16_supported = getattr(torch.xpu, "is_bf16_supported", lambda *args, **kwargs: True) torch.cuda.is_fp16_supported = lambda *args, **kwargs: True torch.backends.cuda.is_built = lambda *args, **kwargs: True torch.version.cuda = "12.1" - torch.cuda.get_arch_list = lambda: ["ats-m150", "pvc"] + torch.cuda.get_arch_list = getattr(torch.xpu, "get_arch_list", lambda: ["ats-m150", "pvc"]) torch.cuda.get_device_capability = lambda *args, **kwargs: (12,1) torch.cuda.get_device_properties.major = 12 torch.cuda.get_device_properties.minor = 1 From 4fce7c279eb5d7aa597e5d6bd268fcc1f258c89e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 18:56:03 +0300 Subject: [PATCH 11/13] IPEX add dg2 to arch list --- modules/intel/ipex/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 9ae19565a..2d82830c1 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -209,7 +209,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.is_fp16_supported = lambda *args, **kwargs: True torch.backends.cuda.is_built = lambda *args, **kwargs: True torch.version.cuda = "12.1" - torch.cuda.get_arch_list = getattr(torch.xpu, "get_arch_list", lambda: ["ats-m150", "pvc"]) + torch.cuda.get_arch_list = getattr(torch.xpu, "get_arch_list", lambda: ["pvc", "dg2", "ats-m150"]) torch.cuda.get_device_capability = lambda *args, **kwargs: (12,1) torch.cuda.get_device_properties.major = 12 torch.cuda.get_device_properties.minor = 1 From c82d5a8a8a29e5ff631bec81fbbc4ebce48285ea Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 19:44:00 +0300 Subject: [PATCH 12/13] Fix get_runpod_limit --- modules/memstats.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/memstats.py b/modules/memstats.py index c4f7f7f49..160492069 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -24,6 +24,8 @@ def get_docker_limit(): docker_limit = float(f.read()) except Exception: docker_limit = sys.float_info.max + if docker_limit == 0: + docker_limit = sys.float_info.max return docker_limit @@ -31,8 +33,8 @@ def get_runpod_limit(): global runpod_limit # pylint: disable=global-statement if runpod_limit is not None: return runpod_limit - runpod_limit = float(os.environ.get('RUNPOD_MEM_GB', sys.float_info.max)) - if runpod_limit == 0: # RUNPOD_MEM_GB can be set to 0 for no limit + runpod_limit = float(os.environ.get('RUNPOD_MEM_GB', 0)) * 1024 * 1024 * 1024 + if runpod_limit == 0: runpod_limit = sys.float_info.max return runpod_limit From fdf4999f40f5842515a6e7decd4c874bc0f3eeec Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 14 Mar 2025 20:03:58 +0300 Subject: [PATCH 13/13] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e85ce44a..32d906a7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - fix circular imports when debug flags are enabled - fix cuda errors with directml - fix memory stats not displaying the ram usage -- fix runpod memory limit reporting when limit is set to 0 +- fix runpod memory limit reporting - fix remote vae not being stored in metadata, thanks @iDeNoh - add --upgrade to torch_command when using --use-nightly for ipex and rocm - **ipex**