From 92f2a2902fc2dc9fc6bf0c52ff5f9ed34d959061 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 23 Sep 2024 11:07:24 -0400
Subject: [PATCH] improve profiling

---
 installer.py                       |  7 +++++
 launch.py                          |  1 +
 modules/call_queue.py              |  1 +
 modules/devices.py                 | 27 ++++++++--------
 modules/processing.py              | 32 +++++++++++++------
 modules/processing_args.py         | 49 ++++++++++++++++--------------
 modules/processing_callbacks.py    |  7 ++++-
 modules/processing_diffusers.py    |  9 +++++-
 modules/prompt_parser_diffusers.py |  6 ++--
 modules/script_loading.py          |  8 +----
 modules/sd_hijack_freeu.py         | 24 +++++++++++++--
 modules/sd_models.py               |  6 ----
 modules/timer.py                   | 20 ++++++++----
 modules/ui_extra_networks.py       |  8 -----
 14 files changed, 128 insertions(+), 77 deletions(-)

diff --git a/installer.py b/installer.py
index 9ddde8d4d..d781e6d0b 100644
--- a/installer.py
+++ b/installer.py
@@ -169,6 +169,7 @@ def print_dict(d):
 
 
 def print_profile(profiler: cProfile.Profile, msg: str):
+    profiler.disable()
     from modules.errors import profile
     profile(profiler, msg)
 
@@ -736,6 +737,7 @@ def check_torch():
     if not args.skip_all:
         install_torch_addons()
     if args.profile:
+        pr.disable()
         print_profile(pr, 'Torch')
 
 
@@ -777,6 +779,7 @@ def install_packages():
     # elif not args.experimental:
     #    uninstall('bitsandbytes')
     if args.profile:
+        pr.disable( )
         print_profile(pr, 'Packages')
 
 
@@ -859,6 +862,7 @@ def install_extensions(force=False):
     if len(extensions_duplicates) > 0:
         log.warning(f'Extensions duplicates: {extensions_duplicates}')
     if args.profile:
+        pr.disable()
         print_profile(pr, 'Extensions')
     return '\n'.join(res)
 
@@ -890,6 +894,7 @@ def install_submodules(force=True):
             log.error(f'Submodule update error: {submodule}')
     setup_logging()
     if args.profile:
+        pr.disable()
         print_profile(pr, 'Submodule')
     return '\n'.join(res)
 
@@ -948,6 +953,7 @@ def install_requirements():
             if not installed(line, quiet=True):
                 _res = install(line)
     if args.profile:
+        pr.disable()
         print_profile(pr, 'Requirements')
 
 
@@ -1234,6 +1240,7 @@ def extensions_preload(parser):
     except Exception:
         log.error('Error running extension preloading')
     if args.profile:
+        pr.disable()
         print_profile(pr, 'Preload')
 
 
diff --git a/launch.py b/launch.py
index a2dbeb740..93230315a 100755
--- a/launch.py
+++ b/launch.py
@@ -182,6 +182,7 @@ def start_server(immediate=True, server=None):
         else:
             uvicorn = server.webui(restart=not immediate)
     if args.profile:
+        pr.disable()
         installer.print_profile(pr, 'WebUI')
     return uvicorn, server
 
diff --git a/modules/call_queue.py b/modules/call_queue.py
index 04dbd81d0..4065d13d9 100644
--- a/modules/call_queue.py
+++ b/modules/call_queue.py
@@ -59,6 +59,7 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None):
             else:
                 res = list(res)
             if shared.cmd_opts.profile:
+                pr.disable()
                 errors.profile(pr, 'Wrap')
         except Exception as e:
             errors.display(e, 'gradio call')
diff --git a/modules/devices.py b/modules/devices.py
index 358937ebf..d761c8b00 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -6,7 +6,7 @@ import contextlib
 from functools import wraps
 import torch
 from modules.errors import log
-from modules import cmd_args, shared, memstats, errors
+from modules import cmd_args, shared, memstats, errors, timer
 
 if sys.platform == "darwin":
     from modules import mac_specific # pylint: disable=ungrouped-imports
@@ -147,19 +147,22 @@ def torch_gc(force=False, fast=False):
         previous_oom = oom
         log.warning(f'GPU out-of-memory error: {mem}')
         force = True
+    if force:
+        # actual gc
+        collected = gc.collect() if not fast else 0 # python gc
+        if cuda_ok:
+            try:
+                with torch.cuda.device(get_cuda_device_string()):
+                    torch.cuda.empty_cache() # cuda gc
+                    torch.cuda.ipc_collect()
+            except Exception:
+                pass
+    t1 = time.time()
+    if 'gc' not in timer.process.records:
+        timer.process.records['gc'] = 0
+    timer.process.records['gc'] += t1 - t0
     if not force:
         return
-
-    # actual gc
-    collected = gc.collect() if not fast else 0 # python gc
-    if cuda_ok:
-        try:
-            with torch.cuda.device(get_cuda_device_string()):
-                torch.cuda.empty_cache() # cuda gc
-                torch.cuda.ipc_collect()
-        except Exception:
-            pass
-    t1 = time.time()
     mem = memstats.memory_stats()
     saved = round(gpu.get('used', 0) - mem.get('gpu', {}).get('used', 0), 2)
     before = { 'gpu': gpu.get('used', 0), 'ram': ram.get('used', 0) }
diff --git a/modules/processing.py b/modules/processing.py
index 5488539e1..bf704a83a 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -4,7 +4,7 @@ import time
 from contextlib import nullcontext
 import numpy as np
 from PIL import Image, ImageOps
-from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, face_restoration, sd_hijack_freeu, sd_models, sd_vae, processing_helpers
+from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, face_restoration, sd_hijack_freeu, sd_models, sd_vae, processing_helpers, timer
 from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet
 from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl # pylint: disable=unused-import
 from modules.processing_info import create_infotext
@@ -116,6 +116,7 @@ class Processed:
 
 
 def process_images(p: StableDiffusionProcessing) -> Processed:
+    timer.process.reset()
     debug(f'Process images: {vars(p)}')
     if not hasattr(p.sd_model, 'sd_checkpoint_info'):
         return None
@@ -168,11 +169,9 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
             p.height = 8 * int(p.height / 8)
 
         script_callbacks.before_process_callback(p)
+        timer.process.record('pre')
 
         if shared.cmd_opts.profile:
-            import cProfile
-            profile_python = cProfile.Profile()
-            profile_python.enable()
             with context_hypertile_vae(p), context_hypertile_unet(p):
                 import torch.profiler # pylint: disable=redefined-outer-name
                 activities=[torch.profiler.ProfilerActivity.CPU]
@@ -180,12 +179,23 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
                     activities.append(torch.profiler.ProfilerActivity.CUDA)
                 shared.log.debug(f'Torch profile: activities={activities}')
                 if shared.profiler is None:
-                    shared.profiler = torch.profiler.profile(activities=activities, profile_memory=True, with_modules=True)
+                    profile_args = {
+                        'activities': activities,
+                        'profile_memory': True,
+                        'with_modules': True,
+                        'with_stack': os.environ.get('SD_PROFILE_STACK', None) is not None,
+                        'experimental_config': torch._C._profiler._ExperimentalConfig(verbose=True) if os.environ.get('SD_PROFILE_STACK', None) is not None else None, # pylint: disable=protected-access
+                        'with_flops': os.environ.get('SD_PROFILE_FLOPS', None) is not None,
+                        'record_shapes': os.environ.get('SD_PROFILE_SHAPES', None) is not None,
+                        'on_trace_ready': torch.profiler.tensorboard_trace_handler(os.environ.get('SD_PROFILE_FOLDER', None)) if os.environ.get('SD_PROFILE_FOLDER', None) is not None else None,
+                    }
+                    shared.log.debug(f'Torch profile: {profile_args}')
+                    shared.profiler = torch.profiler.profile(**profile_args)
                 shared.profiler.start()
-                shared.profiler.step()
+                if not shared.native:
+                    shared.profiler.step()
                 processed = process_images_inner(p)
                 errors.profile_torch(shared.profiler, 'Process')
-            errors.profile(profile_python, 'Process')
         else:
             with context_hypertile_vae(p), context_hypertile_unet(p):
                 processed = process_images_inner(p)
@@ -206,6 +216,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
                     sd_models.reload_model_weights()
                 if k == 'sd_vae':
                     sd_vae.reload_vae_weights()
+        timer.process.record('post')
     return processed
 
 
@@ -302,6 +313,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
 
             x_samples_ddim = None
+            timer.process.record('init')
             if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
                 x_samples_ddim = p.scripts.process_images(p)
             if x_samples_ddim is None:
@@ -313,6 +325,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                     x_samples_ddim = process_diffusers(p)
                 else:
                     raise ValueError(f"Unknown backend {shared.backend}")
+            timer.process.record('process')
 
             if not shared.opts.keep_incomplete and shared.state.interrupted:
                 x_samples_ddim = []
@@ -385,6 +398,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                         output_images.append(image_mask)
                     if shared.opts.return_mask_composite:
                         output_images.append(image_mask_composite)
+            timer.process.record('post')
             del x_samples_ddim
             devices.torch_gc()
 
@@ -394,8 +408,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
 
         t1 = time.time()
-        shared.log.info(f'Processed: images={len(output_images)} time={t1 - t0:.2f} its={(p.steps * len(output_images)) / (t1 - t0):.2f} memory={memstats.memory_stats()}')
-        from modules import timer
 
         p.color_corrections = None
         index_of_first_image = 0
@@ -441,4 +453,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
     )
     if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner) and not (shared.state.interrupted or shared.state.skipped):
         p.scripts.postprocess(p, processed)
+    timer.process.record('post')
+    shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct(min_time=0.02)} memory={memstats.memory_stats()}')
     return processed
diff --git a/modules/processing_args.py b/modules/processing_args.py
index adb0992f2..7cb380ee7 100644
--- a/modules/processing_args.py
+++ b/modules/processing_args.py
@@ -6,7 +6,7 @@ import time
 import inspect
 import torch
 import numpy as np
-from modules import shared, errors, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, prompt_parser_diffusers
+from modules import shared, errors, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, prompt_parser_diffusers, timer
 from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p
 from modules.processing_helpers import resize_hires, fix_prompts, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, get_generator, set_latents, apply_circular # pylint: disable=unused-import
 
@@ -96,24 +96,9 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
     debug(f'Diffusers pipeline possible: {possible}')
     prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2)
     parser = 'Fixed attention'
+    steps = kwargs.get("num_inference_steps", None) or len(getattr(p, 'timesteps', ['1']))
     clip_skip = kwargs.pop("clip_skip", 1)
 
-    steps = kwargs.get("num_inference_steps", None) or len(getattr(p, 'timesteps', ['1']))
-    if 'timesteps' in possible:
-        timesteps = re.split(',| ', shared.opts.schedulers_timesteps)
-        timesteps = [int(x) for x in timesteps if x.isdigit()]
-        if len(timesteps) > 0:
-            if hasattr(model.scheduler, 'set_timesteps') and "timesteps" in set(inspect.signature(model.scheduler.set_timesteps).parameters.keys()):
-                try:
-                    args['timesteps'] = timesteps
-                    p.steps = len(timesteps)
-                    p.timesteps = timesteps
-                    steps = p.steps
-                    shared.log.debug(f'Sampler: steps={len(timesteps)} timesteps={timesteps}')
-                except Exception as e:
-                    shared.log.error(f'Sampler timesteps: {e}')
-            else:
-                shared.log.warning(f'Sampler: sampler={model.scheduler.__class__.__name__} timesteps not supported')
     if shared.opts.prompt_attention != 'Fixed attention' and 'Onnx' not in model.__class__.__name__ and (
         'StableDiffusion' in model.__class__.__name__ or
         'StableCascade' in model.__class__.__name__ or
@@ -126,11 +111,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
             shared.log.error(f'Prompt parser encode: {e}')
             if os.environ.get('SD_PROMPT_DEBUG', None) is not None:
                 errors.display(e, 'Prompt parser encode')
-    if 'clip_skip' in possible and parser == 'Fixed attention':
-        if clip_skip == 1:
-            pass # clip_skip = None
-        else:
-            args['clip_skip'] = clip_skip - 1
+        timer.process.record('encode', reset=False)
+
     if 'prompt' in possible:
         if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None:
             args['prompt_embeds'] = p.prompt_embeds[0]
@@ -158,6 +140,29 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
                 args['negative_prompt'] = negative_prompts[0]
             else:
                 args['negative_prompt'] = negative_prompts
+
+    if 'clip_skip' in possible and parser == 'Fixed attention':
+        if clip_skip == 1:
+            pass # clip_skip = None
+        else:
+            args['clip_skip'] = clip_skip - 1
+
+    if 'timesteps' in possible:
+        timesteps = re.split(',| ', shared.opts.schedulers_timesteps)
+        timesteps = [int(x) for x in timesteps if x.isdigit()]
+        if len(timesteps) > 0:
+            if hasattr(model.scheduler, 'set_timesteps') and "timesteps" in set(inspect.signature(model.scheduler.set_timesteps).parameters.keys()):
+                try:
+                    args['timesteps'] = timesteps
+                    p.steps = len(timesteps)
+                    p.timesteps = timesteps
+                    steps = p.steps
+                    shared.log.debug(f'Sampler: steps={len(timesteps)} timesteps={timesteps}')
+                except Exception as e:
+                    shared.log.error(f'Sampler timesteps: {e}')
+            else:
+                shared.log.warning(f'Sampler: sampler={model.scheduler.__class__.__name__} timesteps not supported')
+
     if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'):
         model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values
         model.scheduler.noise_sampler_seed = p.seeds # some schedulers have internal noise generator and do not use pipeline generator
diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py
index 33416f103..47c8e8827 100644
--- a/modules/processing_callbacks.py
+++ b/modules/processing_callbacks.py
@@ -3,7 +3,7 @@ import os
 import time
 import torch
 import numpy as np
-from modules import shared, processing_correction, extra_networks
+from modules import shared, processing_correction, extra_networks, timer
 
 
 p = None
@@ -34,6 +34,7 @@ def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[to
 
 
 def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict):
+    t0 = time.time()
     if p is None:
         return kwargs
     latents = kwargs.get('latents', None)
@@ -98,4 +99,8 @@ def diffusers_callback(pipe, step: int, timestep: int, kwargs: dict):
         shared.log.error(f'Callback: {e}')
     if shared.cmd_opts.profile and shared.profiler is not None:
         shared.profiler.step()
+    t1 = time.time()
+    if 'callback' not in timer.process.records:
+        timer.process.records['callback'] = 0
+    timer.process.records['callback'] += t1 - t0
     return kwargs
diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index 6fa8cec24..0c458c9dc 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -4,7 +4,7 @@ import time
 import numpy as np
 import torch
 import torchvision.transforms.functional as TF
-from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion
+from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer
 from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler
 from modules.processing_args import set_pipeline_args
 from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
@@ -72,6 +72,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
     shared.sd_model = update_pipeline(shared.sd_model, p)
     shared.log.info(f'Base: class={shared.sd_model.__class__.__name__}')
     update_sampler(p, shared.sd_model)
+    timer.process.record('prepare')
     base_args = set_pipeline_args(
         p=p,
         model=shared.sd_model,
@@ -89,6 +90,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         clip_skip=p.clip_skip,
         desc='Base',
     )
+    timer.process.record('args')
     shared.state.sampling_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None)
     if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1:
         p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta
@@ -100,6 +102,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         hidiffusion.apply(p, shared.sd_model_type)
         # if 'image' in base_args:
         #    base_args['image'] = set_latents(p)
+        timer.process.record('move')
         if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0:
             base_args['gate_step'] = p.gate_step
             output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable
@@ -107,6 +110,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             output = shared.sd_model(**base_args)
         if isinstance(output, dict):
             output = SimpleNamespace(**output)
+        timer.process.record('pipeline')
         hidiffusion.unapply()
         sd_models_compile.openvino_post_compile(op="base") # only executes on compiled vino models
         sd_models_compile.check_deepcache(enable=False)
@@ -220,6 +224,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         shared.state.job = prev_job
         shared.state.nextjob()
         p.is_hr_pass = False
+        timer.process.record('hires')
 
     # optional refiner pass or decode
     if is_refiner_enabled():
@@ -297,6 +302,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         shared.state.job = prev_job
         shared.state.nextjob()
         p.is_refiner_pass = False
+        timer.process.record('refine')
 
     # final decode since there is no refiner
     if not is_refiner_enabled():
@@ -321,5 +327,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             shared.log.warning('Processing returned no results')
             results = []
 
+    timer.process.record('decode')
     shared.sd_model = orig_pipeline
     return results
diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py
index 73993a14f..b36146a1d 100644
--- a/modules/prompt_parser_diffusers.py
+++ b/modules/prompt_parser_diffusers.py
@@ -177,9 +177,11 @@ def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, c
             prompt_embed, positive_pooled, negative_embed, negative_pooled = None, None, None, None
             if last_prompt == prompt and last_negative == negative:
                 prompt_embeds.append(prompt_embeds[-1])
-                positive_pooleds.append(positive_pooleds[-1])
                 negative_embeds.append(negative_embeds[-1])
-                negative_pooleds.append(negative_pooleds[-1])
+                if len(positive_pooleds) > 0:
+                    positive_pooleds.append(positive_pooleds[-1])
+                if len(negative_pooleds) > 0:
+                    negative_pooleds.append(negative_pooleds[-1])
                 continue
             positive_schedule, scheduled = get_prompt_schedule(prompt, steps)
             negative_schedule, neg_scheduled = get_prompt_schedule(negative, steps)
diff --git a/modules/script_loading.py b/modules/script_loading.py
index 03c5fc5d9..f39394625 100644
--- a/modules/script_loading.py
+++ b/modules/script_loading.py
@@ -3,7 +3,7 @@ import os
 import contextlib
 import importlib.util
 import modules.errors as errors
-from installer import setup_logging, args
+from installer import setup_logging
 
 
 preloaded = []
@@ -13,10 +13,6 @@ debug = os.environ.get('SD_SCRIPT_DEBUG', None)
 def load_module(path):
     module_spec = importlib.util.spec_from_file_location(os.path.basename(path), path)
     module = importlib.util.module_from_spec(module_spec)
-    if args.profile:
-        import cProfile
-        pr = cProfile.Profile()
-        pr.enable()
     try:
         if '/sd-extension-' in path or '/Lora' in path: # safe extensions without stdout intercept
             module_spec.loader.exec_module(module)
@@ -33,8 +29,6 @@ def load_module(path):
                     errors.log.info(f"Extension: script='{os.path.relpath(path)}' {line.strip()}")
     except Exception as e:
         errors.display(e, f'Module load: {path}')
-    if args.profile:
-        errors.profile(pr, f'Scripts: {path}')
     return module
 
 
diff --git a/modules/sd_hijack_freeu.py b/modules/sd_hijack_freeu.py
index a1e560485..a62dc46a3 100644
--- a/modules/sd_hijack_freeu.py
+++ b/modules/sd_hijack_freeu.py
@@ -75,6 +75,24 @@ def free_u_cat_hijack(hs, *args, original_function, **kwargs):
     return original_function([h, h_skip], *args, **kwargs)
 
 
+torch_fft_device = None
+def get_fft_device():
+    global torch_fft_device # pylint: disable=global-statement
+    if torch_fft_device is None:
+        from modulkes import devices
+        try:
+            tensor = torch.randn(4, 4)
+            tensor = tensor.to(devices.device)
+            _fft_result = torch.fft.fftn(tensor)
+            _ifft_result = torch.fft.ifftn(_fft_result)
+            _shifted_tensor = torch.fft.fftshift(tensor)
+            _ishifted_tensor = torch.fft.ifftshift(_shifted_tensor)
+            torch_fft_device = devices.device
+        except Exception:
+            torch_fft_device = devices.cpu
+    return torch_fft_device
+
+
 def no_gpu_complex_support():
     mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
     try:
@@ -89,9 +107,9 @@ def no_gpu_complex_support():
 def filter_skip(x, threshold, scale, scale_high):
     if scale == 1 and scale_high == 1:
         return x
-    fft_device = x.device
-    if no_gpu_complex_support():
-        fft_device = "cpu"
+    fft_device = get_fft_device()
+    # if no_gpu_complex_support():
+    #    fft_device = "cpu"
     # FFT
     x_freq = torch.fft.fftn(x.to(fft_device).float(), dim=(-2, -1)) # pylint: disable=E1102
     x_freq = torch.fft.fftshift(x_freq, dim=(-2, -1)) # pylint: disable=E1102
diff --git a/modules/sd_models.py b/modules/sd_models.py
index 5c1997b93..702223498 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -1017,10 +1017,6 @@ def patch_diffuser_config(sd_model, model_file):
 
 
 def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument
-    if shared.cmd_opts.profile:
-        import cProfile
-        pr = cProfile.Profile()
-        pr.enable()
     if timer is None:
         timer = Timer()
     logging.getLogger("diffusers").setLevel(logging.ERROR)
@@ -1357,8 +1353,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
         errors.display(e, "Model")
 
     devices.torch_gc(force=True)
-    if shared.cmd_opts.profile:
-        errors.profile(pr, 'Load')
     script_callbacks.model_loaded_callback(sd_model)
     shared.log.info(f"Load {op}: time={timer.summary()} native={get_native(sd_model)} memory={memory_stats()}")
 
diff --git a/modules/timer.py b/modules/timer.py
index da3996f59..8a5db726d 100644
--- a/modules/timer.py
+++ b/modules/timer.py
@@ -1,4 +1,5 @@
 import time
+import sys
 
 
 class Timer:
@@ -7,29 +8,36 @@ class Timer:
         self.records = {}
         self.total = 0
 
-    def elapsed(self):
+    def elapsed(self, reset=True):
         end = time.time()
         res = end - self.start
-        self.start = end
+        if reset:
+            self.start = end
         return res
 
-    def record(self, category, extra_time=0):
-        e = self.elapsed()
+    def record(self, category=None, extra_time=0, reset=True):
+        e = self.elapsed(reset)
+        if category is None:
+            category = sys._getframe(1).f_code.co_name # pylint: disable=protected-access
         if category not in self.records:
             self.records[category] = 0
 
         self.records[category] += e + extra_time
         self.total += e + extra_time
 
-    def summary(self, min_time=0.05):
-        res = f"{self.total:.2f} "
+    def summary(self, min_time=0.05, total=True):
+        res = f"{self.total:.2f} " if total else ''
         additions = [x for x in self.records.items() if x[1] >= min_time]
         if not additions:
             return res
         res += " ".join([f"{category}={time_taken:.2f}" for category, time_taken in additions])
         return res
 
+    def dct(self, min_time=0.05):
+        return {k: round(v, 2) for k, v in self.records.items() if v >= min_time}
+
     def reset(self):
         self.__init__()
 
 startup = Timer()
+process = Timer()
diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py
index 430775661..7b585ff75 100644
--- a/modules/ui_extra_networks.py
+++ b/modules/ui_extra_networks.py
@@ -518,10 +518,6 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
     ui.tabs = gr.Tabs(elem_id=f"{tabname}_extra_tabs")
     ui.button_details = gr.Button('Details', elem_id=f"{tabname}_extra_details_btn", visible=False)
     state = {}
-    if shared.cmd_opts.profile:
-        import cProfile
-        pr = cProfile.Profile()
-        pr.enable()
 
     def get_item(state, params = None):
         if params is not None and type(params) == dict:
@@ -635,10 +631,6 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
                 page_html = gr.HTML(page.patch(page.html, tabname), elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page")
                 ui.pages.append(page_html)
                 tab.select(ui_tab_change, _js="getENActivePage", inputs=[ui.button_details], outputs=[ui.button_scan, ui.button_save, ui.button_model])
-    if shared.cmd_opts.profile:
-        errors.profile(pr, 'ExtraNetworks')
-        pr.disable()
-        # ui.tabs.change(fn=ui_tab_change, inputs=[], outputs=[ui.button_scan, ui.button_save])
 
     def fn_save_img(image):
         if ui.last_item is None or ui.last_item.local_preview is None: