diff --git a/CHANGELOG.md b/CHANGELOG.md index cfd929c02..1cc4f754d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,17 +45,19 @@ Upgrades are still possible and supported, but above is recommended for best exp faster loading, wider compatibility and support for embeddings with multiple vectors information about used embedding is now also added to image metadata - **Upscalers**: + - fix long outstanding memory leak in legacy code, amazing this went undetected for so long - more high quality upscalers available by default *SwinIR:2, ESRGAN:12, RealESRGAN:6, SCUNet:2* - two additional latent upscalers based on SD upscale models when using Diffusers backend *SD Upscale 2x, SD Upscale 4x* Note: Recommended usage for *SD Upscale* is by using second pass instead of upscaler as it allows for tuning of prompt, seed, sampler settings which are used to guide upscaler - - unified init/download/execute/progress code - - easier installation - - available in **xyz grid** + - upscalers are available in **xyz grid** + - simplified *settings->postprocessing->upscalers* - allow upscale-only as part of **txt2img** and **img2img** workflows simply set *denoising strength* to 0 so hires does not get triggered + - unified init/download/execute/progress code + - easier installation - **Samplers**: - moved ui options to submenu - default list for new installs is now all samplers, list can be modified in settings diff --git a/extensions-builtin/LDSR/ldsr_model_arch.py b/extensions-builtin/LDSR/ldsr_model_arch.py index 4153139ba..407241e65 100644 --- a/extensions-builtin/LDSR/ldsr_model_arch.py +++ b/extensions-builtin/LDSR/ldsr_model_arch.py @@ -20,18 +20,18 @@ cached_ldsr_model: torch.nn.Module = None # Create LDSR Class class LDSR: def load_model_from_config(self, half_attention): - global cached_ldsr_model + global cached_ldsr_model # pylint: disable=global-statement - if shared.opts.ldsr_cached and cached_ldsr_model is not None: - shared.log.info("LDSR Loading model from cache") + if cached_ldsr_model is not None: + shared.log.info(f"Upscaler cached: type=LDSR model={self.modelPath}") model: torch.nn.Module = cached_ldsr_model else: - shared.log.info(f"LDSR Loading model from {self.modelPath}") _, extension = os.path.splitext(self.modelPath) if extension.lower() == ".safetensors": pl_sd = safetensors.torch.load_file(self.modelPath, device="cpu") else: pl_sd = torch.load(self.modelPath, map_location="cpu") + shared.log.info(f"Upscaler loaded: type=LDSR model={self.modelPath}") sd = pl_sd["state_dict"] if "state_dict" in pl_sd else pl_sd config = OmegaConf.load(self.yamlPath) config.model.target = "ldm.models.diffusion.ddpm.LatentDiffusionV1" @@ -42,13 +42,9 @@ class LDSR: model = model.half() if shared.cmd_opts.opt_channelslast: model = model.to(memory_format=torch.channels_last) - sd_hijack.model_hijack.hijack(model) # apply optimization model.eval() - - if shared.opts.ldsr_cached: - cached_ldsr_model = model - + cached_ldsr_model = model return {"model": model} def __init__(self, model_path, yaml_path): @@ -58,18 +54,15 @@ class LDSR: @staticmethod def run(model, selected_path, custom_steps, eta): example = get_cond(selected_path) - n_runs = 1 guider = None ckwargs = None ddim_use_x0_pred = False temperature = 1. - eta = eta + eta = eta # pylint: disable=self-assigning-variable custom_shape = None - height, width = example["image"].shape[1:3] split_input = height >= 128 and width >= 128 - if split_input: ks = 128 stride = 64 @@ -105,14 +98,9 @@ class LDSR: def super_resolution(self, image, steps=100, target_scale=2, half_attention=False): model = self.load_model_from_config(half_attention) - # Run settings diffusion_steps = int(steps) eta = 1.0 - - gc.collect() - devices.torch_gc() - im_og = image width_og, height_og = im_og.size # If we can adjust the max upscale size, then the 4 below should be our variable @@ -121,7 +109,6 @@ class LDSR: hd = height_og * down_sample_rate width_downsampled_pre = int(np.ceil(wd)) height_downsampled_pre = int(np.ceil(hd)) - if down_sample_rate != 1: shared.log.info(f'LDSR Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]') im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS) @@ -141,13 +128,14 @@ class LDSR: sample = sample.numpy().astype(np.uint8) sample = np.transpose(sample, (0, 2, 3, 1)) a = Image.fromarray(sample[0]) - # remove padding a = a.crop((0, 0) + tuple(np.array(im_og.size) * 4)) - del model - gc.collect() - devices.torch_gc() + if shared.opts.upscaler_unload: + del model + cached_ldsr_model = None + shared.log.debug(f"Upscaler unloaded: type=LDSR model={self.modelPath}") + devices.torch_gc(force=True) return a diff --git a/extensions-builtin/LDSR/scripts/ldsr_model.py b/extensions-builtin/LDSR/scripts/ldsr_model.py index e2c671b15..50eb9f2b7 100644 --- a/extensions-builtin/LDSR/scripts/ldsr_model.py +++ b/extensions-builtin/LDSR/scripts/ldsr_model.py @@ -67,9 +67,6 @@ class UpscalerLDSR(Upscaler): def on_ui_settings(): import gradio as gr - - shared.opts.add_option("ldsr_steps", shared.OptionInfo(100, "LDSR processing steps. Lower = faster", gr.Slider, {"minimum": 1, "maximum": 200, "step": 1}, section=('postprocessing', "Postprocessing"))) - shared.opts.add_option("ldsr_cached", shared.OptionInfo(False, "Cache LDSR model in memory", gr.Checkbox, {"interactive": True}, section=('postprocessing', "Postprocessing"))) - + shared.opts.add_option("ldsr_steps", shared.OptionInfo(100, "LDSR processing steps", gr.Slider, {"minimum": 1, "maximum": 200, "step": 1}, section=('postprocessing', "Postprocessing"))) script_callbacks.on_ui_settings(on_ui_settings) diff --git a/modules/postprocess/esrgan_model.py b/modules/postprocess/esrgan_model.py index 8f1291b13..8af099ea0 100644 --- a/modules/postprocess/esrgan_model.py +++ b/modules/postprocess/esrgan_model.py @@ -188,10 +188,10 @@ def upscale_without_tiling(model, img): def esrgan_upscale(model, img): - if opts.ESRGAN_tile == 0: + if opts.upscaler_tile_size == 0: return upscale_without_tiling(model, img) - grid = images.split_grid(img, opts.ESRGAN_tile, opts.ESRGAN_tile, opts.ESRGAN_tile_overlap) + grid = images.split_grid(img, opts.upscaler_tile_size, opts.upscaler_tile_size, opts.upscaler_tile_overlap) newtiles = [] scale_factor = 1 diff --git a/modules/postprocess/realesrgan_model.py b/modules/postprocess/realesrgan_model.py index 92cf375d2..af6bb4705 100644 --- a/modules/postprocess/realesrgan_model.py +++ b/modules/postprocess/realesrgan_model.py @@ -55,8 +55,8 @@ class UpscalerRealESRGAN(Upscaler): model_path=info.local_data_path, model=info.model(), half=not opts.no_half and not opts.upcast_sampling, - tile=opts.ESRGAN_tile, - tile_pad=opts.ESRGAN_tile_overlap, + tile=opts.upscaler_tile_size, + tile_pad=opts.upscaler_tile_overlap, device=device, ) self.models[info.local_data_path] = upsampler diff --git a/modules/postprocess/scunet_model.py b/modules/postprocess/scunet_model.py index 3b21006af..dcb8ff496 100644 --- a/modules/postprocess/scunet_model.py +++ b/modules/postprocess/scunet_model.py @@ -39,8 +39,8 @@ class UpscalerSCUNet(Upscaler): def tiled_inference(img, model): # test the image tile by tile h, w = img.shape[2:] - tile = opts.SCUNET_tile - tile_overlap = opts.SCUNET_tile_overlap + tile = opts.upscaler_tile_size + tile_overlap = opts.upscaler_tile_overlap if tile == 0: return model(img) assert tile % 8 == 0, "tile size should be a multiple of window_size" @@ -72,7 +72,7 @@ class UpscalerSCUNet(Upscaler): model = self.load_model(selected_file) if model is None: return img - tile = opts.SCUNET_tile + tile = opts.upscaler_tile_size h, w = img.height, img.width np_img = np.array(img) np_img = np_img[:, :, ::-1] # RGB to BGR @@ -95,13 +95,3 @@ class UpscalerSCUNet(Upscaler): log.debug(f"Upscaler unloaded: type={self.name} model={selected_file}") devices.torch_gc(force=True) return img - - -def on_ui_settings(): - import gradio as gr - from modules import shared - shared.opts.add_option("SCUNET_tile", shared.OptionInfo(256, "Tile size for SCUNET upscalers", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}, section=('postprocessing', "Postprocessing")).info("0 = no tiling")) - shared.opts.add_option("SCUNET_tile_overlap", shared.OptionInfo(8, "Tile overlap for SCUNET upscalers", gr.Slider, {"minimum": 0, "maximum": 64, "step": 1}, section=('postprocessing', "Postprocessing")).info("Low values = visible seam")) - - -script_callbacks.on_ui_settings(on_ui_settings) diff --git a/modules/postprocess/swinir_model.py b/modules/postprocess/swinir_model.py index 2ccfab042..fd0240ce4 100644 --- a/modules/postprocess/swinir_model.py +++ b/modules/postprocess/swinir_model.py @@ -85,8 +85,8 @@ def upscale( window_size=8, scale=4, ): - tile = tile or shared.opts.SWIN_tile - tile_overlap = tile_overlap or shared.opts.SWIN_tile_overlap + tile = tile or shared.opts.upscaler_tile_size + tile_overlap = tile_overlap or shared.opts.upscaler_tile_overlap img = np.array(img) img = img[:, :, ::-1] img = np.moveaxis(img, 2, 0) / 255 @@ -140,12 +140,3 @@ def inference(img, model, tile, tile_overlap, window_size, scale): progress.update(task, advance=1, description="Upscaling") output = E.div_(W) return output - - -def on_ui_settings(): - import gradio as gr - shared.opts.add_option("SWIN_tile", shared.OptionInfo(192, "Tile size for SwinIR upscaler", gr.Slider, {"minimum": 16, "maximum": 512, "step": 16}, section=('postprocessing', "Postprocessing"))) - shared.opts.add_option("SWIN_tile_overlap", shared.OptionInfo(8, "Tile overlap for SwinIR upscaler", gr.Slider, {"minimum": 0, "maximum": 48, "step": 1}, section=('postprocessing', "Postprocessing"))) - - -script_callbacks.on_ui_settings(on_ui_settings) diff --git a/modules/shared.py b/modules/shared.py index 4470f757c..58daa6726 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -614,13 +614,10 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "postprocessing_sep_upscalers": OptionInfo("

Upscaling

", "", gr.HTML), "upscaler_unload": OptionInfo(False, "Unload upscaler after processing"), - 'upscaling_max_images_in_cache': OptionInfo(5, "Maximum number of images in upscaling cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": False}), + # 'upscaling_max_images_in_cache': OptionInfo(5, "Maximum number of images in upscaling cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": False}), "upscaler_for_img2img": OptionInfo("None", "Default upscaler for image resize operations", gr.Dropdown, lambda: {"choices": [x.name for x in sd_upscalers]}), - # "realesrgan_enabled_models": OptionInfo(["R-ESRGAN 4x+", "R-ESRGAN 4x+ Anime6B"], "Real-ESRGAN available models", gr.CheckboxGroup, lambda: {"choices": shared_items.realesrgan_models_names()}), - "ESRGAN_tile": OptionInfo(192, "Tile size for ESRGAN upscalers", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}), - "ESRGAN_tile_overlap": OptionInfo(8, "Tile overlap in pixels for ESRGAN upscalers", gr.Slider, {"minimum": 0, "maximum": 48, "step": 1}), - "SCUNET_tile": OptionInfo(256, "Tile size for SCUNET upscalers", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}), - "SCUNET_tile_overlap": OptionInfo(8, "Tile overlap for SCUNET upscalers", gr.Slider, {"minimum": 0, "maximum": 64, "step": 1}), + "upscaler_tile_size": OptionInfo(192, "Upscaler tile size", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}), + "upscaler_tile_overlap": OptionInfo(8, "Upscaler tile overlap", gr.Slider, {"minimum": 0, "maximum": 64, "step": 1}), })) options_templates.update(options_section(('training', "Training"), { diff --git a/modules/upscaler.py b/modules/upscaler.py index c008dbca5..6a8c723c5 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -28,8 +28,8 @@ class Upscaler: if models is None: models = modules.shared.readfile('html/upscalers.json') self.mod_pad_h = None - self.tile_size = modules.shared.opts.ESRGAN_tile - self.tile_pad = modules.shared.opts.ESRGAN_tile_overlap + self.tile_size = modules.shared.opts.upscaler_tile_size + self.tile_pad = modules.shared.opts.upscaler_tile_overlap self.device = modules.shared.device self.img = None self.output = None