From ffcb102ce59e45383c567286408c776e2ea341b5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 25 Jul 2025 08:12:04 -0400 Subject: [PATCH] fix instantir Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 ++-- scripts/instantir/sdxl_instantir.py | 6 ++-- scripts/instantir_ext.py | 51 +++++++++++++++++------------ 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce678621..dbc49faff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2025-07-24 +## Update for 2025-07-25 -### Highlights for 2025-07-24 +### Highlights for 2025-07-25 Feature highlights include: - **ModernUI** layout redesign which should make it more user friendly and easier to navigate @@ -29,7 +29,7 @@ For details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-07-24 +### Details for 2025-07-25 - **License** - SD.Next [license](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) switched from **aGPL-v3.0** to **Apache-v2.0** @@ -142,6 +142,7 @@ For details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master - fix control batch-input processing - fix modules merge save model - fix torchvision bicubic upsample with ipex + - fix instantir pipeline - cleanup control infotext - allow upscaling with models that have implicit VAE processing - framepack improve offloading diff --git a/scripts/instantir/sdxl_instantir.py b/scripts/instantir/sdxl_instantir.py index 9e18567e0..5e5df3ece 100644 --- a/scripts/instantir/sdxl_instantir.py +++ b/scripts/instantir/sdxl_instantir.py @@ -65,6 +65,7 @@ if is_invisible_watermark_available(): from peft import LoraConfig, set_peft_model_state_dict from .aggregator import Aggregator +from modules import sd_models, devices logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -922,11 +923,8 @@ class InstantIRPipeline( else: # image batch size is the same as prompt batch size repeat_by = num_images_per_prompt - image = image.repeat_interleave(repeat_by, dim=0) - image = image.to(device=device, dtype=dtype) - return image @torch.no_grad() @@ -1513,6 +1511,8 @@ class InstantIRPipeline( # prepare time_embeds in advance as adapter input cross_attention_t_emb = self.unet.get_time_embed(sample=latent_model_input, timestep=t) + sd_models.move_model(self.unet, devices.device, force=True) # instantir does not handle offloading nicely + cross_attention_emb = self.unet.time_embedding(cross_attention_t_emb, timestep_cond) cross_attention_aug_emb = None diff --git a/scripts/instantir_ext.py b/scripts/instantir_ext.py index a64cc6349..0cb2cc5a0 100644 --- a/scripts/instantir_ext.py +++ b/scripts/instantir_ext.py @@ -25,11 +25,13 @@ class Script(scripts_manager.Script): end = gr.Slider(label='Preview end', minimum=0.0, maximum=1.0, step=0.01, value=1.0) with gr.Row(): hq = gr.Checkbox(label='HQ init latents', value=False) + unload = gr.Checkbox(label='Unload after processing', value=False, visible=False) + with gr.Row(): multistep = gr.Checkbox(label='Multistep restore', value=False) adastep = gr.Checkbox(label='Adaptive restore', value=False) with gr.Row(): image = gr.Image(label='Override guidance image') - return [start, end, hq, multistep, adastep, image] + return [start, end, hq, multistep, adastep, image, unload] def run(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ supported_model_list = ['sdxl'] @@ -39,18 +41,18 @@ class Script(scripts_manager.Script): if shared.sd_model_type not in supported_model_list and shared.sd_model.__class__.__name__ != "InstantIRPipeline": shared.log.warning(f'InstantIR: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') return None - start, end, hq, multistep, adastep, image = args - from scripts import instantir as ir + start, end, hq, multistep, adastep, image, _unload = args + from scripts import instantir if shared.sd_model_type == "sdxl": if shared.sd_model.__class__.__name__ != "InstantIRPipeline": self.orig_pipe = shared.sd_model self.orig_ip_unapply = ipadapter.unapply - shared.sd_model = sd_models.switch_pipe(ir.InstantIRPipeline, shared.sd_model) adapter_file = hf_hub_download('InstantX/InstantIR', subfolder='models', filename='adapter.pt', cache_dir=shared.opts.hfcache_dir) aggregator_file = hf_hub_download('InstantX/InstantIR', subfolder='models', filename='aggregator.pt', cache_dir=shared.opts.hfcache_dir) previewer_file = hf_hub_download('InstantX/InstantIR', subfolder='models', filename='previewer_lora_weights.bin', cache_dir=shared.opts.hfcache_dir) shared.log.debug(f'InstantIR: adapter="{adapter_file}" aggregator="{aggregator_file}" previewer="{previewer_file}"') - ir.load_adapter_to_pipe( + shared.sd_model = sd_models.switch_pipe(instantir.InstantIRPipeline, shared.sd_model) + instantir.load_adapter_to_pipe( pipe=shared.sd_model, pretrained_model_path_or_dict=adapter_file, image_encoder_or_path='facebook/dinov2-large', @@ -62,11 +64,14 @@ class Script(scripts_manager.Script): pretrained_state_dict = torch.load(aggregator_file) shared.sd_model.aggregator.load_state_dict(pretrained_state_dict) shared.sd_model.aggregator.to(device=devices.device, dtype=devices.dtype) + ipadapter.unapply = self.dummy_unapply # disable as main processing unloads ipadapter as it thinks its not needed + sd_models.clear_caches() + sd_models.apply_balanced_offload(shared.sd_model) shared.log.info(f'InstantIR: class={shared.sd_model.__class__.__name__} start={start} end={end} multistep={multistep} adastep={adastep} hq={hq} cache={shared.opts.hfcache_dir}') p.sampler_name = 'Default' # ir has its own sampler p.init() # run init early to take care of resizing - p.task_args['previewer_scheduler'] = ir.LCMSingleStepScheduler.from_config(shared.sd_model.scheduler.config) + p.task_args['previewer_scheduler'] = instantir.LCMSingleStepScheduler.from_config(shared.sd_model.scheduler.config) p.task_args['image'] = p.init_images p.task_args['save_preview_row'] = False p.task_args['init_latents_with_lq'] = not hq @@ -76,22 +81,26 @@ class Script(scripts_manager.Script): p.task_args['preview_end'] = end p.task_args['ip_adapter_image'] = image p.extra_generation_params["InstantIR"] = f'Start={start} End={end} HQ={hq} Multistep={multistep} Adastep={adastep}' - ipadapter.unapply = lambda x: x # disable as main processing unloads ipadapter as it thinks its not needed devices.torch_gc() + def dummy_unapply(self, pipe, unload): # pylint: disable=unused-argument + pass + def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=arguments-differ, unused-argument - # TODO instantir: a mess to unload - """ - if self.orig_pipe is None: - return processed - if hasattr(shared.sd_model, 'aggregator'): - shared.sd_model.aggregator = None - shared.log.debug(f'InstantIR restore: class={shared.sd_model.__class__.__name__}') - shared.sd_model = self.orig_pipe - self.orig_pipe = None - shared.sd_model.unet.register_to_config(encoder_hid_dim_type=None) - ipadapter.unapply = self.orig_ip_unapply - ipadapter.unapply(shared.sd_model) - devices.torch_gc() - """ + _start, _end, _hq, _multistep, _adastep, _image, unload = args + if unload: + shared.log.info('InstantIR: unloading adapter') + if self.orig_ip_unapply is not None: + ipadapter.unapply = self.orig_ip_unapply + self.orig_ip_unapply = None + ipadapter.unapply(shared.sd_model) + if hasattr(shared.sd_model, 'aggregator'): + shared.sd_model.aggregator = None + if self.orig_pipe is not None: + shared.sd_model = self.orig_pipe + self.orig_pipe = None + shared.sd_model.unet.register_to_config(encoder_hid_dim_type=None) + sd_models.apply_balanced_offload(shared.sd_model) + shared.log.debug(f'InstantIR restore: class={shared.sd_model.__class__.__name__}') + devices.torch_gc() return processed