fix instantir

Signed-off-by: Vladimir Mandic <mandic00@live.com>
2026-09-18 16:54:33 +02:00 · 2025-07-25 08:12:04 -04:00
parent 5179255ff7
commit ffcb102ce5
3 changed files with 37 additions and 27 deletions
@@ -1,8 +1,8 @@
 # Change Log for SD.Next

-## Update for 2025-07-24
+## Update for 2025-07-25

-### Highlights for 2025-07-24
+### Highlights for 2025-07-25

 Feature highlights include:  
 - **ModernUI** layout redesign which should make it more user friendly and easier to navigate  
@@ -29,7 +29,7 @@ For details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master

 [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867)

-### Details for 2025-07-24
+### Details for 2025-07-25

 - **License**  
  - SD.Next [license](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) switched from **aGPL-v3.0** to **Apache-v2.0**  
@@ -142,6 +142,7 @@ For details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master
  - fix control batch-input processing  
  - fix modules merge save model  
  - fix torchvision bicubic upsample with ipex  
+  - fix instantir pipeline  
  - cleanup control infotext  
  - allow upscaling with models that have implicit VAE processing  
  - framepack improve offloading  
@@ -65,6 +65,7 @@ if is_invisible_watermark_available():

 from peft import LoraConfig, set_peft_model_state_dict
 from .aggregator import Aggregator
+from modules import sd_models, devices


 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
@@ -922,11 +923,8 @@ class InstantIRPipeline(
        else:
            # image batch size is the same as prompt batch size
            repeat_by = num_images_per_prompt
-
        image = image.repeat_interleave(repeat_by, dim=0)
-
        image = image.to(device=device, dtype=dtype)
-
        return image

    @torch.no_grad()
@@ -1513,6 +1511,8 @@ class InstantIRPipeline(

                # prepare time_embeds in advance as adapter input
                cross_attention_t_emb = self.unet.get_time_embed(sample=latent_model_input, timestep=t)
+                sd_models.move_model(self.unet, devices.device, force=True) # instantir does not handle offloading nicely
+
                cross_attention_emb = self.unet.time_embedding(cross_attention_t_emb, timestep_cond)
                cross_attention_aug_emb = None

@@ -25,11 +25,13 @@ class Script(scripts_manager.Script):
            end = gr.Slider(label='Preview end', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
        with gr.Row():
            hq = gr.Checkbox(label='HQ init latents', value=False)
+            unload = gr.Checkbox(label='Unload after processing', value=False, visible=False)
+        with gr.Row():
            multistep = gr.Checkbox(label='Multistep restore', value=False)
            adastep = gr.Checkbox(label='Adaptive restore', value=False)
        with gr.Row():
            image = gr.Image(label='Override guidance image')
-        return [start, end, hq, multistep, adastep, image]
+        return [start, end, hq, multistep, adastep, image, unload]

    def run(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ
        supported_model_list = ['sdxl']
@@ -39,18 +41,18 @@ class Script(scripts_manager.Script):
        if shared.sd_model_type not in supported_model_list and shared.sd_model.__class__.__name__ != "InstantIRPipeline":
            shared.log.warning(f'InstantIR: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}')
            return None
-        start, end, hq, multistep, adastep, image = args
-        from scripts import instantir as ir
+        start, end, hq, multistep, adastep, image, _unload = args
+        from scripts import instantir
        if shared.sd_model_type == "sdxl":
            if shared.sd_model.__class__.__name__ != "InstantIRPipeline":
                self.orig_pipe = shared.sd_model
                self.orig_ip_unapply = ipadapter.unapply
-                shared.sd_model = sd_models.switch_pipe(ir.InstantIRPipeline, shared.sd_model)
                adapter_file = hf_hub_download('InstantX/InstantIR', subfolder='models', filename='adapter.pt', cache_dir=shared.opts.hfcache_dir)
                aggregator_file = hf_hub_download('InstantX/InstantIR', subfolder='models', filename='aggregator.pt', cache_dir=shared.opts.hfcache_dir)
                previewer_file = hf_hub_download('InstantX/InstantIR', subfolder='models', filename='previewer_lora_weights.bin', cache_dir=shared.opts.hfcache_dir)
                shared.log.debug(f'InstantIR: adapter="{adapter_file}" aggregator="{aggregator_file}" previewer="{previewer_file}"')
-                ir.load_adapter_to_pipe(
+                shared.sd_model = sd_models.switch_pipe(instantir.InstantIRPipeline, shared.sd_model)
+                instantir.load_adapter_to_pipe(
                    pipe=shared.sd_model,
                    pretrained_model_path_or_dict=adapter_file,
                    image_encoder_or_path='facebook/dinov2-large',
@@ -62,11 +64,14 @@ class Script(scripts_manager.Script):
                pretrained_state_dict = torch.load(aggregator_file)
                shared.sd_model.aggregator.load_state_dict(pretrained_state_dict)
                shared.sd_model.aggregator.to(device=devices.device, dtype=devices.dtype)
+                ipadapter.unapply = self.dummy_unapply # disable as main processing unloads ipadapter as it thinks its not needed
+                sd_models.clear_caches()
+                sd_models.apply_balanced_offload(shared.sd_model)

        shared.log.info(f'InstantIR: class={shared.sd_model.__class__.__name__} start={start} end={end} multistep={multistep} adastep={adastep} hq={hq} cache={shared.opts.hfcache_dir}')
        p.sampler_name = 'Default' # ir has its own sampler
        p.init() # run init early to take care of resizing
-        p.task_args['previewer_scheduler'] = ir.LCMSingleStepScheduler.from_config(shared.sd_model.scheduler.config)
+        p.task_args['previewer_scheduler'] = instantir.LCMSingleStepScheduler.from_config(shared.sd_model.scheduler.config)
        p.task_args['image'] = p.init_images
        p.task_args['save_preview_row'] = False
        p.task_args['init_latents_with_lq'] = not hq
@@ -76,22 +81,26 @@ class Script(scripts_manager.Script):
        p.task_args['preview_end'] = end
        p.task_args['ip_adapter_image'] = image
        p.extra_generation_params["InstantIR"] = f'Start={start} End={end} HQ={hq} Multistep={multistep} Adastep={adastep}'
-        ipadapter.unapply = lambda x: x # disable as main processing unloads ipadapter as it thinks its not needed
        devices.torch_gc()

+    def dummy_unapply(self, pipe, unload): # pylint: disable=unused-argument
+        pass
+
    def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=arguments-differ, unused-argument
-        # TODO instantir: a mess to unload
-        """
-        if self.orig_pipe is None:
-            return processed
-        if hasattr(shared.sd_model, 'aggregator'):
-            shared.sd_model.aggregator = None
-        shared.log.debug(f'InstantIR restore: class={shared.sd_model.__class__.__name__}')
-        shared.sd_model = self.orig_pipe
-        self.orig_pipe = None
-        shared.sd_model.unet.register_to_config(encoder_hid_dim_type=None)
-        ipadapter.unapply = self.orig_ip_unapply
-        ipadapter.unapply(shared.sd_model)
-        devices.torch_gc()
-        """
+        _start, _end, _hq, _multistep, _adastep, _image, unload = args
+        if unload:
+            shared.log.info('InstantIR: unloading adapter')
+            if self.orig_ip_unapply is not None:
+                ipadapter.unapply = self.orig_ip_unapply
+                self.orig_ip_unapply = None
+                ipadapter.unapply(shared.sd_model)
+            if hasattr(shared.sd_model, 'aggregator'):
+                shared.sd_model.aggregator = None
+            if self.orig_pipe is not None:
+                shared.sd_model = self.orig_pipe
+                self.orig_pipe = None
+            shared.sd_model.unet.register_to_config(encoder_hid_dim_type=None)
+            sd_models.apply_balanced_offload(shared.sd_model)
+            shared.log.debug(f'InstantIR restore: class={shared.sd_model.__class__.__name__}')
+            devices.torch_gc()
        return processed