Composable LoRA

modules/extra_networks.py

@@ -1,6 +1,6 @@
 import re
 from collections import defaultdict
-from modules import errors
+from modules import errors, shared
 
 
 extra_network_registry = {}
@@ -62,17 +62,29 @@ class ExtraNetwork:
         raise NotImplementedError
 
 
-def activate(p, extra_network_data):
+def is_stepwise(en_obj):
+    all_args = []
+    for en in en_obj:
+        all_args.extend(en.positional[1:])
+        all_args.extend(en.named.values())
+    return any([len(str(x).split("@")) > 1 for x in all_args])
+
+def activate(p, extra_network_data, step=0):
     """call activate for extra networks in extra_network_data in specified order, then call activate for all remaining registered networks with an empty argument list"""
     if extra_network_data is None:
         return
+    stepwise = False
+    for extra_network_args in extra_network_data.values():
+        stepwise = stepwise or is_stepwise(extra_network_args)
+    functional = shared.opts.lora_functional
+    shared.opts.lora_functional = stepwise or functional
     for extra_network_name, extra_network_args in extra_network_data.items():
         extra_network = extra_network_registry.get(extra_network_name, None)
         if extra_network is None:
             errors.log.warning(f"Skipping unknown extra network: {extra_network_name}")
             continue
         try:
-            extra_network.activate(p, extra_network_args)
+            extra_network.activate(p, extra_network_args, step=step)
         except Exception as e:
             errors.display(e, f"activating extra network: name={extra_network_name} args:{extra_network_args}")
 
@@ -84,6 +96,9 @@ def activate(p, extra_network_data):
             extra_network.activate(p, [])
         except Exception as e:
             errors.display(e, f"activating extra network: name={extra_network_name}")
+    if stepwise:
+        p.extra_network_data = extra_network_data
+        shared.opts.lora_functional = functional
 
 
 def deactivate(p, extra_network_data):

modules/processing_diffusers.py

@@ -8,7 +8,7 @@ import numpy as np
 import torch
 import torchvision.transforms.functional as TF
 import diffusers
-from modules import shared, devices, processing, sd_samplers, sd_models, images, errors, prompt_parser_diffusers, sd_hijack_hypertile, processing_correction, processing_vae, sd_models_compile
+from modules import shared, devices, processing, sd_samplers, sd_models, images, errors, prompt_parser_diffusers, sd_hijack_hypertile, processing_correction, processing_vae, sd_models_compile, extra_networks
 from modules.processing_helpers import resize_init_images, resize_hires, fix_prompts, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps
 from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
 
@@ -73,6 +73,14 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
                 if shared.state.interrupted or shared.state.skipped:
                     raise AssertionError('Interrupted...')
                 time.sleep(0.1)
+        if hasattr(p, "extra_network_data"):
+            if shared.opts.lora_force_diffusers:
+                shared.log.warning("Composable LoRA not compatible with 'lora_force_diffusers'")
+            else:
+                functional = shared.opts.lora_functional
+                shared.opts.lora_functional = True
+                extra_networks.activate(p, p.extra_network_data, step=step)
+                shared.opts.lora_functional = functional
         if latents is None:
             return kwargs
         elif shared.opts.nan_skip: