xyzgrid with control tab

Signed-off-by: Vladimir Mandic <mandic00@live.com>

modules/control/processors.py

@@ -235,6 +235,8 @@ class Processor():
         if image_input is None:
             # log.error('Control Processor: no input')
             return image_process
+        if isinstance(image_input, list):
+            image_input = image_input[0]
         if self.processor_id not in config:
             return image_process
         if config[self.processor_id].get('dirty', False):

modules/control/processors_list.py

@@ -0,0 +1,23 @@
+processors = [
+    'None',
+    'OpenPose',
+    'DWPose',
+    'MediaPipe Face',
+    'Canny',
+    'Edge',
+    'LineArt Realistic',
+    'LineArt Anime',
+    'HED',
+    'PidiNet',
+    'Midas Depth Hybrid',
+    'Leres Depth',
+    'Zoe Depth',
+    'Marigold Depth',
+    'Normal Bae',
+    'SegmentAnything',
+    'MLSD',
+    'Shuffle',
+    'DPT Depth Hybrid',
+    'GLPN Depth',
+    'Depth Anything',
+]

modules/control/run.py

@@ -51,8 +51,7 @@ def is_unified_model():
     return shared.sd_model.__class__.__name__ in unified_models
 
 
-def set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits):
-    print('HERE SET PIPE')
+def set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits=None):
     global pipe, instance # pylint: disable=global-statement
     pipe = None
     if has_models:
@@ -119,12 +118,13 @@ def set_pipe(p, has_models, unit_type, selected_models, active_model, active_str
             p.strength = active_strength[0]
         pipe = shared.sd_model
         instance = None
+    if (pipe is not None) and (pipe.__class__.__name__ != shared.sd_model.__class__.__name__):
+        sd_models.copy_diffuser_options(pipe, shared.sd_model) # copy options from original pipeline
     debug_log(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}')
     return pipe
 
 
 def check_active(p, unit_type, units):
-    print('HERE CHECK ACTIVE')
     active_process: List[processors.Processor] = [] # all active preprocessors
     active_model: List[Union[controlnet.ControlNet, xs.ControlNetXS, t2iadapter.Adapter]] = [] # all active models
     active_strength: List[float] = [] # strength factors for all active models
@@ -194,7 +194,6 @@ def check_active(p, unit_type, units):
 
 
 def check_enabled(p, unit_type, units, active_model, active_strength, active_start, active_end):
-    print('HERE CHECK ENABLED')
     has_models = False
     selected_models: List[Union[controlnet.ControlNetModel, xs.ControlNetXSModel, t2iadapter.AdapterModel]] = None
     control_conditioning = None
@@ -233,7 +232,6 @@ def control_set(kwargs):
 
 
 def init_units(units: List[unit.Unit]):
-    print('HERE INIT UNITS')
     for u in units:
         if not u.enabled:
             continue
@@ -397,8 +395,6 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
     elif p.enable_hr and (p.hr_upscale_to_x == 0 or p.hr_upscale_to_y == 0):
         p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.hr_resize_x / 8), 8 * int(hr_resize_y / 8)
 
-    if is_unified_model():
-        p.init_images = inputs
 
     global p_extra_args # pylint: disable=global-statement
     for k, v in p_extra_args.items():
@@ -420,6 +416,8 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
     info_txt = []
 
     p.is_tile = p.is_tile and has_models
+    if is_unified_model():
+        p.init_images = inputs
 
     pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits)
     debug_log(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}')

modules/control/unit.py

@@ -18,9 +18,9 @@ unit_types = ['t2i adapter', 'controlnet', 'xs', 'lite', 'reference', 'ip']
 class Unit(): # mashup of gradio controls and mapping to actual implementation classes
     def update_choices(self, model_id=None):
         name = model_id or self.model_name
-        if name == 'InstantX Union':
+        if name == 'InstantX Union F1':
             self.choices = ['canny', 'tile', 'depth', 'blur', 'pose', 'gray', 'lq']
-        elif name == 'Shakker-Labs Union':
+        elif name == 'Shakker-Labs Union F1':
             self.choices = ['canny', 'tile', 'depth', 'blur', 'pose', 'gray', 'lq']
         elif name == 'Xinsir Union XL':
             self.choices = ['openpose', 'depth', 'scribble', 'canny', 'normal']

modules/control/units/controlnet.py

@@ -77,19 +77,19 @@ predefined_sdxl = {
     # 'StabilityAI Sketch R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-sketch-rank256.safetensors',
 }
 predefined_f1 = {
-    "InstantX Union": 'InstantX/FLUX.1-dev-Controlnet-Union',
-    "InstantX Canny": 'InstantX/FLUX.1-dev-Controlnet-Canny',
-    "JasperAI Depth": 'jasperai/Flux.1-dev-Controlnet-Depth',
-    "BlackForrestLabs Canny LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora/flux1-canny-dev-lora.safetensors',
-    "BlackForrestLabs Depth LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Depth-dev-lora/flux1-depth-dev-lora.safetensors',
-    "JasperAI Surface Normals": 'jasperai/Flux.1-dev-Controlnet-Surface-Normals',
-    "JasperAI Upscaler": 'jasperai/Flux.1-dev-Controlnet-Upscaler',
-    "Shakker-Labs Union": 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro',
-    "Shakker-Labs Pose": 'Shakker-Labs/FLUX.1-dev-ControlNet-Pose',
-    "Shakker-Labs Depth": 'Shakker-Labs/FLUX.1-dev-ControlNet-Depth',
-    "XLabs-AI Canny": 'XLabs-AI/flux-controlnet-canny-diffusers',
-    "XLabs-AI Depth": 'XLabs-AI/flux-controlnet-depth-diffusers',
-    "XLabs-AI HED": 'XLabs-AI/flux-controlnet-hed-diffusers'
+    "InstantX Union F1": 'InstantX/FLUX.1-dev-Controlnet-Union',
+    "InstantX Canny F1": 'InstantX/FLUX.1-dev-Controlnet-Canny',
+    "JasperAI Depth F1": 'jasperai/Flux.1-dev-Controlnet-Depth',
+    "BlackForrestLabs Canny LoRA F1": '/huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora/flux1-canny-dev-lora.safetensors',
+    "BlackForrestLabs Depth LoRA F1": '/huggingface.co/black-forest-labs/FLUX.1-Depth-dev-lora/flux1-depth-dev-lora.safetensors',
+    "JasperAI Surface Normals F1": 'jasperai/Flux.1-dev-Controlnet-Surface-Normals',
+    "JasperAI Upscaler F1": 'jasperai/Flux.1-dev-Controlnet-Upscaler',
+    "Shakker-Labs Union F1": 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro',
+    "Shakker-Labs Pose F1": 'Shakker-Labs/FLUX.1-dev-ControlNet-Pose',
+    "Shakker-Labs Depth F1": 'Shakker-Labs/FLUX.1-dev-ControlNet-Depth',
+    "XLabs-AI Canny F1": 'XLabs-AI/flux-controlnet-canny-diffusers',
+    "XLabs-AI Depth F1": 'XLabs-AI/flux-controlnet-depth-diffusers',
+    "XLabs-AI HED F1": 'XLabs-AI/flux-controlnet-hed-diffusers'
 }
 predefined_sd3 = {
     "StabilityAI Canny SD35": 'diffusers-internal-dev/sd35-controlnet-canny-8b',
@@ -452,6 +452,7 @@ class ControlNetPipeline():
         debug_log(f'Control {what} pipeline: class={self.pipeline.__class__.__name__} time={t1-t0:.2f}')
 
     def restore(self):
-        self.pipeline.unload_lora_weights()
+        if self.pipeline is not None:
+            self.pipeline.unload_lora_weights()
         self.pipeline = None
         return self.orig_pipeline

modules/control/units/detect.py

@@ -1,31 +1,51 @@
 import diffusers.pipelines as p
 
 
-def is_sd15(model):
+def is_compatible(model, compatible):
     if model is None:
         return False
-    if hasattr(model, '__name__'):
-        return model.__name__ == p.StableDiffusionPipeline.__name__ or model.__name__ == p.StableDiffusionImg2ImgPipeline.__name__ or model.__name__ == p.StableDiffusionInpaintPipeline.__name__
-    return isinstance(model, p.StableDiffusionPipeline) or isinstance(model, p.StableDiffusionImg2ImgPipeline) or isinstance(model, p.StableDiffusionInpaintPipeline)
+    if hasattr(model, '__class__'):
+        return any(model.__class__.__name__ == c.__name__ for c in compatible)
+    return any(isinstance(model, c) for c in compatible)
+
+
+def is_sd15(model):
+    compatible = [
+        p.StableDiffusionPipeline,
+        p.StableDiffusionImg2ImgPipeline,
+        p.StableDiffusionInpaintPipeline,
+        p.StableDiffusionControlNetPipeline,
+    ]
+    return is_compatible(model, compatible)
 
 
 def is_sdxl(model):
-    if model is None:
-        return False
-    if hasattr(model, '__name__'):
-        return model.__name__ == p.StableDiffusionXLPipeline.__name__ or model.__name__ == p.StableDiffusionXLImg2ImgPipeline.__name__ or model.__name__ == p.StableDiffusionXLInpaintPipeline.__name__
-    return isinstance(model, p.StableDiffusionXLPipeline) or isinstance(model, p.StableDiffusionXLImg2ImgPipeline) or isinstance(model, p.StableDiffusionXLInpaintPipeline)
+    compatible = [
+        p.StableDiffusionXLPipeline,
+        p.StableDiffusionXLImg2ImgPipeline,
+        p.StableDiffusionXLInpaintPipeline,
+        p.StableDiffusionXLControlNetPipeline,
+        p.StableDiffusionXLControlNetImg2ImgPipeline,
+        p.StableDiffusionXLControlNetUnionPipeline,
+    ]
+    return is_compatible(model, compatible)
+
 
 def is_f1(model):
-    if model is None:
-        return False
-    if hasattr(model, '__name__'):
-        return model.__name__ == p.FluxPipeline.__name__ or model.__name__ == p.FluxImg2ImgPipeline.__name__ or model.__name__ == p.FluxInpaintPipeline.__name__
-    return isinstance(model, p.FluxPipeline) or isinstance(model, p.FluxImg2ImgPipeline) or isinstance(model, p.FluxInpaintPipeline)
+    compatible = [
+        p.FluxPipeline,
+        p.FluxImg2ImgPipeline,
+        p.FluxInpaintPipeline,
+        p.FluxControlNetPipeline,
+    ]
+    return is_compatible(model, compatible)
+
 
 def is_sd3(model):
-    if model is None:
-        return False
-    if hasattr(model, '__name__'):
-        return model.__name__ == p.StableDiffusion3Pipeline.__name__ or model.__name__ == p.StableDiffusion3Img2ImgPipeline.__name__ or model.__name__ == p.StableDiffusion3InpaintPipeline.__name__
-    return isinstance(model, p.StableDiffusion3Pipeline) or isinstance(model, p.StableDiffusion3Img2ImgPipeline) or isinstance(model, p.StableDiffusion3InpaintPipeline)
+    compatible = [
+        p.StableDiffusion3Pipeline,
+        p.StableDiffusion3Img2ImgPipeline,
+        p.StableDiffusion3InpaintPipeline,
+        p.StableDiffusion3ControlNetPipeline,
+    ]
+    return is_compatible(model, compatible)