update control adapters

modules/control/adapters.py

@@ -9,19 +9,25 @@ what = 'T2I-Adapter'
 debug = log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None
 debug('Trace: CONTROL')
 predefined_sd15 = {
-    'Canny': 'TencentARC/t2iadapter_canny_sd15v2',
-    'Depth': 'TencentARC/t2iadapter_depth_sd15v2',
-    'Depth Zoe': 'TencentARC/t2iadapter_zoedepth_sd15v1',
+    'Segment': 'TencentARC/t2iadapter_seg_sd14v1',
+    'Zoe Depth': 'TencentARC/t2iadapter_zoedepth_sd15v1',
     'OpenPose': 'TencentARC/t2iadapter_openpose_sd14v1',
-    'Sketch': 'TencentARC/t2iadapter_sketch_sd15v2',
+    'KeyPose': 'TencentARC/t2iadapter_keypose_sd14v1',
+    'Color': 'TencentARC/t2iadapter_color_sd14v1',
+    'Depth v1': 'TencentARC/t2iadapter_depth_sd14v1',
+    'Depth v2': 'TencentARC/t2iadapter_depth_sd15v2',
+    'Canny v1': 'TencentARC/t2iadapter_canny_sd14v1',
+    'Canny v2': 'TencentARC/t2iadapter_canny_sd15v2',
+    'Sketch v1': 'TencentARC/t2iadapter_sketch_sd14v1',
+    'Sketch v2': 'TencentARC/t2iadapter_sketch_sd15v2',
 }
 predefined_sdxl = {
     'Canny XL': 'TencentARC/t2i-adapter-canny-sdxl-1.0',
-    'Depth Zoe XL': 'TencentARC/t2i-adapter-depth-zoe-sdxl-1.0',
-    'Depth Midas XL': 'TencentARC/t2i-adapter-depth-midas-sdxl-1.0',
     'LineArt XL': 'TencentARC/t2i-adapter-lineart-sdxl-1.0',
-    'OpenPose XL': 'TencentARC/t2i-adapter-openpose-sdxl-1.0',
     'Sketch XL': 'TencentARC/t2i-adapter-sketch-sdxl-1.0',
+    'Zoe Depth XL': 'TencentARC/t2i-adapter-depth-zoe-sdxl-1.0',
+    'OpenPose XL': 'TencentARC/t2i-adapter-openpose-sdxl-1.0',
+    'Midas Depth XL': 'TencentARC/t2i-adapter-depth-midas-sdxl-1.0',
 }
 models = {}
 all_models = {}
@@ -104,8 +110,9 @@ class AdapterPipeline():
         if pipeline is None:
             log.error(f'Control {what} pipeline: model not loaded')
             return
-        # if isinstance(adapter, list) and len(adapter) > 1: # TODO use MultiAdapter
-        #    adapter = MultiAdapter(adapter)
+        if isinstance(adapter, list) and len(adapter) > 1: # TODO use MultiAdapter
+            adapter = MultiAdapter(adapter)
+        adapter.to(device=pipeline.device, dtype=pipeline.dtype)
         if isinstance(pipeline, StableDiffusionXLPipeline):
             self.pipeline = StableDiffusionXLAdapterPipeline(
                 vae=pipeline.vae,

modules/control/run.py

@@ -166,13 +166,6 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
             active_process.append(u.process)
             # active_model.append(model)
             active_strength.append(u.strength)
-    """
-    if (len(active_process) == 0) and (unit_type != 'reference'):
-        msg = 'Control: no active units'
-        shared.log.warning(msg)
-        restore_pipeline()
-        return msg
-    """
     p.ops.append('control')
 
     has_models = False
@@ -426,8 +419,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo
         image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}'
 
     image_txt += f' | {util.dict2str(p.extra_generation_params)}'
-    debug(f'Control ready: {image_txt}')
     restore_pipeline()
+    debug(f'Control ready: {image_txt}')
     if is_generator:
         yield (output_images, processed_image, f'Control ready {image_txt}', output_filename)
     else:

t2i.py

@@ -1,77 +0,0 @@
-import torch
-import diffusers
-from PIL import Image
-from rich import print
-
-model_id = "runwayml/stable-diffusion-v1-5"
-print(f'torch=={torch.__version__} diffusers=={diffusers.__version__}')
-
-adapters = [
-    'TencentARC/t2iadapter_canny_sd15v2',
-    # 'TencentARC/t2iadapter_depth_sd15v2',
-    # 'TencentARC/t2iadapter_zoedepth_sd15v1',
-    # 'TencentARC/t2iadapter_openpose_sd14v1',
-    # 'TencentARC/t2iadapter_sketch_sd15v2',
-]
-seeds = [42]
-
-print(f'loading: {model_id}')
-base = diffusers.StableDiffusionPipeline.from_pretrained(model_id, variant="fp16", cache_dir='/mnt/d/Models/Diffusers').to('cuda')
-image = Image.new('RGB', (512,512), 0) # input is irrelevant, so just creating blank image
-print('loaded')
-
-def callback(step: int, timestep: int, latents: torch.FloatTensor):
-    print(f'callback: step={step} timestep={timestep} latents={latents.shape}')
-
-for adapter_id in adapters:
-    print(f'loading: {adapter_id}')
-    adapter = diffusers.T2IAdapter.from_pretrained('TencentARC/t2iadapter_depth_sd15v2', cache_dir='/mnt/d/Models/Diffusers')
-    pipe = diffusers.StableDiffusionAdapterPipeline(
-        vae=base.vae,
-        text_encoder=base.text_encoder,
-        tokenizer=base.tokenizer,
-        unet=base.unet,
-        scheduler=base.scheduler,
-        requires_safety_checker=False,
-        safety_checker=None,
-        feature_extractor=None,
-        adapter=adapter,
-    ).to('cuda')
-    output = pipe(prompt=['test'], negative_prompt=['test'], num_inference_steps=20, image=image) # ok
-    print(f'adapter: {adapter_id} {output}')
-    pipe.scheduler = diffusers.EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
-    pipe.scheduler.config['num_train_timesteps'] = 1000
-    pipe.scheduler.config['beta_start'] = 0.00085
-    pipe.scheduler.config['beta_end'] = 0.012
-    pipe.scheduler.config['beta_schedule'] = 'scaled_linear'
-    pipe.scheduler.config['prediction_type'] = 'epsilon'
-    pipe.scheduler.config['rescale_betas_zero_snr'] = False
-    output = pipe(
-        prompt=['test'],
-        negative_prompt=['test'],
-        num_inference_steps=20,
-        image=image,
-        callback=callback,
-        callback_steps=1,
-        output_type='latent',
-        eta=1.0,
-        clip_skip=1,
-        guidance_scale=6,
-        generator=[torch.Generator('cpu').manual_seed(seed) for seed in seeds],
-    )
-    print(f'adapter: {adapter_id} {output}')
-
-"""
-'callback_steps': 1,
-'callback': <function process_diffusers.<locals>.diffusers_callback_legacy at 0x7f4569259a80>,
-
-'guidance_scale': 6,
-'generator': [<torch._C.Generator object at 0x7f4562c72370>],
-'num_inference_steps': 20,
-
-'eta': 1.0,
-'clip_skip': 1,
-'image': <PIL.Image.Image image mode=RGB size=512x512 at 0x7F456A271A50>}
-
-Given groups=1, weight of size [320, 64, 3, 3], expected input[1, 192, 64, 64] to have 64 channels, but got 192 channels instead
-"""