diff --git a/modules/control/adapters.py b/modules/control/adapters.py index 263dcef5a..9e8360567 100644 --- a/modules/control/adapters.py +++ b/modules/control/adapters.py @@ -9,19 +9,25 @@ what = 'T2I-Adapter' debug = log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: CONTROL') predefined_sd15 = { - 'Canny': 'TencentARC/t2iadapter_canny_sd15v2', - 'Depth': 'TencentARC/t2iadapter_depth_sd15v2', - 'Depth Zoe': 'TencentARC/t2iadapter_zoedepth_sd15v1', + 'Segment': 'TencentARC/t2iadapter_seg_sd14v1', + 'Zoe Depth': 'TencentARC/t2iadapter_zoedepth_sd15v1', 'OpenPose': 'TencentARC/t2iadapter_openpose_sd14v1', - 'Sketch': 'TencentARC/t2iadapter_sketch_sd15v2', + 'KeyPose': 'TencentARC/t2iadapter_keypose_sd14v1', + 'Color': 'TencentARC/t2iadapter_color_sd14v1', + 'Depth v1': 'TencentARC/t2iadapter_depth_sd14v1', + 'Depth v2': 'TencentARC/t2iadapter_depth_sd15v2', + 'Canny v1': 'TencentARC/t2iadapter_canny_sd14v1', + 'Canny v2': 'TencentARC/t2iadapter_canny_sd15v2', + 'Sketch v1': 'TencentARC/t2iadapter_sketch_sd14v1', + 'Sketch v2': 'TencentARC/t2iadapter_sketch_sd15v2', } predefined_sdxl = { 'Canny XL': 'TencentARC/t2i-adapter-canny-sdxl-1.0', - 'Depth Zoe XL': 'TencentARC/t2i-adapter-depth-zoe-sdxl-1.0', - 'Depth Midas XL': 'TencentARC/t2i-adapter-depth-midas-sdxl-1.0', 'LineArt XL': 'TencentARC/t2i-adapter-lineart-sdxl-1.0', - 'OpenPose XL': 'TencentARC/t2i-adapter-openpose-sdxl-1.0', 'Sketch XL': 'TencentARC/t2i-adapter-sketch-sdxl-1.0', + 'Zoe Depth XL': 'TencentARC/t2i-adapter-depth-zoe-sdxl-1.0', + 'OpenPose XL': 'TencentARC/t2i-adapter-openpose-sdxl-1.0', + 'Midas Depth XL': 'TencentARC/t2i-adapter-depth-midas-sdxl-1.0', } models = {} all_models = {} @@ -104,8 +110,9 @@ class AdapterPipeline(): if pipeline is None: log.error(f'Control {what} pipeline: model not loaded') return - # if isinstance(adapter, list) and len(adapter) > 1: # TODO use MultiAdapter - # adapter = MultiAdapter(adapter) + if isinstance(adapter, list) and len(adapter) > 1: # TODO use MultiAdapter + adapter = MultiAdapter(adapter) + adapter.to(device=pipeline.device, dtype=pipeline.dtype) if isinstance(pipeline, StableDiffusionXLPipeline): self.pipeline = StableDiffusionXLAdapterPipeline( vae=pipeline.vae, diff --git a/modules/control/run.py b/modules/control/run.py index b28963d16..bf3e9fd7f 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -166,13 +166,6 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo active_process.append(u.process) # active_model.append(model) active_strength.append(u.strength) - """ - if (len(active_process) == 0) and (unit_type != 'reference'): - msg = 'Control: no active units' - shared.log.warning(msg) - restore_pipeline() - return msg - """ p.ops.append('control') has_models = False @@ -426,8 +419,8 @@ def control_run(units: List[unit.Unit], inputs, unit_type: str, is_generator: bo image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}' image_txt += f' | {util.dict2str(p.extra_generation_params)}' - debug(f'Control ready: {image_txt}') restore_pipeline() + debug(f'Control ready: {image_txt}') if is_generator: yield (output_images, processed_image, f'Control ready {image_txt}', output_filename) else: diff --git a/t2i.py b/t2i.py deleted file mode 100644 index 0c596169c..000000000 --- a/t2i.py +++ /dev/null @@ -1,77 +0,0 @@ -import torch -import diffusers -from PIL import Image -from rich import print - -model_id = "runwayml/stable-diffusion-v1-5" -print(f'torch=={torch.__version__} diffusers=={diffusers.__version__}') - -adapters = [ - 'TencentARC/t2iadapter_canny_sd15v2', - # 'TencentARC/t2iadapter_depth_sd15v2', - # 'TencentARC/t2iadapter_zoedepth_sd15v1', - # 'TencentARC/t2iadapter_openpose_sd14v1', - # 'TencentARC/t2iadapter_sketch_sd15v2', -] -seeds = [42] - -print(f'loading: {model_id}') -base = diffusers.StableDiffusionPipeline.from_pretrained(model_id, variant="fp16", cache_dir='/mnt/d/Models/Diffusers').to('cuda') -image = Image.new('RGB', (512,512), 0) # input is irrelevant, so just creating blank image -print('loaded') - -def callback(step: int, timestep: int, latents: torch.FloatTensor): - print(f'callback: step={step} timestep={timestep} latents={latents.shape}') - -for adapter_id in adapters: - print(f'loading: {adapter_id}') - adapter = diffusers.T2IAdapter.from_pretrained('TencentARC/t2iadapter_depth_sd15v2', cache_dir='/mnt/d/Models/Diffusers') - pipe = diffusers.StableDiffusionAdapterPipeline( - vae=base.vae, - text_encoder=base.text_encoder, - tokenizer=base.tokenizer, - unet=base.unet, - scheduler=base.scheduler, - requires_safety_checker=False, - safety_checker=None, - feature_extractor=None, - adapter=adapter, - ).to('cuda') - output = pipe(prompt=['test'], negative_prompt=['test'], num_inference_steps=20, image=image) # ok - print(f'adapter: {adapter_id} {output}') - pipe.scheduler = diffusers.EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) - pipe.scheduler.config['num_train_timesteps'] = 1000 - pipe.scheduler.config['beta_start'] = 0.00085 - pipe.scheduler.config['beta_end'] = 0.012 - pipe.scheduler.config['beta_schedule'] = 'scaled_linear' - pipe.scheduler.config['prediction_type'] = 'epsilon' - pipe.scheduler.config['rescale_betas_zero_snr'] = False - output = pipe( - prompt=['test'], - negative_prompt=['test'], - num_inference_steps=20, - image=image, - callback=callback, - callback_steps=1, - output_type='latent', - eta=1.0, - clip_skip=1, - guidance_scale=6, - generator=[torch.Generator('cpu').manual_seed(seed) for seed in seeds], - ) - print(f'adapter: {adapter_id} {output}') - -""" -'callback_steps': 1, -'callback': .diffusers_callback_legacy at 0x7f4569259a80>, - -'guidance_scale': 6, -'generator': [], -'num_inference_steps': 20, - -'eta': 1.0, -'clip_skip': 1, -'image': } - -Given groups=1, weight of size [320, 64, 3, 3], expected input[1, 192, 64, 64] to have 64 channels, but got 192 channels instead -"""