diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b05eac4f..c24484113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-12 +## Update for 2024-12-13 ### New models and integrations @@ -33,11 +33,18 @@ style-aligned applies selected attention layers uniformly to all images to achive consistency can be used with or without input image in which case first prompt is used to establish baseline *note:* all prompts are processes as a single batch, so vram is limiting factor +- **ControlNet** + - improved support for `Union` controlnets with granular control mode type + - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet + - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) + *note*: when selecting tiles in control settings, you can also specify non-square ratios + in which case it will use context-aware image resize to maintain overall composition ### UI and workflow improvements - **Docs**: - New documentation site! + - Additional Wiki content: Styles, Wildcards, etc. - **LoRA** handler rewrite: - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step @@ -82,7 +89,6 @@ ### Updates -- Additional Wiki content: Styles, Wildcards, etc. - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements @@ -108,9 +114,10 @@ - simplify img2img/inpaint/sketch canvas handling - fix prompt caching - fix xyz grid skip final pass -- fix sd upscale script -- fix cogvideox-i2v -- lora auto-apply tags remove duplicates +- fix sd upscale script +- fix cogvideox-i2v +- lora auto-apply tags remove duplicates +- control load model on-demand if not already loaded ## Update for 2024-11-21 diff --git a/installer.py b/installer.py index 70c655ae3..18a8ad1f1 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '914a585be8187ec0ad92fab4f072c992f8c297cd' # diffusers commit hash + sha = '63243406ba5510c10d5cac931882918ceba926f9' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/control/run.py b/modules/control/run.py index 6ae7fb20c..ac1ff233d 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -7,6 +7,7 @@ from PIL import Image from modules.control import util # helper functions from modules.control import unit # control units from modules.control import processors # image preprocessors +from modules.control import tile # tiling module from modules.control.units import controlnet # lllyasviel ControlNet from modules.control.units import xs # VisLearn ControlNet-XS from modules.control.units import lite # Kohya ControlLLLite @@ -83,6 +84,7 @@ def control_run(state: str = '', u.adapter.load(u.model_name, force=False) else: u.controlnet.load(u.model_name, force=False) + u.update_choices(u.model_name) if u.process is not None and u.process.override is None and u.override is not None: u.process.override = u.override @@ -149,6 +151,7 @@ def control_run(state: str = '', outpath_grids=shared.opts.outdir_grids or shared.opts.outdir_control_grids, ) p.state = state + p.is_tile = False # processing.process_init(p) resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0 @@ -242,7 +245,7 @@ def control_run(state: str = '', active_model.append(u.adapter) active_strength.append(float(u.strength)) p.adapter_conditioning_factor = u.factor - shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process={u.process.processor_id} model={u.adapter.model_id} strength={u.strength} factor={u.factor}') + shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process="{u.process.processor_id}" model="{u.adapter.model_id}" strength={u.strength} factor={u.factor}') elif unit_type == 'controlnet' and u.controlnet.model is not None: active_process.append(u.process) active_model.append(u.controlnet) @@ -250,8 +253,12 @@ def control_run(state: str = '', active_start.append(float(u.start)) active_end.append(float(u.end)) p.guess_mode = u.guess - p.control_mode = u.mode - shared.log.debug(f'Control ControlNet unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}') + if isinstance(u.mode, str): + p.control_mode = u.choices.index(u.mode) if u.mode in u.choices else 0 + p.is_tile = p.is_tile or 'tile' in u.mode.lower() + p.control_tile = u.tile + p.extra_generation_params["Control mode"] = u.mode + shared.log.debug(f'Control ControlNet unit: i={num_units} process="{u.process.processor_id}" model="{u.controlnet.model_id}" strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}') elif unit_type == 'xs' and u.controlnet.model is not None: active_process.append(u.process) active_model.append(u.controlnet) @@ -291,6 +298,7 @@ def control_run(state: str = '', selected_models = None elif len(active_model) == 1: selected_models = active_model[0].model if active_model[0].model is not None else None + p.is_tile = p.is_tile or 'tile' in active_model[0].model_id.lower() has_models = selected_models is not None control_conditioning = active_strength[0] if len(active_strength) > 0 else 1 # strength or list[strength] control_guidance_start = active_start[0] if len(active_start) > 0 else 0 @@ -305,29 +313,30 @@ def control_run(state: str = '', has_models = any(u.enabled for u in units if u.type == 'reference') else: pass + p.is_tile = p.is_tile and has_models def set_pipe(): global pipe, instance # pylint: disable=global-statement pipe = None if has_models: p.ops.append('control') - p.extra_generation_params["Control mode"] = unit_type # overriden later with pretty-print + p.extra_generation_params["Control type"] = unit_type # overriden later with pretty-print + p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning] p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start] p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end] - p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]]) p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']]) p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']]) if unit_type == 't2i adapter' and has_models: - p.extra_generation_params["Control mode"] = 'T2I-Adapter' + p.extra_generation_params["Control type"] = 'T2I-Adapter' p.task_args['adapter_conditioning_scale'] = control_conditioning instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: T2I-Adapter does not support separate init image') elif unit_type == 'controlnet' and has_models: - p.extra_generation_params["Control mode"] = 'ControlNet' + p.extra_generation_params["Control type"] = 'ControlNet' p.task_args['controlnet_conditioning_scale'] = control_conditioning p.task_args['control_guidance_start'] = control_guidance_start p.task_args['control_guidance_end'] = control_guidance_end @@ -335,7 +344,7 @@ def control_run(state: str = '', instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p) pipe = instance.pipeline elif unit_type == 'xs' and has_models: - p.extra_generation_params["Control mode"] = 'ControlNet-XS' + p.extra_generation_params["Control type"] = 'ControlNet-XS' p.controlnet_conditioning_scale = control_conditioning p.control_guidance_start = control_guidance_start p.control_guidance_end = control_guidance_end @@ -344,14 +353,14 @@ def control_run(state: str = '', if inits is not None: shared.log.warning('Control: ControlNet-XS does not support separate init image') elif unit_type == 'lite' and has_models: - p.extra_generation_params["Control mode"] = 'ControlLLLite' + p.extra_generation_params["Control type"] = 'ControlLLLite' p.controlnet_conditioning_scale = control_conditioning instance = lite.ControlLLitePipeline(shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: ControlLLLite does not support separate init image') elif unit_type == 'reference' and has_models: - p.extra_generation_params["Control mode"] = 'Reference' + p.extra_generation_params["Control type"] = 'Reference' p.extra_generation_params["Control attention"] = p.attention p.task_args['reference_attn'] = 'Attention' in p.attention p.task_args['reference_adain'] = 'Adain' in p.attention @@ -393,6 +402,8 @@ def control_run(state: str = '', else: original_pipeline = None + possible = sd_models.get_call(pipe).keys() + try: with devices.inference_context(): if isinstance(inputs, str): # only video, the rest is a list @@ -562,19 +573,29 @@ def control_run(state: str = '', return [], '', '', 'Reference mode without image' elif unit_type == 'controlnet' and has_models: if input_type == 0: # Control only - if shared.sd_model_type in ['f1', 'sd3'] and 'control_image' not in p.task_args: - p.task_args['control_image'] = p.init_images # some controlnets mandate this + if 'control_image' in possible: + p.task_args['control_image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images + elif 'image' in possible: + p.task_args['image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images + if 'control_mode' in possible: + p.task_args['control_mode'] = p.control_mode + if 'strength' in possible: p.task_args['strength'] = p.denoising_strength + p.init_images = None elif input_type == 1: # Init image same as control - p.task_args['control_image'] = p.init_images # switch image and control_image - p.task_args['strength'] = p.denoising_strength + if 'control_image' in possible: + p.task_args['control_image'] = p.init_images # switch image and control_image + if 'strength' in possible: + p.task_args['strength'] = p.denoising_strength p.init_images = [p.override or input_image] * len(active_model) elif input_type == 2: # Separate init image if init_image is None: shared.log.warning('Control: separate init image not provided') init_image = input_image - p.task_args['control_image'] = p.init_images # switch image and control_image - p.task_args['strength'] = p.denoising_strength + if 'control_image' in possible: + p.task_args['control_image'] = p.init_images # switch image and control_image + if 'strength' in possible: + p.task_args['strength'] = p.denoising_strength p.init_images = [init_image] * len(active_model) if is_generator: @@ -607,11 +628,11 @@ def control_run(state: str = '', p.task_args['strength'] = denoising_strength p.image_mask = mask shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # only controlnet supports inpaint - elif 'control_image' in p.task_args: + if hasattr(p, 'init_images') and p.init_images is not None: shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # only controlnet supports img2img else: shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) - if hasattr(p, 'init_images') and p.init_images is not None: + if hasattr(p, 'init_images') and p.init_images is not None and 'image' in possible: p.task_args['image'] = p.init_images # need to set explicitly for txt2img del p.init_images if unit_type == 'lite': @@ -624,9 +645,14 @@ def control_run(state: str = '', # final check if has_models: - if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] and p.task_args.get('image', None) is None and getattr(p, 'init_images', None) is None: + if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] \ + and p.task_args.get('image', None) is None \ + and p.task_args.get('control_image', None) is None \ + and getattr(p, 'init_images', None) is None \ + and getattr(p, 'image', None) is None: if is_generator: - yield terminate(f'Mode={p.extra_generation_params.get("Control mode", None)} input image is none') + shared.log.debug(f'Control args: {p.task_args}') + yield terminate(f'Mode={p.extra_generation_params.get("Control type", None)} input image is none') return [], '', '', 'Error: Input image is none' # resize mask @@ -656,11 +682,17 @@ def control_run(state: str = '', script_runner.initialize_scripts(False) p.script_args = script.init_default_script_args(script_runner) - processed = p.scripts.run(p, *p.script_args) + # actual processing + if p.is_tile: + processed: processing.Processed = tile.run_tiling(p, input_image) + if processed is None: + processed = p.scripts.run(p, *p.script_args) if processed is None: processed: processing.Processed = processing.process_images(p) # run actual pipeline else: script_run = True + + # postprocessing processed = p.scripts.after(p, processed, *p.script_args) output = None if processed is not None: diff --git a/modules/control/tile.py b/modules/control/tile.py new file mode 100644 index 000000000..5dc104e47 --- /dev/null +++ b/modules/control/tile.py @@ -0,0 +1,75 @@ +from PIL import Image +from modules import shared, processing, images, sd_models + + +def get_tile(image: Image.Image, x: int, y: int, sx: int, sy: int) -> Image.Image: + return image.crop(( + (x + 0) * image.width // sx, + (y + 0) * image.height // sy, + (x + 1) * image.width // sx, + (y + 1) * image.height // sy + )) + + +def set_tile(image: Image.Image, x: int, y: int, tiled: Image.Image): + image.paste(tiled, (x * tiled.width, y * tiled.height)) + return image + + +def run_tiling(p: processing.StableDiffusionProcessing, input_image: Image.Image) -> processing.Processed: + # prepare images + sx, sy = p.control_tile.split('x') + sx = int(sx) + sy = int(sy) + if sx <= 0 or sy <= 0: + raise ValueError('Control: invalid tile size') + control_image = p.task_args.get('control_image', None) or p.task_args.get('image', None) + control_upscaled = None + if isinstance(control_image, list) and len(control_image) > 0: + control_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, + im=control_image[0], + width=8 * int(sx * control_image[0].width) // 8, + height=8 * int(sy * control_image[0].height) // 8, + context='add with forward' + ) + init_image = p.override or input_image + init_upscaled = None + if init_image is not None: + init_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, + im=init_image, + width=8 * int(sx * init_image.width) // 8, + height=8 * int(sy * init_image.height) // 8, + context='add with forward' + ) + + # stop processing from restoring pipeline on each iteration + orig_restore_pipeline = getattr(shared.sd_model, 'restore_pipeline', None) + shared.sd_model.restore_pipeline = None + + # run tiling + for x in range(sx): + for y in range(sy): + shared.log.info(f'Control Tile: tile={x+1}-{sx}/{y+1}-{sy} target={control_upscaled}') + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + p.init_images = None + p.task_args['control_mode'] = p.control_mode + p.task_args['strength'] = p.denoising_strength + if init_upscaled is not None: + p.task_args['image'] = [get_tile(init_upscaled, x, y, sx, sy)] + if control_upscaled is not None: + p.task_args['control_image'] = [get_tile(control_upscaled, x, y, sx, sy)] + processed: processing.Processed = processing.process_images(p) # run actual pipeline + if processed is None or len(processed.images) == 0: + continue + control_upscaled = set_tile(control_upscaled, x, y, processed.images[0]) + + # post-process + p.width = control_upscaled.width + p.height = control_upscaled.height + processed.images = [control_upscaled] + processed.info = processed.infotext(p, 0) + processed.infotexts = [processed.info] + shared.sd_model.restore_pipeline = orig_restore_pipeline + if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: + shared.sd_model.restore_pipeline() + return processed diff --git a/modules/control/unit.py b/modules/control/unit.py index 7dc5528a6..eeb729740 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -16,6 +16,22 @@ unit_types = ['t2i adapter', 'controlnet', 'xs', 'lite', 'reference', 'ip'] class Unit(): # mashup of gradio controls and mapping to actual implementation classes + def update_choices(self, model_id=None): + name = model_id or self.model_name + if name == 'InstantX Union': + self.choices = ['canny', 'tile', 'depth', 'blur', 'pose', 'gray', 'lq'] + elif name == 'Shakker-Labs Union': + self.choices = ['canny', 'tile', 'depth', 'blur', 'pose', 'gray', 'lq'] + elif name == 'Xinsir Union XL': + self.choices = ['openpose', 'depth', 'scribble', 'canny', 'normal'] + elif name == 'Xinsir ProMax XL': + self.choices = ['openpose', 'depth', 'scribble', 'canny', 'normal', 'segment', 'tile', 'repaint'] + else: + self.choices = ['default'] + + def __str__(self): + return f'Unit: type={self.type} enabled={self.enabled} strength={self.strength} start={self.start} end={self.end} mode={self.mode} tile={self.tile}' + def __init__(self, # values index: int = None, @@ -38,6 +54,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c control_start = None, control_end = None, control_mode = None, + control_tile = None, result_txt = None, extra_controls: list = [], ): @@ -70,6 +87,10 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c self.fidelity = 0.5 self.query_weight = 1.0 self.adain_weight = 1.0 + # control mode + self.choices = ['default'] + # control tile + self.tile = '1x1' def reset(): if self.process is not None: @@ -92,10 +113,16 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c self.end = max(start, end) def control_mode_change(mode): - self.mode = mode - 1 if mode > 0 else None + self.mode = self.choices.index(mode) if mode is not None and mode in self.choices else 0 - def control_mode_show(model_id): - return gr.update(visible='union' in model_id.lower()) + def control_tile_change(tile): + self.tile = tile + + def control_choices(model_id): + self.update_choices(model_id) + mode_visible = 'union' in model_id.lower() or 'promax' in model_id.lower() + tile_visible = 'union' in model_id.lower() or 'promax' in model_id.lower() or 'tile' in model_id.lower() + return [gr.update(visible=mode_visible, choices=self.choices), gr.update(visible=tile_visible)] def adapter_extra(c1): self.factor = c1 @@ -172,7 +199,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c else: self.controls.append(model_id) model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True) - model_id.change(fn=control_mode_show, inputs=[model_id], outputs=[control_mode], show_progress=False) + model_id.change(fn=control_choices, inputs=[model_id], outputs=[control_mode, control_tile], show_progress=False) if extra_controls is not None and len(extra_controls) > 0: extra_controls[0].change(fn=controlnet_extra, inputs=extra_controls) elif self.type == 'xs': @@ -231,3 +258,6 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c if control_mode is not None: self.controls.append(control_mode) control_mode.change(fn=control_mode_change, inputs=[control_mode]) + if control_tile is not None: + self.controls.append(control_tile) + control_tile.change(fn=control_tile_change, inputs=[control_tile]) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 3fa2d90eb..7361638c6 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -52,17 +52,20 @@ predefined_sdxl = { 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0/bin', 'Xinsir Union XL': 'xinsir/controlnet-union-sdxl-1.0', + 'Xinsir ProMax XL': 'brad-twinkl/controlnet-union-sdxl-1.0-promax', 'Xinsir OpenPose XL': 'xinsir/controlnet-openpose-sdxl-1.0', 'Xinsir Canny XL': 'xinsir/controlnet-canny-sdxl-1.0', 'Xinsir Depth XL': 'xinsir/controlnet-depth-sdxl-1.0', 'Xinsir Scribble XL': 'xinsir/controlnet-scribble-sdxl-1.0', 'Xinsir Anime Painter XL': 'xinsir/anime-painter', + 'Xinsir Tile XL': 'xinsir/controlnet-tile-sdxl-1.0', 'NoobAI Canny XL': 'Eugeoter/noob-sdxl-controlnet-canny', 'NoobAI Lineart Anime XL': 'Eugeoter/noob-sdxl-controlnet-lineart_anime', 'NoobAI Depth XL': 'Eugeoter/noob-sdxl-controlnet-depth', 'NoobAI Normal XL': 'Eugeoter/noob-sdxl-controlnet-normal', 'NoobAI SoftEdge XL': 'Eugeoter/noob-sdxl-controlnet-softedge_hed', 'NoobAI OpenPose XL': 'einar77/noob-openpose', + 'TTPlanet Tile Realistic XL': 'Yakonrus/SDXL_Controlnet_Tile_Realistic_v2', # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', @@ -166,30 +169,30 @@ class ControlNet(): self.model_id = None def get_class(self, model_id:str=''): - import modules.shared - if modules.shared.sd_model_type == 'sd': + from modules import shared + if shared.sd_model_type == 'none': + _load = shared.sd_model # trigger a load + if shared.sd_model_type == 'sd': from diffusers import ControlNetModel as cls # pylint: disable=reimported config = 'lllyasviel/control_v11p_sd15_canny' - elif modules.shared.sd_model_type == 'sdxl': - # TODO ControlNetUnion - """ + elif shared.sd_model_type == 'sdxl': if 'union' in model_id.lower(): from diffusers import ControlNetUnionModel as cls config = 'xinsir/controlnet-union-sdxl-1.0' + elif 'promax' in model_id.lower(): + from diffusers import ControlNetUnionModel as cls + config = 'brad-twinkl/controlnet-union-sdxl-1.0-promax' else: from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class config = 'Eugeoter/noob-sdxl-controlnet-canny' - """ - from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class - config = 'Eugeoter/noob-sdxl-controlnet-canny' - elif modules.shared.sd_model_type == 'f1': + elif shared.sd_model_type == 'f1': from diffusers import FluxControlNetModel as cls config = 'InstantX/FLUX.1-dev-Controlnet-Union' - elif modules.shared.sd_model_type == 'sd3': + elif shared.sd_model_type == 'sd3': from diffusers import SD3ControlNetModel as cls config = 'InstantX/SD3-Controlnet-Canny' else: - log.error(f'Control {what}: type={modules.shared.sd_model_type} unsupported model') + log.error(f'Control {what}: type={shared.sd_model_type} unsupported model') return None, None return cls, config @@ -299,7 +302,7 @@ class ControlNetPipeline(): controlnet: Union[ControlNetModel, list[ControlNetModel]], pipeline: Union[StableDiffusionXLPipeline, StableDiffusionPipeline, FluxPipeline, StableDiffusion3Pipeline], dtype = None, - p: StableDiffusionProcessingControl = None, + p: StableDiffusionProcessingControl = None, # pylint: disable=unused-argument ): t0 = time.time() self.orig_pipeline = pipeline @@ -314,14 +317,11 @@ class ControlNetPipeline(): return elif detect.is_sdxl(pipeline) and len(controlnets) > 0: from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetUnionPipeline - # TODO ControlNetUnion - """ if controlnet.__class__.__name__ == 'ControlNetUnionModel': cls = StableDiffusionXLControlNetUnionPipeline + controlnets = controlnets[0] # using only first one else: cls = StableDiffusionXLControlNetPipeline - """ - cls = StableDiffusionXLControlNetPipeline self.pipeline = cls( vae=pipeline.vae, text_encoder=pipeline.text_encoder, diff --git a/modules/images_resize.py b/modules/images_resize.py index d86ff6f22..5cf3e57e4 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -5,7 +5,7 @@ from PIL import Image from modules import shared -def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type='image', context=None): +def resize_image(resize_mode: int, im: Image.Image, width: int, height: int, upscaler_name: str=None, output_type: str='image', context: str=None): upscaler_name = upscaler_name or shared.opts.upscaler_for_img2img def latent(im, w, h, upscaler): @@ -79,18 +79,18 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type def context_aware(im, width, height, context): import seam_carving # https://github.com/li-plus/seam-carving - if 'forward' in context: + if 'forward' in context.lower(): energy_mode = "forward" - elif 'backward' in context: + elif 'backward' in context.lower(): energy_mode = "backward" else: return im - if 'Add' in context: + if 'add' in context.lower(): src_ratio = min(width / im.width, height / im.height) src_w = int(im.width * src_ratio) src_h = int(im.height * src_ratio) src_image = resize(im, src_w, src_h) - elif 'Remove' in context: + elif 'remove' in context.lower(): ratio = width / height src_ratio = im.width / im.height src_w = width if ratio > src_ratio else im.width * height // im.height diff --git a/modules/sd_models.py b/modules/sd_models.py index c5875c61f..5d42e314b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1057,6 +1057,11 @@ def get_signature(cls): return signature.parameters +def get_call(cls): + signature = inspect.signature(cls.__call__, follow_wrapped=True, eval_str=True) + return signature.parameters + + def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionPipeline = None, force = False, args = {}): """ args: diff --git a/modules/ui_control.py b/modules/ui_control.py index 59db12fc5..5a146a8fc 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -138,7 +138,7 @@ def create_ui(_blocks: gr.Blocks=None): show_input = gr.Checkbox(label="Show input", value=True, elem_id="control_show_input") show_preview = gr.Checkbox(label="Show preview", value=False, elem_id="control_show_preview") with gr.Row(): - input_type = gr.Radio(label="Input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type') + input_type = gr.Radio(label="Control input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type') with gr.Row(): denoising_strength = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Denoising strength', value=0.30, elem_id="control_input_denoising_strength") @@ -251,9 +251,10 @@ def create_ui(_blocks: gr.Blocks=None): model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'refresh_controlnet_models_{i}') model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - control_mode = gr.Dropdown(label="CN Mode", choices=['', 'Canny', 'Tile', 'Depth', 'Blur', 'Pose', 'Gray', 'LQ'], value=0, type='index', visible=False, elem_id=f'control_unit-{i}-mode') + control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') + control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') + control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') + control_tile = gr.Dropdown(label="CN Tiles", choices=['1x1', '1x2', '1x3', '1x4', '2x1', '2x1', '2x2', '2x3', '2x4', '3x1', '3x2', '3x3', '3x4', '4x1', '4x2', '4x3', '4x4'], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) image_reuse= ui_components.ToolButton(value=ui_symbols.reuse) @@ -278,6 +279,7 @@ def create_ui(_blocks: gr.Blocks=None): control_start = control_start, control_end = control_end, control_mode = control_mode, + control_tile = control_tile, extra_controls = extra_controls, ) ) diff --git a/scripts/regional_prompting.py b/scripts/regional_prompting.py index 08b84dd94..48309704e 100644 --- a/scripts/regional_prompting.py +++ b/scripts/regional_prompting.py @@ -82,6 +82,7 @@ class Script(scripts.Script): } # run pipeline shared.log.debug(f'Regional: args={p.task_args}') + p.task_args['prompt'] = p.prompt processed: processing.Processed = processing.process_images(p) # runs processing using main loop # restore pipeline and params