From f963891046484f44c026a206aede3791fd04e319 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 26 Apr 2024 09:20:49 -0400 Subject: [PATCH] multiple control fixes --- CHANGELOG.md | 3 ++- modules/control/run.py | 29 +++++++++++++++++------------ modules/control/unit.py | 10 ++++++++++ modules/control/util.py | 8 ++++++-- wiki | 2 +- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6342f88e..e17d6efae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Given huge number of changes with *+3443/-3342 commits diff over the past year and complete focus on different backend/engine, it is time to give credit to original [author](https://github.com/auTOMATIC1111), and move on! -## Update for 2024-04-25 +## Update for 2024-04-26 - **Features**: - **Gallery**: list, preview, search through all your images and videos! @@ -120,6 +120,7 @@ it is time to give credit to original [author](https://github.com/auTOMATIC1111) - Noise sampler seed, thanks @leppie - Control module with ADetailer and active ControlNet - Control module restore button full functionality + - Control improved handling with multiple control units and different init images - Control add correct metadata to image - MOTD exception handling diff --git a/modules/control/run.py b/modules/control/run.py index 67d707b04..c90ea9675 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -432,6 +432,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini process.model = None debug(f'Control processed: {len(processed_images)}') + blended_image = None if len(processed_images) > 0: try: if len(p.extra_generation_params["Control process"]) == 0: @@ -443,12 +444,18 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini if any(img is None for img in processed_images): yield terminate('Control: attempting process but output is none') return - if len(processed_images) > 1: + if len(processed_images) > 1 and len(active_process) != len(active_model): processed_image = [np.array(i) for i in processed_images] processed_image = util.blend(processed_image) # blend all processed images into one processed_image = Image.fromarray(processed_image) + blended_image = processed_image + elif len(processed_images) == 1: + processed_image = processed_images + blended_image = processed_image[0] else: - processed_image = processed_images[0] + blended_image = [np.array(i) for i in processed_images] + blended_image = util.blend(blended_image) # blend all processed images into one + blended_image = Image.fromarray(blended_image) if isinstance(selected_models, list) and len(processed_images) == len(selected_models): debug(f'Control: inputs match: input={len(processed_images)} models={len(selected_models)}') p.init_images = processed_images @@ -456,9 +463,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini yield terminate(f'Control: number of inputs does not match: input={len(processed_images)} models={len(selected_models)}') return elif selected_models is not None: - if len(processed_images) > 1: - debug('Control: using blended image for single model') - p.init_images = [processed_image] + p.init_images = processed_image else: debug('Control processed: using input direct') processed_image = input_image @@ -484,11 +489,11 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini p.init_images = [init_image] * len(active_model) if is_generator: - image_txt = f'{processed_image.width}x{processed_image.height}' if processed_image is not None else 'None' + image_txt = f'{blended_image.width}x{blended_image.height}' if blended_image is not None else 'None' msg = f'process | {index} of {frames if video is not None else len(inputs)} | {"Image" if video is None else "Frame"} {image_txt}' debug(f'Control yield: {msg}') if is_generator: - yield (None, processed_image, f'Control {msg}') + yield (None, blended_image, f'Control {msg}') t2 += time.time() - t2 # determine txt2img, img2img, inpaint pipeline @@ -499,10 +504,10 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini if mask is not None: p.task_args['strength'] = p.denoising_strength p.image_mask = mask - p.init_images = [input_image] + p.init_images = input_image if isinstance(input_image, list) else [input_image] shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) elif processed_image is not None: - p.init_images = [processed_image] + p.init_images = processed_image if isinstance(processed_image, list) else [processed_image] shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) else: p.init_hr(p.scale_by, p.resize_name, force=True) @@ -595,7 +600,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini else: msg = f'Control output | {index} of {len(inputs)} | Image {image_txt}' if is_generator: - yield (output_image, processed_image, msg) # result is control_output, proces_output + yield (output_image, blended_image, msg) # result is control_output, proces_output if video is not None and frame is not None: status, frame = video.read() @@ -630,7 +635,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini restore_pipeline() debug(f'Control ready: {image_txt}') if is_generator: - yield (output_images, processed_image, f'Control ready {image_txt}', output_filename) + yield (output_images, blended_image, f'Control ready {image_txt}', output_filename) else: - yield (output_images, processed_image, f'Control ready {image_txt}', output_filename) + yield (output_images, blended_image, f'Control ready {image_txt}', output_filename) return diff --git a/modules/control/unit.py b/modules/control/unit.py index d9962e933..b2b102b9b 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -100,6 +100,9 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c def upload_image(image_file): if image_file is None: + self.process.override = None + self.override = None + log.debug('Control process clear image') return gr.update(value=None) try: self.process.override = Image.open(image_file.name) @@ -116,6 +119,11 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c self.override = self.process.override return gr.update(visible=self.process.override is not None, value=self.process.override) + def set_image(image): + self.process.override = image + self.override = image + return gr.update(visible=image is not None) + # actual init if self.type == 't2i adapter': self.adapter = t2iadapter.Adapter(device=default_device, dtype=default_dtype) @@ -189,6 +197,8 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c image_upload.upload(fn=upload_image, inputs=[image_upload], outputs=[image_preview]) # return list of images for gallery if image_reuse is not None: image_reuse.click(fn=reuse_image, inputs=[preview_process], outputs=[image_preview]) # return list of images for gallery + if image_preview is not None: + image_preview.change(fn=set_image, inputs=[image_preview], outputs=[image_preview]) if control_start is not None and control_end is not None: control_start.change(fn=control_change, inputs=[control_start, control_end]) control_end.change(fn=control_change, inputs=[control_start, control_end]) diff --git a/modules/control/util.py b/modules/control/util.py index 8a6a1ac12..bc1a9f1ba 100644 --- a/modules/control/util.py +++ b/modules/control/util.py @@ -146,10 +146,14 @@ def ade_palette(): def blend(images): if images is None or len(images) == 0: return images - y = np.zeros(images[0].shape, dtype=np.float32) + y = np.zeros((images[0].shape[0], images[0].shape[1], 3), dtype=np.float32) for img in images: - if img.shape != y.shape: + if img.shape[0] != y.shape[0] or img.shape[1] != y.shape[1]: img = cv2.resize(img, (y.shape[1], y.shape[0]), interpolation=cv2.INTER_CUBIC) + if len(img.shape) == 3 and img.shape[2] == 4: # rgba to rgb + img = cv2.cvtColor(img, cv2.COLOR_RGBA2RGB) + if len(img.shape) == 2: # grayscale to rgb + img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) y = cv2.add(y, img.astype(np.float32)) y = y.clip(0, 255).astype(np.uint8) return y diff --git a/wiki b/wiki index 97d02a14d..1be8ff636 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 97d02a14dff135b092252c7bab3eff9b45cba58f +Subproject commit 1be8ff636b6e8b2c4c1360f4471ef5dada981690