diff --git a/javascript/control.js b/javascript/control.js index e0bb550ba..1db434778 100644 --- a/javascript/control.js +++ b/javascript/control.js @@ -3,7 +3,14 @@ function controlInputMode(inputMode, ...args) { if (updateEl) updateEl.click(); const tab = gradioApp().querySelector('#control-tab-input button.selected'); if (!tab) return ['Image', ...args]; - inputMode = tab.innerText; + let inputTab = tab.innerText; + log('controlInputMode', inputMode, inputTab); + if ((inputTab === 'Image') && ('kanvas' in window)) { + inputTab = 'Kanvas'; + // const imageData = window.kanvas.getImageData(); + const imageData = window.kanvas.getImage(); + args[0] = imageData; + } return [inputMode, ...args]; } diff --git a/modules/gr_hijack.py b/modules/gr_hijack.py index b4607cc11..7fa1cecdd 100644 --- a/modules/gr_hijack.py +++ b/modules/gr_hijack.py @@ -1,3 +1,4 @@ +import time from PIL import Image import gradio as gr import gradio.processing_utils @@ -11,13 +12,49 @@ original_BlockContext_init = None original_Blocks_get_config_file = None +def process_kanvas(self, x): # only used when kanvas overrides gr.Image object + import numpy as np + from modules import errors + t0 = time.time() + image_data = list(x.get('image', {}).values()) + image = None + mask = None + if image_data: + width = x['imageWidth'] + height = x['imageHeight'] + array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4)) + image = Image.fromarray(array, 'RGBA') + image = image.convert('RGB') + mask_data = list(x.get('mask', {}).values()) + if mask_data: + width = x['maskWidth'] + height = x['maskHeight'] + array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4)) + mask = Image.fromarray(array, 'RGBA') + # alpha = mask.getchannel("A").convert("L") + # mask = Image.merge("RGB", [alpha, alpha, alpha]) + mask = mask.convert('L') + t1 = time.time() + errors.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}') + if image is None: + return None + if mask is None: + return self._format_image(image) # pylint: disable=protected-access + return { "image": self._format_image(image), "mask": self._format_image(mask) } # pylint: disable=protected-access + + def gr_image_preprocess(self, x): if x is None: return x mask = None - if isinstance(x, dict): + if isinstance(x, dict) and "kanvas" in x: + return process_kanvas(self, x) + if isinstance(x, dict) and "image" in x: x, mask = x["image"], x["mask"] - im = gradio.processing_utils.decode_base64_to_image(x) + if isinstance(x, str): + im = gradio.processing_utils.decode_base64_to_image(x) + else: + im = x im = im.convert(self.image_mode) if self.shape is not None: im = gradio.processing_utils.resize_and_crop(im, self.shape) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 877740a76..7c93c56f3 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -178,7 +178,7 @@ def fastvlm(question: str, image: Image.Image, repo: str = None): def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str = None): global processor, model, loaded # pylint: disable=global-statement - if model is None or loaded != repo: + if (model is None) or (loaded != repo): shared.log.debug(f'Interrogate load: vlm="{repo}"') model = None if 'Qwen3-VL' in repo or 'Qwen3VL' in repo: @@ -633,8 +633,7 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image: global quant_args # pylint: disable=global-statement jobid = shared.state.begin('Interrogate LLM') t0 = time.time() - if quant_args is None: - quant_args = model_quant.create_config(module='LLM') + quant_args = model_quant.create_config(module='LLM') model_name = model_name or shared.opts.interrogate_vlm_model if isinstance(image, list): image = image[0] if len(image) > 0 else None diff --git a/modules/model_quant.py b/modules/model_quant.py index 93f3accc4..e844a0ccf 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -246,10 +246,10 @@ def check_nunchaku(module: str = ''): def create_config(kwargs = None, allow: bool = True, module: str = 'Model', modules_to_not_convert: list = None, modules_dtype_dict: dict = None): - if dont_quant(): - return kwargs if kwargs is None: kwargs = {} + if module == 'Model' and dont_quant(): + return kwargs kwargs = create_sdnq_config(kwargs, allow=allow, module=module, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict) if kwargs is not None and 'quantization_config' in kwargs: if debug: diff --git a/modules/ui_control.py b/modules/ui_control.py index f474a6ee2..217bff925 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -185,7 +185,9 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'): input_mode = gr.Label(value='select', visible=False) with gr.Tab('Image', id='in-image') as tab_image: - input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image']) + input_image = gr.HTML(value="Kanvas placeholder", elem_id='control_input_select') + input_changed = gr.Button('Kanvas change', elem_id='control_input_change', visible=False) + # input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image']) btn_interrogate = ui_sections.create_interrogate_button('control', what='input') with gr.Tab('Video', id='in-video') as tab_video: input_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height, elem_classes=['control-image']) @@ -242,7 +244,6 @@ def create_ui(_blocks: gr.Blocks=None): input_type.change(fn=lambda x: gr.update(visible=x == 2), inputs=[input_type], outputs=[column_init]) btn_prompt_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[prompt], outputs=[prompt_counter], show_progress = False) btn_negative_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[negative], outputs=[negative_counter], show_progress = False) - btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt]) select_dict = dict( fn=helpers.select_input, @@ -253,6 +254,11 @@ def create_ui(_blocks: gr.Blocks=None): queue=False, ) + input_changed.click(**select_dict) + btn_interrogate.click(**select_dict) # need to fetch input first + btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt]) + + prompt.submit(**select_dict) negative.submit(**select_dict) btn_generate.click(**select_dict) @@ -403,7 +409,7 @@ def create_ui(_blocks: gr.Blocks=None): generation_parameters_copypaste.add_paste_fields("control", input_image, paste_fields, override_settings) bindings = generation_parameters_copypaste.ParamBinding(paste_button=btn_paste, tabname="control", source_text_component=prompt, source_image_component=output_gallery) generation_parameters_copypaste.register_paste_params_button(bindings) - masking.bind_controls([input_image], preview_process, output_image) + # masking.bind_controls([input_image], preview_process, output_image) if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs, test_lite diff --git a/modules/ui_control_helpers.py b/modules/ui_control_helpers.py index 40a6a55e4..2d79e17a7 100644 --- a/modules/ui_control_helpers.py +++ b/modules/ui_control_helpers.py @@ -1,4 +1,5 @@ import os +import time import gradio as gr from PIL import Image from modules import shared, scripts_manager, masking, video # pylint: disable=ungrouped-imports @@ -48,12 +49,13 @@ def initialize(): def interrogate(): - prompt = None + if input_source is None or len(input_source) == 0: + shared.log.warning('Interrogate: no input source') try: from modules.interrogate.interrogate import interrogate as interrogate_fn prompt = interrogate_fn(input_source[0]) - except Exception: - pass + except Exception as e: + shared.log.error(f'Interrogate: {e}') return prompt @@ -74,25 +76,69 @@ def get_video(filepath: str): return msg +def process_kanvas(x): # only used when kanvas overrides gr.Image object + image = None + mask = None + try: # try base64 decode + t0 = time.time() + image_data = x.get('image', '') + image_bytes = len(image_data) + if image_bytes > 0: + from modules.api import helpers + image = helpers.decode_base64_to_image(image_data) + image = image.convert('RGB') + mask_data = x.get('mask', '') + mask_bytes = len(mask_data) + if mask_bytes > 0: + from modules.api import helpers + mask = helpers.decode_base64_to_image(mask_data) + mask = mask.convert('L') + t1 = time.time() + shared.log.debug(f'Kanvas: image={image}:{image_bytes} mask={mask}:{mask_bytes} time={t1-t0:.2f}') + return image, mask + except Exception: + pass + try: # try raw pixel data + import numpy as np + t0 = time.time() + image_data = list(x.get('image', {}).values()) + if image_data: + width = x['imageWidth'] + height = x['imageHeight'] + array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4)) + image = Image.fromarray(array, 'RGBA') + image = image.convert('RGB') + mask_data = list(x.get('mask', {}).values()) + if mask_data: + width = x['maskWidth'] + height = x['maskHeight'] + array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4)) + mask = Image.fromarray(array, 'RGBA') + # alpha = mask.getchannel("A").convert("L") + # mask = Image.merge("RGB", [alpha, alpha, alpha]) + mask = mask.convert('L') + t1 = time.time() + shared.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}') + except Exception: + pass + return image, mask + + def select_input(input_mode, input_image, init_image, init_type, input_video, input_batch, input_folder): global busy, input_source, input_init, input_mask # pylint: disable=global-statement + t0 = time.time() busy = True - selected_input = input_image - if input_mode == 'Kanvas': - selected_input = input_image - elif input_mode == 'Video': + selected_input = input_image # default: Image or Kanvas + if input_mode == 'Video': selected_input = input_video elif input_mode == 'Batch': selected_input = input_batch elif input_mode == 'Folder': selected_input = input_folder - else: - selected_input = None size = [gr.update(), gr.update()] if selected_input is None: input_source = None busy = False - # debug('Control input: none') return [gr.Tabs.update(), None, ''] + size input_type = type(selected_input) input_mask = None @@ -105,15 +151,23 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in selected_input, input_mask = masking.outpaint(input_image=selected_input) input_source = [selected_input] input_type = 'PIL.Image' - status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' + status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}' size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)] res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] - elif isinstance(selected_input, dict): # inpaint -> dict image+mask + elif isinstance(selected_input, dict) and 'kanvas' in selected_input: # kanvas via js -> kanvas dict + selected_input, input_mask = process_kanvas(selected_input) + input_source = [selected_input] + input_type = 'Kanvas' + status = f'Control input | Kanvas | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}' + if selected_input: + size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)] + res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] + elif isinstance(selected_input, dict) and 'mask' in selected_input: # inpaint -> dict image+mask input_mask = selected_input['mask'] selected_input = selected_input['image'] input_source = [selected_input] input_type = 'PIL.Image' - status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' + status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}' res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] elif isinstance(selected_input, gr.components.image.Image): # not likely input_source = [selected_input.value] @@ -135,14 +189,14 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] else: # unknown input_source = None - # init inputs: optional if init_type == 0: # Control only input_init = None elif init_type == 1: # Init image same as control assigned during runtime input_init = None elif init_type == 2: # Separate init image input_init = [init_image] - debug_log(f'Control select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode}') + t1 = time.time() + shared.log.debug(f'Select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode} time={t1-t0:.2f}') busy = False return res + size