kanvas bindings

Signed-off-by: Vladimir Mandic <mandic00@live.com>

javascript/control.js

@@ -3,7 +3,14 @@ function controlInputMode(inputMode, ...args) {
   if (updateEl) updateEl.click();
   const tab = gradioApp().querySelector('#control-tab-input button.selected');
   if (!tab) return ['Image', ...args];
-  inputMode = tab.innerText;
+  let inputTab = tab.innerText;
+  log('controlInputMode', inputMode, inputTab);
+  if ((inputTab === 'Image') && ('kanvas' in window)) {
+    inputTab = 'Kanvas';
+    // const imageData = window.kanvas.getImageData();
+    const imageData = window.kanvas.getImage();
+    args[0] = imageData;
+  }
   return [inputMode, ...args];
 }
 

modules/gr_hijack.py

@@ -1,3 +1,4 @@
+import time
 from PIL import Image
 import gradio as gr
 import gradio.processing_utils
@@ -11,13 +12,49 @@ original_BlockContext_init = None
 original_Blocks_get_config_file = None
 
 
+def process_kanvas(self, x): # only used when kanvas overrides gr.Image object
+    import numpy as np
+    from modules import errors
+    t0 = time.time()
+    image_data = list(x.get('image', {}).values())
+    image = None
+    mask = None
+    if image_data:
+        width = x['imageWidth']
+        height = x['imageHeight']
+        array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4))
+        image = Image.fromarray(array, 'RGBA')
+        image = image.convert('RGB')
+    mask_data = list(x.get('mask', {}).values())
+    if mask_data:
+        width = x['maskWidth']
+        height = x['maskHeight']
+        array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4))
+        mask = Image.fromarray(array, 'RGBA')
+        # alpha = mask.getchannel("A").convert("L")
+        # mask = Image.merge("RGB", [alpha, alpha, alpha])
+        mask = mask.convert('L')
+    t1 = time.time()
+    errors.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}')
+    if image is None:
+        return None
+    if mask is None:
+        return self._format_image(image) # pylint: disable=protected-access
+    return { "image": self._format_image(image), "mask": self._format_image(mask) } # pylint: disable=protected-access
+
+
 def gr_image_preprocess(self, x):
     if x is None:
         return x
     mask = None
-    if isinstance(x, dict):
+    if isinstance(x, dict) and "kanvas" in x:
+        return process_kanvas(self, x)
+    if isinstance(x, dict) and "image" in x:
         x, mask = x["image"], x["mask"]
-    im = gradio.processing_utils.decode_base64_to_image(x)
+    if isinstance(x, str):
+        im = gradio.processing_utils.decode_base64_to_image(x)
+    else:
+        im = x
     im = im.convert(self.image_mode)
     if self.shape is not None:
         im = gradio.processing_utils.resize_and_crop(im, self.shape)

modules/interrogate/vqa.py

@@ -178,7 +178,7 @@ def fastvlm(question: str, image: Image.Image, repo: str = None):
 
 def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str = None):
     global processor, model, loaded # pylint: disable=global-statement
-    if model is None or loaded != repo:
+    if (model is None) or (loaded != repo):
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
         model = None
         if 'Qwen3-VL' in repo or 'Qwen3VL' in repo:
@@ -633,8 +633,7 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image:
     global quant_args # pylint: disable=global-statement
     jobid = shared.state.begin('Interrogate LLM')
     t0 = time.time()
-    if quant_args is None:
-        quant_args = model_quant.create_config(module='LLM')
+    quant_args = model_quant.create_config(module='LLM')
     model_name = model_name or shared.opts.interrogate_vlm_model
     if isinstance(image, list):
         image = image[0] if len(image) > 0 else None

modules/model_quant.py

@@ -246,10 +246,10 @@ def check_nunchaku(module: str = ''):
 
 
 def create_config(kwargs = None, allow: bool = True, module: str = 'Model', modules_to_not_convert: list = None, modules_dtype_dict: dict = None):
-    if dont_quant():
-        return kwargs
     if kwargs is None:
         kwargs = {}
+    if module == 'Model' and dont_quant():
+        return kwargs
     kwargs = create_sdnq_config(kwargs, allow=allow, module=module, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict)
     if kwargs is not None and 'quantization_config' in kwargs:
         if debug:

modules/ui_control.py

@@ -185,7 +185,9 @@ def create_ui(_blocks: gr.Blocks=None):
                     with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'):
                         input_mode = gr.Label(value='select', visible=False)
                         with gr.Tab('Image', id='in-image') as tab_image:
-                            input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image'])
+                            input_image = gr.HTML(value="Kanvas placeholder", elem_id='control_input_select')
+                            input_changed = gr.Button('Kanvas change', elem_id='control_input_change', visible=False)
+                            # input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image'])
                             btn_interrogate = ui_sections.create_interrogate_button('control', what='input')
                         with gr.Tab('Video', id='in-video') as tab_video:
                             input_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height, elem_classes=['control-image'])
@@ -242,7 +244,6 @@ def create_ui(_blocks: gr.Blocks=None):
             input_type.change(fn=lambda x: gr.update(visible=x == 2), inputs=[input_type], outputs=[column_init])
             btn_prompt_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[prompt], outputs=[prompt_counter], show_progress = False)
             btn_negative_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[negative], outputs=[negative_counter], show_progress = False)
-            btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt])
 
             select_dict = dict(
                 fn=helpers.select_input,
@@ -253,6 +254,11 @@ def create_ui(_blocks: gr.Blocks=None):
                 queue=False,
             )
 
+            input_changed.click(**select_dict)
+            btn_interrogate.click(**select_dict) # need to fetch input first
+            btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt])
+
+
             prompt.submit(**select_dict)
             negative.submit(**select_dict)
             btn_generate.click(**select_dict)
@@ -403,7 +409,7 @@ def create_ui(_blocks: gr.Blocks=None):
             generation_parameters_copypaste.add_paste_fields("control", input_image, paste_fields, override_settings)
             bindings = generation_parameters_copypaste.ParamBinding(paste_button=btn_paste, tabname="control", source_text_component=prompt, source_image_component=output_gallery)
             generation_parameters_copypaste.register_paste_params_button(bindings)
-            masking.bind_controls([input_image], preview_process, output_image)
+            # masking.bind_controls([input_image], preview_process, output_image)
 
             if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only
                 from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs, test_lite

modules/ui_control_helpers.py

@@ -1,4 +1,5 @@
 import os
+import time
 import gradio as gr
 from PIL import Image
 from modules import shared, scripts_manager, masking, video # pylint: disable=ungrouped-imports
@@ -48,12 +49,13 @@ def initialize():
 
 
 def interrogate():
-    prompt = None
+    if input_source is None or len(input_source) == 0:
+        shared.log.warning('Interrogate: no input source')
     try:
         from modules.interrogate.interrogate import interrogate as interrogate_fn
         prompt = interrogate_fn(input_source[0])
-    except Exception:
-        pass
+    except Exception as e:
+        shared.log.error(f'Interrogate: {e}')
     return prompt
 
 
@@ -74,25 +76,69 @@ def get_video(filepath: str):
         return msg
 
 
+def process_kanvas(x): # only used when kanvas overrides gr.Image object
+    image = None
+    mask = None
+    try: # try base64 decode
+        t0 = time.time()
+        image_data = x.get('image', '')
+        image_bytes = len(image_data)
+        if image_bytes > 0:
+            from modules.api import helpers
+            image = helpers.decode_base64_to_image(image_data)
+            image = image.convert('RGB')
+        mask_data = x.get('mask', '')
+        mask_bytes = len(mask_data)
+        if mask_bytes > 0:
+            from modules.api import helpers
+            mask = helpers.decode_base64_to_image(mask_data)
+            mask = mask.convert('L')
+        t1 = time.time()
+        shared.log.debug(f'Kanvas: image={image}:{image_bytes} mask={mask}:{mask_bytes} time={t1-t0:.2f}')
+        return image, mask
+    except Exception:
+        pass
+    try: # try raw pixel data
+        import numpy as np
+        t0 = time.time()
+        image_data = list(x.get('image', {}).values())
+        if image_data:
+            width = x['imageWidth']
+            height = x['imageHeight']
+            array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4))
+            image = Image.fromarray(array, 'RGBA')
+            image = image.convert('RGB')
+        mask_data = list(x.get('mask', {}).values())
+        if mask_data:
+            width = x['maskWidth']
+            height = x['maskHeight']
+            array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4))
+            mask = Image.fromarray(array, 'RGBA')
+            # alpha = mask.getchannel("A").convert("L")
+            # mask = Image.merge("RGB", [alpha, alpha, alpha])
+            mask = mask.convert('L')
+        t1 = time.time()
+        shared.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}')
+    except Exception:
+        pass
+    return image, mask
+
+
 def select_input(input_mode, input_image, init_image, init_type, input_video, input_batch, input_folder):
     global busy, input_source, input_init, input_mask # pylint: disable=global-statement
+    t0 = time.time()
     busy = True
-    selected_input = input_image
-    if input_mode == 'Kanvas':
-        selected_input = input_image
-    elif input_mode == 'Video':
+    selected_input = input_image # default: Image or Kanvas
+    if input_mode == 'Video':
         selected_input = input_video
     elif input_mode == 'Batch':
         selected_input = input_batch
     elif input_mode == 'Folder':
         selected_input = input_folder
-    else:
-        selected_input = None
     size = [gr.update(), gr.update()]
     if selected_input is None:
         input_source = None
         busy = False
-        # debug('Control input: none')
         return [gr.Tabs.update(), None, ''] + size
     input_type = type(selected_input)
     input_mask = None
@@ -105,15 +151,23 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in
             selected_input, input_mask = masking.outpaint(input_image=selected_input)
         input_source = [selected_input]
         input_type = 'PIL.Image'
-        status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}'
+        status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}'
         size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)]
         res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
-    elif isinstance(selected_input, dict): # inpaint -> dict image+mask
+    elif isinstance(selected_input, dict) and 'kanvas' in selected_input: # kanvas via js -> kanvas dict
+        selected_input, input_mask = process_kanvas(selected_input)
+        input_source = [selected_input]
+        input_type = 'Kanvas'
+        status = f'Control input | Kanvas | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}'
+        if selected_input:
+            size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)]
+        res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
+    elif isinstance(selected_input, dict) and 'mask' in selected_input: # inpaint -> dict image+mask
         input_mask = selected_input['mask']
         selected_input = selected_input['image']
         input_source = [selected_input]
         input_type = 'PIL.Image'
-        status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}'
+        status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}'
         res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
     elif isinstance(selected_input, gr.components.image.Image): # not likely
         input_source = [selected_input.value]
@@ -135,14 +189,14 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in
         res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
     else: # unknown
         input_source = None
-    # init inputs: optional
     if init_type == 0: # Control only
         input_init = None
     elif init_type == 1: # Init image same as control assigned during runtime
         input_init = None
     elif init_type == 2: # Separate init image
         input_init = [init_image]
-    debug_log(f'Control select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode}')
+    t1 = time.time()
+    shared.log.debug(f'Select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode} time={t1-t0:.2f}')
     busy = False
     return res + size