Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4388 from vladmandic/kanvas

Browse Source 
merge kanvas to dev
This commit is contained in:
Vladimir Mandic
2025-11-09 07:57:54 -05:00
committed by GitHub
parent 97e92d5eff f491955991
commit 3a3c984411
13 changed files with 161 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/devices.py
+3 -3
View File
@@ -458,9 +458,9 @@ def set_sdpa_params():
log.warning(f'Torch attention: type="sdpa" {err}')
try:
torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options)
torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options)
torch.backends.cuda.enable_math_sdp('Math' in opts.sdp_options)
torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options or 'Flash attention' in opts.sdp_options)
torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options or 'Memory attention' in opts.sdp_options)
torch.backends.cuda.enable_math_sdp('Math' in opts.sdp_options or 'Math attention' in opts.sdp_options)
if hasattr(torch.backends.cuda, "allow_fp16_bf16_reduction_math_sdp"): # only valid for torch >= 2.5
torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True)
log.debug(f'Torch attention: type="sdpa" kernels={opts.sdp_options} overrides={opts.sdp_overrides}')
modules/gr_hijack.py
+39 -2
View File
@@ -1,3 +1,4 @@
import time
from PIL import Image
import gradio as gr
import gradio.processing_utils
@@ -11,13 +12,49 @@ original_BlockContext_init = None
original_Blocks_get_config_file = None
def process_kanvas(self, x): # only used when kanvas overrides gr.Image object
import numpy as np
from modules import errors
t0 = time.time()
image_data = list(x.get('image', {}).values())
image = None
mask = None
if image_data:
width = x['imageWidth']
height = x['imageHeight']
array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4))
image = Image.fromarray(array, 'RGBA')
image = image.convert('RGB')
mask_data = list(x.get('mask', {}).values())
if mask_data:
width = x['maskWidth']
height = x['maskHeight']
array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4))
mask = Image.fromarray(array, 'RGBA')
# alpha = mask.getchannel("A").convert("L")
# mask = Image.merge("RGB", [alpha, alpha, alpha])
mask = mask.convert('L')
t1 = time.time()
errors.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}')
if image is None:
return None
if mask is None:
return self._format_image(image) # pylint: disable=protected-access
return { "image": self._format_image(image), "mask": self._format_image(mask) } # pylint: disable=protected-access
def gr_image_preprocess(self, x):
if x is None:
return x
mask = None
if isinstance(x, dict):
if isinstance(x, dict) and "kanvas" in x:
return process_kanvas(self, x)
if isinstance(x, dict) and "image" in x:
x, mask = x["image"], x["mask"]
im = gradio.processing_utils.decode_base64_to_image(x)
if isinstance(x, str):
im = gradio.processing_utils.decode_base64_to_image(x)
else:
im = x
im = im.convert(self.image_mode)
if self.shape is not None:
im = gradio.processing_utils.resize_and_crop(im, self.shape)
modules/images_resize.py
+2 -1
View File
@@ -154,5 +154,6 @@ def resize_image(resize_mode: int, im: Union[Image.Image, torch.Tensor], width:
shared.log.error(f'Invalid resize mode: {resize_mode}')
t1 = time.time()
fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
shared.log.debug(f'Image resize: source={im.width}:{im.height} target={width}:{height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" type={output_type} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access
if im.width != width or im.height != height:
shared.log.debug(f'Image resize: source={im.width}:{im.height} target={width}:{height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" type={output_type} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access
return np.array(res) if output_type == 'np' else res
modules/interrogate/vqa.py
+2 -3
View File
@@ -178,7 +178,7 @@ def fastvlm(question: str, image: Image.Image, repo: str = None):
def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str = None):
global processor, model, loaded # pylint: disable=global-statement
if model is None or loaded != repo:
if (model is None) or (loaded != repo):
shared.log.debug(f'Interrogate load: vlm="{repo}"')
model = None
if 'Qwen3-VL' in repo or 'Qwen3VL' in repo:
@@ -633,8 +633,7 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image:
global quant_args # pylint: disable=global-statement
jobid = shared.state.begin('Interrogate LLM')
t0 = time.time()
if quant_args is None:
quant_args = model_quant.create_config(module='LLM')
quant_args = model_quant.create_config(module='LLM')
model_name = model_name or shared.opts.interrogate_vlm_model
if isinstance(image, list):
image = image[0] if len(image) > 0 else None
modules/model_quant.py
+2 -2
View File
@@ -246,10 +246,10 @@ def check_nunchaku(module: str = ''):
def create_config(kwargs = None, allow: bool = True, module: str = 'Model', modules_to_not_convert: list = None, modules_dtype_dict: dict = None):
if dont_quant():
return kwargs
if kwargs is None:
kwargs = {}
if module == 'Model' and dont_quant():
return kwargs
kwargs = create_sdnq_config(kwargs, allow=allow, module=module, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict)
if kwargs is not None and 'quantization_config' in kwargs:
if debug:
modules/ui_control.py
+12 -3
View File
@@ -5,6 +5,7 @@ from modules.control import unit
from modules import errors, shared, progress, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports
from modules import ui_common, ui_sections, ui_guidance
from modules import ui_control_helpers as helpers
import installer
gr_height = 512
@@ -185,7 +186,11 @@ def create_ui(_blocks: gr.Blocks=None):
with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'):
input_mode = gr.Label(value='select', visible=False)
with gr.Tab('Image', id='in-image') as tab_image:
input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image'])
if (installer.version['kanvas'] == 'disabled') or (installer.version['kanvas'] == 'unavailable'):
input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image'])
else:
input_image = gr.HTML(value='<h1 style="text-align:center;color:var(--color-error);margin:1em;">Kanvas not initialized</h1>', elem_id='kanvas-container')
input_changed = gr.Button('Kanvas change', elem_id='kanvas-change-button', visible=False)
btn_interrogate = ui_sections.create_interrogate_button('control', what='input')
with gr.Tab('Video', id='in-video') as tab_video:
input_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height, elem_classes=['control-image'])
@@ -242,7 +247,6 @@ def create_ui(_blocks: gr.Blocks=None):
input_type.change(fn=lambda x: gr.update(visible=x == 2), inputs=[input_type], outputs=[column_init])
btn_prompt_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[prompt], outputs=[prompt_counter], show_progress = False)
btn_negative_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[negative], outputs=[negative_counter], show_progress = False)
btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt])
select_dict = dict(
fn=helpers.select_input,
@@ -253,6 +257,11 @@ def create_ui(_blocks: gr.Blocks=None):
queue=False,
)
input_changed.click(**select_dict)
btn_interrogate.click(**select_dict) # need to fetch input first
btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt])
prompt.submit(**select_dict)
negative.submit(**select_dict)
btn_generate.click(**select_dict)
@@ -402,7 +411,7 @@ def create_ui(_blocks: gr.Blocks=None):
generation_parameters_copypaste.add_paste_fields("control", input_image, paste_fields, override_settings)
bindings = generation_parameters_copypaste.ParamBinding(paste_button=btn_paste, tabname="control", source_text_component=prompt, source_image_component=output_gallery)
generation_parameters_copypaste.register_paste_params_button(bindings)
masking.bind_controls([input_image], preview_process, output_image)
# masking.bind_controls([input_image], preview_process, output_image)
if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only
from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs, test_lite
modules/ui_control_helpers.py
+70 -14
View File
@@ -1,4 +1,5 @@
import os
import time
import gradio as gr
from PIL import Image
from modules import shared, scripts_manager, masking, video # pylint: disable=ungrouped-imports
@@ -49,11 +50,14 @@ def initialize():
def interrogate():
prompt = None
if input_source is None or len(input_source) == 0:
shared.log.warning('Interrogate: no input source')
return prompt
try:
from modules.interrogate.interrogate import interrogate as interrogate_fn
prompt = interrogate_fn(input_source[0])
except Exception:
pass
except Exception as e:
shared.log.error(f'Interrogate: {e}')
return prompt
@@ -74,25 +78,69 @@ def get_video(filepath: str):
return msg
def process_kanvas(x): # only used when kanvas overrides gr.Image object
image = None
mask = None
try: # try base64 decode
t0 = time.time()
image_data = x.get('image', '')
image_bytes = len(image_data)
if image_bytes > 0:
from modules.api import helpers
image = helpers.decode_base64_to_image(image_data)
image = image.convert('RGB')
mask_data = x.get('mask', '')
mask_bytes = len(mask_data)
if mask_bytes > 0:
from modules.api import helpers
mask = helpers.decode_base64_to_image(mask_data)
mask = mask.convert('L')
t1 = time.time()
shared.log.debug(f'Kanvas: image={image}:{image_bytes} mask={mask}:{mask_bytes} time={t1-t0:.2f}')
return image, mask
except Exception:
pass
try: # try raw pixel data
import numpy as np
t0 = time.time()
image_data = list(x.get('image', {}).values())
if image_data:
width = x['imageWidth']
height = x['imageHeight']
array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4))
image = Image.fromarray(array, 'RGBA')
image = image.convert('RGB')
mask_data = list(x.get('mask', {}).values())
if mask_data:
width = x['maskWidth']
height = x['maskHeight']
array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4))
mask = Image.fromarray(array, 'RGBA')
# alpha = mask.getchannel("A").convert("L")
# mask = Image.merge("RGB", [alpha, alpha, alpha])
mask = mask.convert('L')
t1 = time.time()
shared.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}')
except Exception:
pass
return image, mask
def select_input(input_mode, input_image, init_image, init_type, input_video, input_batch, input_folder):
global busy, input_source, input_init, input_mask # pylint: disable=global-statement
t0 = time.time()
busy = True
selected_input = input_image
if input_mode == 'Kanvas':
selected_input = input_image
elif input_mode == 'Video':
selected_input = input_image # default: Image or Kanvas
if input_mode == 'Video':
selected_input = input_video
elif input_mode == 'Batch':
selected_input = input_batch
elif input_mode == 'Folder':
selected_input = input_folder
else:
selected_input = None
size = [gr.update(), gr.update()]
if selected_input is None:
input_source = None
busy = False
# debug('Control input: none')
return [gr.Tabs.update(), None, ''] + size
input_type = type(selected_input)
input_mask = None
@@ -105,15 +153,23 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in
selected_input, input_mask = masking.outpaint(input_image=selected_input)
input_source = [selected_input]
input_type = 'PIL.Image'
status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}'
status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}'
size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)]
res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
elif isinstance(selected_input, dict): # inpaint -> dict image+mask
elif isinstance(selected_input, dict) and 'kanvas' in selected_input: # kanvas via js -> kanvas dict
selected_input, input_mask = process_kanvas(selected_input)
input_source = [selected_input]
input_type = 'Kanvas'
status = f'Control input | Kanvas | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}'
if selected_input:
size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)]
res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
elif isinstance(selected_input, dict) and 'mask' in selected_input: # inpaint -> dict image+mask
input_mask = selected_input['mask']
selected_input = selected_input['image']
input_source = [selected_input]
input_type = 'PIL.Image'
status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}'
status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}'
res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
elif isinstance(selected_input, gr.components.image.Image): # not likely
input_source = [selected_input.value]
@@ -135,14 +191,14 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in
res = [gr.Tabs.update(selected='out-gallery'), input_mask, status]
else: # unknown
input_source = None
# init inputs: optional
if init_type == 0: # Control only
input_init = None
elif init_type == 1: # Init image same as control assigned during runtime
input_init = None
elif init_type == 2: # Separate init image
input_init = [init_image]
debug_log(f'Control select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode}')
t1 = time.time()
shared.log.debug(f'Select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode} time={t1-t0:.2f}')
busy = False
return res + size
modules/ui_javascript.py
+1 -2
View File
@@ -102,8 +102,7 @@ def reload_javascript():
css_base = theme.reload_gradio_theme()
css_timesheet = "timesheet.css"
css_kanvas = "kanvas.css"
css = html_css([css_base, css_timesheet, css_kanvas])
css = html_css([css_base, css_timesheet])
body = html_body()
def template_response(*args, **kwargs):