diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index f14464f3f..bbb5f99e6 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit f14464f3f7bb0de9ef854a40f7f56f8dd0378f42 +Subproject commit bbb5f99e6e103a1fe32ca43ca8f2fd4552c5dd09 diff --git a/installer.py b/installer.py index fb195112f..d3d842fa5 100644 --- a/installer.py +++ b/installer.py @@ -1445,21 +1445,29 @@ def get_version(force=False): version = { 'app': 'sd.next', 'version': 'unknown', 'branch': 'unknown' } cwd = os.getcwd() try: - os.chdir('extensions-builtin/sdnext-modernui') - res = subprocess.run('git rev-parse --abbrev-ref HEAD', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) - branch_ui = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' - branch_ui = 'dev' if 'dev' in branch_ui else 'main' - version['ui'] = branch_ui + if os.path.exists('extensions-builtin/sdnext-modernui'): + os.chdir('extensions-builtin/sdnext-modernui') + res = subprocess.run('git rev-parse --abbrev-ref HEAD', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + branch_ui = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + branch_ui = 'dev' if 'dev' in branch_ui else 'main' + version['ui'] = branch_ui + else: + version['ui'] = 'unavailable' except Exception: version['ui'] = 'unknown' finally: os.chdir(cwd) try: - os.chdir('extensions-builtin/sdnext-kanvas') - res = subprocess.run('git rev-parse --abbrev-ref HEAD', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) - branch_kanvas = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' - branch_kanvas = 'dev' if 'dev' in branch_kanvas else 'main' - version['kanvas'] = branch_kanvas + if os.environ.get('SD_KANVAS_DISABLE', None) is not None: + version['kanvas'] = 'disabled' + elif os.path.exists('extensions-builtin/sdnext-kanvas'): + os.chdir('extensions-builtin/sdnext-kanvas') + res = subprocess.run('git rev-parse --abbrev-ref HEAD', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + branch_kanvas = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + branch_kanvas = 'dev' if 'dev' in branch_kanvas else 'main' + version['kanvas'] = branch_kanvas + else: + version['kanvas'] = 'unavailable' except Exception: version['kanvas'] = 'unknown' finally: diff --git a/javascript/control.js b/javascript/control.js index e0bb550ba..1db434778 100644 --- a/javascript/control.js +++ b/javascript/control.js @@ -3,7 +3,14 @@ function controlInputMode(inputMode, ...args) { if (updateEl) updateEl.click(); const tab = gradioApp().querySelector('#control-tab-input button.selected'); if (!tab) return ['Image', ...args]; - inputMode = tab.innerText; + let inputTab = tab.innerText; + log('controlInputMode', inputMode, inputTab); + if ((inputTab === 'Image') && ('kanvas' in window)) { + inputTab = 'Kanvas'; + // const imageData = window.kanvas.getImageData(); + const imageData = window.kanvas.getImage(); + args[0] = imageData; + } return [inputMode, ...args]; } diff --git a/modules/devices.py b/modules/devices.py index e6f21194c..76d9a8652 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -458,9 +458,9 @@ def set_sdpa_params(): log.warning(f'Torch attention: type="sdpa" {err}') try: - torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options) - torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options) - torch.backends.cuda.enable_math_sdp('Math' in opts.sdp_options) + torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options or 'Flash attention' in opts.sdp_options) + torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options or 'Memory attention' in opts.sdp_options) + torch.backends.cuda.enable_math_sdp('Math' in opts.sdp_options or 'Math attention' in opts.sdp_options) if hasattr(torch.backends.cuda, "allow_fp16_bf16_reduction_math_sdp"): # only valid for torch >= 2.5 torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True) log.debug(f'Torch attention: type="sdpa" kernels={opts.sdp_options} overrides={opts.sdp_overrides}') diff --git a/modules/gr_hijack.py b/modules/gr_hijack.py index b4607cc11..7fa1cecdd 100644 --- a/modules/gr_hijack.py +++ b/modules/gr_hijack.py @@ -1,3 +1,4 @@ +import time from PIL import Image import gradio as gr import gradio.processing_utils @@ -11,13 +12,49 @@ original_BlockContext_init = None original_Blocks_get_config_file = None +def process_kanvas(self, x): # only used when kanvas overrides gr.Image object + import numpy as np + from modules import errors + t0 = time.time() + image_data = list(x.get('image', {}).values()) + image = None + mask = None + if image_data: + width = x['imageWidth'] + height = x['imageHeight'] + array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4)) + image = Image.fromarray(array, 'RGBA') + image = image.convert('RGB') + mask_data = list(x.get('mask', {}).values()) + if mask_data: + width = x['maskWidth'] + height = x['maskHeight'] + array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4)) + mask = Image.fromarray(array, 'RGBA') + # alpha = mask.getchannel("A").convert("L") + # mask = Image.merge("RGB", [alpha, alpha, alpha]) + mask = mask.convert('L') + t1 = time.time() + errors.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}') + if image is None: + return None + if mask is None: + return self._format_image(image) # pylint: disable=protected-access + return { "image": self._format_image(image), "mask": self._format_image(mask) } # pylint: disable=protected-access + + def gr_image_preprocess(self, x): if x is None: return x mask = None - if isinstance(x, dict): + if isinstance(x, dict) and "kanvas" in x: + return process_kanvas(self, x) + if isinstance(x, dict) and "image" in x: x, mask = x["image"], x["mask"] - im = gradio.processing_utils.decode_base64_to_image(x) + if isinstance(x, str): + im = gradio.processing_utils.decode_base64_to_image(x) + else: + im = x im = im.convert(self.image_mode) if self.shape is not None: im = gradio.processing_utils.resize_and_crop(im, self.shape) diff --git a/modules/images_resize.py b/modules/images_resize.py index 03646484e..1f2ba0cea 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -154,5 +154,6 @@ def resize_image(resize_mode: int, im: Union[Image.Image, torch.Tensor], width: shared.log.error(f'Invalid resize mode: {resize_mode}') t1 = time.time() fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - shared.log.debug(f'Image resize: source={im.width}:{im.height} target={width}:{height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" type={output_type} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access + if im.width != width or im.height != height: + shared.log.debug(f'Image resize: source={im.width}:{im.height} target={width}:{height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" type={output_type} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access return np.array(res) if output_type == 'np' else res diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 877740a76..7c93c56f3 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -178,7 +178,7 @@ def fastvlm(question: str, image: Image.Image, repo: str = None): def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str = None): global processor, model, loaded # pylint: disable=global-statement - if model is None or loaded != repo: + if (model is None) or (loaded != repo): shared.log.debug(f'Interrogate load: vlm="{repo}"') model = None if 'Qwen3-VL' in repo or 'Qwen3VL' in repo: @@ -633,8 +633,7 @@ def interrogate(question:str='', system_prompt:str=None, prompt:str=None, image: global quant_args # pylint: disable=global-statement jobid = shared.state.begin('Interrogate LLM') t0 = time.time() - if quant_args is None: - quant_args = model_quant.create_config(module='LLM') + quant_args = model_quant.create_config(module='LLM') model_name = model_name or shared.opts.interrogate_vlm_model if isinstance(image, list): image = image[0] if len(image) > 0 else None diff --git a/modules/model_quant.py b/modules/model_quant.py index 93f3accc4..e844a0ccf 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -246,10 +246,10 @@ def check_nunchaku(module: str = ''): def create_config(kwargs = None, allow: bool = True, module: str = 'Model', modules_to_not_convert: list = None, modules_dtype_dict: dict = None): - if dont_quant(): - return kwargs if kwargs is None: kwargs = {} + if module == 'Model' and dont_quant(): + return kwargs kwargs = create_sdnq_config(kwargs, allow=allow, module=module, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict) if kwargs is not None and 'quantization_config' in kwargs: if debug: diff --git a/modules/ui_control.py b/modules/ui_control.py index 0220412bf..9671f01d5 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -5,6 +5,7 @@ from modules.control import unit from modules import errors, shared, progress, generation_parameters_copypaste, call_queue, scripts_manager, masking, images, processing_vae, timer # pylint: disable=ungrouped-imports from modules import ui_common, ui_sections, ui_guidance from modules import ui_control_helpers as helpers +import installer gr_height = 512 @@ -185,7 +186,11 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Tabs(elem_classes=['control-tabs'], elem_id='control-tab-input'): input_mode = gr.Label(value='select', visible=False) with gr.Tab('Image', id='in-image') as tab_image: - input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image']) + if (installer.version['kanvas'] == 'disabled') or (installer.version['kanvas'] == 'unavailable'): + input_image = gr.Image(label="Input", show_label=False, type="pil", interactive=True, tool="editor", height=gr_height, image_mode='RGB', elem_id='control_input_select', elem_classes=['control-image']) + else: + input_image = gr.HTML(value='

Kanvas not initialized

', elem_id='kanvas-container') + input_changed = gr.Button('Kanvas change', elem_id='kanvas-change-button', visible=False) btn_interrogate = ui_sections.create_interrogate_button('control', what='input') with gr.Tab('Video', id='in-video') as tab_video: input_video = gr.Video(label="Input", show_label=False, interactive=True, height=gr_height, elem_classes=['control-image']) @@ -242,7 +247,6 @@ def create_ui(_blocks: gr.Blocks=None): input_type.change(fn=lambda x: gr.update(visible=x == 2), inputs=[input_type], outputs=[column_init]) btn_prompt_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[prompt], outputs=[prompt_counter], show_progress = False) btn_negative_counter.click(fn=call_queue.wrap_queued_call(ui_common.update_token_counter), inputs=[negative], outputs=[negative_counter], show_progress = False) - btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt]) select_dict = dict( fn=helpers.select_input, @@ -253,6 +257,11 @@ def create_ui(_blocks: gr.Blocks=None): queue=False, ) + input_changed.click(**select_dict) + btn_interrogate.click(**select_dict) # need to fetch input first + btn_interrogate.click(fn=helpers.interrogate, inputs=[], outputs=[prompt]) + + prompt.submit(**select_dict) negative.submit(**select_dict) btn_generate.click(**select_dict) @@ -402,7 +411,7 @@ def create_ui(_blocks: gr.Blocks=None): generation_parameters_copypaste.add_paste_fields("control", input_image, paste_fields, override_settings) bindings = generation_parameters_copypaste.ParamBinding(paste_button=btn_paste, tabname="control", source_text_component=prompt, source_image_component=output_gallery) generation_parameters_copypaste.register_paste_params_button(bindings) - masking.bind_controls([input_image], preview_process, output_image) + # masking.bind_controls([input_image], preview_process, output_image) if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs, test_lite diff --git a/modules/ui_control_helpers.py b/modules/ui_control_helpers.py index 40a6a55e4..99504491a 100644 --- a/modules/ui_control_helpers.py +++ b/modules/ui_control_helpers.py @@ -1,4 +1,5 @@ import os +import time import gradio as gr from PIL import Image from modules import shared, scripts_manager, masking, video # pylint: disable=ungrouped-imports @@ -49,11 +50,14 @@ def initialize(): def interrogate(): prompt = None + if input_source is None or len(input_source) == 0: + shared.log.warning('Interrogate: no input source') + return prompt try: from modules.interrogate.interrogate import interrogate as interrogate_fn prompt = interrogate_fn(input_source[0]) - except Exception: - pass + except Exception as e: + shared.log.error(f'Interrogate: {e}') return prompt @@ -74,25 +78,69 @@ def get_video(filepath: str): return msg +def process_kanvas(x): # only used when kanvas overrides gr.Image object + image = None + mask = None + try: # try base64 decode + t0 = time.time() + image_data = x.get('image', '') + image_bytes = len(image_data) + if image_bytes > 0: + from modules.api import helpers + image = helpers.decode_base64_to_image(image_data) + image = image.convert('RGB') + mask_data = x.get('mask', '') + mask_bytes = len(mask_data) + if mask_bytes > 0: + from modules.api import helpers + mask = helpers.decode_base64_to_image(mask_data) + mask = mask.convert('L') + t1 = time.time() + shared.log.debug(f'Kanvas: image={image}:{image_bytes} mask={mask}:{mask_bytes} time={t1-t0:.2f}') + return image, mask + except Exception: + pass + try: # try raw pixel data + import numpy as np + t0 = time.time() + image_data = list(x.get('image', {}).values()) + if image_data: + width = x['imageWidth'] + height = x['imageHeight'] + array = np.array(image_data, dtype=np.uint8).reshape((height, width, 4)) + image = Image.fromarray(array, 'RGBA') + image = image.convert('RGB') + mask_data = list(x.get('mask', {}).values()) + if mask_data: + width = x['maskWidth'] + height = x['maskHeight'] + array = np.array(mask_data, dtype=np.uint8).reshape((height, width, 4)) + mask = Image.fromarray(array, 'RGBA') + # alpha = mask.getchannel("A").convert("L") + # mask = Image.merge("RGB", [alpha, alpha, alpha]) + mask = mask.convert('L') + t1 = time.time() + shared.log.debug(f'Kanvas: image={image} mask={mask} time={t1-t0:.2f}') + except Exception: + pass + return image, mask + + def select_input(input_mode, input_image, init_image, init_type, input_video, input_batch, input_folder): global busy, input_source, input_init, input_mask # pylint: disable=global-statement + t0 = time.time() busy = True - selected_input = input_image - if input_mode == 'Kanvas': - selected_input = input_image - elif input_mode == 'Video': + selected_input = input_image # default: Image or Kanvas + if input_mode == 'Video': selected_input = input_video elif input_mode == 'Batch': selected_input = input_batch elif input_mode == 'Folder': selected_input = input_folder - else: - selected_input = None size = [gr.update(), gr.update()] if selected_input is None: input_source = None busy = False - # debug('Control input: none') return [gr.Tabs.update(), None, ''] + size input_type = type(selected_input) input_mask = None @@ -105,15 +153,23 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in selected_input, input_mask = masking.outpaint(input_image=selected_input) input_source = [selected_input] input_type = 'PIL.Image' - status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' + status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}' size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)] res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] - elif isinstance(selected_input, dict): # inpaint -> dict image+mask + elif isinstance(selected_input, dict) and 'kanvas' in selected_input: # kanvas via js -> kanvas dict + selected_input, input_mask = process_kanvas(selected_input) + input_source = [selected_input] + input_type = 'Kanvas' + status = f'Control input | Kanvas | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}' + if selected_input: + size = [gr.update(value=selected_input.width), gr.update(value=selected_input.height)] + res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] + elif isinstance(selected_input, dict) and 'mask' in selected_input: # inpaint -> dict image+mask input_mask = selected_input['mask'] selected_input = selected_input['image'] input_source = [selected_input] input_type = 'PIL.Image' - status = f'Control input | Image | Size {selected_input.width}x{selected_input.height} | Mode {selected_input.mode}' + status = f'Control input | Image | Size {selected_input.width if selected_input else 0}x{selected_input.height if selected_input else 0} | Mode {selected_input.mode if selected_input else "Unknown"}' res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] elif isinstance(selected_input, gr.components.image.Image): # not likely input_source = [selected_input.value] @@ -135,14 +191,14 @@ def select_input(input_mode, input_image, init_image, init_type, input_video, in res = [gr.Tabs.update(selected='out-gallery'), input_mask, status] else: # unknown input_source = None - # init inputs: optional if init_type == 0: # Control only input_init = None elif init_type == 1: # Init image same as control assigned during runtime input_init = None elif init_type == 2: # Separate init image input_init = [init_image] - debug_log(f'Control select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode}') + t1 = time.time() + shared.log.debug(f'Select input: type={input_type} source={input_source} init={input_init} mask={input_mask} mode={input_mode} time={t1-t0:.2f}') busy = False return res + size diff --git a/modules/ui_javascript.py b/modules/ui_javascript.py index 1897d1b58..49ff5d157 100644 --- a/modules/ui_javascript.py +++ b/modules/ui_javascript.py @@ -102,8 +102,7 @@ def reload_javascript(): css_base = theme.reload_gradio_theme() css_timesheet = "timesheet.css" - css_kanvas = "kanvas.css" - css = html_css([css_base, css_timesheet, css_kanvas]) + css = html_css([css_base, css_timesheet]) body = html_body() def template_response(*args, **kwargs): diff --git a/webui.py b/webui.py index 8dc29d5bc..0b6e2b109 100644 --- a/webui.py +++ b/webui.py @@ -12,7 +12,7 @@ from threading import Thread import modules.loader import modules.hashes -from installer import log, git_commit, custom_excepthook +from installer import log, git_commit, custom_excepthook, version from modules import timer, paths, shared, extensions, gr_tempdir, modelloader, modeldata from modules.call_queue import queue_lock, wrap_queued_call, wrap_gradio_gpu_call # pylint: disable=unused-import import modules.devices @@ -263,6 +263,7 @@ def mount_subpath(app): def start_ui(): log.debug('UI start sequence') + log.debug(f'UI image support: kanvas={version["kanvas"]}') modules.script_callbacks.before_ui_callback() timer.startup.record("before-ui") shared.demo = modules.ui.create_ui(timer.startup) diff --git a/wiki b/wiki index 3e40abb7e..9e9871c5a 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 3e40abb7ea2530d91f376c676a61d119434f9551 +Subproject commit 9e9871c5acda37d2101a09d456c7fccc0f56a518