diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fc35937..8d195bbc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2023-01-02 +## Update for 2023-01-03 Following-up on a major release, some more functionality in new Control module And it also includes fixes for all reported issues so far @@ -18,8 +18,12 @@ And it also includes fixes for all reported issues so far - auto-refresh available models on tab activate - reduce usage of temp files - add context menu to action buttons + - resize by now applies to input image or frame individually + allows for processing where input images are of different sizes - fix input image size + - fix video color mode - fix correct image mode + - fix batch/folder/video modes - **Improvements** - allow deployment without git clone for example, you can now deploy a zip of the sdnext folder @@ -29,6 +33,8 @@ And it also includes fixes for all reported issues so far - cli: sdapi.py allow manual api invoke example: `python cli/sdapi.py /sdapi/v1/sd-models` - memory: add ram usage monitoring in addition to gpu memory usage monitoring + - vae: enable taesd batch decode + enable/disable with settings -> diffusers > vae slicing - updated core requirements - **Compile** - new option: **fused projections** @@ -55,6 +61,7 @@ And it also includes fixes for all reported issues so far - sampler: guard against invalid sampler index - config: reset default cfg scale to 6.0 - processing: correct display metadata + - live preview: fix when using `bfloat16` - upscale: fix ldsr ## Update for 2023-12-29 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 4cf15d1c9..8bd8057cc 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 4cf15d1c9c565b8d0c5f782a89c5a6286dc6e6ff +Subproject commit 8bd8057cca51fb0d951a9b8be42bc9873b6087da diff --git a/javascript/control.js b/javascript/control.js index 95d288eae..495e3a771 100644 --- a/javascript/control.js +++ b/javascript/control.js @@ -1,7 +1,12 @@ function controlInputMode(inputMode, ...args) { - if (!gradioApp().getElementById('control_input_select').classList.contains('hidden')) inputMode = 'Select'; - else if (!gradioApp().getElementById('control_input_resize').classList.contains('hidden')) inputMode = 'Outpaint'; - else if (!gradioApp().getElementById('control_input_inpaint').classList.contains('hidden')) inputMode = 'Inpaint'; + const tab = gradioApp().querySelector('#control-tab-input button.selected'); + if (!tab) return ['Select', ...args]; + inputMode = tab.innerText; + if (inputMode === 'Image') { + if (!gradioApp().getElementById('control_input_select').classList.contains('hidden')) inputMode = 'Select'; + else if (!gradioApp().getElementById('control_input_resize').classList.contains('hidden')) inputMode = 'Outpaint'; + else if (!gradioApp().getElementById('control_input_inpaint').classList.contains('hidden')) inputMode = 'Inpaint'; + } return [inputMode, ...args]; } diff --git a/javascript/progressBar.js b/javascript/progressBar.js index 85c339f26..238a2988e 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -85,7 +85,7 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres let livePreview; let img; - const init = () => { + const initLivePreview = () => { img = new Image(); if (parentGallery) { livePreview = document.createElement('div'); @@ -123,7 +123,7 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres return; } setProgress(res); - if (res.live_preview && !livePreview) init(); + if (res.live_preview && !livePreview) initLivePreview(); if (res.live_preview && galleryEl) img.src = res.live_preview; if (onProgress) onProgress(res); setTimeout(() => start(id_task, id_live_preview), opts.live_preview_refresh_period || 500); diff --git a/modules/control/run.py b/modules/control/run.py index ee04231f8..147363722 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -130,9 +130,6 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if resize_mode != 0 or inputs is None or inputs == [None]: p.width = width # pylint: disable=attribute-defined-outside-init p.height = height # pylint: disable=attribute-defined-outside-init - if selected_scale_tab == 1: - width = int(width * scale_by) - height = int(height * scale_by) else: del p.width del p.height @@ -293,6 +290,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ w, h = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)), int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) codec = util.decode_fourcc(video.get(cv2.CAP_PROP_FOURCC)) status, frame = video.read() + if status: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) shared.log.debug(f'Control: input video: path={inputs} frames={frames} fps={fps} size={w}x{h} codec={codec}') except Exception as e: msg = f'Control: video open failed: path={inputs} {e}' @@ -344,6 +343,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ # resize if p.resize_mode != 0 and input_image is not None: p.extra_generation_params["Control resize"] = f'{resize_time}: {resize_name}' + if selected_scale_tab == 1: + width = int(input_image.width * scale_by) + height = int(input_image.height * scale_by) if p.resize_mode != 0 and input_image is not None and resize_time == 'Before': debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}') input_image = images.resize_image(p.resize_mode, input_image, width, height, resize_name) @@ -502,6 +504,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if video is not None and frame is not None: status, frame = video.read() + if status: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) debug(f'Control: video frame={index} frames={frames} status={status} skip={index % (video_skip_frames + 1)} progress={index/frames:.2f}') else: status = False diff --git a/modules/devices.py b/modules/devices.py index cc5c8eda2..956d8fb93 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -181,7 +181,7 @@ def test_fp16(): if shared.cmd_opts.experimental: return True try: - x = torch.tensor([[1.5,.0,.0,.0]]).to(device).half() + x = torch.tensor([[1.5,.0,.0,.0]]).to(device=device, dtype=torch.float16) layerNorm = torch.nn.LayerNorm(4, eps=0.00001, elementwise_affine=True, dtype=torch.float16, device=device) _y = layerNorm(x) return True @@ -228,20 +228,20 @@ def set_cuda_params(): dtype = torch.float32 dtype_vae = torch.float32 dtype_unet = torch.float32 - if shared.opts.cuda_dtype == 'BF16' or dtype == torch.bfloat16: + fp16_ok = None + bf16_ok = None + elif shared.opts.cuda_dtype == 'BF16' or dtype == torch.bfloat16: + fp16_ok = test_fp16() bf16_ok = test_bf16() dtype = torch.bfloat16 if bf16_ok else torch.float16 dtype_vae = torch.bfloat16 if bf16_ok else torch.float16 dtype_unet = torch.bfloat16 if bf16_ok else torch.float16 - else: - bf16_ok = False - if shared.opts.cuda_dtype == 'FP16' or dtype == torch.float16: + elif shared.opts.cuda_dtype == 'FP16' or dtype == torch.float16: fp16_ok = test_fp16() + bf16_ok = None dtype = torch.float16 if fp16_ok else torch.float32 dtype_vae = torch.float16 if fp16_ok else torch.float32 dtype_unet = torch.float16 if fp16_ok else torch.float32 - else: - fp16_ok = False if shared.opts.no_half: log.info('Torch override dtype: no-half set') dtype = torch.float32 diff --git a/modules/processing_vae.py b/modules/processing_vae.py index bd9425de7..7e98f44ad 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -80,12 +80,15 @@ def full_vae_encode(image, model): def taesd_vae_decode(latents): - debug(f'VAE decode: name=TAESD images={len(latents)} latents={latents.shape}') + debug(f'VAE decode: name=TAESD images={len(latents)} latents={latents.shape} slicing={shared.opts.diffusers_vae_slicing}') if len(latents) == 0: return [] - decoded = torch.zeros((len(latents), 3, latents.shape[2] * 8, latents.shape[3] * 8), dtype=devices.dtype_vae, device=devices.device) - for i in range(latents.shape[0]): - decoded[i] = sd_vae_taesd.decode(latents[i]) + if shared.opts.diffusers_vae_slicing: + decoded = torch.zeros((len(latents), 3, latents.shape[2] * 8, latents.shape[3] * 8), dtype=devices.dtype_vae, device=devices.device) + for i in range(latents.shape[0]): + decoded[i] = sd_vae_taesd.decode(latents[i]) + else: + decoded = sd_vae_taesd.decode(latents) return decoded diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 88f511a5d..e749aa378 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -31,13 +31,15 @@ def setup_img2img_steps(p, steps=None): def single_sample_to_image(sample, approximation=None): - # sample should be [4,64,64] if approximation is None: approximation = approximation_indexes.get(shared.opts.show_progress_type, None) if approximation is None: warn_once('Unknown decode type, please reset preview method') approximation = 0 + # normal sample is [4,64,64] + if sample.dtype == torch.bfloat16: + sample = sample.to(torch.float16) if len(sample.shape) > 4: # likely unknown video latent (e.g. svd) return Image.new(mode="RGB", size=(512, 512)) if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent @@ -47,7 +49,7 @@ def single_sample_to_image(sample, approximation=None): elif approximation == 1: # Approximate x_sample = sd_vae_approx.nn_approximation(sample) * 0.5 + 0.5 if shared.sd_model_type == "sdxl": - x_sample = x_sample[[2,1,0],:,:] # BGR to RGB + x_sample = x_sample[[2,1,0], :, :] # BGR to RGB elif approximation == 2: # TAESD x_sample = sd_vae_taesd.decode(sample) x_sample = (1.0 + x_sample) / 2.0 # preview requires smaller range @@ -58,10 +60,12 @@ def single_sample_to_image(sample, approximation=None): return Image.new(mode="RGB", size=(512, 512)) try: + if x_sample.dtype == torch.bfloat16: + x_sample.to(torch.float16) transform = T.ToPILImage() image = transform(x_sample) except Exception as e: - warn_once(f'Transform tensor to image: {e}') + warn_once(f'Live preview: {e}') image = Image.new(mode="RGB", size=(512, 512)) return image diff --git a/modules/sd_vae_approx.py b/modules/sd_vae_approx.py index 2a7b0c9ef..14a67fca1 100644 --- a/modules/sd_vae_approx.py +++ b/modules/sd_vae_approx.py @@ -42,10 +42,10 @@ def nn_approximation(sample): # Approximate NN approx_weights = torch.load(model_path, map_location='cpu' if devices.device.type != 'cuda' else None) sd_vae_approx_model.load_state_dict(approx_weights) sd_vae_approx_model.eval() - sd_vae_approx_model.to(devices.device, devices.dtype) + sd_vae_approx_model.to(devices.device, sample.dtype) shared.log.debug(f'Load VAE decode approximate: model="{model_path}"') try: - in_sample = sample.to(devices.device, devices.dtype).unsqueeze(0) + in_sample = sample.to(devices.device).unsqueeze(0) x_sample = sd_vae_approx_model(in_sample) x_sample = x_sample[0] return x_sample @@ -71,7 +71,9 @@ def cheap_approximation(sample): # Approximate simple ]).reshape(3, 4, 1, 1) simple_bias = None try: - x_sample = nn.functional.conv2d(sample, simple_weights.to(sample.device, sample.dtype), simple_bias.to(sample.device, sample.dtype) if simple_bias is not None else None) # pylint: disable=not-callable + weights = simple_weights.to(sample.device, sample.dtype) + bias = simple_bias.to(sample.device, sample.dtype) if simple_bias is not None else None + x_sample = nn.functional.conv2d(sample, weights, bias) # pylint: disable=not-callable return x_sample except Exception as e: shared.log.error(f'Decode simple: {e}') diff --git a/modules/shared.py b/modules/shared.py index c3d5b3120..6008d0af5 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -367,7 +367,7 @@ options_templates.update(options_section(('advanced', "Inference Settings"), { "inference_other_sep": OptionInfo("