From 9ef41df6f9043d58fbbeea1f06be8e5c8622248b Mon Sep 17 00:00:00 2001 From: Josh R Date: Sat, 14 Jan 2023 15:26:45 -0800 Subject: [PATCH 01/12] add inpaint masking controls to orderable section that the settings can order --- modules/shared.py | 1 + modules/ui.py | 58 +++++++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 51df056c3..7ce8003f6 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -116,6 +116,7 @@ restricted_opts = { } ui_reorder_categories = [ + "masking", "sampler", "dimensions", "cfg", diff --git a/modules/ui.py b/modules/ui.py index 2425c66f2..174930ab3 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -867,35 +867,6 @@ def create_ui(): outputs=[], ) - with FormGroup(elem_id="inpaint_controls", visible=False) as inpaint_controls: - with FormRow(): - mask_blur = gr.Slider(label='Mask blur', minimum=0, maximum=64, step=1, value=4, elem_id="img2img_mask_blur") - mask_alpha = gr.Slider(label="Mask transparency", visible=False, elem_id="img2img_mask_alpha") - - with FormRow(): - inpainting_mask_invert = gr.Radio(label='Mask mode', choices=['Inpaint masked', 'Inpaint not masked'], value='Inpaint masked', type="index", elem_id="img2img_mask_mode") - - with FormRow(): - inpainting_fill = gr.Radio(label='Masked content', choices=['fill', 'original', 'latent noise', 'latent nothing'], value='original', type="index", elem_id="img2img_inpainting_fill") - - with FormRow(): - with gr.Column(): - inpaint_full_res = gr.Radio(label="Inpaint area", choices=["Whole picture", "Only masked"], type="index", value="Whole picture", elem_id="img2img_inpaint_full_res") - - with gr.Column(scale=4): - inpaint_full_res_padding = gr.Slider(label='Only masked padding, pixels', minimum=0, maximum=256, step=4, value=32, elem_id="img2img_inpaint_full_res_padding") - - def select_img2img_tab(tab): - return gr.update(visible=tab in [2, 3, 4]), gr.update(visible=tab == 3), - - for i, elem in enumerate([tab_img2img, tab_sketch, tab_inpaint, tab_inpaint_color, tab_inpaint_upload, tab_batch]): - elem.select( - fn=lambda tab=i: select_img2img_tab(tab), - inputs=[], - outputs=[inpaint_controls, mask_alpha], - ) - - with FormRow(): resize_mode = gr.Radio(label="Resize mode", elem_id="resize_mode", choices=["Just resize", "Crop and resize", "Resize and fill", "Just resize (latent upscale)"], type="index", value="Just resize") @@ -937,6 +908,35 @@ def create_ui(): with FormGroup(elem_id="img2img_script_container"): custom_inputs = modules.scripts.scripts_img2img.setup_ui() + elif category == "masking": + with FormGroup(elem_id="inpaint_controls", visible=False) as inpaint_controls: + with FormRow(): + mask_blur = gr.Slider(label='Mask blur', minimum=0, maximum=64, step=1, value=4, elem_id="img2img_mask_blur") + mask_alpha = gr.Slider(label="Mask transparency", visible=False, elem_id="img2img_mask_alpha") + + with FormRow(): + inpainting_mask_invert = gr.Radio(label='Mask mode', choices=['Inpaint masked', 'Inpaint not masked'], value='Inpaint masked', type="index", elem_id="img2img_mask_mode") + + with FormRow(): + inpainting_fill = gr.Radio(label='Masked content', choices=['fill', 'original', 'latent noise', 'latent nothing'], value='original', type="index", elem_id="img2img_inpainting_fill") + + with FormRow(): + with gr.Column(): + inpaint_full_res = gr.Radio(label="Inpaint area", choices=["Whole picture", "Only masked"], type="index", value="Whole picture", elem_id="img2img_inpaint_full_res") + + with gr.Column(scale=4): + inpaint_full_res_padding = gr.Slider(label='Only masked padding, pixels', minimum=0, maximum=256, step=4, value=32, elem_id="img2img_inpaint_full_res_padding") + + def select_img2img_tab(tab): + return gr.update(visible=tab in [2, 3, 4]), gr.update(visible=tab == 3), + + for i, elem in enumerate([tab_img2img, tab_sketch, tab_inpaint, tab_inpaint_color, tab_inpaint_upload, tab_batch]): + elem.select( + fn=lambda tab=i: select_img2img_tab(tab), + inputs=[], + outputs=[inpaint_controls, mask_alpha], + ) + img2img_gallery, generation_info, html_info, html_log = create_output_panel("img2img", opts.outdir_img2img_samples) parameters_copypaste.bind_buttons({"img2img": img2img_paste}, None, img2img_prompt) From 388708f7b13dfbc890135cad678bfbcebd7baf37 Mon Sep 17 00:00:00 2001 From: pangbo13 <373108669@qq.com> Date: Mon, 16 Jan 2023 00:56:24 +0800 Subject: [PATCH 02/12] fix when show_progress_every_n_steps == -1 --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index de99aca97..f857ccde9 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -228,7 +228,7 @@ class State: if not parallel_processing_allowed: return - if self.sampling_step - self.current_image_sampling_step >= opts.show_progress_every_n_steps and opts.live_previews_enable: + if self.sampling_step - self.current_image_sampling_step >= opts.show_progress_every_n_steps and opts.live_previews_enable and opts.show_progress_every_n_steps != -1: self.do_set_current_image() def do_set_current_image(self): From 16f410893eb96c7810cbbd812541ba35e0e92524 Mon Sep 17 00:00:00 2001 From: AngelBottomless <35677394+aria1th@users.noreply.github.com> Date: Mon, 16 Jan 2023 02:08:47 +0900 Subject: [PATCH 03/12] fix missing 'mean loss' for tensorboard integration --- modules/hypernetworks/hypernetwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index ae6af5169..bbd1f673c 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -644,7 +644,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi if shared.opts.training_enable_tensorboard: epoch_num = hypernetwork.step // len(ds) epoch_step = hypernetwork.step - (epoch_num * len(ds)) + 1 - + mean_loss = sum(sum(x) for x in loss_dict.values()) / sum(len(x) for x in loss_dict.values()) textual_inversion.tensorboard_add(tensorboard_writer, loss=mean_loss, global_step=hypernetwork.step, step=epoch_step, learn_rate=scheduler.learn_rate, epoch_num=epoch_num) textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, steps_per_epoch, { From f6aac4c65a681383616f6e72e3865002600f476f Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sun, 15 Jan 2023 20:20:29 +0300 Subject: [PATCH 04/12] eliminate flicker for live previews --- javascript/progressbar.js | 14 +++++++------- style.css | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/javascript/progressbar.js b/javascript/progressbar.js index b7524ef7c..8f22c0181 100644 --- a/javascript/progressbar.js +++ b/javascript/progressbar.js @@ -184,15 +184,15 @@ function requestProgress(id_task, progressbarContainer, gallery, atEnd, onProgre if(res.live_preview){ + + var rect = gallery.getBoundingClientRect() + if(rect.width){ + livePreview.style.width = rect.width + "px" + livePreview.style.height = rect.height + "px" + } + var img = new Image(); img.onload = function() { - var rect = gallery.getBoundingClientRect() - if(rect.width){ - livePreview.style.width = rect.width + "px" - livePreview.style.height = rect.height + "px" - } - - livePreview.innerHTML = '' livePreview.appendChild(img) if(livePreview.childElementCount > 2){ livePreview.removeChild(livePreview.firstElementChild) diff --git a/style.css b/style.css index 786b71d18..5bf1c6f94 100644 --- a/style.css +++ b/style.css @@ -338,6 +338,7 @@ input[type="range"]{ } .livePreview img{ + position: absolute; object-fit: contain; width: 100%; height: 100%; From a534bdfc801e0c83e378dfaa2d04cf865d7109f9 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sun, 15 Jan 2023 20:27:39 +0300 Subject: [PATCH 05/12] add setting for progressbar update period --- javascript/progressbar.js | 2 +- modules/shared.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/javascript/progressbar.js b/javascript/progressbar.js index 8f22c0181..59173c832 100644 --- a/javascript/progressbar.js +++ b/javascript/progressbar.js @@ -208,7 +208,7 @@ function requestProgress(id_task, progressbarContainer, gallery, atEnd, onProgre setTimeout(() => { fun(id_task, res.id_live_preview); - }, 500) + }, opts.live_preview_refresh_period || 500) }, function(){ removeProgressBar() }) diff --git a/modules/shared.py b/modules/shared.py index de99aca97..3483db1c2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -455,6 +455,7 @@ options_templates.update(options_section(('ui', "Live previews"), { "show_progress_every_n_steps": OptionInfo(10, "Show new live preview image every N sampling steps. Set to -1 to show after completion of batch.", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "show_progress_type": OptionInfo("Approx NN", "Image creation progress preview mode", gr.Radio, {"choices": ["Full", "Approx NN", "Approx cheap"]}), "live_preview_content": OptionInfo("Prompt", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}), + "live_preview_refresh_period": OptionInfo(1000, "Progressbar/preview update period, in milliseconds") })) options_templates.update(options_section(('sampler-params', "Sampler parameters"), { From b6ce041cdf722b400df9b5eac306d0cb049923d7 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sun, 15 Jan 2023 20:29:48 +0300 Subject: [PATCH 06/12] put interrupt and skip buttons back where they were --- modules/ui.py | 2 +- style.css | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/ui.py b/modules/ui.py index ff33236bb..7a357f9af 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -357,8 +357,8 @@ def create_toprow(is_img2img): with gr.Column(scale=1): with gr.Row(elem_id=f"{id_part}_generate_box"): - skip = gr.Button('Skip', elem_id=f"{id_part}_skip") interrupt = gr.Button('Interrupt', elem_id=f"{id_part}_interrupt") + skip = gr.Button('Skip', elem_id=f"{id_part}_skip") submit = gr.Button('Generate', elem_id=f"{id_part}_generate", variant='primary') skip.click( diff --git a/style.css b/style.css index 5bf1c6f94..750fe3152 100644 --- a/style.css +++ b/style.css @@ -480,13 +480,13 @@ input[type="range"]{ } #txt2img_interrupt, #img2img_interrupt{ - right: 0; - border-radius: 0 0.5rem 0.5rem 0; -} -#txt2img_skip, #img2img_skip{ left: 0; border-radius: 0.5rem 0 0 0.5rem; } +#txt2img_skip, #img2img_skip{ + right: 0; + border-radius: 0 0.5rem 0.5rem 0; +} .red { color: red; From 110d1a2d598bcfacffe3d524df1a3422b4cbd8ec Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 15 Jan 2023 12:41:00 -0500 Subject: [PATCH 07/12] add fields to settings file --- modules/textual_inversion/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/textual_inversion/logging.py b/modules/textual_inversion/logging.py index 31e50b64e..734a4b6f4 100644 --- a/modules/textual_inversion/logging.py +++ b/modules/textual_inversion/logging.py @@ -2,7 +2,7 @@ import datetime import json import os -saved_params_shared = {"model_name", "model_hash", "initial_step", "num_of_dataset_images", "learn_rate", "batch_size", "clip_grad_mode", "clip_grad_value", "gradient_step", "data_root", "log_directory", "training_width", "training_height", "steps", "create_image_every", "template_file"} +saved_params_shared = {"model_name", "model_hash", "initial_step", "num_of_dataset_images", "learn_rate", "batch_size", "clip_grad_mode", "clip_grad_value", "gradient_step", "data_root", "log_directory", "training_width", "training_height", "steps", "create_image_every", "template_file", "gradient_step", "latent_sampling_method"} saved_params_ti = {"embedding_name", "num_vectors_per_token", "save_embedding_every", "save_image_with_stored_embedding"} saved_params_hypernet = {"hypernetwork_name", "layer_structure", "activation_func", "weight_init", "add_layer_norm", "use_dropout", "save_hypernetwork_every"} saved_params_all = saved_params_shared | saved_params_ti | saved_params_hypernet From 598f7fcd84f655dd204ad5e258dc1c41cc806cde Mon Sep 17 00:00:00 2001 From: aria1th <35677394+aria1th@users.noreply.github.com> Date: Mon, 16 Jan 2023 02:46:21 +0900 Subject: [PATCH 08/12] Fix loss_dict problem --- modules/hypernetworks/hypernetwork.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index bbd1f673c..438e3e9f5 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -561,6 +561,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi _loss_step = 0 #internal # size = len(ds.indexes) # loss_dict = defaultdict(lambda : deque(maxlen = 1024)) + loss_logging = [] # losses = torch.zeros((size,)) # previous_mean_losses = [0] # previous_mean_loss = 0 @@ -601,6 +602,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi else: c = stack_conds(batch.cond).to(devices.device, non_blocking=pin_memory) loss = shared.sd_model(x, c)[0] / gradient_step + loss_logging.append(loss.item()) del x del c @@ -644,7 +646,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi if shared.opts.training_enable_tensorboard: epoch_num = hypernetwork.step // len(ds) epoch_step = hypernetwork.step - (epoch_num * len(ds)) + 1 - mean_loss = sum(sum(x) for x in loss_dict.values()) / sum(len(x) for x in loss_dict.values()) + mean_loss = sum(loss_logging) / len(loss_logging) textual_inversion.tensorboard_add(tensorboard_writer, loss=mean_loss, global_step=hypernetwork.step, step=epoch_step, learn_rate=scheduler.learn_rate, epoch_num=epoch_num) textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, steps_per_epoch, { From 13445738d974edcca5ff2f4f8f3833c1f3433e5e Mon Sep 17 00:00:00 2001 From: aria1th <35677394+aria1th@users.noreply.github.com> Date: Mon, 16 Jan 2023 03:02:54 +0900 Subject: [PATCH 09/12] Fix tensorboard related functions --- modules/hypernetworks/hypernetwork.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 438e3e9f5..c963fc404 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -561,7 +561,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi _loss_step = 0 #internal # size = len(ds.indexes) # loss_dict = defaultdict(lambda : deque(maxlen = 1024)) - loss_logging = [] + loss_logging = deque(maxlen=len(ds) * 3) # this should be configurable parameter, this is 3 * epoch(dataset size) # losses = torch.zeros((size,)) # previous_mean_losses = [0] # previous_mean_loss = 0 @@ -602,7 +602,6 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi else: c = stack_conds(batch.cond).to(devices.device, non_blocking=pin_memory) loss = shared.sd_model(x, c)[0] / gradient_step - loss_logging.append(loss.item()) del x del c @@ -612,7 +611,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi # go back until we reach gradient accumulation steps if (j + 1) % gradient_step != 0: continue - + loss_logging.append(_loss_step) if clip_grad: clip_grad(weights, clip_grad_sched.learn_rate) @@ -690,9 +689,6 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi processed = processing.process_images(p) image = processed.images[0] if len(processed.images) > 0 else None - - if shared.opts.training_enable_tensorboard and shared.opts.training_tensorboard_save_images: - textual_inversion.tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}", image, hypernetwork.step) if unload: shared.sd_model.cond_stage_model.to(devices.cpu) @@ -703,7 +699,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi hypernetwork.train() if image is not None: shared.state.assign_current_image(image) - + if shared.opts.training_enable_tensorboard and shared.opts.training_tensorboard_save_images: + textual_inversion.tensorboard_add_image(tensorboard_writer, + f"Validation at epoch {epoch_num}", image, + hypernetwork.step) last_saved_image, last_text_info = images.save_image(image, images_dir, "", p.seed, p.prompt, shared.opts.samples_format, processed.infotexts[0], p=p, forced_filename=forced_filename, save_to_dirs=False) last_saved_image += f", prompt: {preview_text}" From 8e2aeee4a127b295bfc880800e4a312e0f049b85 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sun, 15 Jan 2023 22:29:53 +0300 Subject: [PATCH 10/12] add BREAK keyword to end current text chunk and start the next --- modules/prompt_parser.py | 7 ++++++- modules/sd_hijack_clip.py | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index 870218db8..696653725 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -274,6 +274,7 @@ re_attention = re.compile(r""" : """, re.X) +re_break = re.compile(r"\s*\bBREAK\b\s*", re.S) def parse_prompt_attention(text): """ @@ -339,7 +340,11 @@ def parse_prompt_attention(text): elif text == ']' and len(square_brackets) > 0: multiply_range(square_brackets.pop(), square_bracket_multiplier) else: - res.append([text, 1.0]) + parts = re.split(re_break, text) + for i, part in enumerate(parts): + if i > 0: + res.append(["BREAK", -1]) + res.append([part, 1.0]) for pos in round_brackets: multiply_range(pos, round_bracket_multiplier) diff --git a/modules/sd_hijack_clip.py b/modules/sd_hijack_clip.py index 852afc665..9fa5c5c50 100644 --- a/modules/sd_hijack_clip.py +++ b/modules/sd_hijack_clip.py @@ -96,13 +96,18 @@ class FrozenCLIPEmbedderWithCustomWordsBase(torch.nn.Module): token_count = 0 last_comma = -1 - def next_chunk(): - """puts current chunk into the list of results and produces the next one - empty""" + def next_chunk(is_last=False): + """puts current chunk into the list of results and produces the next one - empty; + if is_last is true, tokens tokens at the end won't add to token_count""" nonlocal token_count nonlocal last_comma nonlocal chunk - token_count += len(chunk.tokens) + if is_last: + token_count += len(chunk.tokens) + else: + token_count += self.chunk_length + to_add = self.chunk_length - len(chunk.tokens) if to_add > 0: chunk.tokens += [self.id_end] * to_add @@ -116,6 +121,10 @@ class FrozenCLIPEmbedderWithCustomWordsBase(torch.nn.Module): chunk = PromptChunk() for tokens, (text, weight) in zip(tokenized, parsed): + if text == 'BREAK' and weight == -1: + next_chunk() + continue + position = 0 while position < len(tokens): token = tokens[position] @@ -159,7 +168,7 @@ class FrozenCLIPEmbedderWithCustomWordsBase(torch.nn.Module): position += embedding_length_in_tokens if len(chunk.tokens) > 0 or len(chunks) == 0: - next_chunk() + next_chunk(is_last=True) return chunks, token_count From fc25af3939f0366f147892a8ae5b9da56495352b Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sun, 15 Jan 2023 23:22:51 +0300 Subject: [PATCH 11/12] remove unneeded log from progressbar --- javascript/progressbar.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/javascript/progressbar.js b/javascript/progressbar.js index 59173c832..5072c13fd 100644 --- a/javascript/progressbar.js +++ b/javascript/progressbar.js @@ -139,8 +139,6 @@ function requestProgress(id_task, progressbarContainer, gallery, atEnd, onProgre var fun = function(id_task, id_live_preview){ request("/internal/progress", {"id_task": id_task, "id_live_preview": id_live_preview}, function(res){ - console.log(res) - if(res.completed){ removeProgressBar() return From 89314e79da21ac71ad3133ccf5ac3e85d4c24052 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sun, 15 Jan 2023 23:23:16 +0300 Subject: [PATCH 12/12] fix an error that happens when you send an empty image from txt2img to img2img --- modules/generation_parameters_copypaste.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index 593d99ef5..a381ff599 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -37,6 +37,9 @@ def quote(text): def image_from_url_text(filedata): + if filedata is None: + return None + if type(filedata) == list and len(filedata) > 0 and type(filedata[0]) == dict and filedata[0].get("is_file", False): filedata = filedata[0]