From 29e74d6e71826da9a3fe3c5790fed1329fc4d1e8 Mon Sep 17 00:00:00 2001
From: Melan <alexleander91@gmail.com>
Date: Thu, 20 Oct 2022 16:26:16 +0200
Subject: [PATCH 01/15] Add support for Tensorboard for training embeddings

---
 modules/shared.py                             |  4 +++
 .../textual_inversion/textual_inversion.py    | 31 ++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/modules/shared.py b/modules/shared.py
index faede8214..2c6341f75 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -254,6 +254,10 @@ options_templates.update(options_section(('training', "Training"), {
     "dataset_filename_join_string": OptionInfo(" ", "Filename join string"),
     "training_image_repeats_per_epoch": OptionInfo(1, "Number of repeats for a single input image per epoch; used only for displaying epoch number", gr.Number, {"precision": 0}),
     "training_write_csv_every": OptionInfo(500, "Save an csv containing the loss to log directory every N steps, 0 to disable"),
+    "training_enable_tensorboard": OptionInfo(False, "Enable tensorboard logging."),
+    "training_tensorboard_save_images": OptionInfo(False, "Save generated images within tensorboard."),
+    "training_tensorboard_flush_every": OptionInfo(120, "How often, in seconds, to flush the pending tensorboard events and summaries to disk."),
+
 }))
 
 options_templates.update(options_section(('sd', "Stable Diffusion"), {
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 3be69562b..c57d3aced 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -7,9 +7,11 @@ import tqdm
 import html
 import datetime
 import csv
+import numpy as np
 
+import torchvision.transforms
 from PIL import Image, PngImagePlugin
-
+from torch.utils.tensorboard import SummaryWriter
 from modules import shared, devices, sd_hijack, processing, sd_models
 import modules.textual_inversion.dataset
 from modules.textual_inversion.learn_schedule import LearnRateScheduler
@@ -199,6 +201,19 @@ def write_loss(log_directory, filename, step, epoch_len, values):
             **values,
         })
 
+def tensorboard_add_scaler(tensorboard_writer, tag, value, step):
+    if shared.opts.training_enable_tensorboard:
+        tensorboard_writer.add_scalar(tag=tag, 
+            scalar_value=value, global_step=step)
+
+def tensorboard_add_image(tensorboard_writer, tag, pil_image, step):
+    if shared.opts.training_enable_tensorboard:
+        # Convert a pil image to a torch tensor
+        img_tensor = torch.as_tensor(np.array(pil_image, copy=True))
+        img_tensor = img_tensor.view(pil_image.size[1], pil_image.size[0], len(pil_image.getbands()))
+        img_tensor = img_tensor.permute((2, 0, 1))
+                
+        tensorboard_writer.add_image(tag, img_tensor, global_step=step)
 
 def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_directory, training_width, training_height, steps, create_image_every, save_embedding_every, template_file, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     assert embedding_name, 'embedding not selected'
@@ -252,6 +267,12 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
     scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)
     optimizer = torch.optim.AdamW([embedding.vec], lr=scheduler.learn_rate)
 
+    if shared.opts.training_enable_tensorboard:
+        os.makedirs(os.path.join(log_directory, "tensorboard"), exist_ok=True)
+        tensorboard_writer = SummaryWriter(
+                log_dir=os.path.join(log_directory, "tensorboard"),
+                flush_secs=shared.opts.training_tensorboard_flush_every)
+
     pbar = tqdm.tqdm(enumerate(ds), total=steps-ititial_step)
     for i, entries in pbar:
         embedding.step = i + ititial_step
@@ -270,6 +291,7 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
             del x
 
             losses[embedding.step % losses.shape[0]] = loss.item()
+            
 
             optimizer.zero_grad()
             loss.backward()
@@ -285,6 +307,12 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
             embedding.save(last_saved_file)
             embedding_yet_to_be_embedded = True
 
+        if shared.opts.training_enable_tensorboard:
+            tensorboard_add_scaler(tensorboard_writer, "Loss/train", losses.mean(), embedding.step)
+            tensorboard_add_scaler(tensorboard_writer, f"Loss/train/epoch-{epoch_num}", losses.mean(), epoch_step)
+            tensorboard_add_scaler(tensorboard_writer, "Learn rate/train", scheduler.learn_rate, embedding.step)
+            tensorboard_add_scaler(tensorboard_writer, f"Learn rate/train/epoch-{epoch_num}", scheduler.learn_rate, epoch_step)
+
         write_loss(log_directory, "textual_inversion_loss.csv", embedding.step, len(ds), {
             "loss": f"{losses.mean():.7f}",
             "learn_rate": scheduler.learn_rate
@@ -349,6 +377,7 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
                 embedding_yet_to_be_embedded = False
 
             image.save(last_saved_image)
+            tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}", image, embedding.step)
 
             last_saved_image += f", prompt: {preview_text}"
 

From a6d593a6b51dc6a8443f2aa5c24caa391a04cd56 Mon Sep 17 00:00:00 2001
From: Melan <alexleander91@gmail.com>
Date: Thu, 20 Oct 2022 19:43:21 +0200
Subject: [PATCH 02/15] Fixed a typo in a variable

---
 modules/textual_inversion/textual_inversion.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index c57d3aced..ec8176bfa 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -260,11 +260,11 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
     last_saved_image = "<none>"
     embedding_yet_to_be_embedded = False
 
-    ititial_step = embedding.step or 0
-    if ititial_step > steps:
+    initial_step = embedding.step or 0
+    if initial_step > steps:
         return embedding, filename
 
-    scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)
+    scheduler = LearnRateScheduler(learn_rate, steps, initial_step)
     optimizer = torch.optim.AdamW([embedding.vec], lr=scheduler.learn_rate)
 
     if shared.opts.training_enable_tensorboard:
@@ -273,9 +273,9 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
                 log_dir=os.path.join(log_directory, "tensorboard"),
                 flush_secs=shared.opts.training_tensorboard_flush_every)
 
-    pbar = tqdm.tqdm(enumerate(ds), total=steps-ititial_step)
+    pbar = tqdm.tqdm(enumerate(ds), total=steps-initial_step)
     for i, entries in pbar:
-        embedding.step = i + ititial_step
+        embedding.step = i + initial_step
 
         scheduler.apply(optimizer, embedding.step)
         if scheduler.finished:

From 8f5912984794c4c69e429c4636e984854d911b6a Mon Sep 17 00:00:00 2001
From: Melan <alexleander91@gmail.com>
Date: Thu, 20 Oct 2022 22:37:16 +0200
Subject: [PATCH 03/15] Some changes to the tensorboard code and hypernetwork
 support

---
 modules/hypernetworks/hypernetwork.py         | 18 +++++++-
 .../textual_inversion/textual_inversion.py    | 45 +++++++++++--------
 2 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 74300122b..5e919775b 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -4,6 +4,7 @@ import html
 import os
 import sys
 import traceback
+import tensorboard
 import tqdm
 import csv
 
@@ -18,7 +19,6 @@ import modules.textual_inversion.dataset
 from modules.textual_inversion import textual_inversion
 from modules.textual_inversion.learn_schedule import LearnRateScheduler
 
-
 class HypernetworkModule(torch.nn.Module):
     multiplier = 1.0
 
@@ -291,6 +291,9 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
     scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)
     optimizer = torch.optim.AdamW(weights, lr=scheduler.learn_rate)
 
+    if shared.opts.training_enable_tensorboard:
+        tensorboard_writer = textual_inversion.tensorboard_setup(log_directory)
+
     pbar = tqdm.tqdm(enumerate(ds), total=steps - ititial_step)
     for i, entries in pbar:
         hypernetwork.step = i + ititial_step
@@ -315,6 +318,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
             optimizer.zero_grad()
             loss.backward()
             optimizer.step()
+
         mean_loss = losses.mean()
         if torch.isnan(mean_loss):
             raise RuntimeError("Loss diverged.")
@@ -323,6 +327,14 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
         if hypernetwork.step > 0 and hypernetwork_dir is not None and hypernetwork.step % save_hypernetwork_every == 0:
             last_saved_file = os.path.join(hypernetwork_dir, f'{hypernetwork_name}-{hypernetwork.step}.pt')
             hypernetwork.save(last_saved_file)
+        
+        if shared.opts.training_enable_tensorboard:
+            epoch_num = hypernetwork.step // len(ds)
+            epoch_step = hypernetwork.step - (epoch_num * len(ds)) + 1
+            
+            textual_inversion.tensorboard_add(tensorboard_writer, loss=mean_loss,
+                global_step=hypernetwork.step, step=epoch_step, 
+                learn_rate=scheduler.learn_rate, epoch_num=epoch_num)
 
         textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, len(ds), {
             "loss": f"{mean_loss:.7f}",
@@ -360,6 +372,10 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
             processed = processing.process_images(p)
             image = processed.images[0] if len(processed.images)>0 else None
 
+            if shared.opts.training_enable_tensorboard and shared.opts.training_tensorboard_save_images:
+                textual_inversion.tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}",
+                      image, hypernetwork.step)
+
             if unload:
                 shared.sd_model.cond_stage_model.to(devices.cpu)
                 shared.sd_model.first_stage_model.to(devices.cpu)
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index ec8176bfa..b1dc25962 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -201,19 +201,30 @@ def write_loss(log_directory, filename, step, epoch_len, values):
             **values,
         })
 
+def tensorboard_setup(log_directory):
+    os.makedirs(os.path.join(log_directory, "tensorboard"), exist_ok=True)
+    return SummaryWriter(
+            log_dir=os.path.join(log_directory, "tensorboard"),
+            flush_secs=shared.opts.training_tensorboard_flush_every)
+
+def tensorboard_add(tensorboard_writer, loss, global_step, step, learn_rate, epoch_num):
+    tensorboard_add_scaler(tensorboard_writer, "Loss/train", loss, global_step)
+    tensorboard_add_scaler(tensorboard_writer, f"Loss/train/epoch-{epoch_num}", loss, step)
+    tensorboard_add_scaler(tensorboard_writer, "Learn rate/train", learn_rate, global_step)
+    tensorboard_add_scaler(tensorboard_writer, f"Learn rate/train/epoch-{epoch_num}", learn_rate, step)
+
 def tensorboard_add_scaler(tensorboard_writer, tag, value, step):
-    if shared.opts.training_enable_tensorboard:
-        tensorboard_writer.add_scalar(tag=tag, 
-            scalar_value=value, global_step=step)
+    tensorboard_writer.add_scalar(tag=tag, 
+        scalar_value=value, global_step=step)
 
 def tensorboard_add_image(tensorboard_writer, tag, pil_image, step):
-    if shared.opts.training_enable_tensorboard:
-        # Convert a pil image to a torch tensor
-        img_tensor = torch.as_tensor(np.array(pil_image, copy=True))
-        img_tensor = img_tensor.view(pil_image.size[1], pil_image.size[0], len(pil_image.getbands()))
-        img_tensor = img_tensor.permute((2, 0, 1))
+    # Convert a pil image to a torch tensor
+    img_tensor = torch.as_tensor(np.array(pil_image, copy=True))
+    img_tensor = img_tensor.view(pil_image.size[1], pil_image.size[0], 
+        len(pil_image.getbands()))
+    img_tensor = img_tensor.permute((2, 0, 1))
                 
-        tensorboard_writer.add_image(tag, img_tensor, global_step=step)
+    tensorboard_writer.add_image(tag, img_tensor, global_step=step)
 
 def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_directory, training_width, training_height, steps, create_image_every, save_embedding_every, template_file, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     assert embedding_name, 'embedding not selected'
@@ -268,10 +279,7 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
     optimizer = torch.optim.AdamW([embedding.vec], lr=scheduler.learn_rate)
 
     if shared.opts.training_enable_tensorboard:
-        os.makedirs(os.path.join(log_directory, "tensorboard"), exist_ok=True)
-        tensorboard_writer = SummaryWriter(
-                log_dir=os.path.join(log_directory, "tensorboard"),
-                flush_secs=shared.opts.training_tensorboard_flush_every)
+        tensorboard_writer = tensorboard_setup(log_directory)
 
     pbar = tqdm.tqdm(enumerate(ds), total=steps-initial_step)
     for i, entries in pbar:
@@ -308,10 +316,8 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
             embedding_yet_to_be_embedded = True
 
         if shared.opts.training_enable_tensorboard:
-            tensorboard_add_scaler(tensorboard_writer, "Loss/train", losses.mean(), embedding.step)
-            tensorboard_add_scaler(tensorboard_writer, f"Loss/train/epoch-{epoch_num}", losses.mean(), epoch_step)
-            tensorboard_add_scaler(tensorboard_writer, "Learn rate/train", scheduler.learn_rate, embedding.step)
-            tensorboard_add_scaler(tensorboard_writer, f"Learn rate/train/epoch-{epoch_num}", scheduler.learn_rate, epoch_step)
+            tensorboard_add(tensorboard_writer, loss=losses.mean(), global_step=embedding.step, 
+                step=epoch_step, learn_rate=scheduler.learn_rate, epoch_num=epoch_num)
 
         write_loss(log_directory, "textual_inversion_loss.csv", embedding.step, len(ds), {
             "loss": f"{losses.mean():.7f}",
@@ -377,7 +383,10 @@ def train_embedding(embedding_name, learn_rate, batch_size, data_root, log_direc
                 embedding_yet_to_be_embedded = False
 
             image.save(last_saved_image)
-            tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}", image, embedding.step)
+
+            if shared.opts.training_enable_tensorboard and shared.opts.training_tensorboard_save_images:
+                tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}", 
+                    image, embedding.step)
 
             last_saved_image += f", prompt: {preview_text}"
 

From 7543cf5e3b5eaced00582da257801227d1ff2a6e Mon Sep 17 00:00:00 2001
From: Melan <alexleander91@gmail.com>
Date: Thu, 20 Oct 2022 22:43:08 +0200
Subject: [PATCH 04/15] Fixed some typos in the code

---
 modules/hypernetworks/hypernetwork.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 5e919775b..0cd94f49b 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -284,19 +284,19 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log
     last_saved_file = "<none>"
     last_saved_image = "<none>"
 
-    ititial_step = hypernetwork.step or 0
-    if ititial_step > steps:
+    initial_step = hypernetwork.step or 0
+    if initial_step > steps:
         return hypernetwork, filename
 
-    scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)
+    scheduler = LearnRateScheduler(learn_rate, steps, initial_step)
     optimizer = torch.optim.AdamW(weights, lr=scheduler.learn_rate)
 
     if shared.opts.training_enable_tensorboard:
         tensorboard_writer = textual_inversion.tensorboard_setup(log_directory)
 
-    pbar = tqdm.tqdm(enumerate(ds), total=steps - ititial_step)
+    pbar = tqdm.tqdm(enumerate(ds), total=steps - initial_step)
     for i, entries in pbar:
-        hypernetwork.step = i + ititial_step
+        hypernetwork.step = i + initial_step
 
         scheduler.apply(optimizer, hypernetwork.step)
         if scheduler.finished:

From 18f86e41f6f289042c075bff1498e620ab997b8c Mon Sep 17 00:00:00 2001
From: Melan <alexleander91@gmail.com>
Date: Mon, 24 Oct 2022 17:21:18 +0200
Subject: [PATCH 05/15] Removed two unused imports

---
 modules/hypernetworks/hypernetwork.py          | 1 -
 modules/textual_inversion/textual_inversion.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 0cd94f49b..2263e95e8 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -4,7 +4,6 @@ import html
 import os
 import sys
 import traceback
-import tensorboard
 import tqdm
 import csv
 
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index b1dc25962..589314fee 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -9,7 +9,6 @@ import datetime
 import csv
 import numpy as np
 
-import torchvision.transforms
 from PIL import Image, PngImagePlugin
 from torch.utils.tensorboard import SummaryWriter
 from modules import shared, devices, sd_hijack, processing, sd_models

From 6be644fa04ce1542f3a01804310cbbc0a4a91620 Mon Sep 17 00:00:00 2001
From: dan <guaneec@gmail.com>
Date: Wed, 11 Jan 2023 05:31:58 +0800
Subject: [PATCH 06/15] Enable batch_size>1 for mixed-sized training

---
 modules/textual_inversion/dataset.py | 36 ++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index fa48708e4..b47414f34 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -3,8 +3,10 @@ import numpy as np
 import PIL
 import torch
 from PIL import Image
-from torch.utils.data import Dataset, DataLoader
+from torch.utils.data import Dataset, DataLoader, Sampler
 from torchvision import transforms
+from collections import defaultdict
+from random import shuffle, choices
 
 import random
 import tqdm
@@ -45,12 +47,12 @@ class PersonalizedBase(Dataset):
         assert data_root, 'dataset directory not specified'
         assert os.path.isdir(data_root), "Dataset directory doesn't exist"
         assert os.listdir(data_root), "Dataset directory is empty"
-        assert batch_size == 1 or not varsize, 'variable img size must have batch size 1'
 
         self.image_paths = [os.path.join(data_root, file_path) for file_path in os.listdir(data_root)]
 
         self.shuffle_tags = shuffle_tags
         self.tag_drop_out = tag_drop_out
+        groups = defaultdict(list)
 
         print("Preparing dataset...")
         for path in tqdm.tqdm(self.image_paths):
@@ -103,13 +105,14 @@ class PersonalizedBase(Dataset):
             if include_cond and not (self.tag_drop_out != 0 or self.shuffle_tags):
                 with devices.autocast():
                     entry.cond = cond_model([entry.cond_text]).to(devices.cpu).squeeze(0)
-
+            groups[image.size].append(len(self.dataset))
             self.dataset.append(entry)
             del torchdata
             del latent_dist
             del latent_sample
 
         self.length = len(self.dataset)
+        self.groups = list(groups.values())
         assert self.length > 0, "No images have been found in the dataset."
         self.batch_size = min(batch_size, self.length)
         self.gradient_step = min(gradient_step, self.length // self.batch_size)
@@ -137,9 +140,34 @@ class PersonalizedBase(Dataset):
             entry.latent_sample = shared.sd_model.get_first_stage_encoding(entry.latent_dist).to(devices.cpu)
         return entry
 
+class GroupedBatchSampler(Sampler):
+    def __init__(self, data_source: PersonalizedBase, batch_size: int):
+        n = len(data_source)
+        self.groups = data_source.groups
+        self.len = n_batch = n // batch_size
+        expected = [len(g) / n * n_batch * batch_size for g in data_source.groups]
+        self.base = [int(e) // batch_size for e in expected]
+        self.n_rand_batches = nrb = n_batch - sum(self.base)
+        self.probs = [e%batch_size/nrb/batch_size if nrb>0 else 0 for e in expected]
+        self.batch_size = batch_size
+    def __len__(self):
+        return self.len
+    def __iter__(self):
+        b = self.batch_size
+        for g in self.groups:
+            shuffle(g)
+        batches = []
+        for g in self.groups:
+            batches.extend(g[i*b:(i+1)*b] for i in range(len(g) // b))
+        for _ in range(self.n_rand_batches):
+            rand_group = choices(self.groups, self.probs)[0]
+            batches.append(choices(rand_group, k=b))
+        shuffle(batches)
+        yield from batches
+
 class PersonalizedDataLoader(DataLoader):
     def __init__(self, dataset, latent_sampling_method="once", batch_size=1, pin_memory=False):
-        super(PersonalizedDataLoader, self).__init__(dataset, shuffle=True, drop_last=True, batch_size=batch_size, pin_memory=pin_memory)
+        super(PersonalizedDataLoader, self).__init__(dataset, batch_sampler=GroupedBatchSampler(dataset, batch_size), pin_memory=pin_memory)
         if latent_sampling_method == "random":
             self.collate_fn = collate_wrapper_random
         else:

From d52a80f7f7da160c73afd067c8f1bf491391f994 Mon Sep 17 00:00:00 2001
From: Shondoit <shondoit@gmail.com>
Date: Thu, 12 Jan 2023 09:22:29 +0100
Subject: [PATCH 07/15] Allow creation of zero vectors for TI

---
 modules/textual_inversion/textual_inversion.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index b915b091c..853246a6d 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -248,11 +248,14 @@ def create_embedding(name, num_vectors_per_token, overwrite_old, init_text='*'):
     with devices.autocast():
         cond_model([""])  # will send cond model to GPU if lowvram/medvram is active
 
-    embedded = cond_model.encode_embedding_init_text(init_text, num_vectors_per_token)
+    #cond_model expects at least some text, so we provide '*' as backup.
+    embedded = cond_model.encode_embedding_init_text(init_text or '*', num_vectors_per_token)
     vec = torch.zeros((num_vectors_per_token, embedded.shape[1]), device=devices.device)
 
-    for i in range(num_vectors_per_token):
-        vec[i] = embedded[i * int(embedded.shape[0]) // num_vectors_per_token]
+    #Only copy if we provided an init_text, otherwise keep vectors as zeros
+    if init_text:
+        for i in range(num_vectors_per_token):
+            vec[i] = embedded[i * int(embedded.shape[0]) // num_vectors_per_token]
 
     # Remove illegal characters from name.
     name = "".join( x for x in name if (x.isalnum() or x in "._- "))

From d48dcbd2b29eab492d53d78f482356d78e5beb19 Mon Sep 17 00:00:00 2001
From: Shondoit <shondoit@gmail.com>
Date: Thu, 12 Jan 2023 09:53:35 +0100
Subject: [PATCH 08/15] Add zero vector feature to hints.js

Also add the note that some tokens might be skipped. Not everyone is aware of this.
---
 javascript/hints.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/javascript/hints.js b/javascript/hints.js
index 856e1389a..244bfde21 100644
--- a/javascript/hints.js
+++ b/javascript/hints.js
@@ -92,6 +92,7 @@ titles = {
     "Weighted sum": "Result = A * (1 - M) + B * M",
     "Add difference": "Result = A + (B - C) * M",
 
+	"Initialization text": "If the number of tokens is more than the number of vectors, some may be skipped.\nLeave the textbox empty to start with zeroed out vectors",
     "Learning rate": "How fast should training go. Low values will take longer to train, high values may fail to converge (not generate accurate results) and/or may break the embedding (This has happened if you see Loss: nan in the training info textbox. If this happens, you need to manually restore your embedding from an older not-broken backup).\n\nYou can set a single numeric value, or multiple learning rates using the syntax:\n\n   rate_1:max_steps_1, rate_2:max_steps_2, ...\n\nEG:   0.005:100, 1e-3:1000, 1e-5\n\nWill train with rate of 0.005 for first 100 steps, then 1e-3 until 1000 steps, then 1e-5 for all remaining steps.",
 
     "Clip skip": "Early stopping parameter for CLIP model; 1 is stop at last layer as usual, 2 is stop at penultimate layer, etc.",

From 5623a3e7b1beed61f3ae6829a05b7b861d70e203 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Thu, 12 Jan 2023 19:47:33 +0300
Subject: [PATCH 09/15] fix send to inpaint sending you to wrong place

---
 javascript/ui.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/javascript/ui.js b/javascript/ui.js
index ee226927e..a41dd26f5 100644
--- a/javascript/ui.js
+++ b/javascript/ui.js
@@ -54,7 +54,7 @@ function switch_to_img2img(){
 
 function switch_to_inpaint(){
     gradioApp().querySelector('#tabs').querySelectorAll('button')[1].click();
-    gradioApp().getElementById('mode_img2img').querySelectorAll('button')[1].click();
+    gradioApp().getElementById('mode_img2img').querySelectorAll('button')[2].click();
 
     return args_to_array(arguments);
 }

From 6ffefdcc9f47b66cbc543690d97cbf8327f4ba58 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Thu, 12 Jan 2023 19:47:44 +0300
Subject: [PATCH 10/15] fix js errors when restarting UI

---
 script.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/script.js b/script.js
index 0e117d067..21960d914 100644
--- a/script.js
+++ b/script.js
@@ -1,5 +1,6 @@
 function gradioApp() {
-    const gradioShadowRoot = document.getElementsByTagName('gradio-app')[0].shadowRoot
+    const elems = document.getElementsByTagName('gradio-app')
+    const gradioShadowRoot = elems.length == 0 ? null : elems[0].shadowRoot
     return !!gradioShadowRoot ? gradioShadowRoot : document;
 }
 

From 88416ab5ff787eec3b9962b43b5e544bb75fbad6 Mon Sep 17 00:00:00 2001
From: space-nuko <24979496+space-nuko@users.noreply.github.com>
Date: Thu, 12 Jan 2023 13:46:59 -0800
Subject: [PATCH 11/15] Fix extension parameters not being saved to last used
 parameters

---
 modules/processing.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/processing.py b/modules/processing.py
index f04a0e1e4..ae04cab78 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -531,16 +531,16 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
     def infotext(iteration=0, position_in_batch=0):
         return create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds, comments, iteration, position_in_batch)
 
-    with open(os.path.join(shared.script_path, "params.txt"), "w", encoding="utf8") as file:
-        processed = Processed(p, [], p.seed, "")
-        file.write(processed.infotext(p, 0))
-
     if os.path.exists(cmd_opts.embeddings_dir) and not p.do_not_reload_embeddings:
         model_hijack.embedding_db.load_textual_inversion_embeddings()
 
     if p.scripts is not None:
         p.scripts.process(p)
 
+    with open(os.path.join(shared.script_path, "params.txt"), "w", encoding="utf8") as file:
+        processed = Processed(p, [], p.seed, "")
+        file.write(processed.infotext(p, 0))
+
     infotexts = []
     output_images = []
 

From 6c88eaed4f5efca54a882eb1f8f30f01f350332a Mon Sep 17 00:00:00 2001
From: space-nuko <24979496+space-nuko@users.noreply.github.com>
Date: Thu, 12 Jan 2023 13:50:09 -0800
Subject: [PATCH 12/15] Add script callback for fixing infotext parameters

---
 modules/generation_parameters_copypaste.py |  3 ++-
 modules/script_callbacks.py                | 20 +++++++++++++++++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index 620aa6069..593d99ef5 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -7,7 +7,7 @@ from pathlib import Path
 
 import gradio as gr
 from modules.shared import script_path
-from modules import shared, ui_tempdir
+from modules import shared, ui_tempdir, script_callbacks
 import tempfile
 from PIL import Image
 
@@ -298,6 +298,7 @@ def connect_paste(button, paste_fields, input_comp, jsfunc=None):
                     prompt = file.read()
 
         params = parse_generation_parameters(prompt)
+        script_callbacks.infotext_pasted_callback(prompt, params)
         res = []
 
         for output, key in paste_fields:
diff --git a/modules/script_callbacks.py b/modules/script_callbacks.py
index 608c5300e..a9e192369 100644
--- a/modules/script_callbacks.py
+++ b/modules/script_callbacks.py
@@ -2,7 +2,7 @@ import sys
 import traceback
 from collections import namedtuple
 import inspect
-from typing import Optional
+from typing import Optional, Dict, Any
 
 from fastapi import FastAPI
 from gradio import Blocks
@@ -71,6 +71,7 @@ callback_map = dict(
     callbacks_before_component=[],
     callbacks_after_component=[],
     callbacks_image_grid=[],
+    callbacks_infotext_pasted=[],
     callbacks_script_unloaded=[],
 )
 
@@ -172,6 +173,14 @@ def image_grid_callback(params: ImageGridLoopParams):
             report_exception(c, 'image_grid')
 
 
+def infotext_pasted_callback(infotext: str, params: Dict[str, Any]):
+    for c in callback_map['callbacks_infotext_pasted']:
+        try:
+            c.callback(infotext, params)
+        except Exception:
+            report_exception(c, 'infotext_pasted')
+
+
 def script_unloaded_callback():
     for c in reversed(callback_map['callbacks_script_unloaded']):
         try:
@@ -290,6 +299,15 @@ def on_image_grid(callback):
     add_callback(callback_map['callbacks_image_grid'], callback)
 
 
+def on_infotext_pasted(callback):
+    """register a function to be called before applying an infotext.
+    The callback is called with two arguments:
+       - infotext: str - raw infotext.
+       - result: Dict[str, any] - parsed infotext parameters.
+    """
+    add_callback(callback_map['callbacks_infotext_pasted'], callback)
+
+
 def on_script_unloaded(callback):
     """register a function to be called before the script is unloaded. Any hooks/hijacks/monkeying about that
     the script did should be reverted here"""

From 0b262802b86a55c4f71faf377f2cb1aee2960b63 Mon Sep 17 00:00:00 2001
From: Josh R <josh@joshrubenstein.com>
Date: Thu, 12 Jan 2023 17:31:05 -0800
Subject: [PATCH 13/15] add gradient settings to training settings log files

---
 modules/textual_inversion/logging.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/textual_inversion/logging.py b/modules/textual_inversion/logging.py
index 8b1981d53..31e50b64e 100644
--- a/modules/textual_inversion/logging.py
+++ b/modules/textual_inversion/logging.py
@@ -2,7 +2,7 @@ import datetime
 import json
 import os
 
-saved_params_shared = {"model_name", "model_hash", "initial_step", "num_of_dataset_images", "learn_rate", "batch_size", "data_root", "log_directory", "training_width", "training_height", "steps", "create_image_every", "template_file"}
+saved_params_shared = {"model_name", "model_hash", "initial_step", "num_of_dataset_images", "learn_rate", "batch_size", "clip_grad_mode", "clip_grad_value", "gradient_step", "data_root", "log_directory", "training_width", "training_height", "steps", "create_image_every", "template_file"}
 saved_params_ti = {"embedding_name", "num_vectors_per_token", "save_embedding_every", "save_image_with_stored_embedding"}
 saved_params_hypernet = {"hypernetwork_name", "layer_structure", "activation_func", "weight_init", "add_layer_norm", "use_dropout", "save_hypernetwork_every"}
 saved_params_all = saved_params_shared | saved_params_ti | saved_params_hypernet

From a176d89487d92f5a5b152401e5c424b34ff43b96 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Fri, 13 Jan 2023 14:32:15 +0300
Subject: [PATCH 14/15] print bucket sizes for training without resizing images
 #6620 fix an error when generating a picture with embedding in it

---
 modules/textual_inversion/dataset.py           | 16 ++++++++++++++++
 modules/textual_inversion/image_embedding.py   |  4 ++--
 modules/textual_inversion/textual_inversion.py |  2 +-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index b47414f34..d31963d48 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -118,6 +118,12 @@ class PersonalizedBase(Dataset):
         self.gradient_step = min(gradient_step, self.length // self.batch_size)
         self.latent_sampling_method = latent_sampling_method
 
+        if len(groups) > 1:
+            print("Buckets:")
+            for (w, h), ids in sorted(groups.items(), key=lambda x: x[0]):
+                print(f"  {w}x{h}: {len(ids)}")
+            print()
+
     def create_text(self, filename_text):
         text = random.choice(self.lines)
         tags = filename_text.split(',')
@@ -140,8 +146,11 @@ class PersonalizedBase(Dataset):
             entry.latent_sample = shared.sd_model.get_first_stage_encoding(entry.latent_dist).to(devices.cpu)
         return entry
 
+
 class GroupedBatchSampler(Sampler):
     def __init__(self, data_source: PersonalizedBase, batch_size: int):
+        super().__init__(data_source)
+
         n = len(data_source)
         self.groups = data_source.groups
         self.len = n_batch = n // batch_size
@@ -150,21 +159,28 @@ class GroupedBatchSampler(Sampler):
         self.n_rand_batches = nrb = n_batch - sum(self.base)
         self.probs = [e%batch_size/nrb/batch_size if nrb>0 else 0 for e in expected]
         self.batch_size = batch_size
+
     def __len__(self):
         return self.len
+
     def __iter__(self):
         b = self.batch_size
+
         for g in self.groups:
             shuffle(g)
+
         batches = []
         for g in self.groups:
             batches.extend(g[i*b:(i+1)*b] for i in range(len(g) // b))
         for _ in range(self.n_rand_batches):
             rand_group = choices(self.groups, self.probs)[0]
             batches.append(choices(rand_group, k=b))
+
         shuffle(batches)
+
         yield from batches
 
+
 class PersonalizedDataLoader(DataLoader):
     def __init__(self, dataset, latent_sampling_method="once", batch_size=1, pin_memory=False):
         super(PersonalizedDataLoader, self).__init__(dataset, batch_sampler=GroupedBatchSampler(dataset, batch_size), pin_memory=pin_memory)
diff --git a/modules/textual_inversion/image_embedding.py b/modules/textual_inversion/image_embedding.py
index ea653806f..5593f88c7 100644
--- a/modules/textual_inversion/image_embedding.py
+++ b/modules/textual_inversion/image_embedding.py
@@ -76,10 +76,10 @@ def insert_image_data_embed(image, data):
     next_size = data_np_low.shape[0] + (h-(data_np_low.shape[0] % h))
     next_size = next_size + ((h*d)-(next_size % (h*d)))
 
-    data_np_low.resize(next_size)
+    data_np_low = np.resize(data_np_low, next_size)
     data_np_low = data_np_low.reshape((h, -1, d))
 
-    data_np_high.resize(next_size)
+    data_np_high = np.resize(data_np_high, next_size)
     data_np_high = data_np_high.reshape((h, -1, d))
 
     edge_style = list(data['string_to_param'].values())[0].cpu().detach().numpy().tolist()[0][:1024]
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 853246a6d..e23906cab 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -479,7 +479,7 @@ def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_
                 epoch_num = embedding.step // steps_per_epoch
                 epoch_step = embedding.step % steps_per_epoch
 
-                description = f"Training textual inversion [Epoch {epoch_num}: {epoch_step+1}/{steps_per_epoch}]loss: {loss_step:.7f}"
+                description = f"Training textual inversion [Epoch {epoch_num}: {epoch_step+1}/{steps_per_epoch}] loss: {loss_step:.7f}"
                 pbar.set_description(description)
                 shared.state.textinfo = description
                 if embedding_dir is not None and steps_done % save_embedding_every == 0:

From 82725f0ac439f7e3b67858d55900e95330bbd326 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Fri, 13 Jan 2023 15:04:37 +0300
Subject: [PATCH 15/15] fix a bug caused by merge

---
 modules/textual_inversion/textual_inversion.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 85210b0e7..6939efcc9 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -11,6 +11,7 @@ import datetime
 import csv
 import safetensors.torch
 
+import numpy as np
 from PIL import Image, PngImagePlugin
 from torch.utils.tensorboard import SummaryWriter