From a36e2744e2b18a2582247bc5b95bfa0339dfa629 Mon Sep 17 00:00:00 2001
From: Taithrah <Taithrah@users.noreply.github.com>
Date: Sat, 7 Jan 2023 04:09:02 -0500
Subject: [PATCH 01/22] Update hints.js

Small touch up to hints
---
 javascript/hints.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/javascript/hints.js b/javascript/hints.js
index dda66e097..73ab4a266 100644
--- a/javascript/hints.js
+++ b/javascript/hints.js
@@ -4,7 +4,7 @@ titles = {
     "Sampling steps": "How many times to improve the generated image iteratively; higher values take longer; very low values can produce bad results",
     "Sampling method": "Which algorithm to use to produce the image",
 	"GFPGAN": "Restore low quality faces using GFPGAN neural network",
-	"Euler a": "Euler Ancestral - very creative, each can get a completely different picture depending on step count, setting steps to higher than 30-40 does not help",
+	"Euler a": "Euler Ancestral - very creative, each can get a completely different picture depending on step count, setting steps higher than 30-40 does not help",
 	"DDIM": "Denoising Diffusion Implicit Models - best at inpainting",
 	"DPM adaptive": "Ignores step count - uses a number of steps determined by the CFG and resolution", 
 
@@ -12,8 +12,8 @@ titles = {
 	"Batch size": "How many image to create in a single batch",
     "CFG Scale": "Classifier Free Guidance Scale - how strongly the image should conform to prompt - lower values produce more creative results",
     "Seed": "A value that determines the output of random number generator - if you create an image with same parameters and seed as another image, you'll get the same result",
-    "\u{1f3b2}\ufe0f": "Set seed to -1, which will cause a new random number to be used every time",
-    "\u267b\ufe0f": "Reuse seed from last generation, mostly useful if it was randomed",
+    "\u{1f3b2}\ufe0f": "Set seed to -1 will set a new random number every time.",
+    "\u267b\ufe0f": "Reuse seed from last generation, most useful if it was randomized.",
     "\u{1f3a8}": "Add a random artist to the prompt.",
     "\u2199\ufe0f": "Read generation parameters from prompt or last generation if prompt is empty into user interface.",
     "\u{1f4c2}": "Open images output directory",
@@ -74,7 +74,7 @@ titles = {
     "Style 1": "Style to apply; styles have components for both positive and negative prompts and apply to both",
     "Style 2": "Style to apply; styles have components for both positive and negative prompts and apply to both",
     "Apply style": "Insert selected styles into prompt fields",
-    "Create style": "Save current prompts as a style. If you add the token {prompt} to the text, the style use that as placeholder for your prompt when you use the style in the future.",
+    "Create style": "Save current prompts as a style. If you add the token {prompt} to the text, the style uses that as a placeholder for your prompt when you use the style in the future.",
 
     "Checkpoint name": "Loads weights from checkpoint before making images. You can either use hash or a part of filename (as seen in settings) for checkpoint name. Recommended to use with Y axis for less switching.",
     "Inpainting conditioning mask strength": "Only applies to inpainting models. Determines how strongly to mask off the original image for inpainting and img2img. 1.0 means fully masked, which is the default behaviour. 0.0 means a fully unmasked conditioning. Lower values will help preserve the overall composition of the image, but will struggle with large changes.",
@@ -92,12 +92,12 @@ titles = {
     "Weighted sum": "Result = A * (1 - M) + B * M",
     "Add difference": "Result = A + (B - C) * M",
 
-    "Learning rate": "how fast should the training go. Low values will take longer to train, high values may fail to converge (not generate accurate results) and/or may break the embedding (This has happened if you see Loss: nan in the training info textbox. If this happens, you need to manually restore your embedding from an older not-broken backup).\n\nYou can set a single numeric value, or multiple learning rates using the syntax:\n\n   rate_1:max_steps_1, rate_2:max_steps_2, ...\n\nEG:   0.005:100, 1e-3:1000, 1e-5\n\nWill train with rate of 0.005 for first 100 steps, then 1e-3 until 1000 steps, then 1e-5 for all remaining steps.",
+    "Learning rate": "How fast should training go. Low values will take longer to train, high values may fail to converge (not generate accurate results) and/or may break the embedding (This has happened if you see Loss: nan in the training info textbox. If this happens, you need to manually restore your embedding from an older not-broken backup).\n\nYou can set a single numeric value, or multiple learning rates using the syntax:\n\n   rate_1:max_steps_1, rate_2:max_steps_2, ...\n\nEG:   0.005:100, 1e-3:1000, 1e-5\n\nWill train with rate of 0.005 for first 100 steps, then 1e-3 until 1000 steps, then 1e-5 for all remaining steps.",
 
     "Clip skip": "Early stopping parameter for CLIP model; 1 is stop at last layer as usual, 2 is stop at penultimate layer, etc.",
 
-    "Approx NN": "Cheap neural network approximation. Very fast compared to VAE, but produces pictures with 4 times smaller horizontal/vertical resoluton and lower quality.",
-    "Approx cheap": "Very cheap approximation. Very fast compared to VAE, but produces pictures with 8 times smaller horizontal/vertical resoluton and extremely low quality.",
+    "Approx NN": "Cheap neural network approximation. Very fast compared to VAE, but produces pictures with 4 times smaller horizontal/vertical resolution and lower quality.",
+    "Approx cheap": "Very cheap approximation. Very fast compared to VAE, but produces pictures with 8 times smaller horizontal/vertical resolution and extremely low quality.",
 
     "Hires. fix": "Use a two step process to partially create an image at smaller resolution, upscale, and then improve details in it without changing composition",
     "Hires steps": "Number of sampling steps for upscaled picture. If 0, uses same as for original.",

From 0fc1848e40dbd46c93753a2937403e1139ecd366 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Sat, 7 Jan 2023 11:25:41 +0200
Subject: [PATCH 02/22] CI: Use native actions/setup-python caching

---
 .github/workflows/on_pull_request.yaml | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index b097d1805..a168be5b8 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -19,22 +19,19 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v3
       - name: Set up Python 3.10
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
-      - uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          cache: pip
+          cache-dependency-path: |
+            **/requirements*txt
       - name: Install PyLint
         run: | 
           python -m pip install --upgrade pip
           pip install pylint
       # This lets PyLint check to see if it can resolve imports
       - name: Install dependencies
-        run : |
+        run: |
           export COMMANDLINE_ARGS="--skip-torch-cuda-test --exit"
           python launch.py
       - name: Analysing the code with pylint

From a77873974b97618351791ea3015639be7d9f98d1 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Sat, 7 Jan 2023 11:34:02 +0200
Subject: [PATCH 03/22] ... also for tests.

---
 .github/workflows/run_tests.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 49dc92bd9..ecb9012ab 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -14,11 +14,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: ${{ runner.os }}-pip-
+          cache: pip
+          cache-dependency-path: |
+            **/requirements*txt
       - name: Run tests
         run: python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
       - name: Upload main app stdout-stderr

From 8a27730da5d5b25e28370e8ad94844856a839af9 Mon Sep 17 00:00:00 2001
From: Taithrah <Taithrah@users.noreply.github.com>
Date: Sat, 7 Jan 2023 06:11:57 -0500
Subject: [PATCH 04/22] Update hints.js

Partial revert
---
 javascript/hints.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/javascript/hints.js b/javascript/hints.js
index 73ab4a266..856e1389a 100644
--- a/javascript/hints.js
+++ b/javascript/hints.js
@@ -12,8 +12,8 @@ titles = {
 	"Batch size": "How many image to create in a single batch",
     "CFG Scale": "Classifier Free Guidance Scale - how strongly the image should conform to prompt - lower values produce more creative results",
     "Seed": "A value that determines the output of random number generator - if you create an image with same parameters and seed as another image, you'll get the same result",
-    "\u{1f3b2}\ufe0f": "Set seed to -1 will set a new random number every time.",
-    "\u267b\ufe0f": "Reuse seed from last generation, most useful if it was randomized.",
+    "\u{1f3b2}\ufe0f": "Set seed to -1, which will cause a new random number to be used every time",
+    "\u267b\ufe0f": "Reuse seed from last generation, mostly useful if it was randomed",
     "\u{1f3a8}": "Add a random artist to the prompt.",
     "\u2199\ufe0f": "Read generation parameters from prompt or last generation if prompt is empty into user interface.",
     "\u{1f4c2}": "Open images output directory",

From 47534577eda63b0db1eeb8921c2a161773ec434c Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Sat, 7 Jan 2023 07:51:35 -0500
Subject: [PATCH 05/22] api-get-memory

---
 modules/api/api.py    | 37 +++++++++++++++++++++++++++++++++++++
 modules/api/models.py |  4 ++++
 2 files changed, 41 insertions(+)

diff --git a/modules/api/api.py b/modules/api/api.py
index 2103709b0..d2222b18f 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -130,6 +130,7 @@ class Api:
         self.add_api_route("/sdapi/v1/preprocess", self.preprocess, methods=["POST"], response_model=PreprocessResponse)
         self.add_api_route("/sdapi/v1/train/embedding", self.train_embedding, methods=["POST"], response_model=TrainResponse)
         self.add_api_route("/sdapi/v1/train/hypernetwork", self.train_hypernetwork, methods=["POST"], response_model=TrainResponse)
+        self.add_api_route("/sdapi/v1/memory", self.get_memory, methods=["GET"], response_model=MemoryResponse)
 
     def add_api_route(self, path: str, endpoint, **kwargs):
         if shared.cmd_opts.api_auth:
@@ -465,6 +466,42 @@ class Api:
             shared.state.end()
             return TrainResponse(info = "train embedding error: {error}".format(error = error))
 
+    def get_memory(self):
+        def gb(val: float):
+            return round(val / 1024 / 1024 / 1024, 2)
+        try:
+            import os, psutil
+            process = psutil.Process(os.getpid())
+            res = process.memory_info()
+            ram_total = 100 * res.rss / process.memory_percent()
+            ram = { 'free': gb(ram_total - res.rss), 'used': gb(res.rss), 'total': gb(ram_total) }
+        except Exception as err:
+            ram = { 'error': f'{err}' }
+        try:
+            import torch
+            if torch.cuda.is_available():
+                s = torch.cuda.mem_get_info()
+                system = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+                s = dict(torch.cuda.memory_stats(shared.device))
+                allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) }
+                reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) }
+                active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) }
+                inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) }
+                warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] }
+                cuda = {
+                    'system': system,
+                    'active': active,
+                    'allocated': allocated,
+                    'reserved': reserved,
+                    'inactive': inactive,
+                    'events': warnings,
+                }
+            else:
+                cuda = { 'error': 'unavailable' }
+        except Exception as err:
+            cuda = { 'error': f'{err}' }
+        return MemoryResponse(ram = ram, cuda = cuda)
+
     def launch(self, server_name, port):
         self.app.include_router(self.router)
         uvicorn.run(self.app, host=server_name, port=port)
diff --git a/modules/api/models.py b/modules/api/models.py
index 5fa637745..49bf1e7a9 100644
--- a/modules/api/models.py
+++ b/modules/api/models.py
@@ -260,3 +260,7 @@ class EmbeddingItem(BaseModel):
 class EmbeddingsResponse(BaseModel):
     loaded: Dict[str, EmbeddingItem] = Field(title="Loaded", description="Embeddings loaded for the current model")
     skipped: Dict[str, EmbeddingItem] = Field(title="Skipped", description="Embeddings skipped for the current model (likely due to architecture incompatibility)")
+
+class MemoryResponse(BaseModel):
+    ram: dict[str, str] | dict[str, float] = Field(title="RAM", description="System memory stats")
+    cuda: dict[str, str] | dict[str, dict] = Field(title="CUDA", description="nVidia CUDA memory stats")

From 448b9cedab66e05b5b2800513ca334a769b42aa7 Mon Sep 17 00:00:00 2001
From: dan <guaneec@gmail.com>
Date: Sat, 7 Jan 2023 21:07:27 +0800
Subject: [PATCH 06/22] Allow variable img size

---
 modules/textual_inversion/dataset.py           | 18 +++++++++++-------
 modules/textual_inversion/textual_inversion.py |  4 ++--
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index 88d68c76c..375178ede 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -17,7 +17,7 @@ re_numbers_at_start = re.compile(r"^[-\d]+\s*")
 
 
 class DatasetEntry:
-    def __init__(self, filename=None, filename_text=None, latent_dist=None, latent_sample=None, cond=None, cond_text=None, pixel_values=None):
+    def __init__(self, filename=None, filename_text=None, latent_dist=None, latent_sample=None, cond=None, cond_text=None, pixel_values=None, img_shape=None):
         self.filename = filename
         self.filename_text = filename_text
         self.latent_dist = latent_dist
@@ -25,6 +25,7 @@ class DatasetEntry:
         self.cond = cond
         self.cond_text = cond_text
         self.pixel_values = pixel_values
+        self.img_shape = img_shape
 
 
 class PersonalizedBase(Dataset):
@@ -33,8 +34,6 @@ class PersonalizedBase(Dataset):
 
         self.placeholder_token = placeholder_token
 
-        self.width = width
-        self.height = height
         self.flip = transforms.RandomHorizontalFlip(p=flip_p)
 
         self.dataset = []
@@ -59,7 +58,11 @@ class PersonalizedBase(Dataset):
             if shared.state.interrupted:
                 raise Exception("interrupted")
             try:
-                image = Image.open(path).convert('RGB').resize((self.width, self.height), PIL.Image.BICUBIC)
+                image = Image.open(path).convert('RGB')
+                if width < 2000:
+                    image = image.resize((width, height), PIL.Image.BICUBIC)
+                else:
+                    assert batch_size == 1, 'variable img size must have batch size 1'
             except Exception:
                 continue
 
@@ -88,14 +91,14 @@ class PersonalizedBase(Dataset):
             if latent_sampling_method == "once" or (latent_sampling_method == "deterministic" and not isinstance(latent_dist, DiagonalGaussianDistribution)):
                 latent_sample = model.get_first_stage_encoding(latent_dist).squeeze().to(devices.cpu)
                 latent_sampling_method = "once"
-                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample)
+                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample, img_shape=image.size)
             elif latent_sampling_method == "deterministic":
                 # Works only for DiagonalGaussianDistribution
                 latent_dist.std = 0
                 latent_sample = model.get_first_stage_encoding(latent_dist).squeeze().to(devices.cpu)
-                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample)
+                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample, img_shape=image.size)
             elif latent_sampling_method == "random":
-                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_dist=latent_dist)
+                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_dist=latent_dist, img_shape=image.size)
 
             if not (self.tag_drop_out != 0 or self.shuffle_tags):
                 entry.cond_text = self.create_text(filename_text)
@@ -151,6 +154,7 @@ class BatchLoader:
         self.cond_text = [entry.cond_text for entry in data]
         self.cond = [entry.cond for entry in data]
         self.latent_sample = torch.stack([entry.latent_sample for entry in data]).squeeze(1)
+        self.img_shape = [entry.img_shape for entry in data]
         #self.emb_index = [entry.emb_index for entry in data]
         #print(self.latent_sample.device)
 
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 45882ed68..9f96d0fda 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -451,8 +451,8 @@ def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_
                     else:
                         p.prompt = batch.cond_text[0]
                         p.steps = 20
-                        p.width = training_width
-                        p.height = training_height
+                        p.width = batch.img_shape[0][0]
+                        p.height = batch.img_shape[0][1]
 
                     preview_text = p.prompt
 

From 669fb18d5222f53ae48abe0f30393d846c50ad91 Mon Sep 17 00:00:00 2001
From: dan <guaneec@gmail.com>
Date: Sun, 8 Jan 2023 01:34:52 +0800
Subject: [PATCH 07/22] Add checkbox for variable training dims

---
 modules/hypernetworks/hypernetwork.py          | 2 +-
 modules/textual_inversion/dataset.py           | 4 ++--
 modules/textual_inversion/textual_inversion.py | 4 ++--
 modules/ui.py                                  | 3 +++
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index b0cfbe71a..dba52841a 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -403,7 +403,7 @@ def create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure=None,
     shared.reload_hypernetworks()
 
 
-def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_hypernetwork_every, template_file, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
+def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_hypernetwork_every, template_file, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     # images allows training previews to have infotext. Importing it at the top causes a circular import problem.
     from modules import images
 
diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index 375178ede..7f8a314fa 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -29,7 +29,7 @@ class DatasetEntry:
 
 
 class PersonalizedBase(Dataset):
-    def __init__(self, data_root, width, height, repeats, flip_p=0.5, placeholder_token="*", model=None, cond_model=None, device=None, template_file=None, include_cond=False, batch_size=1, gradient_step=1, shuffle_tags=False, tag_drop_out=0, latent_sampling_method='once'):
+    def __init__(self, data_root, width, height, repeats, flip_p=0.5, placeholder_token="*", model=None, cond_model=None, device=None, template_file=None, include_cond=False, batch_size=1, gradient_step=1, shuffle_tags=False, tag_drop_out=0, latent_sampling_method='once', varsize=False):
         re_word = re.compile(shared.opts.dataset_filename_word_regex) if len(shared.opts.dataset_filename_word_regex) > 0 else None
 
         self.placeholder_token = placeholder_token
@@ -59,7 +59,7 @@ class PersonalizedBase(Dataset):
                 raise Exception("interrupted")
             try:
                 image = Image.open(path).convert('RGB')
-                if width < 2000:
+                if not varsize:
                     image = image.resize((width, height), PIL.Image.BICUBIC)
                 else:
                     assert batch_size == 1, 'variable img size must have batch size 1'
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 9f96d0fda..110efd19c 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -255,7 +255,7 @@ def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, dat
     if save_model_every or create_image_every:
         assert log_directory, "Log directory is empty"
 
-def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_embedding_every, template_file, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
+def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_embedding_every, template_file, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     save_embedding_every = save_embedding_every or 0
     create_image_every = create_image_every or 0
     validate_train_inputs(embedding_name, learn_rate, batch_size, gradient_step, data_root, template_file, steps, save_embedding_every, create_image_every, log_directory, name="embedding")
@@ -309,7 +309,7 @@ def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_
 
     pin_memory = shared.opts.pin_memory
 
-    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=embedding_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method)
+    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=embedding_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize)
 
     if shared.opts.save_training_settings_to_txt:
         save_settings_to_file(log_directory, {**dict(model_name=checkpoint.model_name, model_hash=checkpoint.hash, num_of_dataset_images=len(ds), num_vectors_per_token=len(embedding.vec)), **locals()})
diff --git a/modules/ui.py b/modules/ui.py
index 99483130c..4e709a71f 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -1343,6 +1343,7 @@ def create_ui():
                     template_file = gr.Textbox(label='Prompt template file', value=os.path.join(script_path, "textual_inversion_templates", "style_filewords.txt"), elem_id="train_template_file")
                     training_width = gr.Slider(minimum=64, maximum=2048, step=8, label="Width", value=512, elem_id="train_training_width")
                     training_height = gr.Slider(minimum=64, maximum=2048, step=8, label="Height", value=512, elem_id="train_training_height")
+                    varsize = gr.Checkbox(label="Ignore dimension settings and do not resize images", value=False, elem_id="train_varsize")
                     steps = gr.Number(label='Max steps', value=100000, precision=0, elem_id="train_steps")
 
                     with FormRow():
@@ -1449,6 +1450,7 @@ def create_ui():
                 log_directory,
                 training_width,
                 training_height,
+                varsize,
                 steps,
                 clip_grad_mode,
                 clip_grad_value,
@@ -1480,6 +1482,7 @@ def create_ui():
                 log_directory,
                 training_width,
                 training_height,
+                varsize,
                 steps,
                 clip_grad_mode,
                 clip_grad_value,

From 72497895b9b1948f86d9309fe897cbb70c20ba7e Mon Sep 17 00:00:00 2001
From: dan <guaneec@gmail.com>
Date: Sun, 8 Jan 2023 01:36:00 +0800
Subject: [PATCH 08/22] Move batchsize check

---
 modules/hypernetworks/hypernetwork.py | 2 +-
 modules/textual_inversion/dataset.py  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index dba52841a..32c67ccce 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -456,7 +456,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, gradient_step,
 
     pin_memory = shared.opts.pin_memory
 
-    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method)
+    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize)
 
     if shared.opts.save_training_settings_to_txt:
         saved_params = dict(
diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index 7f8a314fa..bcad68481 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -46,6 +46,8 @@ class PersonalizedBase(Dataset):
         assert data_root, 'dataset directory not specified'
         assert os.path.isdir(data_root), "Dataset directory doesn't exist"
         assert os.listdir(data_root), "Dataset directory is empty"
+        if varsize:
+            assert batch_size == 1, 'variable img size must have batch size 1'
 
         self.image_paths = [os.path.join(data_root, file_path) for file_path in os.listdir(data_root)]
 
@@ -61,8 +63,6 @@ class PersonalizedBase(Dataset):
                 image = Image.open(path).convert('RGB')
                 if not varsize:
                     image = image.resize((width, height), PIL.Image.BICUBIC)
-                else:
-                    assert batch_size == 1, 'variable img size must have batch size 1'
             except Exception:
                 continue
 

From 984b86dd0abf0da7f6b116864c791a2bfe8859ef Mon Sep 17 00:00:00 2001
From: ProGamerGov <ProGamerGov@users.noreply.github.com>
Date: Sat, 7 Jan 2023 13:08:21 -0700
Subject: [PATCH 09/22] Add fallback for Protocol import

---
 modules/sub_quadratic_attention.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/modules/sub_quadratic_attention.py b/modules/sub_quadratic_attention.py
index fea7aaacc..93381bae4 100644
--- a/modules/sub_quadratic_attention.py
+++ b/modules/sub_quadratic_attention.py
@@ -15,7 +15,13 @@ import torch
 from torch import Tensor
 from torch.utils.checkpoint import checkpoint
 import math
-from typing import Optional, NamedTuple, Protocol, List
+
+try:
+    from typing import Protocol
+except:
+    from typing_extensions import Protocol
+    
+from typing import Optional, NamedTuple, List
 
 def narrow_trunc(
     input: Tensor,

From cb255faec6e5f6b47b7632e6b7d450b9e2f6678b Mon Sep 17 00:00:00 2001
From: Lee Bousfield <ljbousfield@gmail.com>
Date: Sun, 8 Jan 2023 10:17:50 -0700
Subject: [PATCH 10/22] Add support for loading VAEs from safetensor files

---
 modules/sd_vae.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/modules/sd_vae.py b/modules/sd_vae.py
index ac71d62db..9fcfd9dbb 100644
--- a/modules/sd_vae.py
+++ b/modules/sd_vae.py
@@ -1,4 +1,5 @@
 import torch
+import safetensors.torch
 import os
 import collections
 from collections import namedtuple
@@ -72,8 +73,10 @@ def refresh_vae_list(vae_path=vae_path, model_path=model_path):
     candidates = [
         *glob.iglob(os.path.join(model_path, '**/*.vae.ckpt'), recursive=True),
         *glob.iglob(os.path.join(model_path, '**/*.vae.pt'), recursive=True),
+        *glob.iglob(os.path.join(model_path, '**/*.vae.safetensors'), recursive=True),
         *glob.iglob(os.path.join(vae_path, '**/*.ckpt'), recursive=True),
-        *glob.iglob(os.path.join(vae_path, '**/*.pt'), recursive=True)
+        *glob.iglob(os.path.join(vae_path, '**/*.pt'), recursive=True),
+        *glob.iglob(os.path.join(vae_path, '**/*.safetensors'), recursive=True),
     ]
     if shared.cmd_opts.vae_path is not None and os.path.isfile(shared.cmd_opts.vae_path):
         candidates.append(shared.cmd_opts.vae_path)
@@ -137,6 +140,12 @@ def resolve_vae(checkpoint_file=None, vae_file="auto"):
         if os.path.isfile(vae_file_try):
             vae_file = vae_file_try
             print(f"Using VAE found similar to selected model: {vae_file}")
+    # if still not found, try look for ".vae.safetensors" beside model
+    if vae_file == "auto":
+        vae_file_try = model_path + ".vae.safetensors"
+        if os.path.isfile(vae_file_try):
+            vae_file = vae_file_try
+            print(f"Using VAE found similar to selected model: {vae_file}")
     # No more fallbacks for auto
     if vae_file == "auto":
         vae_file = None
@@ -163,8 +172,14 @@ def load_vae(model, vae_file=None):
             assert os.path.isfile(vae_file), f"VAE file doesn't exist: {vae_file}"
             print(f"Loading VAE weights from: {vae_file}")
             store_base_vae(model)
-            vae_ckpt = torch.load(vae_file, map_location=shared.weight_load_location)
-            vae_dict_1 = {k: v for k, v in vae_ckpt["state_dict"].items() if k[0:4] != "loss" and k not in vae_ignore_keys}
+            _, extension = os.path.splitext(vae_file)
+            if extension.lower() == ".safetensors":
+                vae_ckpt = safetensors.torch.load_file(vae_file, device=shared.weight_load_location)
+            else:
+                vae_ckpt = torch.load(vae_file, map_location=shared.weight_load_location)
+            if "state_dict" in vae_ckpt:
+                vae_ckpt = vae_ckpt["state_dict"]
+            vae_dict_1 = {k: v for k, v in vae_ckpt.items() if k[0:4] != "loss" and k not in vae_ignore_keys}
             _load_vae_dict(model, vae_dict_1)
 
             if cache_enabled:

From 1d663a04da900b79132063e94c94ab379ebd14a8 Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 9 Jan 2023 14:11:37 +0300
Subject: [PATCH 11/22] make tests runnable without specifying subdirectory

---
 test/server_poll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/server_poll.py b/test/server_poll.py
index d4df697b2..42d56a4ca 100644
--- a/test/server_poll.py
+++ b/test/server_poll.py
@@ -15,7 +15,7 @@ def run_tests(proc, test_dir):
                 break
     if proc.poll() is None:
         if test_dir is None:
-            test_dir = ""
+            test_dir = "test"
         suite = unittest.TestLoader().discover(test_dir, pattern="*_test.py", top_level_dir="test")
         result = unittest.TextTestRunner(verbosity=2).run(suite)
         return len(result.failures) + len(result.errors)

From 3af488bdff983efc8e77f49b26c18847413754f4 Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 9 Jan 2023 14:29:28 +0300
Subject: [PATCH 12/22] try all tests

---
 .github/workflows/run_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 49dc92bd9..110a6a758 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -20,7 +20,7 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: ${{ runner.os }}-pip-
       - name: Run tests
-        run: python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
+        run: python launch.py --tests --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
       - name: Upload main app stdout-stderr
         uses: actions/upload-artifact@v3
         if: always()

From d4fd2418efb0986a8226add0b800fb5c73ffb58c Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Mon, 9 Jan 2023 14:57:47 +0300
Subject: [PATCH 13/22] add an option to use old hiresfix width/height behavior
 add a visual effect to inactive hires fix elements

---
 javascript/hires_fix.js                    | 25 +++++++++++++++++++++
 modules/generation_parameters_copypaste.py | 17 +++++++++-----
 modules/processing.py                      | 26 ++++++++++++++++++++--
 modules/shared.py                          |  1 +
 modules/ui.py                              | 23 +++++++++++--------
 style.css                                  |  4 ++++
 6 files changed, 79 insertions(+), 17 deletions(-)
 create mode 100644 javascript/hires_fix.js

diff --git a/javascript/hires_fix.js b/javascript/hires_fix.js
new file mode 100644
index 000000000..07fba5493
--- /dev/null
+++ b/javascript/hires_fix.js
@@ -0,0 +1,25 @@
+
+function setInactive(elem, inactive){
+    console.log(elem)
+    if(inactive){
+        elem.classList.add('inactive')
+    } else{
+        elem.classList.remove('inactive')
+    }
+}
+
+function onCalcResolutionHires(enable, width, height, hr_scale, hr_resize_x, hr_resize_y){
+    console.log(enable, width, height, hr_scale, hr_resize_x, hr_resize_y)
+
+    hrUpscaleBy = gradioApp().getElementById('txt2img_hr_scale')
+    hrResizeX = gradioApp().getElementById('txt2img_hr_resize_x')
+    hrResizeY = gradioApp().getElementById('txt2img_hr_resize_y')
+
+    gradioApp().getElementById('txt2img_hires_fix_row2').style.display = opts.use_old_hires_fix_width_height ? "none" : ""
+
+    setInactive(hrUpscaleBy, opts.use_old_hires_fix_width_height || hr_resize_x > 0 || hr_resize_y > 0)
+    setInactive(hrResizeX, opts.use_old_hires_fix_width_height || hr_resize_x == 0)
+    setInactive(hrResizeY, opts.use_old_hires_fix_width_height || hr_resize_y == 0)
+
+    return [enable, width, height, hr_scale, hr_resize_x, hr_resize_y]
+}
diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index 12a9de3dc..f7f68b67e 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -197,6 +197,15 @@ def restore_old_hires_fix_params(res):
     firstpass_width = res.get('First pass size-1', None)
     firstpass_height = res.get('First pass size-2', None)
 
+    if shared.opts.use_old_hires_fix_width_height:
+        hires_width = int(res.get("Hires resize-1", None))
+        hires_height = int(res.get("Hires resize-2", None))
+
+        if hires_width is not None and hires_height is not None:
+            res['Size-1'] = hires_width
+            res['Size-2'] = hires_height
+            return
+
     if firstpass_width is None or firstpass_height is None:
         return
 
@@ -205,12 +214,8 @@ def restore_old_hires_fix_params(res):
     height = int(res.get("Size-2", 512))
 
     if firstpass_width == 0 or firstpass_height == 0:
-        # old algorithm for auto-calculating first pass size
-        desired_pixel_count = 512 * 512
-        actual_pixel_count = width * height
-        scale = math.sqrt(desired_pixel_count / actual_pixel_count)
-        firstpass_width = math.ceil(scale * width / 64) * 64
-        firstpass_height = math.ceil(scale * height / 64) * 64
+        from modules import processing
+        firstpass_width, firstpass_height = processing.old_hires_fix_first_pass_dimensions(width, height)
 
     res['Size-1'] = firstpass_width
     res['Size-2'] = firstpass_height
diff --git a/modules/processing.py b/modules/processing.py
index 1d23b15fd..f04a0e1e4 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -687,6 +687,18 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
     return res
 
 
+def old_hires_fix_first_pass_dimensions(width, height):
+    """old algorithm for auto-calculating first pass size"""
+
+    desired_pixel_count = 512 * 512
+    actual_pixel_count = width * height
+    scale = math.sqrt(desired_pixel_count / actual_pixel_count)
+    width = math.ceil(scale * width / 64) * 64
+    height = math.ceil(scale * height / 64) * 64
+
+    return width, height
+
+
 class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
     sampler = None
 
@@ -703,16 +715,26 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         self.hr_upscale_to_y = hr_resize_y
 
         if firstphase_width != 0 or firstphase_height != 0:
-            print("firstphase_width/firstphase_height no longer supported; use hr_scale", file=sys.stderr)
-            self.hr_scale = self.width / firstphase_width
+            self.hr_upscale_to_x = self.width
+            self.hr_upscale_to_y = self.height
             self.width = firstphase_width
             self.height = firstphase_height
 
         self.truncate_x = 0
         self.truncate_y = 0
+        self.applied_old_hires_behavior_to = None
 
     def init(self, all_prompts, all_seeds, all_subseeds):
         if self.enable_hr:
+            if opts.use_old_hires_fix_width_height and self.applied_old_hires_behavior_to != (self.width, self.height):
+                self.hr_resize_x = self.width
+                self.hr_resize_y = self.height
+                self.hr_upscale_to_x = self.width
+                self.hr_upscale_to_y = self.height
+
+                self.width, self.height = old_hires_fix_first_pass_dimensions(self.width, self.height)
+                self.applied_old_hires_behavior_to = (self.width, self.height)
+
             if self.hr_resize_x == 0 and self.hr_resize_y == 0:
                 self.extra_generation_params["Hires upscale"] = self.hr_scale
                 self.hr_upscale_to_x = int(self.width * self.hr_scale)
diff --git a/modules/shared.py b/modules/shared.py
index a6712dae9..a1e102019 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -398,6 +398,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
 options_templates.update(options_section(('compatibility', "Compatibility"), {
     "use_old_emphasis_implementation": OptionInfo(False, "Use old emphasis implementation. Can be useful to reproduce old seeds."),
     "use_old_karras_scheduler_sigmas": OptionInfo(False, "Use old karras scheduler sigmas (0.1 to 10)."),
+    "use_old_hires_fix_width_height": OptionInfo(False, "For hires fix, use width/height sliders to set final resolution rather than first pass (disables Upscale by, Resize width/height to)."),
 }))
 
 options_templates.update(options_section(('interrogate', "Interrogate Options"), {
diff --git a/modules/ui.py b/modules/ui.py
index 99483130c..719c26b32 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -267,7 +267,7 @@ def calc_resolution_hires(enable, width, height, hr_scale, hr_resize_x, hr_resiz
     with devices.autocast():
         p.init([""], [0], [0])
 
-    return f"resize: from <span class='resolution'>{width}x{height}</span> to <span class='resolution'>{p.hr_upscale_to_x}x{p.hr_upscale_to_y}</span>"
+    return f"resize: from <span class='resolution'>{p.width}x{p.height}</span> to <span class='resolution'>{p.hr_resize_x or p.hr_upscale_to_x}x{p.hr_resize_y or p.hr_upscale_to_y}</span>"
 
 
 def apply_styles(prompt, prompt_neg, style1_name, style2_name):
@@ -745,15 +745,20 @@ def create_ui():
                             custom_inputs = modules.scripts.scripts_txt2img.setup_ui()
 
             hr_resolution_preview_inputs = [enable_hr, width, height, hr_scale, hr_resize_x, hr_resize_y]
-            hr_resolution_preview_args = dict(
-                fn=calc_resolution_hires,
-                inputs=hr_resolution_preview_inputs,
-                outputs=[hr_final_resolution],
-                show_progress=False
-            )
-
             for input in hr_resolution_preview_inputs:
-                input.change(**hr_resolution_preview_args)
+                input.change(
+                    fn=calc_resolution_hires,
+                    inputs=hr_resolution_preview_inputs,
+                    outputs=[hr_final_resolution],
+                    show_progress=False,
+                )
+                input.change(
+                    None,
+                    _js="onCalcResolutionHires",
+                    inputs=hr_resolution_preview_inputs,
+                    outputs=[],
+                    show_progress=False,
+                )
 
             txt2img_gallery, generation_info, html_info, html_log = create_output_panel("txt2img", opts.outdir_txt2img_samples)
             parameters_copypaste.bind_buttons({"txt2img": txt2img_paste}, None, txt2img_prompt)
diff --git a/style.css b/style.css
index d796cbe99..ec5e41827 100644
--- a/style.css
+++ b/style.css
@@ -670,6 +670,10 @@ footer {
     min-width: auto;
 }
 
+.inactive{
+    opacity: 0.5;
+}
+
 /* The following handles localization for right-to-left (RTL) languages like Arabic.
 The rtl media type will only be activated by the logic in javascript/localization.js.
 If you change anything above, you need to make sure it is RTL compliant by just running

From 7d2bb86cce10ee6a8e81aaad810544a4ca38cec9 Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 9 Jan 2023 19:39:06 +0300
Subject: [PATCH 14/22] combine tests together, return set options test

---
 test/advanced_features/__init__.py            |  0
 test/advanced_features/txt2img_test.py        | 47 -------------------
 .../extras_test.py                            |  0
 test/basic_features/txt2img_test.py           |  4 ++
 test/basic_features/utils_test.py             | 14 ++++++
 5 files changed, 18 insertions(+), 47 deletions(-)
 delete mode 100644 test/advanced_features/__init__.py
 delete mode 100644 test/advanced_features/txt2img_test.py
 rename test/{advanced_features => basic_features}/extras_test.py (100%)

diff --git a/test/advanced_features/__init__.py b/test/advanced_features/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/test/advanced_features/txt2img_test.py b/test/advanced_features/txt2img_test.py
deleted file mode 100644
index 36ed7b9a9..000000000
--- a/test/advanced_features/txt2img_test.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import unittest
-import requests
-
-
-class TestTxt2ImgWorking(unittest.TestCase):
-    def setUp(self):
-        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
-        self.simple_txt2img = {
-            "enable_hr": False,
-            "denoising_strength": 0,
-            "firstphase_width": 0,
-            "firstphase_height": 0,
-            "prompt": "example prompt",
-            "styles": [],
-            "seed": -1,
-            "subseed": -1,
-            "subseed_strength": 0,
-            "seed_resize_from_h": -1,
-            "seed_resize_from_w": -1,
-            "batch_size": 1,
-            "n_iter": 1,
-            "steps": 3,
-            "cfg_scale": 7,
-            "width": 64,
-            "height": 64,
-            "restore_faces": False,
-            "tiling": False,
-            "negative_prompt": "",
-            "eta": 0,
-            "s_churn": 0,
-            "s_tmax": 0,
-            "s_tmin": 0,
-            "s_noise": 1,
-            "sampler_index": "Euler a"
-        }
-
-    def test_txt2img_with_restore_faces_performed(self):
-        self.simple_txt2img["restore_faces"] = True
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-
-class TestTxt2ImgCorrectness(unittest.TestCase):
-    pass
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/advanced_features/extras_test.py b/test/basic_features/extras_test.py
similarity index 100%
rename from test/advanced_features/extras_test.py
rename to test/basic_features/extras_test.py
diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
index 1c2674b2a..bbc846eda 100644
--- a/test/basic_features/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -63,6 +63,10 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.simple_txt2img["n_iter"] = 2
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
+    def test_txt2img_with_restore_faces_performed(self):
+        self.simple_txt2img["restore_faces"] = True
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
index 765470c90..b3c4045aa 100644
--- a/test/basic_features/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -18,6 +18,20 @@ class UtilsTests(unittest.TestCase):
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
 
+  def test_options_write(self):
+    response = requests.get(self.url_options)
+    self.assertEqual(response.status_code, 200)
+
+    pre_value = response.json()["send_seed"]
+
+    self.assertEqual(requests.post(self.url_options, json={"send_seed":not pre_value}).status_code, 200)
+
+    response = requests.get(self.url_options)
+    self.assertEqual(response.status_code, 200)
+    self.assertEqual(response.json()["send_seed"], not pre_value)
+
+    requests.post(self.url_options, json={"send_seed": pre_value})
+
   def test_cmd_flags(self):
     self.assertEqual(requests.get(self.url_cmd_flags).status_code, 200)
 

From 49c4509ce2302350210ff650fd26373518c46a79 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Mon, 9 Jan 2023 19:58:35 +0300
Subject: [PATCH 15/22] use existing function for loading VAE weights from file

---
 modules/sd_vae.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/modules/sd_vae.py b/modules/sd_vae.py
index 9fcfd9dbb..0a49daa16 100644
--- a/modules/sd_vae.py
+++ b/modules/sd_vae.py
@@ -3,7 +3,7 @@ import safetensors.torch
 import os
 import collections
 from collections import namedtuple
-from modules import shared, devices, script_callbacks
+from modules import shared, devices, script_callbacks, sd_models
 from modules.paths import models_path
 import glob
 from copy import deepcopy
@@ -172,13 +172,8 @@ def load_vae(model, vae_file=None):
             assert os.path.isfile(vae_file), f"VAE file doesn't exist: {vae_file}"
             print(f"Loading VAE weights from: {vae_file}")
             store_base_vae(model)
-            _, extension = os.path.splitext(vae_file)
-            if extension.lower() == ".safetensors":
-                vae_ckpt = safetensors.torch.load_file(vae_file, device=shared.weight_load_location)
-            else:
-                vae_ckpt = torch.load(vae_file, map_location=shared.weight_load_location)
-            if "state_dict" in vae_ckpt:
-                vae_ckpt = vae_ckpt["state_dict"]
+
+            vae_ckpt = sd_models.read_state_dict(vae_file, map_location=shared.weight_load_location)
             vae_dict_1 = {k: v for k, v in vae_ckpt.items() if k[0:4] != "loss" and k not in vae_ignore_keys}
             _load_vae_dict(model, vae_dict_1)
 
@@ -210,10 +205,12 @@ def _load_vae_dict(model, vae_dict_1):
     model.first_stage_model.load_state_dict(vae_dict_1)
     model.first_stage_model.to(devices.dtype_vae)
 
+
 def clear_loaded_vae():
     global loaded_vae_file
     loaded_vae_file = None
 
+
 def reload_vae_weights(sd_model=None, vae_file="auto"):
     from modules import lowvram, devices, sd_hijack
 

From cdfcbd995932ffa728db0cc00a5f97665c752103 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Mon, 9 Jan 2023 20:08:48 +0300
Subject: [PATCH 16/22] Remove fallback for Protocol import and remove Protocol
 import and remove instances of Protocol in code add some whitespace between
 functions to be in line with other code in the repo

---
 modules/sub_quadratic_attention.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/modules/sub_quadratic_attention.py b/modules/sub_quadratic_attention.py
index 93381bae4..55052815b 100644
--- a/modules/sub_quadratic_attention.py
+++ b/modules/sub_quadratic_attention.py
@@ -15,14 +15,9 @@ import torch
 from torch import Tensor
 from torch.utils.checkpoint import checkpoint
 import math
-
-try:
-    from typing import Protocol
-except:
-    from typing_extensions import Protocol
-    
 from typing import Optional, NamedTuple, List
 
+
 def narrow_trunc(
     input: Tensor,
     dim: int,
@@ -31,12 +26,14 @@ def narrow_trunc(
 ) -> Tensor:
     return torch.narrow(input, dim, start, length if input.shape[dim] >= start + length else input.shape[dim] - start)
 
+
 class AttnChunk(NamedTuple):
     exp_values: Tensor
     exp_weights_sum: Tensor
     max_score: Tensor
 
-class SummarizeChunk(Protocol):
+
+class SummarizeChunk:
     @staticmethod
     def __call__(
         query: Tensor,
@@ -44,7 +41,8 @@ class SummarizeChunk(Protocol):
         value: Tensor,
     ) -> AttnChunk: ...
 
-class ComputeQueryChunkAttn(Protocol):
+
+class ComputeQueryChunkAttn:
     @staticmethod
     def __call__(
         query: Tensor,
@@ -52,6 +50,7 @@ class ComputeQueryChunkAttn(Protocol):
         value: Tensor,
     ) -> Tensor: ...
 
+
 def _summarize_chunk(
     query: Tensor,
     key: Tensor,
@@ -72,6 +71,7 @@ def _summarize_chunk(
     max_score = max_score.squeeze(-1)
     return AttnChunk(exp_values, exp_weights.sum(dim=-1), max_score)
 
+
 def _query_chunk_attention(
     query: Tensor,
     key: Tensor,
@@ -112,6 +112,7 @@ def _query_chunk_attention(
     all_weights = torch.unsqueeze(chunk_weights, -1).sum(dim=0)
     return all_values / all_weights
 
+
 # TODO: refactor CrossAttention#get_attention_scores to share code with this
 def _get_attention_scores_no_kv_chunking(
     query: Tensor,
@@ -131,10 +132,12 @@ def _get_attention_scores_no_kv_chunking(
     hidden_states_slice = torch.bmm(attn_probs, value)
     return hidden_states_slice
 
+
 class ScannedChunk(NamedTuple):
     chunk_idx: int
     attn_chunk: AttnChunk
 
+
 def efficient_dot_product_attention(
     query: Tensor,
     key: Tensor,

From 00005ac9af10d58a75f7ce0aa04db78775808e93 Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 9 Jan 2023 21:01:28 +0300
Subject: [PATCH 17/22] add more tests

---
 test/basic_features/extras_test.py  | 37 ++++++++++++++++++++++++-----
 test/basic_features/img2img_test.py |  7 +++++-
 test/basic_features/txt2img_test.py | 11 +++++++--
 test/basic_features/utils_test.py   |  3 +++
 4 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/test/basic_features/extras_test.py b/test/basic_features/extras_test.py
index 8763f8ed1..0170c511f 100644
--- a/test/basic_features/extras_test.py
+++ b/test/basic_features/extras_test.py
@@ -1,10 +1,12 @@
 import unittest
-
+import requests
+from gradio.processing_utils import encode_pil_to_base64
+from PIL import Image
 
 class TestExtrasWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://localhost:7860/sdapi/v1/extra-single-image"
-        self.simple_extras = {
+        self.url_extras_single = "http://localhost:7860/sdapi/v1/extra-single-image"
+        self.extras_single = {
             "resize_mode": 0,
             "show_extras_results": True,
             "gfpgan_visibility": 0,
@@ -17,12 +19,35 @@ class TestExtrasWorking(unittest.TestCase):
             "upscaler_1": "None",
             "upscaler_2": "None",
             "extras_upscaler_2_visibility": 0,
-            "image": ""
+            "image": encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png"))
             }
 
+    def test_simple_upscaling_performed(self):
+        self.extras_single["upscaler_1"] = "Lanczos"
+        self.assertEqual(requests.post(self.url_extras_single, json=self.extras_single).status_code, 200)
 
-class TestExtrasCorrectness(unittest.TestCase):
-    pass
+
+class TestPngInfoWorking(unittest.TestCase):
+    def setUp(self):
+        self.url_png_info = "http://localhost:7860/sdapi/v1/extra-single-image"
+        self.png_info = {
+            "image": encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png"))
+        }
+
+    def test_png_info_performed(self):
+        self.assertEqual(requests.post(self.url_png_info, json=self.png_info).status_code, 200)
+
+
+class TestInterrogateWorking(unittest.TestCase):
+    def setUp(self):
+        self.url_interrogate = "http://localhost:7860/sdapi/v1/extra-single-image"
+        self.interrogate = {
+            "image": encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png")),
+            "model": "clip"
+        }
+
+    def test_interrogate_performed(self):
+        self.assertEqual(requests.post(self.url_interrogate, json=self.interrogate).status_code, 200)
 
 
 if __name__ == "__main__":
diff --git a/test/basic_features/img2img_test.py b/test/basic_features/img2img_test.py
index bd520b139..08c5c903e 100644
--- a/test/basic_features/img2img_test.py
+++ b/test/basic_features/img2img_test.py
@@ -16,7 +16,7 @@ class TestImg2ImgWorking(unittest.TestCase):
             "inpainting_fill": 0,
             "inpaint_full_res": False,
             "inpaint_full_res_padding": 0,
-            "inpainting_mask_invert": 0,
+            "inpainting_mask_invert": False,
             "prompt": "example prompt",
             "styles": [],
             "seed": -1,
@@ -50,6 +50,11 @@ class TestImg2ImgWorking(unittest.TestCase):
         self.simple_img2img["mask"] = encode_pil_to_base64(Image.open(r"test/test_files/mask_basic.png"))
         self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
 
+    def test_inpainting_with_inverted_masked_performed(self):
+        self.simple_img2img["mask"] = encode_pil_to_base64(Image.open(r"test/test_files/mask_basic.png"))
+        self.simple_img2img["inpainting_mask_invert"] = True
+        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
+
     def test_img2img_sd_upscale_performed(self):
         self.simple_img2img["script_name"] = "sd upscale"
         self.simple_img2img["script_args"] = ["", 8, "Lanczos", 2.0]
diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
index bbc846eda..5b27a7ec9 100644
--- a/test/basic_features/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -41,6 +41,9 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.simple_txt2img["negative_prompt"] = "example negative prompt"
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
+    def test_txt2img_with_complex_prompt_performed(self):
+        self.simple_txt2img["prompt"] = "((emphasis)), (emphasis1:1.1), [to:1], [from::2], [from:to:0.3], [alt|alt1]"
+
     def test_txt2img_not_square_image_performed(self):
         self.simple_txt2img["height"] = 128
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
@@ -53,6 +56,10 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.simple_txt2img["tiling"] = True
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
+    def test_txt2img_with_restore_faces_performed(self):
+        self.simple_txt2img["restore_faces"] = True
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
     def test_txt2img_with_vanilla_sampler_performed(self):
         self.simple_txt2img["sampler_index"] = "PLMS"
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
@@ -63,8 +70,8 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.simple_txt2img["n_iter"] = 2
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
-    def test_txt2img_with_restore_faces_performed(self):
-        self.simple_txt2img["restore_faces"] = True
+    def test_txt2img_batch_performed(self):
+        self.simple_txt2img["batch_size"] = 2
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
 
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
index b3c4045aa..94e002530 100644
--- a/test/basic_features/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -14,6 +14,7 @@ class UtilsTests(unittest.TestCase):
     self.url_prompt_styles = "http://localhost:7860/sdapi/v1/prompt-styles"
     self.url_artist_categories = "http://localhost:7860/sdapi/v1/artist-categories"
     self.url_artists = "http://localhost:7860/sdapi/v1/artists"
+    self.url_embeddings = "http://localhost:7860/sdapi/v1/embeddings"
 
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
@@ -62,6 +63,8 @@ class UtilsTests(unittest.TestCase):
   def test_artists(self):
     self.assertEqual(requests.get(self.url_artists).status_code, 200)
 
+  def test_embeddings(self):
+    self.assertEqual(requests.get(self.url_artists).status_code, 200)
 
 if __name__ == "__main__":
     unittest.main()

From 43bb5190fc9e7ae479a5dc6640be202c9a71e464 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Mon, 9 Jan 2023 22:52:23 +0300
Subject: [PATCH 18/22] remove/simplify some changes from #6481

---
 modules/textual_inversion/dataset.py           | 14 +++++---------
 modules/textual_inversion/textual_inversion.py |  4 ++--
 modules/ui.py                                  |  2 +-
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/modules/textual_inversion/dataset.py b/modules/textual_inversion/dataset.py
index bcad68481..fa48708e4 100644
--- a/modules/textual_inversion/dataset.py
+++ b/modules/textual_inversion/dataset.py
@@ -17,7 +17,7 @@ re_numbers_at_start = re.compile(r"^[-\d]+\s*")
 
 
 class DatasetEntry:
-    def __init__(self, filename=None, filename_text=None, latent_dist=None, latent_sample=None, cond=None, cond_text=None, pixel_values=None, img_shape=None):
+    def __init__(self, filename=None, filename_text=None, latent_dist=None, latent_sample=None, cond=None, cond_text=None, pixel_values=None):
         self.filename = filename
         self.filename_text = filename_text
         self.latent_dist = latent_dist
@@ -25,7 +25,6 @@ class DatasetEntry:
         self.cond = cond
         self.cond_text = cond_text
         self.pixel_values = pixel_values
-        self.img_shape = img_shape
 
 
 class PersonalizedBase(Dataset):
@@ -46,12 +45,10 @@ class PersonalizedBase(Dataset):
         assert data_root, 'dataset directory not specified'
         assert os.path.isdir(data_root), "Dataset directory doesn't exist"
         assert os.listdir(data_root), "Dataset directory is empty"
-        if varsize:
-            assert batch_size == 1, 'variable img size must have batch size 1'
+        assert batch_size == 1 or not varsize, 'variable img size must have batch size 1'
 
         self.image_paths = [os.path.join(data_root, file_path) for file_path in os.listdir(data_root)]
 
-
         self.shuffle_tags = shuffle_tags
         self.tag_drop_out = tag_drop_out
 
@@ -91,14 +88,14 @@ class PersonalizedBase(Dataset):
             if latent_sampling_method == "once" or (latent_sampling_method == "deterministic" and not isinstance(latent_dist, DiagonalGaussianDistribution)):
                 latent_sample = model.get_first_stage_encoding(latent_dist).squeeze().to(devices.cpu)
                 latent_sampling_method = "once"
-                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample, img_shape=image.size)
+                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample)
             elif latent_sampling_method == "deterministic":
                 # Works only for DiagonalGaussianDistribution
                 latent_dist.std = 0
                 latent_sample = model.get_first_stage_encoding(latent_dist).squeeze().to(devices.cpu)
-                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample, img_shape=image.size)
+                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_sample=latent_sample)
             elif latent_sampling_method == "random":
-                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_dist=latent_dist, img_shape=image.size)
+                entry = DatasetEntry(filename=path, filename_text=filename_text, latent_dist=latent_dist)
 
             if not (self.tag_drop_out != 0 or self.shuffle_tags):
                 entry.cond_text = self.create_text(filename_text)
@@ -154,7 +151,6 @@ class BatchLoader:
         self.cond_text = [entry.cond_text for entry in data]
         self.cond = [entry.cond for entry in data]
         self.latent_sample = torch.stack([entry.latent_sample for entry in data]).squeeze(1)
-        self.img_shape = [entry.img_shape for entry in data]
         #self.emb_index = [entry.emb_index for entry in data]
         #print(self.latent_sample.device)
 
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index ad76297e7..14be2c967 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -492,8 +492,8 @@ def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_
                     else:
                         p.prompt = batch.cond_text[0]
                         p.steps = 20
-                        p.width = batch.img_shape[0][0]
-                        p.height = batch.img_shape[0][1]
+                        p.width = training_width
+                        p.height = training_height
 
                     preview_text = p.prompt
 
diff --git a/modules/ui.py b/modules/ui.py
index 9d6b141e5..ddfe1b1a1 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -1348,7 +1348,7 @@ def create_ui():
                     template_file = gr.Textbox(label='Prompt template file', value=os.path.join(script_path, "textual_inversion_templates", "style_filewords.txt"), elem_id="train_template_file")
                     training_width = gr.Slider(minimum=64, maximum=2048, step=8, label="Width", value=512, elem_id="train_training_width")
                     training_height = gr.Slider(minimum=64, maximum=2048, step=8, label="Height", value=512, elem_id="train_training_height")
-                    varsize = gr.Checkbox(label="Ignore dimension settings and do not resize images", value=False, elem_id="train_varsize")
+                    varsize = gr.Checkbox(label="Do not resize images", value=False, elem_id="train_varsize")
                     steps = gr.Number(label='Max steps', value=100000, precision=0, elem_id="train_steps")
 
                     with FormRow():

From 56ed085edf25482a957b08479206154359f9185d Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 9 Jan 2023 23:19:08 +0300
Subject: [PATCH 19/22] move template so github could use it

---
 .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md (94%)

diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/pull_request_template.md
similarity index 94%
rename from .github/PULL_REQUEST_TEMPLATE/pull_request_template.md
rename to .github/pull_request_template.md
index 86009613e..69056331b 100644
--- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -18,8 +18,8 @@ More technical discussion about your changes go here, plus anything that a maint
 
 List the environment you have developed / tested this on. As per the contributing page, changes should be able to work on Windows out of the box.
  - OS: [e.g. Windows, Linux]
- - Browser [e.g. chrome, safari]
- - Graphics card [e.g. NVIDIA RTX 2080 8GB, AMD RX 6600 8GB]
+ - Browser: [e.g. chrome, safari]
+ - Graphics card: [e.g. NVIDIA RTX 2080 8GB, AMD RX 6600 8GB]
 
 **Screenshots or videos of your changes**
 

From 1fbb6f9ebe48326a3b12ecf611105dbc4a46891e Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Mon, 9 Jan 2023 23:35:40 +0300
Subject: [PATCH 20/22] make a dropdown for prompt template selection

---
 modules/hypernetworks/hypernetwork.py         |  7 ++--
 modules/shared.py                             |  1 +
 .../textual_inversion/textual_inversion.py    | 35 ++++++++++++++-----
 modules/ui.py                                 | 11 ++++--
 webui.py                                      |  3 ++
 5 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 32c67ccce..ea3f1db94 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -24,6 +24,7 @@ from statistics import stdev, mean
 
 optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"}
 
+
 class HypernetworkModule(torch.nn.Module):
     multiplier = 1.0
     activation_dict = {
@@ -403,13 +404,15 @@ def create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure=None,
     shared.reload_hypernetworks()
 
 
-def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_hypernetwork_every, template_file, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
+def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_hypernetwork_every, template_filename, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     # images allows training previews to have infotext. Importing it at the top causes a circular import problem.
     from modules import images
 
     save_hypernetwork_every = save_hypernetwork_every or 0
     create_image_every = create_image_every or 0
-    textual_inversion.validate_train_inputs(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, template_file, steps, save_hypernetwork_every, create_image_every, log_directory, name="hypernetwork")
+    template_file = textual_inversion.textual_inversion_templates.get(template_filename, None)
+    textual_inversion.validate_train_inputs(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, template_file, template_filename, steps, save_hypernetwork_every, create_image_every, log_directory, name="hypernetwork")
+    template_file = template_file.path
 
     path = shared.hypernetworks.get(hypernetwork_name, None)
     shared.loaded_hypernetwork = Hypernetwork()
diff --git a/modules/shared.py b/modules/shared.py
index a1e102019..aa37c8ce8 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -33,6 +33,7 @@ parser.add_argument("--no-half-vae", action='store_true', help="do not switch th
 parser.add_argument("--no-progressbar-hiding", action='store_true', help="do not hide progressbar in gradio UI (we hide it because it slows down ML if you have hardware acceleration in browser)")
 parser.add_argument("--max-batch-count", type=int, default=16, help="maximum batch count value for the UI")
 parser.add_argument("--embeddings-dir", type=str, default=os.path.join(script_path, 'embeddings'), help="embeddings directory for textual inversion (default: embeddings)")
+parser.add_argument("--textual-inversion-templates-dir", type=str, default=os.path.join(script_path, 'textual_inversion_templates'), help="directory with textual inversion templates")
 parser.add_argument("--hypernetwork-dir", type=str, default=os.path.join(models_path, 'hypernetworks'), help="hypernetwork directory")
 parser.add_argument("--localizations-dir", type=str, default=os.path.join(script_path, 'localizations'), help="localizations directory")
 parser.add_argument("--allow-code", action='store_true', help="allow custom script execution from webui")
diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py
index 14be2c967..5420903fd 100644
--- a/modules/textual_inversion/textual_inversion.py
+++ b/modules/textual_inversion/textual_inversion.py
@@ -2,6 +2,7 @@ import os
 import sys
 import traceback
 import inspect
+from collections import namedtuple
 
 import torch
 import tqdm
@@ -15,12 +16,26 @@ from modules import shared, devices, sd_hijack, processing, sd_models, images, s
 import modules.textual_inversion.dataset
 from modules.textual_inversion.learn_schedule import LearnRateScheduler
 
-from modules.textual_inversion.image_embedding import (embedding_to_b64, embedding_from_b64,
-                                                       insert_image_data_embed, extract_image_data_embed,
-                                                       caption_image_overlay)
+from modules.textual_inversion.image_embedding import embedding_to_b64, embedding_from_b64, insert_image_data_embed, extract_image_data_embed, caption_image_overlay
 from modules.textual_inversion.logging import save_settings_to_file
 
 
+TextualInversionTemplate = namedtuple("TextualInversionTemplate", ["name", "path"])
+textual_inversion_templates = {}
+
+
+def list_textual_inversion_templates():
+    textual_inversion_templates.clear()
+
+    for root, dirs, fns in os.walk(shared.cmd_opts.textual_inversion_templates_dir):
+        for fn in fns:
+            path = os.path.join(root, fn)
+
+            textual_inversion_templates[fn] = TextualInversionTemplate(fn, path)
+
+    return textual_inversion_templates
+
+
 class Embedding:
     def __init__(self, vec, name, step=None):
         self.vec = vec
@@ -274,7 +289,7 @@ def write_loss(log_directory, filename, step, epoch_len, values):
         })
 
 
-def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, data_root, template_file, steps, save_model_every, create_image_every, log_directory, name="embedding"):
+def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, data_root, template_file, template_filename, steps, save_model_every, create_image_every, log_directory, name="embedding"):
     assert model_name, f"{name} not selected"
     assert learn_rate, "Learning rate is empty or 0"
     assert isinstance(batch_size, int), "Batch size must be integer"
@@ -284,8 +299,9 @@ def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, dat
     assert data_root, "Dataset directory is empty"
     assert os.path.isdir(data_root), "Dataset directory doesn't exist"
     assert os.listdir(data_root), "Dataset directory is empty"
-    assert template_file, "Prompt template file is empty"
-    assert os.path.isfile(template_file), "Prompt template file doesn't exist"
+    assert template_filename, "Prompt template file not selected"
+    assert template_file, f"Prompt template file {template_filename} not found"
+    assert os.path.isfile(template_file.path), f"Prompt template file {template_filename} doesn't exist"
     assert steps, "Max steps is empty or 0"
     assert isinstance(steps, int), "Max steps must be integer"
     assert steps > 0, "Max steps must be positive"
@@ -296,10 +312,13 @@ def validate_train_inputs(model_name, learn_rate, batch_size, gradient_step, dat
     if save_model_every or create_image_every:
         assert log_directory, "Log directory is empty"
 
-def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_embedding_every, template_file, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
+
+def train_embedding(embedding_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_embedding_every, template_filename, save_image_with_stored_embedding, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     save_embedding_every = save_embedding_every or 0
     create_image_every = create_image_every or 0
-    validate_train_inputs(embedding_name, learn_rate, batch_size, gradient_step, data_root, template_file, steps, save_embedding_every, create_image_every, log_directory, name="embedding")
+    template_file = textual_inversion_templates.get(template_filename, None)
+    validate_train_inputs(embedding_name, learn_rate, batch_size, gradient_step, data_root, template_file, template_filename, steps, save_embedding_every, create_image_every, log_directory, name="embedding")
+    template_file = template_file.path
 
     shared.state.job = "train-embedding"
     shared.state.textinfo = "Initializing textual inversion training..."
diff --git a/modules/ui.py b/modules/ui.py
index ddfe1b1a1..b6079aeca 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -37,7 +37,7 @@ from modules import prompt_parser
 from modules.images import save_image
 from modules.sd_hijack import model_hijack
 from modules.sd_samplers import samplers, samplers_for_img2img
-import modules.textual_inversion.ui
+from modules.textual_inversion import textual_inversion
 import modules.hypernetworks.ui
 from modules.generation_parameters_copypaste import image_from_url_text
 
@@ -1322,6 +1322,9 @@ def create_ui():
                         outputs=[process_focal_crop_row],
                     )
 
+                def get_textual_inversion_template_names():
+                    return sorted([x for x in textual_inversion.textual_inversion_templates])
+
                 with gr.Tab(label="Train"):
                     gr.HTML(value="<p style='margin-bottom: 0.7em'>Train an embedding or Hypernetwork; you must specify a directory with a set of 1:1 ratio images <a href=\"https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Textual-Inversion\" style=\"font-weight:bold;\">[wiki]</a></p>")
                     with FormRow():
@@ -1345,7 +1348,11 @@ def create_ui():
 
                     dataset_directory = gr.Textbox(label='Dataset directory', placeholder="Path to directory with input images", elem_id="train_dataset_directory")
                     log_directory = gr.Textbox(label='Log directory', placeholder="Path to directory where to write outputs", value="textual_inversion", elem_id="train_log_directory")
-                    template_file = gr.Textbox(label='Prompt template file', value=os.path.join(script_path, "textual_inversion_templates", "style_filewords.txt"), elem_id="train_template_file")
+
+                    with FormRow():
+                        template_file = gr.Dropdown(label='Prompt template', value="style_filewords.txt", elem_id="train_template_file", choices=get_textual_inversion_template_names())
+                        create_refresh_button(template_file, textual_inversion.list_textual_inversion_templates, lambda: {"choices": get_textual_inversion_template_names()}, "refrsh_train_template_file")
+
                     training_width = gr.Slider(minimum=64, maximum=2048, step=8, label="Width", value=512, elem_id="train_training_width")
                     training_height = gr.Slider(minimum=64, maximum=2048, step=8, label="Height", value=512, elem_id="train_training_height")
                     varsize = gr.Checkbox(label="Do not resize images", value=False, elem_id="train_varsize")
diff --git a/webui.py b/webui.py
index 8737e593f..47d372c79 100644
--- a/webui.py
+++ b/webui.py
@@ -33,6 +33,7 @@ import modules.sd_models
 import modules.sd_vae
 import modules.txt2img
 import modules.script_callbacks
+import modules.textual_inversion.textual_inversion
 
 import modules.ui
 from modules import modelloader
@@ -67,6 +68,8 @@ def initialize():
 
     modules.sd_vae.refresh_vae_list()
 
+    modules.textual_inversion.textual_inversion.list_textual_inversion_templates()
+
     try:
         modules.sd_models.load_model()
     except Exception as e:

From 95727312ca5913876aa1c74f47d1ff6d93bb6b1f Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 9 Jan 2023 16:54:12 -0500
Subject: [PATCH 21/22] remove bytes -> gb conversion

---
 modules/api/api.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/modules/api/api.py b/modules/api/api.py
index d2222b18f..1c121ff02 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -467,26 +467,24 @@ class Api:
             return TrainResponse(info = "train embedding error: {error}".format(error = error))
 
     def get_memory(self):
-        def gb(val: float):
-            return round(val / 1024 / 1024 / 1024, 2)
         try:
             import os, psutil
             process = psutil.Process(os.getpid())
-            res = process.memory_info()
-            ram_total = 100 * res.rss / process.memory_percent()
-            ram = { 'free': gb(ram_total - res.rss), 'used': gb(res.rss), 'total': gb(ram_total) }
+            res = process.memory_info() # only rss is cross-platform guaranteed so we dont rely on other values
+            ram_total = 100 * res.rss / process.memory_percent() # and total memory is calculated as actual value is not cross-platform safe
+            ram = { 'free': ram_total - res.rss, 'used': res.rss, 'total': ram_total }
         except Exception as err:
             ram = { 'error': f'{err}' }
         try:
             import torch
             if torch.cuda.is_available():
                 s = torch.cuda.mem_get_info()
-                system = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) }
+                system = { 'free': s[0], 'used': s[1] - s[0], 'total': s[1] }
                 s = dict(torch.cuda.memory_stats(shared.device))
-                allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) }
-                reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) }
-                active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) }
-                inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) }
+                allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] }
+                reserved = { 'current': s['reserved_bytes.all.current'], 'peak': s['reserved_bytes.all.peak'] }
+                active = { 'current': s['active_bytes.all.current'], 'peak': s['active_bytes.all.peak'] }
+                inactive = { 'current': s['inactive_split_bytes.all.current'], 'peak': s['inactive_split_bytes.all.peak'] }
                 warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] }
                 cuda = {
                     'system': system,

From 3fe9e9e54dcfc41d7c5ee6976f83b0de29fd3dda Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Tue, 10 Jan 2023 02:17:33 +0300
Subject: [PATCH 22/22] fix broken resolution detection when pasting parameters
 with old hires fix enabled

---
 modules/generation_parameters_copypaste.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index f7f68b67e..620aa6069 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -198,10 +198,10 @@ def restore_old_hires_fix_params(res):
     firstpass_height = res.get('First pass size-2', None)
 
     if shared.opts.use_old_hires_fix_width_height:
-        hires_width = int(res.get("Hires resize-1", None))
-        hires_height = int(res.get("Hires resize-2", None))
+        hires_width = int(res.get("Hires resize-1", 0))
+        hires_height = int(res.get("Hires resize-2", 0))
 
-        if hires_width is not None and hires_height is not None:
+        if hires_width and hires_height:
             res['Size-1'] = hires_width
             res['Size-2'] = hires_height
             return