From 550b7056ac877f161b742672653d8b4c33bf4430 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Mon, 18 Sep 2023 15:36:14 +0300
Subject: [PATCH 01/27] IPEX fix SDPA and reduce torch_gc force to %90

---
 modules/devices.py              |  2 +-
 modules/intel/ipex/attention.py | 52 ++++++++++++++++++++++++---------
 2 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/modules/devices.py b/modules/devices.py
index 8a7db4244..2808760b5 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -115,7 +115,7 @@ def torch_gc(force=False):
     if oom > previous_oom:
         previous_oom = oom
         shared.log.warning(f'GPU out-of-memory error: {mem}')
-    if used > 95:
+    if used > 90:
         shared.log.info(f'GPU high memory utilization: {used}% {mem}')
         force = True
     if not force:
diff --git a/modules/intel/ipex/attention.py b/modules/intel/ipex/attention.py
index fc4ab6e26..e38689f21 100644
--- a/modules/intel/ipex/attention.py
+++ b/modules/intel/ipex/attention.py
@@ -64,7 +64,13 @@ def torch_bmm(input, mat2, *, out=None):
 original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention
 def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False):
     #ARC GPUs can't allocate more than 4GB to a single block, Slice it:
-    shape_one, batch_size_attention, query_tokens, shape_four = query.shape
+    if len(query.shape) == 3:
+        batch_size_attention, query_tokens, shape_four = query.shape
+        shape_one = 1
+        no_shape_one = True
+    else:
+        shape_one, batch_size_attention, query_tokens, shape_four = query.shape
+        no_shape_one = False
     block_multiply = 3.6 if query.dtype == torch.float32 else 1.8
     block_size = (shape_one * batch_size_attention * query_tokens * shape_four) / 1024 * block_multiply #MB
     split_slice_size = batch_size_attention
@@ -101,21 +107,39 @@ def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.
                 for i2 in range(query_tokens // split_2_slice_size): # pylint: disable=invalid-name
                     start_idx_2 = i2 * split_2_slice_size
                     end_idx_2 = (i2 + 1) * split_2_slice_size
-                    hidden_states[:, start_idx:end_idx, start_idx_2:end_idx_2] = original_scaled_dot_product_attention(
-                        query[:, start_idx:end_idx, start_idx_2:end_idx_2],
-                        key[:, start_idx:end_idx, start_idx_2:end_idx_2],
-                        value[:, start_idx:end_idx, start_idx_2:end_idx_2],
-                        attn_mask=attn_mask[:, start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask,
+                    if no_shape_one:
+                        hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = original_scaled_dot_product_attention(
+                            query[start_idx:end_idx, start_idx_2:end_idx_2],
+                            key[start_idx:end_idx, start_idx_2:end_idx_2],
+                            value[start_idx:end_idx, start_idx_2:end_idx_2],
+                            attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask,
+                            dropout_p=dropout_p, is_causal=is_causal
+                        )
+                    else:
+                        hidden_states[:, start_idx:end_idx, start_idx_2:end_idx_2] = original_scaled_dot_product_attention(
+                            query[:, start_idx:end_idx, start_idx_2:end_idx_2],
+                            key[:, start_idx:end_idx, start_idx_2:end_idx_2],
+                            value[:, start_idx:end_idx, start_idx_2:end_idx_2],
+                            attn_mask=attn_mask[:, start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask,
+                            dropout_p=dropout_p, is_causal=is_causal
+                        )
+            else:
+                if no_shape_one:
+                    hidden_states[start_idx:end_idx] = original_scaled_dot_product_attention(
+                        query[start_idx:end_idx],
+                        key[start_idx:end_idx],
+                        value[start_idx:end_idx],
+                        attn_mask=attn_mask[start_idx:end_idx] if attn_mask is not None else attn_mask,
+                        dropout_p=dropout_p, is_causal=is_causal
+                    )
+                else:
+                    hidden_states[:, start_idx:end_idx] = original_scaled_dot_product_attention(
+                        query[:, start_idx:end_idx],
+                        key[:, start_idx:end_idx],
+                        value[:, start_idx:end_idx],
+                        attn_mask=attn_mask[:, start_idx:end_idx] if attn_mask is not None else attn_mask,
                         dropout_p=dropout_p, is_causal=is_causal
                     )
-            else:
-                hidden_states[:, start_idx:end_idx] = original_scaled_dot_product_attention(
-                    query[:, start_idx:end_idx],
-                    key[:, start_idx:end_idx],
-                    value[:, start_idx:end_idx],
-                    attn_mask=attn_mask[:, start_idx:end_idx] if attn_mask is not None else attn_mask,
-                    dropout_p=dropout_p, is_causal=is_causal
-                )
     else:
         return original_scaled_dot_product_attention(
             query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal

From f7901c8d53b42354074ae0bdc2a8d27cfe412bdf Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 08:38:02 -0400
Subject: [PATCH 02/27] add changelog to ui

---
 javascript/style.css | 7 +++++++
 modules/ui.py        | 5 +++++
 2 files changed, 12 insertions(+)

diff --git a/javascript/style.css b/javascript/style.css
index 52554cca3..2070ca44b 100644
--- a/javascript/style.css
+++ b/javascript/style.css
@@ -281,6 +281,12 @@ div.controlnet_main_options { display: grid; grid-template-columns: 1fr 1fr; gri
 .log-monitor { display: none; justify-content: unset !important; overflow: hidden; padding: 0; margin-top: auto; font-family: monospace; font-size: 0.85em; }
 .log-monitor td, .log-monitor th { padding-left: 1em; }
 
+/* changelog */
+.md h2 { background-color: var(--background-fill-primary); padding: 0.5em; }
+.md ul { list-style-type: square !important; text-indent: 1em; margin-left: 4em; }
+.md li { list-style-position: outside !important; text-indent: 0; }
+.md p { margin-left: 2em; }
+
 /* custom component */
 .folder-selector textarea { height: 2em !important; padding: 6px !important; }
 
@@ -295,6 +301,7 @@ div.controlnet_main_options { display: grid; grid-template-columns: 1fr 1fr; gri
 .loader::before { border-top-color: var(--primary-900); animation: 3s spin linear infinite; }
 .loader::after { border-top-color: var(--primary-300); animation: spin 1.5s linear infinite; }
 
+
 @keyframes move {
   from { background-position-x: 0, -40px; }
   to { background-position-x: 0, 40px; }
diff --git a/modules/ui.py b/modules/ui.py
index d6f7bb62c..8da53b22a 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -1076,6 +1076,11 @@ def create_ui(startup_timer = None):
                 loadsave.create_ui()
                 create_dirty_indicator("tab_defaults", [], interactive=False)
 
+            with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"):
+                with open('CHANGELOG.md', 'r', encoding='utf-8') as f:
+                    md = f.read()
+                gr.Markdown(md)
+
             with gr.TabItem("Licenses", id="system_licenses", elem_id="system_tab_licenses"):
                 gr.HTML(modules.shared.html("licenses.html"), elem_id="licenses", elem_classes="licenses")
                 create_dirty_indicator("tab_licenses", [], interactive=False)

From eee6d3104e227703fee4c5853c1b72d93422ab29 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 09:25:25 -0400
Subject: [PATCH 03/27] rename en

---
 modules/ui_extra_networks.py                   | 2 +-
 modules/ui_extra_networks_checkpoints.py       | 2 +-
 modules/ui_extra_networks_hypernets.py         | 2 +-
 modules/ui_extra_networks_styles.py            | 2 +-
 modules/ui_extra_networks_textual_inversion.py | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py
index 62662b9fb..caf0d39fd 100644
--- a/modules/ui_extra_networks.py
+++ b/modules/ui_extra_networks.py
@@ -128,7 +128,7 @@ class ExtraNetworksPage:
                     shared.log.error(f'Cannot evaluate extra network prompt: {item["prompt"]} {e}')
 
         if not any(self.title in x.label for x in xyz_grid.axis_options):
-            if self.title == 'Checkpoints':
+            if self.title == 'Model':
                 return
             opt = xyz_grid.AxisOption(f"[Network] {self.title}", str, add_prompt, choices=lambda: [x["name"] for x in self.items])
             xyz_grid.axis_options.append(opt)
diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py
index 4a4a18e1d..39ce706be 100644
--- a/modules/ui_extra_networks_checkpoints.py
+++ b/modules/ui_extra_networks_checkpoints.py
@@ -7,7 +7,7 @@ from modules import shared, ui_extra_networks, sd_models
 
 class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
     def __init__(self):
-        super().__init__('Checkpoints')
+        super().__init__('Model')
 
     def refresh(self):
         shared.refresh_checkpoints()
diff --git a/modules/ui_extra_networks_hypernets.py b/modules/ui_extra_networks_hypernets.py
index 66ddca527..a61f5ffb6 100644
--- a/modules/ui_extra_networks_hypernets.py
+++ b/modules/ui_extra_networks_hypernets.py
@@ -5,7 +5,7 @@ from modules import shared, ui_extra_networks
 
 class ExtraNetworksPageHypernetworks(ui_extra_networks.ExtraNetworksPage):
     def __init__(self):
-        super().__init__('Hypernetworks')
+        super().__init__('Hypernetwork')
 
     def refresh(self):
         shared.reload_hypernetworks()
diff --git a/modules/ui_extra_networks_styles.py b/modules/ui_extra_networks_styles.py
index c3e402d70..db772efd1 100644
--- a/modules/ui_extra_networks_styles.py
+++ b/modules/ui_extra_networks_styles.py
@@ -6,7 +6,7 @@ from modules import shared, ui_extra_networks
 
 class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage):
     def __init__(self):
-        super().__init__('Styles')
+        super().__init__('Style')
 
     def refresh(self):
         shared.prompt_styles.reload()
diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py
index 438ce86f1..78e900858 100644
--- a/modules/ui_extra_networks_textual_inversion.py
+++ b/modules/ui_extra_networks_textual_inversion.py
@@ -7,7 +7,7 @@ from modules.textual_inversion.textual_inversion import Embedding
 
 class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage):
     def __init__(self):
-        super().__init__('Textual Inversion')
+        super().__init__('Embedding')
         self.allow_negative_prompt = True
 
     def refresh(self):

From 0f3dc232c13800b05c31550c53d56c348bb05e18 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 10:27:31 -0400
Subject: [PATCH 04/27] add en show/hide/reorder

---
 CHANGELOG.md                 |  5 ++-
 modules/api/api.py           |  3 +-
 modules/shared.py            | 29 ++++++++++-------
 modules/ui_extra_networks.py | 62 +++++++++++++++++++++---------------
 4 files changed, 59 insertions(+), 40 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dd92b2c7e..2917625ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,13 @@
 # Change Log for SD.Next
 
-## Update for 2023-09-15
+## Update for 2023-09-18
 
 Downgrade of `diffusers` to 0.20.2 due to critical issue with model offloading  
 This means that new model **Wuerstchen** is not supported until diffusers issue is resolved  
 
+- Added **change log** to UI, see *System -> Changelog*
+- **Extra networks**: faster search, ability to show/hide/sort networks
+
 ## Update for 2023-09-13
 
 Started as a mostly a service release with quite a few fixes, but then...  
diff --git a/modules/api/api.py b/modules/api/api.py
index 36a22e423..e750fccb4 100644
--- a/modules/api/api.py
+++ b/modules/api/api.py
@@ -504,9 +504,8 @@ class Api:
         }
 
     def get_extra_networks(self, page: Optional[str] = None, name: Optional[str] = None, filename: Optional[str] = None, title: Optional[str] = None, fullname: Optional[str] = None, hash: Optional[str] = None): # pylint: disable=redefined-builtin
-        import modules.ui_extra_networks
         res = []
-        for pg in modules.ui_extra_networks.extra_pages:
+        for pg in shared.extra_networks:
             if page is not None and pg.name != page.lower():
                 continue
             for item in pg.items:
diff --git a/modules/shared.py b/modules/shared.py
index 713074f45..97f15e5e5 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -11,7 +11,7 @@ from enum import Enum
 import gradio as gr
 import fasteners
 from rich.console import Console
-from modules import errors, ui_components, shared_items, cmd_args
+from modules import errors, shared_items, cmd_args, ui_components
 from modules.paths_internal import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611
 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack
 import modules.interrogate
@@ -37,6 +37,7 @@ interrogator = modules.interrogate.InterrogateModels("interrogate")
 sd_upscalers = []
 face_restorers = []
 tab_names = []
+extra_networks = []
 options_templates = {}
 hypernetworks = {}
 loaded_hypernetworks = []
@@ -71,21 +72,12 @@ restricted_opts = {
 compatibility_opts = ['clip_skip', 'uni_pc_lower_order_final', 'uni_pc_order']
 console = Console(log_time=True, log_time_format='%H:%M:%S-%f')
 
-def is_url(string):
-    parsed_url = urlparse(string)
-    return all([parsed_url.scheme, parsed_url.netloc])
-
 
 class Backend(Enum):
     ORIGINAL = 1
     DIFFUSERS = 2
 
 
-def reload_hypernetworks():
-    from modules.hypernetworks import hypernetwork
-    global hypernetworks # pylint: disable=W0603
-    hypernetworks = hypernetwork.list_hypernetworks(opts.hypernetwork_dir)
-
 
 class State:
     skipped = False
@@ -192,6 +184,7 @@ class State:
         self.current_image = image
         self.id_live_preview += 1
 
+
 state = State()
 state.server_start = time.time()
 if not hasattr(cmd_opts, "use_openvino"):
@@ -251,6 +244,17 @@ def list_checkpoint_tiles():
 default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.ckpt"
 
 
+def is_url(string):
+    parsed_url = urlparse(string)
+    return all([parsed_url.scheme, parsed_url.netloc])
+
+
+def reload_hypernetworks():
+    from modules.hypernetworks import hypernetwork
+    global hypernetworks # pylint: disable=W0603
+    hypernetworks = hypernetwork.list_hypernetworks(opts.hypernetwork_dir)
+
+
 def refresh_checkpoints():
     import modules.sd_models # pylint: disable=W0621
     return modules.sd_models.list_models()
@@ -644,6 +648,7 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
 }))
 
 options_templates.update(options_section(('extra_networks', "Extra Networks"), {
+    "extra_networks": OptionInfo(["All"], "Extra networks", ui_components.DropdownMulti, lambda: {"choices": ['All'] + [en.title for en in extra_networks]}),
     "extra_networks_card_cover": OptionInfo("sidebar", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}),
     "extra_networks_height": OptionInfo(53, "UI height (%)", gr.Slider, {"minimum": 10, "maximum": 100, "step": 1}),
     "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}),
@@ -653,8 +658,8 @@ options_templates.update(options_section(('extra_networks', "Extra Networks"), {
     "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"]}),
     "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox),
     "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all LoRA types", gr.Checkbox),
-    "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox),
-    "extra_networks_default_multiplier": OptionInfo(1.0, "Multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
+    "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }),
+    "extra_networks_default_multiplier": OptionInfo(1.0, "Multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01, "visible": False}),
     "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, lambda: { "choices": ["None"] + list(hypernetworks.keys()), "visible": False }, refresh=reload_hypernetworks),
 }))
 
diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py
index caf0d39fd..bf70cf4d7 100644
--- a/modules/ui_extra_networks.py
+++ b/modules/ui_extra_networks.py
@@ -16,10 +16,10 @@ from modules.ui_components import ToolButton
 import modules.ui_symbols as symbols
 
 
-extra_pages = []
 allowed_dirs = []
 dir_cache = {} # key=path, value=(mtime, listdir(path))
 refresh_time = None
+extra_pages = shared.extra_networks
 
 
 def listdir(path):
@@ -37,9 +37,9 @@ def listdir(path):
 
 def register_page(page):
     # registers extra networks page for the UI; recommend doing it in on_before_ui() callback for extensions
-    extra_pages.append(page)
+    shared.extra_networks.append(page)
     allowed_dirs.clear()
-    for page in extra_pages:
+    for page in shared.extra_networks:
         for folder in page.allowed_directories_for_previews():
             if folder not in allowed_dirs:
                 allowed_dirs.append(os.path.abspath(folder))
@@ -58,7 +58,7 @@ def fetch_file(filename: str = ""):
 
 
 def get_metadata(page: str = "", item: str = ""):
-    page = next(iter([x for x in extra_pages if x.name == page]), None)
+    page = next(iter([x for x in shared.extra_networks if x.name == page]), None)
     if page is None:
         return JSONResponse({ 'metadata': 'none' })
     metadata = page.metadata.get(item, 'none')
@@ -69,7 +69,7 @@ def get_metadata(page: str = "", item: str = ""):
 
 
 def get_info(page: str = "", item: str = ""):
-    page = next(iter([x for x in extra_pages if x.name == page]), None)
+    page = next(iter([x for x in shared.extra_networks if x.name == page]), None)
     if page is None:
         return JSONResponse({ 'info': 'none' })
     info = page.info.get(item, 'none')
@@ -308,7 +308,7 @@ class ExtraNetworksPage:
             shared.log.error(f'Extra network save preview: {filename} {e}')
             return
         is_allowed = False
-        for page in extra_pages:
+        for page in shared.extra_networks:
             if any(path_is_parent(x, filename) for x in page.allowed_directories_for_previews()):
                 is_allowed = True
                 break
@@ -339,7 +339,7 @@ class ExtraNetworksPage:
 
 
 def initialize():
-    extra_pages.clear()
+    shared.extra_networks.clear()
 
 
 def register_pages():
@@ -353,6 +353,21 @@ def register_pages():
     register_page(ExtraNetworksPageHypernetworks())
 
 
+def get_pages():
+    pages = []
+    if 'All' in shared.opts.extra_networks:
+        pages = shared.extra_networks
+    else:
+        titles = [page.title for page in shared.extra_networks]
+        for page in shared.opts.extra_networks:
+            try:
+                idx = titles.index(page)
+            except ValueError:
+                continue
+            pages.append(shared.extra_networks[idx])
+    return pages
+
+
 class ExtraNetworksUi:
     def __init__(self):
         self.pages = None
@@ -384,7 +399,7 @@ def create_ui(container, button, tabname, skip_indexing = False):
         if ui.tabname == 'txt2img': # refresh only once
             global refresh_time # pylint: disable=global-statement
             refresh_time = time.time()
-        for page in extra_pages:
+        for page in get_pages():
             page.create_page(ui.tabname, skip_indexing)
             with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab"):
                 hmtl = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page")
@@ -396,16 +411,16 @@ def create_ui(container, button, tabname, skip_indexing = False):
         return is_visible, gr.update(visible=is_visible), gr.update(variant=("secondary-down" if is_visible else "secondary"))
 
     def en_refresh(title):
-        res = []
-        for page in extra_pages:
+        pages = []
+        for page in get_pages():
             if title is None or title == '' or title == page.title or len(page.html) == 0:
                 page.refresh()
                 page.refresh_time = None
                 page.create_page(ui.tabname)
                 shared.log.debug(f"Refreshing Extra networks: page='{page.title}' items={len(page.items)} tab={ui.tabname}")
-            res.append(page.html)
+            pages.append(page.html)
         ui.search.update(value = ui.search.value)
-        return res
+        return pages
 
     state_visible = gr.State(value=False) # pylint: disable=abstract-class-instantiated
     button.click(fn=toggle_visibility, inputs=[state_visible], outputs=[state_visible, container, button])
@@ -423,15 +438,13 @@ def path_is_parent(parent_path, child_path):
 def setup_ui(ui, gallery):
 
     def save_preview(pagename, index, images, filename):
-        res = []
-        for page in extra_pages:
+        pages = []
+        for page in get_pages():
             if pagename is None or pagename == '' or pagename == page.title or len(page.html) == 0:
                 page.save_preview(index, images, filename)
-                res.append(page.create_page(ui.tabname))
-            else:
-                res.append(page.html)
-        return res
-
+                page.create_page(ui.tabname)
+            pages.append(page.html)
+        return pages
 
     ui.button_save_preview.click(
         fn=save_preview,
@@ -441,14 +454,13 @@ def setup_ui(ui, gallery):
     )
 
     def save_description(pagename, filename, desc):
-        res = []
-        for page in extra_pages:
+        pages = []
+        for page in get_pages():
             if pagename is None or pagename == '' or pagename == page.title or len(page.html) == 0:
                 page.save_description(filename, desc)
-                res.append(page.create_page(ui.tabname))
-            else:
-                res.append(page.html)
-        return res
+                page.create_page(ui.tabname)
+            pages.append(page.html)
+        return pages
 
     ui.button_save_description.click(
         fn=save_description,

From c45ac96363e8b922dd758e5d43b3dbe47fac7345 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 11:01:16 -0400
Subject: [PATCH 05/27] remove controlnet custom styling

---
 javascript/style.css | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/javascript/style.css b/javascript/style.css
index 2070ca44b..29c7b7cb2 100644
--- a/javascript/style.css
+++ b/javascript/style.css
@@ -252,7 +252,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 #txt2img_description, #img2img_description { max-height: 63px; overflow-y: auto !important; }
 #txt2img_description > label > textarea, #img2img_description > label > textarea { font-size: 0.9em }
 
-/* controlnet */
+/* controlnet
 .controlnet_control_type .controlnet_control_type_filter_group .wrap:last-of-type { display: grid; grid-auto-flow: row; grid-template-columns: repeat(4, minmax(0, 1fr)); }
 fieldset.controlnet_resize_mode_radio .wrap:last-of-type, fieldset.controlnet_control_mode_radio .wrap:last-of-type { flex-direction: column; }
 div.controlnet_preprocessor_model { display: grid; grid-auto-flow: row; grid-template-columns: 1fr max-content; }
@@ -263,6 +263,7 @@ div.controlnet_image_controls { display: grid; grid-template-columns: repeat(4,
 div.controlnet_image_controls .controlnet_invert_warning { grid-column: 1 / -1; }
 div.controlnet_image_controls button { justify-self: center; }
 div.controlnet_main_options { display: grid; grid-template-columns: 1fr 1fr; grid-auto-flow: row; }
+ */
 
 /* specific elements */
 #modelmerger_interp_description { margin-top: 1em; margin-bottom: 1em; }

From b35a84f505545aee221d49ba74c92379b239b5ed Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 14:48:41 -0400
Subject: [PATCH 06/27] add upscaler logging

---
 .../SwinIR/scripts/swinir_model.py            | 12 +++++---
 modules/esrgan_model.py                       | 30 +++++++++++--------
 modules/gfpgan_model.py                       |  3 +-
 modules/images.py                             |  2 +-
 modules/realesrgan_model.py                   | 21 ++++++-------
 modules/ui_common.py                          |  2 +-
 6 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/extensions-builtin/SwinIR/scripts/swinir_model.py b/extensions-builtin/SwinIR/scripts/swinir_model.py
index e470db874..7df4a50b8 100644
--- a/extensions-builtin/SwinIR/scripts/swinir_model.py
+++ b/extensions-builtin/SwinIR/scripts/swinir_model.py
@@ -2,8 +2,8 @@ import os
 import numpy as np
 import torch
 from PIL import Image
+from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn, TimeElapsedColumn
 from basicsr.utils.download_util import load_file_from_url
-from tqdm.rich import tqdm
 from swinir_model_arch import SwinIR as net
 from swinir_model_arch_v2 import Swin2SR as net2
 from modules import modelloader, devices, script_callbacks, shared
@@ -45,11 +45,12 @@ class UpscalerSwinIR(Upscaler):
 
     def load_model(self, path, scale=4):
         if "http" in path:
-            dl_name = "%s%s" % (self.model_name.replace(" ", "_"), ".pth")
+            dl_name = "%s%s" % (self.model_name.replace(" ", "_"), ".pth") # pylint: disable=consider-using-f-string
             filename = load_file_from_url(url=path, model_dir=self.model_download_path, file_name=dl_name, progress=True)
         else:
             filename = path
         if filename is None or not os.path.exists(filename):
+            shared.log.error(f"Model failed loading: type=SwinIR model={filename}")
             return None
         model_v2 = net2(
             upscale=scale,
@@ -78,6 +79,8 @@ class UpscalerSwinIR(Upscaler):
             resi_connection="3conv",
         )
         pretrained_model = torch.load(filename)
+        shared.log.info(f"Model loaded: type=SwinIR model={filename}")
+
         for model in [model_v1, model_v2]:
             for param in ["params_ema", "params", None]:
                 try:
@@ -140,7 +143,8 @@ def inference(img, model, tile, tile_overlap, window_size, scale):
     E = torch.zeros(b, c, h * sf, w * sf, dtype=devices.dtype, device=device_swinir).type_as(img)
     W = torch.zeros_like(E, dtype=devices.dtype, device=device_swinir)
 
-    with tqdm(total=len(h_idx_list) * len(w_idx_list), desc="Upscaling SwinIR") as pbar:
+    with Progress(TextColumn('[cyan]{task.description}'), BarColumn(), TaskProgressColumn(), TimeRemainingColumn(), TimeElapsedColumn(), console=shared.console) as progress:
+        task = progress.add_task(description="Upscaling Initializing", total=len(h_idx_list) * len(w_idx_list))
         for h_idx in h_idx_list:
             if state.interrupted or state.skipped:
                 break
@@ -159,7 +163,7 @@ def inference(img, model, tile, tile_overlap, window_size, scale):
                 W[
                 ..., h_idx * sf: (h_idx + tile) * sf, w_idx * sf: (w_idx + tile) * sf
                 ].add_(out_patch_mask)
-                pbar.update(1)
+                progress.update(task, advance=1, description="Upscaling")
     output = E.div_(W)
 
     return output
diff --git a/modules/esrgan_model.py b/modules/esrgan_model.py
index 7ed5c33ec..3a4bf599b 100644
--- a/modules/esrgan_model.py
+++ b/modules/esrgan_model.py
@@ -4,11 +4,12 @@ import numpy as np
 import torch
 from PIL import Image
 from basicsr.utils.download_util import load_file_from_url
+from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn, TimeElapsedColumn
 
 import modules.esrgan_model_arch as arch
 from modules import modelloader, images, devices
 from modules.upscaler import Upscaler, UpscalerData
-from modules.shared import opts
+from modules.shared import opts, log, console
 
 
 
@@ -161,10 +162,11 @@ class UpscalerESRGAN(Upscaler):
         else:
             filename = path
         if not os.path.exists(filename) or filename is None:
-            print(f"Unable to load {self.model_path} from {filename}")
+            log.error(f"Model failed loading: type=ESRGAN model={filename}")
             return None
 
         state_dict = torch.load(filename, map_location='cpu' if devices.device_esrgan.type == 'mps' else None)
+        log.info(f"Model loaded: type=ESRGAN model={filename}")
 
         if "params_ema" in state_dict:
             state_dict = state_dict["params_ema"]
@@ -216,16 +218,20 @@ def esrgan_upscale(model, img):
     newtiles = []
     scale_factor = 1
 
-    for y, h, row in grid.tiles:
-        newrow = []
-        for tiledata in row:
-            x, w, tile = tiledata
-
-            output = upscale_without_tiling(model, tile)
-            scale_factor = output.width // tile.width
-
-            newrow.append([x * scale_factor, w * scale_factor, output])
-        newtiles.append([y * scale_factor, h * scale_factor, newrow])
+    with Progress(TextColumn('[cyan]{task.description}'), BarColumn(), TaskProgressColumn(), TimeRemainingColumn(), TimeElapsedColumn(), console=console) as progress:
+        total = 0
+        for y, h, row in grid.tiles:
+            total += len(row)
+        task = progress.add_task(description="Upscaling", total=total)
+        for y, h, row in grid.tiles:
+            newrow = []
+            for tiledata in row:
+                x, w, tile = tiledata
+                output = upscale_without_tiling(model, tile)
+                scale_factor = output.width // tile.width
+                newrow.append([x * scale_factor, w * scale_factor, output])
+                progress.update(task, advance=1, description="Upscaling")
+            newtiles.append([y * scale_factor, h * scale_factor, newrow])
 
     newgrid = images.Grid(newtiles, grid.tile_w * scale_factor, grid.tile_h * scale_factor, grid.image_w * scale_factor, grid.image_h * scale_factor, grid.overlap * scale_factor)
     output = images.combine_grid(newgrid)
diff --git a/modules/gfpgan_model.py b/modules/gfpgan_model.py
index 728df70bf..2988cd407 100644
--- a/modules/gfpgan_model.py
+++ b/modules/gfpgan_model.py
@@ -29,12 +29,13 @@ def gfpgann():
         latest_file = max(models, key=os.path.getctime)
         model_file = latest_file
     else:
-        print("Unable to load gfpgan model!")
+        shared.log.error(f"Model failed loading: type=GFPGAN model={model_file}")
         return None
     if hasattr(facexlib.detection.retinaface, 'device'):
         facexlib.detection.retinaface.device = devices.device_gfpgan
     model = gfpgan_constructor(model_path=model_file, upscale=1, arch='clean', channel_multiplier=2, bg_upsampler=None, device=devices.device_gfpgan)
     loaded_gfpgan_model = model
+    shared.log.info(f"Model loaded: type=GFPGAN model={model_file}")
 
     return model
 
diff --git a/modules/images.py b/modules/images.py
index 39d9e7246..3a2e8ded0 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -320,7 +320,7 @@ class FilenameGenerator:
     }
     default_time_format = '%Y%m%d%H%M%S'
 
-    def __init__(self, p, seed, prompt, image, index):
+    def __init__(self, p, seed, prompt, image, index = 0):
         self.p = p
         self.seed = seed
         self.prompt = prompt
diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py
index 5f5b132e5..318fe21fc 100644
--- a/modules/realesrgan_model.py
+++ b/modules/realesrgan_model.py
@@ -1,12 +1,10 @@
 import os
-import sys
 import numpy as np
 from PIL import Image
 from basicsr.utils.download_util import load_file_from_url
 from modules.upscaler import Upscaler, UpscalerData
-from modules.shared import opts, device
+from modules.shared import opts, device, log
 from modules import modelloader
-import modules.errors as errors
 
 
 class UpscalerRealESRGAN(Upscaler):
@@ -30,9 +28,8 @@ class UpscalerRealESRGAN(Upscaler):
                         scaler.local_data_path = local_model_candidates[0]
                 if scaler.name in opts.realesrgan_enabled_models:
                     self.scalers.append(scaler)
-
         except Exception as e:
-            errors.display(e, 'real-esrgan')
+            log.error(f"Error loading Real-ESRGAN: model={path} {e}")
             self.enable = False
             self.scalers = []
 
@@ -43,12 +40,11 @@ class UpscalerRealESRGAN(Upscaler):
         try:
             from realesrgan import RealESRGANer
         except Exception:
-            print("Error importing Real-ESRGAN:", file=sys.stderr)
+            log.error("Error importing Real-ESRGAN:")
             return img
 
         info = self.load_model(selected_model)
-        if not os.path.exists(info.local_data_path):
-            print(f"Unable to load RealESRGAN model: {info.name}")
+        if info is None or not os.path.exists(info.local_data_path):
             return img
 
         upsampler = RealESRGANer(
@@ -70,13 +66,14 @@ class UpscalerRealESRGAN(Upscaler):
         try:
             info = next(iter([scaler for scaler in self.scalers if scaler.data_path == path]), None)
             if info is None:
-                print(f"Unable to find model info: {path}")
+                log.error(f"Model failed loading: type=R-ESRGAN model={info.name}")
                 return None
             if info.local_data_path.startswith("http"):
                 info.local_data_path = load_file_from_url(url=info.data_path, model_dir=self.model_download_path, progress=True)
+            log.info(f"Model loaded: type=R-ESRGAN model={info.name}")
             return info
         except Exception as e:
-            errors.display(e, 'real-esrgan model list')
+            log.error(f"Model failed loading: type=R-ESRGAN model={info.name} {e}")
         return None
 
     def load_models(self, _):
@@ -132,6 +129,6 @@ def get_realesrgan_models(scaler):
             ),
         ]
         return models
-    except Exception:
-        print("Error creating Real-ESRGAN models list", file=sys.stderr)
+    except Exception as e:
+        log.error(f'Error creating Real-ESRGAN models list: {e}')
         return []
diff --git a/modules/ui_common.py b/modules/ui_common.py
index b4f31734c..cac1363a9 100644
--- a/modules/ui_common.py
+++ b/modules/ui_common.py
@@ -110,7 +110,7 @@ def save_files(js_data, images, html_info, index):
             fullfns.append(fullfn)
             destination = shared.opts.outdir_save
             if shared.opts.use_save_to_dirs_for_ui:
-                namegen = modules.images.FilenameGenerator(p, seed=p.all_seeds[i], prompt=p.all_prompts[i], image=None)  # pylint: disable=no-member
+                namegen = modules.images.FilenameGenerator(p, seed=p.all_seeds[i], prompt=p.all_prompts[i], image=None, index=image_index)  # pylint: disable=no-member
                 dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]").lstrip(' ').rstrip('\\ /')
                 destination = os.path.join(destination, dirname)
                 os.makedirs(destination, exist_ok = True)

From ef919761653eef4f71f08e065d00dff7da1588a9 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 15:21:11 -0400
Subject: [PATCH 07/27] switch to native r-esrgan and modelloader

---
 extensions-builtin/LDSR/scripts/ldsr_model.py |   3 +-
 .../ScuNET/scripts/scunet_model.py            |   3 +-
 .../SwinIR/scripts/swinir_model.py            |   2 +-
 modules/dml/hijack/realesrgan_model.py        |   2 +-
 modules/esrgan_model.py                       |   2 +-
 modules/realesrgan_model.py                   |  12 +-
 modules/realesrgan_model_arch.py              | 384 ++++++++++++++++++
 modules/shared.py                             |   2 +-
 requirements.txt                              |   1 -
 9 files changed, 395 insertions(+), 16 deletions(-)
 create mode 100644 modules/realesrgan_model_arch.py

diff --git a/extensions-builtin/LDSR/scripts/ldsr_model.py b/extensions-builtin/LDSR/scripts/ldsr_model.py
index 9c40740a5..e2c671b15 100644
--- a/extensions-builtin/LDSR/scripts/ldsr_model.py
+++ b/extensions-builtin/LDSR/scripts/ldsr_model.py
@@ -2,8 +2,6 @@ import os
 import sys
 import traceback
 
-from basicsr.utils.download_util import load_file_from_url
-
 from modules.upscaler import Upscaler, UpscalerData
 from ldsr_model_arch import LDSR
 from modules import shared, script_callbacks
@@ -42,6 +40,7 @@ class UpscalerLDSR(Upscaler):
             print("Renaming model from model.pth to model.ckpt")
             os.rename(old_model_path, new_model_path)
 
+        from modules.modelloader import load_file_from_url
         if local_safetensors_path is not None and os.path.exists(local_safetensors_path):
             model = local_safetensors_path
         else:
diff --git a/extensions-builtin/ScuNET/scripts/scunet_model.py b/extensions-builtin/ScuNET/scripts/scunet_model.py
index c1aa25230..120ec0ffc 100644
--- a/extensions-builtin/ScuNET/scripts/scunet_model.py
+++ b/extensions-builtin/ScuNET/scripts/scunet_model.py
@@ -7,8 +7,6 @@ import numpy as np
 import torch
 from tqdm import tqdm
 
-from basicsr.utils.download_util import load_file_from_url
-
 import modules.upscaler
 from modules import devices, modelloader, script_callbacks
 from scunet_model_arch import SCUNet as net
@@ -121,6 +119,7 @@ class UpscalerScuNET(modules.upscaler.Upscaler):
     def load_model(self, path: str):
         device = devices.get_device_for('scunet')
         if "http" in path:
+            from modules.modelloader import load_file_from_url
             filename = load_file_from_url(url=self.model_url, model_dir=self.model_download_path, file_name="%s.pth" % self.name, progress=True)
         else:
             filename = path
diff --git a/extensions-builtin/SwinIR/scripts/swinir_model.py b/extensions-builtin/SwinIR/scripts/swinir_model.py
index 7df4a50b8..4527b4c79 100644
--- a/extensions-builtin/SwinIR/scripts/swinir_model.py
+++ b/extensions-builtin/SwinIR/scripts/swinir_model.py
@@ -3,7 +3,6 @@ import numpy as np
 import torch
 from PIL import Image
 from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn, TimeElapsedColumn
-from basicsr.utils.download_util import load_file_from_url
 from swinir_model_arch import SwinIR as net
 from swinir_model_arch_v2 import Swin2SR as net2
 from modules import modelloader, devices, script_callbacks, shared
@@ -45,6 +44,7 @@ class UpscalerSwinIR(Upscaler):
 
     def load_model(self, path, scale=4):
         if "http" in path:
+            from modules.modelloader import load_file_from_url
             dl_name = "%s%s" % (self.model_name.replace(" ", "_"), ".pth") # pylint: disable=consider-using-f-string
             filename = load_file_from_url(url=path, model_dir=self.model_download_path, file_name=dl_name, progress=True)
         else:
diff --git a/modules/dml/hijack/realesrgan_model.py b/modules/dml/hijack/realesrgan_model.py
index ad3b01cce..b55e14647 100644
--- a/modules/dml/hijack/realesrgan_model.py
+++ b/modules/dml/hijack/realesrgan_model.py
@@ -1,6 +1,6 @@
 import math
 import torch
-from realesrgan import RealESRGANer
+from modules.realesrgan_model_arch import RealESRGANer
 
 
 # DML Solution: Some of contents of output tensor turn to 0 after Extended Slices. Move it to cpu.
diff --git a/modules/esrgan_model.py b/modules/esrgan_model.py
index 3a4bf599b..cbede232f 100644
--- a/modules/esrgan_model.py
+++ b/modules/esrgan_model.py
@@ -3,7 +3,6 @@ import os
 import numpy as np
 import torch
 from PIL import Image
-from basicsr.utils.download_util import load_file_from_url
 from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn, TimeElapsedColumn
 
 import modules.esrgan_model_arch as arch
@@ -153,6 +152,7 @@ class UpscalerESRGAN(Upscaler):
 
     def load_model(self, path: str):
         if "http" in path:
+            from modules.modelloader import load_file_from_url
             filename = load_file_from_url(
                 url=self.model_url,
                 model_dir=self.model_download_path,
diff --git a/modules/realesrgan_model.py b/modules/realesrgan_model.py
index 318fe21fc..5a88f2330 100644
--- a/modules/realesrgan_model.py
+++ b/modules/realesrgan_model.py
@@ -1,7 +1,6 @@
 import os
 import numpy as np
 from PIL import Image
-from basicsr.utils.download_util import load_file_from_url
 from modules.upscaler import Upscaler, UpscalerData
 from modules.shared import opts, device, log
 from modules import modelloader
@@ -14,8 +13,7 @@ class UpscalerRealESRGAN(Upscaler):
         super().__init__()
         try:
             from basicsr.archs.rrdbnet_arch import RRDBNet # pylint: disable=unused-import
-            from realesrgan import RealESRGANer # pylint: disable=unused-import
-            from realesrgan.archs.srvgg_arch import SRVGGNetCompact # pylint: disable=unused-import
+            from modules.realesrgan_model_arch import RealESRGANer, SRVGGNetCompact # pylint: disable=unused-import
             self.enable = True
             self.scalers = []
             scalers = self.load_models(path)
@@ -26,8 +24,7 @@ class UpscalerRealESRGAN(Upscaler):
                     local_model_candidates = [local_model for local_model in local_model_paths if local_model.endswith(f"{filename}.pth")]
                     if local_model_candidates:
                         scaler.local_data_path = local_model_candidates[0]
-                if scaler.name in opts.realesrgan_enabled_models:
-                    self.scalers.append(scaler)
+                self.scalers.append(scaler)
         except Exception as e:
             log.error(f"Error loading Real-ESRGAN: model={path} {e}")
             self.enable = False
@@ -38,7 +35,7 @@ class UpscalerRealESRGAN(Upscaler):
             return img
 
         try:
-            from realesrgan import RealESRGANer
+            from modules.realesrgan_model_arch import RealESRGANer
         except Exception:
             log.error("Error importing Real-ESRGAN:")
             return img
@@ -69,6 +66,7 @@ class UpscalerRealESRGAN(Upscaler):
                 log.error(f"Model failed loading: type=R-ESRGAN model={info.name}")
                 return None
             if info.local_data_path.startswith("http"):
+                from modules.modelloader import load_file_from_url
                 info.local_data_path = load_file_from_url(url=info.data_path, model_dir=self.model_download_path, progress=True)
             log.info(f"Model loaded: type=R-ESRGAN model={info.name}")
             return info
@@ -83,7 +81,7 @@ class UpscalerRealESRGAN(Upscaler):
 def get_realesrgan_models(scaler):
     try:
         from basicsr.archs.rrdbnet_arch import RRDBNet
-        from realesrgan.archs.srvgg_arch import SRVGGNetCompact
+        from modules.realesrgan_model_arch import SRVGGNetCompact # pylint: disable=unused-import
         models = [
             UpscalerData(
                 name="R-ESRGAN General 4xV3",
diff --git a/modules/realesrgan_model_arch.py b/modules/realesrgan_model_arch.py
new file mode 100644
index 000000000..8b991de6d
--- /dev/null
+++ b/modules/realesrgan_model_arch.py
@@ -0,0 +1,384 @@
+import os
+import math
+import queue
+import threading
+import cv2
+import numpy as np
+import torch
+from torch.nn import functional as F
+from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn, TimeElapsedColumn
+from modules.shared import log, console
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+class RealESRGANer():
+    """A helper class for upsampling images with RealESRGAN.
+
+    Args:
+        scale (int): Upsampling scale factor used in the networks. It is usually 2 or 4.
+        model_path (str): The path to the pretrained model. It can be urls (will first download it automatically).
+        model (nn.Module): The defined network. Default: None.
+        tile (int): As too large images result in the out of GPU memory issue, so this tile option will first crop
+            input images into tiles, and then process each of them. Finally, they will be merged into one image.
+            0 denotes for do not use tile. Default: 0.
+        tile_pad (int): The pad size for each tile, to remove border artifacts. Default: 10.
+        pre_pad (int): Pad the input images to avoid border artifacts. Default: 10.
+        half (float): Whether to use half precision during inference. Default: False.
+    """
+
+    def __init__(self,
+                 scale,
+                 model_path,
+                 dni_weight=None,
+                 model=None,
+                 tile=0,
+                 tile_pad=10,
+                 pre_pad=10,
+                 half=False,
+                 device=None,
+                 gpu_id=None):
+        self.scale = scale
+        self.tile_size = tile
+        self.tile_pad = tile_pad
+        self.pre_pad = pre_pad
+        self.mod_scale = None
+        self.half = half
+
+        # initialize model
+        if gpu_id:
+            self.device = torch.device(
+                f'cuda:{gpu_id}' if torch.cuda.is_available() else 'cpu') if device is None else device
+        else:
+            self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device
+
+        if isinstance(model_path, list):
+            # dni
+            assert len(model_path) == len(dni_weight), 'model_path and dni_weight should have the save length.'
+            loadnet = self.dni(model_path[0], model_path[1], dni_weight)
+        else:
+            # if the model_path starts with https, it will first download models to the folder: weights
+            if model_path.startswith('https://'):
+                from modules.modelloader import load_file_from_url
+                model_path = load_file_from_url(url=model_path, model_dir=os.path.join(ROOT_DIR, 'weights'), progress=True, file_name=None)
+            loadnet = torch.load(model_path, map_location=torch.device('cpu'))
+
+        # prefer to use params_ema
+        if 'params_ema' in loadnet:
+            keyname = 'params_ema'
+        else:
+            keyname = 'params'
+        model.load_state_dict(loadnet[keyname], strict=True)
+
+        model.eval()
+        self.model = model.to(self.device)
+        if self.half:
+            self.model = self.model.half()
+
+    def dni(self, net_a, net_b, dni_weight, key='params', loc='cpu'):
+        """Deep network interpolation.
+
+        ``Paper: Deep Network Interpolation for Continuous Imagery Effect Transition``
+        """
+        net_a = torch.load(net_a, map_location=torch.device(loc))
+        net_b = torch.load(net_b, map_location=torch.device(loc))
+        for k, v_a in net_a[key].items():
+            net_a[key][k] = dni_weight[0] * v_a + dni_weight[1] * net_b[key][k]
+        return net_a
+
+    def pre_process(self, img):
+        """Pre-process, such as pre-pad and mod pad, so that the images can be divisible
+        """
+        img = torch.from_numpy(np.transpose(img, (2, 0, 1))).float()
+        self.img = img.unsqueeze(0).to(self.device)
+        if self.half:
+            self.img = self.img.half()
+
+        # pre_pad
+        if self.pre_pad != 0:
+            self.img = F.pad(self.img, (0, self.pre_pad, 0, self.pre_pad), 'reflect')
+        # mod pad for divisible borders
+        if self.scale == 2:
+            self.mod_scale = 2
+        elif self.scale == 1:
+            self.mod_scale = 4
+        if self.mod_scale is not None:
+            self.mod_pad_h, self.mod_pad_w = 0, 0
+            _, _, h, w = self.img.size()
+            if (h % self.mod_scale != 0):
+                self.mod_pad_h = (self.mod_scale - h % self.mod_scale)
+            if (w % self.mod_scale != 0):
+                self.mod_pad_w = (self.mod_scale - w % self.mod_scale)
+            self.img = F.pad(self.img, (0, self.mod_pad_w, 0, self.mod_pad_h), 'reflect')
+
+    def process(self):
+        # model inference
+        self.output = self.model(self.img)
+
+    def tile_process(self):
+        """It will first crop input images to tiles, and then process each tile.
+        Finally, all the processed tiles are merged into one images.
+
+        Modified from: https://github.com/ata4/esrgan-launcher
+        """
+        batch, channel, height, width = self.img.shape
+        output_height = height * self.scale
+        output_width = width * self.scale
+        output_shape = (batch, channel, output_height, output_width)
+
+        # start with black image
+        self.output = self.img.new_zeros(output_shape)
+        tiles_x = math.ceil(width / self.tile_size)
+        tiles_y = math.ceil(height / self.tile_size)
+
+        # loop over all tiles
+        with Progress(TextColumn('[cyan]{task.description}'), BarColumn(), TaskProgressColumn(), TimeRemainingColumn(), TimeElapsedColumn(), console=console) as progress:
+            task = progress.add_task(description="Upscaling", total=tiles_y * tiles_x)
+            with torch.no_grad():
+                for y in range(tiles_y):
+                    for x in range(tiles_x):
+                        # extract tile from input image
+                        ofs_x = x * self.tile_size
+                        ofs_y = y * self.tile_size
+                        # input tile area on total image
+                        input_start_x = ofs_x
+                        input_end_x = min(ofs_x + self.tile_size, width)
+                        input_start_y = ofs_y
+                        input_end_y = min(ofs_y + self.tile_size, height)
+
+                        # input tile area on total image with padding
+                        input_start_x_pad = max(input_start_x - self.tile_pad, 0)
+                        input_end_x_pad = min(input_end_x + self.tile_pad, width)
+                        input_start_y_pad = max(input_start_y - self.tile_pad, 0)
+                        input_end_y_pad = min(input_end_y + self.tile_pad, height)
+
+                        # input tile dimensions
+                        input_tile_width = input_end_x - input_start_x
+                        input_tile_height = input_end_y - input_start_y
+                        tile_idx = y * tiles_x + x + 1
+                        input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
+
+                        # upscale tile
+                        try:
+                            output_tile = self.model(input_tile)
+                        except Exception as e:
+                            log.error(f'Upscale error: type=R-ESRGAN {e}')
+
+                        # output tile area on total image
+                        output_start_x = input_start_x * self.scale
+                        output_end_x = input_end_x * self.scale
+                        output_start_y = input_start_y * self.scale
+                        output_end_y = input_end_y * self.scale
+
+                        # output tile area without padding
+                        output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale
+                        output_end_x_tile = output_start_x_tile + input_tile_width * self.scale
+                        output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale
+                        output_end_y_tile = output_start_y_tile + input_tile_height * self.scale
+
+                        # put tile into output image
+                        self.output[:, :, output_start_y:output_end_y,
+                                    output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile,
+                                                                            output_start_x_tile:output_end_x_tile]
+                        progress.update(task, advance=1, description="Upscaling")
+
+    def post_process(self):
+        # remove extra pad
+        if self.mod_scale is not None:
+            _, _, h, w = self.output.size()
+            self.output = self.output[:, :, 0:h - self.mod_pad_h * self.scale, 0:w - self.mod_pad_w * self.scale]
+        # remove prepad
+        if self.pre_pad != 0:
+            _, _, h, w = self.output.size()
+            self.output = self.output[:, :, 0:h - self.pre_pad * self.scale, 0:w - self.pre_pad * self.scale]
+        return self.output
+
+    @torch.no_grad()
+    def enhance(self, img, outscale=None, alpha_upsampler='realesrgan'):
+        h_input, w_input = img.shape[0:2]
+        # img: numpy
+        img = img.astype(np.float32)
+        if np.max(img) > 256:  # 16-bit image
+            max_range = 65535
+        else:
+            max_range = 255
+        img = img / max_range
+        if len(img.shape) == 2:  # gray image
+            img_mode = 'L'
+            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
+        elif img.shape[2] == 4:  # RGBA image with alpha channel
+            img_mode = 'RGBA'
+            alpha = img[:, :, 3]
+            img = img[:, :, 0:3]
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            if alpha_upsampler == 'realesrgan':
+                alpha = cv2.cvtColor(alpha, cv2.COLOR_GRAY2RGB)
+        else:
+            img_mode = 'RGB'
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+        # ------------------- process image (without the alpha channel) ------------------- #
+        self.pre_process(img)
+        if self.tile_size > 0:
+            self.tile_process()
+        else:
+            self.process()
+        output_img = self.post_process()
+        output_img = output_img.data.squeeze().float().cpu().clamp_(0, 1).numpy()
+        output_img = np.transpose(output_img[[2, 1, 0], :, :], (1, 2, 0))
+        if img_mode == 'L':
+            output_img = cv2.cvtColor(output_img, cv2.COLOR_BGR2GRAY)
+
+        # ------------------- process the alpha channel if necessary ------------------- #
+        if img_mode == 'RGBA':
+            if alpha_upsampler == 'realesrgan':
+                self.pre_process(alpha)
+                if self.tile_size > 0:
+                    self.tile_process()
+                else:
+                    self.process()
+                output_alpha = self.post_process()
+                output_alpha = output_alpha.data.squeeze().float().cpu().clamp_(0, 1).numpy()
+                output_alpha = np.transpose(output_alpha[[2, 1, 0], :, :], (1, 2, 0))
+                output_alpha = cv2.cvtColor(output_alpha, cv2.COLOR_BGR2GRAY)
+            else:  # use the cv2 resize for alpha channel
+                h, w = alpha.shape[0:2]
+                output_alpha = cv2.resize(alpha, (w * self.scale, h * self.scale), interpolation=cv2.INTER_LINEAR)
+
+            # merge the alpha channel
+            output_img = cv2.cvtColor(output_img, cv2.COLOR_BGR2BGRA)
+            output_img[:, :, 3] = output_alpha
+
+        # ------------------------------ return ------------------------------ #
+        if max_range == 65535:  # 16-bit image
+            output = (output_img * 65535.0).round().astype(np.uint16)
+        else:
+            output = (output_img * 255.0).round().astype(np.uint8)
+
+        if outscale is not None and outscale != float(self.scale):
+            output = cv2.resize(
+                output, (
+                    int(w_input * outscale),
+                    int(h_input * outscale),
+                ), interpolation=cv2.INTER_LANCZOS4)
+
+        return output, img_mode
+
+
+class PrefetchReader(threading.Thread):
+    """Prefetch images.
+
+    Args:
+        img_list (list[str]): A image list of image paths to be read.
+        num_prefetch_queue (int): Number of prefetch queue.
+    """
+
+    def __init__(self, img_list, num_prefetch_queue):
+        super().__init__()
+        self.que = queue.Queue(num_prefetch_queue)
+        self.img_list = img_list
+
+    def run(self):
+        for img_path in self.img_list:
+            img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
+            self.que.put(img)
+
+        self.que.put(None)
+
+    def __next__(self):
+        next_item = self.que.get()
+        if next_item is None:
+            raise StopIteration
+        return next_item
+
+    def __iter__(self):
+        return self
+
+
+class IOConsumer(threading.Thread):
+
+    def __init__(self, opt, que, qid):
+        super().__init__()
+        self._queue = que
+        self.qid = qid
+        self.opt = opt
+
+    def run(self):
+        while True:
+            msg = self._queue.get()
+            if isinstance(msg, str) and msg == 'quit':
+                break
+
+            output = msg['output']
+            save_path = msg['save_path']
+            cv2.imwrite(save_path, output)
+
+from basicsr.utils.registry import ARCH_REGISTRY
+from torch import nn as nn
+from torch.nn import functional as F
+
+
+class SRVGGNetCompact(nn.Module):
+    """A compact VGG-style network structure for super-resolution.
+
+    It is a compact network structure, which performs upsampling in the last layer and no convolution is
+    conducted on the HR feature space.
+
+    Args:
+        num_in_ch (int): Channel number of inputs. Default: 3.
+        num_out_ch (int): Channel number of outputs. Default: 3.
+        num_feat (int): Channel number of intermediate features. Default: 64.
+        num_conv (int): Number of convolution layers in the body network. Default: 16.
+        upscale (int): Upsampling factor. Default: 4.
+        act_type (str): Activation type, options: 'relu', 'prelu', 'leakyrelu'. Default: prelu.
+    """
+
+    def __init__(self, num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=16, upscale=4, act_type='prelu'):
+        super(SRVGGNetCompact, self).__init__()
+        self.num_in_ch = num_in_ch
+        self.num_out_ch = num_out_ch
+        self.num_feat = num_feat
+        self.num_conv = num_conv
+        self.upscale = upscale
+        self.act_type = act_type
+
+        self.body = nn.ModuleList()
+        # the first conv
+        self.body.append(nn.Conv2d(num_in_ch, num_feat, 3, 1, 1))
+        # the first activation
+        if act_type == 'relu':
+            activation = nn.ReLU(inplace=True)
+        elif act_type == 'prelu':
+            activation = nn.PReLU(num_parameters=num_feat)
+        elif act_type == 'leakyrelu':
+            activation = nn.LeakyReLU(negative_slope=0.1, inplace=True)
+        self.body.append(activation)
+
+        # the body structure
+        for _ in range(num_conv):
+            self.body.append(nn.Conv2d(num_feat, num_feat, 3, 1, 1))
+            # activation
+            if act_type == 'relu':
+                activation = nn.ReLU(inplace=True)
+            elif act_type == 'prelu':
+                activation = nn.PReLU(num_parameters=num_feat)
+            elif act_type == 'leakyrelu':
+                activation = nn.LeakyReLU(negative_slope=0.1, inplace=True)
+            self.body.append(activation)
+
+        # the last conv
+        self.body.append(nn.Conv2d(num_feat, num_out_ch * upscale * upscale, 3, 1, 1))
+        # upsample
+        self.upsampler = nn.PixelShuffle(upscale)
+
+    def forward(self, x):
+        out = x
+        for i in range(0, len(self.body)):
+            out = self.body[i](out)
+
+        out = self.upsampler(out)
+        # add the nearest upsampled image, so that the network learns the residual
+        base = F.interpolate(x, scale_factor=self.upscale, mode='nearest')
+        out += base
+        return out
+    
\ No newline at end of file
diff --git a/modules/shared.py b/modules/shared.py
index 97f15e5e5..ef89ff9d4 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -610,7 +610,7 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), {
     "postprocessing_sep_upscalers": OptionInfo("<h2>Upscaling</h2>", "", gr.HTML),
     'upscaling_max_images_in_cache': OptionInfo(5, "Maximum number of images in upscaling cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
     "upscaler_for_img2img": OptionInfo("None", "Default upscaler for image resize operations", gr.Dropdown, lambda: {"choices": [x.name for x in sd_upscalers]}),
-    "realesrgan_enabled_models": OptionInfo(["R-ESRGAN 4x+", "R-ESRGAN 4x+ Anime6B"], "Real-ESRGAN available models", gr.CheckboxGroup, lambda: {"choices": shared_items.realesrgan_models_names()}),
+    # "realesrgan_enabled_models": OptionInfo(["R-ESRGAN 4x+", "R-ESRGAN 4x+ Anime6B"], "Real-ESRGAN available models", gr.CheckboxGroup, lambda: {"choices": shared_items.realesrgan_models_names()}),
     "ESRGAN_tile": OptionInfo(192, "Tile size for ESRGAN upscalers", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}),
     "ESRGAN_tile_overlap": OptionInfo(8, "Tile overlap in pixels for ESRGAN upscalers", gr.Slider, {"minimum": 0, "maximum": 48, "step": 1}),
     "SCUNET_tile": OptionInfo(256, "Tile size for SCUNET upscalers", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}),
diff --git a/requirements.txt b/requirements.txt
index d926ae426..59d4a3f55 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,7 +27,6 @@ opencv-contrib-python-headless
 piexif
 psutil
 pyyaml
-realesrgan
 resize-right
 rich
 safetensors

From 7e298b2039ca301a9a11a71918240b11fecde1df Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 15:27:39 -0400
Subject: [PATCH 08/27] reenable sequential lora apply

---
 modules/shared.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/shared.py b/modules/shared.py
index ef89ff9d4..8dd49648d 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -441,7 +441,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
     "diffusers_attention_slicing": OptionInfo(True if devices.backend == "ipex" else False, "Enable attention slicing"),
     "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}),
     "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}),
-    "diffusers_lora_loader": OptionInfo("diffusers", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}),
+    "diffusers_lora_loader": OptionInfo("sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}),
     "diffusers_force_zeros": OptionInfo(True, "Force zeros for prompts when empty"),
     "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"),
 }))
@@ -834,7 +834,7 @@ else:
 opts.data['sd_backend'] = 'diffusers' if backend == Backend.DIFFUSERS else 'original'
 opts.data['uni_pc_lower_order_final'] = opts.schedulers_use_loworder
 opts.data['uni_pc_order'] = opts.schedulers_solver_order
-opts.data['diffusers_lora_loader'] = 'diffusers' # TODO broken in diffusers=0.21
+# opts.data['diffusers_lora_loader'] = 'diffusers' # TODO broken in diffusers=0.21
 log.info(f'Engine: backend={backend} compute={devices.backend} mode={devices.inference_context.__name__} device={devices.get_optimal_device_name()}')
 log.info(f'Device: {print_dict(devices.get_gpu_info())}')
 

From 8797a34e19068f658759d0854b5d304ad0fa5501 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 15:49:04 -0400
Subject: [PATCH 09/27] add tensor to samples method

---
 modules/esrgan_model.py          |  2 +-
 modules/realesrgan_model_arch.py |  5 ++---
 modules/sd_samplers_common.py    | 23 +++++++++++++++++++++++
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/modules/esrgan_model.py b/modules/esrgan_model.py
index cbede232f..a9682bceb 100644
--- a/modules/esrgan_model.py
+++ b/modules/esrgan_model.py
@@ -220,7 +220,7 @@ def esrgan_upscale(model, img):
 
     with Progress(TextColumn('[cyan]{task.description}'), BarColumn(), TaskProgressColumn(), TimeRemainingColumn(), TimeElapsedColumn(), console=console) as progress:
         total = 0
-        for y, h, row in grid.tiles:
+        for _y, _h, row in grid.tiles:
             total += len(row)
         task = progress.add_task(description="Upscaling", total=total)
         for y, h, row in grid.tiles:
diff --git a/modules/realesrgan_model_arch.py b/modules/realesrgan_model_arch.py
index 8b991de6d..50f085255 100644
--- a/modules/realesrgan_model_arch.py
+++ b/modules/realesrgan_model_arch.py
@@ -155,7 +155,7 @@ class RealESRGANer():
                         # input tile dimensions
                         input_tile_width = input_end_x - input_start_x
                         input_tile_height = input_end_y - input_start_y
-                        tile_idx = y * tiles_x + x + 1
+                        tile_idx = y * tiles_x + x + 1 # noqa
                         input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
 
                         # upscale tile
@@ -315,7 +315,7 @@ class IOConsumer(threading.Thread):
 
 from basicsr.utils.registry import ARCH_REGISTRY
 from torch import nn as nn
-from torch.nn import functional as F
+from torch.nn import functional as F # noqa
 
 
 class SRVGGNetCompact(nn.Module):
@@ -381,4 +381,3 @@ class SRVGGNetCompact(nn.Module):
         base = F.interpolate(x, scale_factor=self.upscale, mode='nearest')
         out += base
         return out
-    
\ No newline at end of file
diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py
index 4c2b3655a..b2e2daf36 100644
--- a/modules/sd_samplers_common.py
+++ b/modules/sd_samplers_common.py
@@ -55,6 +55,29 @@ def samples_to_image_grid(samples, approximation=None):
     return images.image_grid([single_sample_to_image(sample, approximation) for sample in samples])
 
 
+def images_tensor_to_samples(image, approximation=None, model=None):
+    '''image[0, 1] -> latent'''
+    if approximation is None:
+        approximation = approximation_indexes.get(shared.opts.show_progress_type, 0)
+    if approximation == 3:
+        image = image.to(devices.device, devices.dtype)
+        x_latent = sd_vae_taesd.encode(image)
+    else:
+        if model is None:
+            model = shared.sd_model
+        model.first_stage_model.to(devices.dtype_vae)
+        image = image.to(shared.device, dtype=devices.dtype_vae)
+        image = image * 2 - 1
+        if len(image) > 1:
+            x_latent = torch.stack([
+                model.get_first_stage_encoding(model.encode_first_stage(torch.unsqueeze(img, 0)))[0]
+                for img in image
+            ])
+        else:
+            x_latent = model.get_first_stage_encoding(model.encode_first_stage(image))
+    return x_latent
+
+
 def store_latent(decoded):
     shared.state.current_latent = decoded
     if shared.opts.live_previews_enable and shared.opts.show_progress_every_n_steps > 0 and shared.state.sampling_step % shared.opts.show_progress_every_n_steps == 0:

From 7ae304f76e72b781bf1fde3eb0280dcd1675fd1a Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 21:24:40 -0400
Subject: [PATCH 10/27] cleanup settings

---
 html/locale_en.json    |  12 ++++++------
 html/logo-bg-dark.jpg  | Bin 18778 -> 18702 bytes
 javascript/loader.js   |   3 ++-
 javascript/script.js   |   2 +-
 javascript/setHints.js |   2 +-
 javascript/style.css   |  11 ++++++++---
 modules/shared.py      |  38 ++++++++++++++++++++------------------
 7 files changed, 38 insertions(+), 30 deletions(-)

diff --git a/html/locale_en.json b/html/locale_en.json
index 692a82929..1436a6786 100644
--- a/html/locale_en.json
+++ b/html/locale_en.json
@@ -365,9 +365,9 @@
   {"id":"","label":"Split attention","localized":"","hint":""},
   {"id":"","label":"xFormers enable flash Attention","localized":"","hint":""},
   {"id":"","label":"SDP disable memory attention","localized":"","hint":""},
-  {"id":"","label":"Sub-quadratic cross-attention query chunk size","localized":"","hint":""},
-  {"id":"","label":"Sub-quadratic cross-attention kv chunk size","localized":"","hint":""},
-  {"id":"","label":"Sub-quadratic cross-attention chunking threshold","localized":"","hint":""},
+  {"id":"","label":"cross-attention query chunk size","localized":"","hint":""},
+  {"id":"","label":"cross-attention kv chunk size","localized":"","hint":""},
+  {"id":"","label":"cross-attention chunking threshold","localized":"","hint":""},
   {"id":"","label":"Full parser","localized":"","hint":""},
   {"id":"","label":"Compel parser","localized":"","hint":""},
   {"id":"","label":"A1111 parser","localized":"","hint":""},
@@ -466,9 +466,9 @@
   {"id":"","label":"Auto","localized":"","hint":""},
   {"id":"","label":"Dark","localized":"","hint":""},
   {"id":"","label":"Light","localized":"","hint":""},
-  {"id":"","label":"Show grid in results for web","localized":"","hint":""},
-  {"id":"","label":"For inpainting, include the greyscale mask in results for web","localized":"","hint":""},
-  {"id":"","label":"For inpainting, include masked composite in results for web","localized":"","hint":""},
+  {"id":"","label":"Show grid in results","localized":"","hint":""},
+  {"id":"","label":"For inpainting, include the greyscale mask in results","localized":"","hint":""},
+  {"id":"","label":"For inpainting, include masked composite in results","localized":"","hint":""},
   {"id":"","label":"Do not change selected model when reading generation parameters","localized":"","hint":""},
   {"id":"","label":"Send seed when sending prompt or image to other interface","localized":"","hint":""},
   {"id":"","label":"Send size when sending prompt or image to another interface","localized":"","hint":""},
diff --git a/html/logo-bg-dark.jpg b/html/logo-bg-dark.jpg
index 20392e1ae68a6249591c8d990c681d88a976ce49..40af4bbbc41fe91365ec82c8cc7f5ac8ba7ff2a9 100644
GIT binary patch
delta 12973
zcma*Oc{r5q`#*k96H~&JHQP*u>_kLahGglHMyZrFAzQX2yIV=hHZjE$X{J<2NZHp4
z*%GphU1QH!W-yE~W`4KldB1<(&*%95^Zmnh9JgZ*?(4kH^L#DmHAAVe<W$(lqkFZ*
zy|hmSW&N<i^5xqfes-SJptvY=XKU0y%nG4v8#Wt9cj6zDaJbTDaSij}V_OCNisXSc
zB5xZ$VDCe`|6U<b|M+h;7EY9~&IfLb38P`)R7ed6_~^|6=2~*$qd|Nc%~w8$>!0pd
z$_(D>ZmC3!Q<ksH!M{h?QMDzOSSlyN2*vi8xnvL@6|~R`qm8n&Lf?*R6>OK3_2i5t
zUsg1UbS`B0nbBl(21~sU7FtF2OY;(zK2Ajm!B+IcG}05I_Ria04=+Nt3_UBih68d&
zi-R117wuw*vb%`gUDl&_ypi@-9DnlLYuHzaw}2V`0<S6lI<GeL!vosAL{XzEClUy9
zXNyBGpO<-FcQu<?JZUf*nG5Hyi=pOxO_DHC2>Ni~y+h*u#EMwtCk`OW+K2N3HdhM0
z5Apf306<jZI{&yjk>`#T;#~w62S}M%Wq&Q=0NF>f|7UD~NSFyAWcAO2z)XO<=Ny&6
z=)$)*^{B_v)mfi%CP%_0ab9!s@L4rsmNcnGziy(5KhL#hITj3w*?vP|B(5zwnRX$*
z2d$A33=W2XP2o!@^vCgXj8{1zf>|swWk;>SR#k%Plo@_Sj0-QXEwcM=oN^ex--g&S
ze$Unm6Eu{3CInnC<ofS|SLHXyALBoC^iqFWx+qD($9s3WV2gjsnb_yJk5V4H#OXzQ
zBlg!;x>nPazyCtvYk8GH?bF3}`WkIY<~9r$bO;mQO9<=$LUFMM2s=BWYMR!UN__6;
z=oE1dP%~;br{0rS<UrixqoVRbC3WcMm~M#lLGua2_t^tH8PPoQ@F~7Pw((HhcBBOd
zcqM+fo&!kpzvTc<x1j{O87+Uo5cQlY$~H^r+~TfhC&uHV`RiBo9KiuC4zT8W3AToJ
z<`)7{K@*)ZRj7(j_J6!I^BX#s(l=Bid&>I)*0knIU@N%CHKK(f=RvhgpOp^iSUh5T
zykTmu=@|6UEt^V2)0X`Lc>@~`H8dZ2cm?$URRW_mh36zh?HL$LucnTC`|i?O`UEeC
zcIg8ztPzp>3RL_Wm_>1ZmWBClDjx_9H4%FY^)JWq8a#z%{wKFkpabDSov1fkaj(Dh
z7FR;xdS<M&KL>CQE1%P!gC_#`XbtaqBu|76l>@}?>hSYh&nNdU{pP+74oZnqXL~b%
z6~8?ZlKrLMEwWrsJahHGTg7G?Ez~#A{aQ#}C9oKB5w}N_VEIZ(^|91r4L>=6=BG$I
zCNI)!jJPWvkU#s+y0dH|8__H3^B{dgjs4Iw!UT%1Y916{JP3vNcrFonj<FB=UDND#
zj1vbiH8K0*w24x~EI`M{)KCPN_`~}p!ME={#iL4Z_J=)tV`HU#DJ3vwULRTOKtzht
zg1=0>+9_jl>j~?6%$Yz|=4z1#5t&?c%Xf|PuC%T!ZRpNccU$b$DDJwCq?+T|j5N{$
zvotZ7$AEGm4!d#nHk1SLYaBqBCDf-`uwlEeF6Y+C74u!L=<_(n^(`yJ-_`=t@fM*O
z3NEfG{9;$H$IO`k8u)Ha75r~a6#lvb7g(}};h!IquGvz>yQuWWEDImJg!<5YkI}{L
z7T)<FYVV&a-}d?g=2b~w#6<q2@SvlXnMkNTSdzir-Y=D3sAk@bi1y#O$SRhb9NFR}
zcrjn^IZ_sT_`XcM%)v`Zr{CiY5q|;X8Z~NMLmgTn;7*)%jblXXmw}w^r++qQs$oNT
z8dB5<nG$gpiljk46y48IbkiHhS2X9x>D$`uhaPZ^cwntF`hVu`|85mTBoPnWFAMSq
zHlet?BJB2o9z%*FlY4_i+ZCCCP-lprn}g3HMIvNK5ju_&-TXN>POdsTjUD0uo~PK-
z(j?)vNFc)9>0HQXa&R2$lH+9C7S}42ctP+04&W1}Jr_9#p9!PrU87Dmq6BYdOpGs{
zI7AEf-FFDR$e+)k%~=As?^xkAc=bVSyn^b{tFMz@n(17=kcuS=0d3F$?S&3V3#&?T
zN-1e~=}aYxjes(&+fbkESookyS9(@jT$(l$yj9#7`!{=6-N&w-v-X0{+A{I(eAFr|
z9o+bP#RJ|RAxqUhtje-Jaq55@e041t-3p#s4+HXZJkx9~Gp_wvFWxh3m?fR$uh@ro
z=0%^5NSXjQ#pxiEoyh@=HJ63KmeCI!fW(3lR6Q52`2Q4F74Y<N3qw@k#BxlAY6Xg7
zw(qUuix)@OGsa6$O*yrYvaJwp2C~$fmE@DG+KWFvxRO-`XX3HhpmEHi(<S?%;jv-;
z(%u7|yJL`&xnftt_J?&Lnzfdqlo^NJ!mB6^)+$e%@!sl3k8*&ZN+l-lEfC`7;eARZ
zhrBiOxlehTzx3%HejVreoNQ!(q1YN;ombgixG%(xyi_|eczL0qCSQ+53Y>Of=@v7i
zbcp;{>i(vte;Y?K?A#j6wqHe6U_7%-g&<(96{Q=j!KPjcGAs%ic(;yUQW}k`Ge7Hf
zaqHGa6aM<goy@urx?~*O!%^S?->ELGR=k>!wv|~t2bl8juU|NfVJCV8C)7{f?&kmt
z(ok+>i$`n7|6hIJGJEg%;2*YTR*~xKa=~kw7oOY82Wyy~PTV*o4ls}WQ-9YL_L}pY
zDoc7tNwaj%8*B=KgS-hZ&{knFONw+L#HJA$Lz^oT74Lr>%znHpJ#C0w=Zan^2Vc#O
zka8l-IY4S?0Z1xl--tG_#gq3O25aAX45gJul!dt^mIszoO^@H3#~SK)3c*Zk%fAue
zK4hh!iO|O7-fP3fdoW{BIdkfCP1tl4n)<SIA{@{C%vt8^d6e}^GkWVs2>N^}-R*5w
zLF^~HYr{=`yIuB5nhV>^aRH&T57_(Wn8vbWW$o`r`On;{xQE=TZ(`6E`5d1a?s=d4
z>Jor3RPexe?>Pn<SkU5F$yLkS4}39eE*BZ?I0?%5x<Ca7=uR>`*t>(~ciDoB{33?C
zTg@lXW1t9iW+50VN@phSRm*zek!XWA)S^ZecvEhkd*IaQq>8t8+vK0YdY;Vysciqb
zrfcVo5WB4qgF&ksu?CtHadQg?pYwShI;&?J=P_f_bK^MZxYzhxAhZuti&r_vv|Kr7
zv)mYUO>lc!4_EXTFbG9XP4Xn}F?t27RnJ_<{IIS&!U0+hp!214=fbDsA+9EVbC;dB
zcS8H4#l`EXp>K3A%0Kz6z6RX(o+t1!)QqVwP>$(0^A8^ke{A6CG(X;TJ=_ZMEX3k>
z5`C$?Uf{>ASMw^)huX%MwGV^VYej)X-fi@m9?gBSpMNd9E`1QRr>2<h0`UM_{9%4w
z5PdYHlWi6m&i-bDhDwx1U?np*wjUHi0kq=Q`Y-i>NC-?nYq6hjJN-(q#&D-tTdUwR
z=d~k5BGQqX)2#I9v@PT1c<1^1E2D-1?3Fv13%FNBhl$9dk}P*il5*PWxtjPTZnpfe
zj=_T<!~m)7)QRdmM%1RC*`4ZEpPP<vk-Yi`?>H}bz$tAjetTJdpK1raC$5@2&AAt}
zXQxg>L;tDMunuFNAir=Qsg4h{&fN)>GLK`FkXB~40RA3Y62E^xkH8H32>9P{G9GgP
znJs+XL0PC<>)-sT=Rm`obPF$*+^#B^o4l>PWVKaJixgcIg3Z~89<w#v5It_jtDe*;
zlPG*CEH%ugZA_N-u5zN4)p3zpV>qocxp@OOSKm9e)y3r}G|~_{(Q#vDu<;2tQZJ=y
z%}qmncNF)hTqqMoLG2o?Nz1@*ftwi)<g2ai!>y|(a(!U#*NqQQJ&JxqA6)bCxk$R^
z`l|>lc6Ym-a1#?eH~zfnWiEW7YBTN|cz|U@)_5{J68XLn<)gaS6~2w{1Nrrhh{zmD
z5|z-%0g#L$w<^nZiS@~a^4+_dw;#@pcMe7CD#`<|qj~Um_Z}2K+4;z61xv4Iu7x>0
z!mJ$fCeUX%KwGrIWhe<Hcn#X6xMaY2(wsc37wT|opgk#GFO<vycuZ}qYLT}Q?cUrd
zYdX4tuCq?$Ppk8%*)-LHjppn9Mce7FSY~ywfeX?!Du?=I_C*(>C?tQvFRn)Umt8An
z*slw0E~wm{H~Y{xVU5yh?4)6ZaGqFxoUxH$@l|I0#z(n?tH`#7?azJn>7@Z{|8iWs
zHvbV7*t`=jTbeTISls33YcTE^cqr-I)bk^gpM#_aW~qf(ay{#*z9_rSY(*1B$E&MJ
zpkY5*Mi7gK+jD>-ZT6))4D^85c~HPS10wPujwgYP#EUYYa)3J*jo7a-8)o=%2?em2
zWi*i`+8{C(RS@yS)$PbnSNwLLBw^Z6Lio3V0%I_L{#HMeM@oZ~li}Z^!(4M}AcO(y
zm;;Pz6KVmGaJl;G=soo>HSKa+`P=7#C6kD};NU>;_J($_fIaqbd#h5~O@lWXz#2xF
zb+ezIEn9}mI{JfFD{;JH!;j2V<CE?sWm*A0t|h3q>Ys%UL>&cx$ME9-vPg*a6{WC}
z(0mXGyqc@0uV7M$NN6Kc5|aGk%^YCm5QRXWmRreg9$z^O9qP0kJH`;Lh&SSw5JkHT
zgUuf>OKK}`N;v?|9HeA$0LA}S2jU`9hfy%OX!q^NulB!^WzuM3b-yM9&h0<;KBQCs
zyjiJRM942SKcABPExT*7_q)!i%9zNQ^8XN7lSFyIn~KzYOC@ruYAe%bLiuga+J8pp
zX>mcYe*zL`yHim*hYq52q39zWpbdzDYl3FU&<>1AAmzK$_U<l3+a)N?#KjN;Wgu-5
zv3=WV`m8etI7IPdP7qqGxOO_93<!^=xqp*FG)gQf@AnMO$)Xl5jc|aHEX)Di_i84v
zY2C=)LyC|+>F`%q=57D<3lZIz$$>3Napp^p8x^syGPF{)54r|CF{>aVlPKd5f5KYy
zX^(mcsAk>{T{$QSC(IBYnw0_}Qr_NUqWv|;1E%Kt1%{`{uF~=xVCm!T-(B36P+wHX
zw@d1vuZJwN&*%rUYh*b<m!)nqdgX|B{W=n&Hinw8m00L=;wacU3VJ@&?gZ_Ex#vv}
zW=hzT1K10403#uo7lgdZ;2z%sm7;=|LrY^yk7N#pOdGZ}jG#)jfKWs{oBOMLc+tMj
zsaDcLFz;XB7j!Q3L>OI|vIbk>FUa<?mEF2Fe^+DZe9n@&ddfWVw0^TN{qE;ie_kFc
z_SV1nQ#TiDNNFQn);HRK6t+|tBsKRq4t*XC@pTDCT6-SnN&!a?`zA4dZ+pO7y}a}7
z5|q-*tIJczJbt1Stnw8$5GjWHyYctbkD_5c5!w*Q$GwK=S_|70_Ylf7t2HO)Cc32y
z(Yw<k#8KR$$(X4#Bya((&5rh<HM1T;U>gH<l`cLY;?+*ANk+!J1zV1^vuk^HB9LLm
zy7!@;GKSgSZN;_YL!59v#xnlBjcgK`P_b>2bvt<0W;(?n@nmy3)@y`_B<R^toou^f
zM&>>Y735C6E<9?)UZ#OxLf4CkbMQ&iE2fHH^5HGgZzD}6>f`Frv>Te~X?2!k26nl}
zK1Jzg(ul}(sMjAh$@x@V-CLN}|9!~r`S?<^fQSo&2N?ff1r~;FSbiXyVVdn-=pF26
z*;E~xVtd;7ItM5l#lSYZ5QW|N3;g1T4?5~VNi7N`^~VT^#lmtzNbEB{;q0#@G*cxP
zJ~h9C*cNcgwglTS4NYG}v+pa-jPGTc(-02^58wYiI*?qJzWWo5-8x?%haz?%dgx6I
z>_mE8jrAv__LjlsM}v;rUwknO+NRscQsC}WDAFhQSb`Pfe*uhs`~?Luuo|;BLY0Py
zzi%H3UjKLtT{5Du8Mn2suij|cP|{1<oP5CIxFo=RwkWqoiMnM(jeo3Mz#6Wt@UE=$
zL>B7GGowxtk@8f+S0AS<D@&#QKB~Q|(=Nv=n0K(vkbv41t#Jp<JI59;-7>(tNmUVQ
zzPOEV&*)|J|0_)KqWi^;@M0m6*^*QQ4xF0g04Aj<O(-g8M<^;j&FlooVL1i8>LGBE
zr9^XTbQB--Q?4Jnq6qfdtB+)Fr_W;`vt&XZK9AvHouO_d2)mS2?RQA&_%Q3umdd0S
zrNa(QNggmn-?ZsO2{D=rJ$$6fa+@!ImW-koYard1*KTc_0FlV6C^1@L%tQ<PPjK`P
zZpR^P+9hTYnA@^J<YWF9_!PihCx5C;VHVKSmeo3+|K<QaC)k%@w2uG5Wp)t9W3Eah
zXJTI;#ERZjJ*5xm7|l1o#swOR^)1yMU`V^%xVYSSyI^h9q{Q{n)2){UOk@ENihUR%
zT*&z3pW*iWSWH|ce)_RN!T6j;Wd3|Yc!<w+a}~1r4f#Z80WF^Q`hb)?JQmUg`uiH1
zREsWo1vg}^$}M$qfKOQ%LBcH?5Gp%`BM0k_|F1ymxpROcrnMa4`Y>)=q!+g01GO1B
zP{2s9tVgEnFUTB|hj=J9SNg4)A&Nl8CS|;P{%P1LiLM*a{G#I86Ht!jA4&Nk52yGb
z8kyRUeQLIkGIgU7KcdM!5DAUnJpFT>f~~a3RC?}{WBe>tPO*U|1Y1`LNq@lB899Bi
zr)mEt?IqU&54>orMQ4Y{5s^Oc#ZF!M{El+hBJ`bOQ!tFJlld=?TQwAga7h}JC-Q8t
zJV_XO6SS9i^WX|kH3uj}asXlmz*vVYj-5nO>Si6L23{kHsg}k@s<u4&*CokW+O~th
ze6~L=U3){p$J0-?aYa(*!r_}P3tW+x$>WH*m%?Q$nweUU|GsfHb0!s=51xP73!APJ
zWJm?*SQH8VynbHcb=&B@K=$ey+URAiNq7)&;a7=|_L=`UU!n|Yd)I%RFGu`={mKwc
z<dSNFi0)(YjqMmnDcD_@#B2&^EpKs;fOBO;Bp!c3UK;uo(z>gg-hn+`Cz|1~R)*&<
z5F^LElG?MbcO?>tRHD{@KNXjF$KrXZ{OfB4ZU;mk9jH&7N^bshjlgwB1#8mfmwEA1
zTG!~gB7wFOpXD7Z<}u>H+Iy&59I;ZEpB2}lae$mq%$lHOf3l{EA(W;i1m<2M@`=Yz
zu-0(*(x{!2&xfw2nm+tGrqQ02RHvRY=WpIwnP;#Iszh2Slp-WZ*`!z@y71jnYIrN&
zJ!gf5n|_Eq(Qo?E-N5Uj_8%T=#ch)aA+b5BenA3~kgtboW}C+kYPLeov8Zv};_G?Q
z`*t~P-(TIcE(vmJoxMjVA_XBvq6AV{a#T_yN=ZTU1P5>}_KoWE=gCK|n_Xe)KycGy
z(_<jWaDW)&zFUO0t;m&C$T&kl;O88sHV^q1%@In}>_$gfg#cx6uuO#ms4NLBIkVe>
zFv*D8NF_tboY*?45b)chB2S;$vE=f==uX0VxEh%;Q&sg(#^BeHhfnivM@%+UrL2{t
z@1XAZ0m%^e%b2y<9sGe$+3pm*zS?hX$kYbc`S6e@W@h%qZxf{o$z=&Sc4Bh0!98dz
z$G|DpHJXGG^@&HJ+?M5%ft~ptH4d;#UnnwkjXU~O+h)B3Z9Ku?&`24<jW@-%;%iCC
zPo7z}t#$zY%Q_Kxp61lFjW86j^Zco0K~5<<y8R2iPN?q=>?sYXLNrKWETf$m@pqng
z`FfQ5JjCR&)w&`*;Do9wT2bRLaaGhCG#rQcg^iDBPcH`;x3n9YZIV#YhW=zpa?m3m
z0d$&dvmr4E{R{{klVt=2kQe#1xT;0ZJ=ohP{lzdGXN3%Od`#AlLW?p&heCRO0!@J;
zI;~Oxb7pkN<m4_O2!}-Z@q4EZp2Z*7B!kO`|FF@qfXn~!j`0?d@p@(BVT35uy+AdK
z&o>`Q>NgUU&(i&%?v)GY{>MGf5;fZ!RKV5ehtL%Z*oAgut1(1T-=g(!4QOPI$wY0D
z%T1Ruxy(Gad2<)t?~6Dy<?@Boxd4B(6#`DiJ}^LcofsJ-_Nf{D-0a!#00++#9(R^0
z)&GkkyYL71%kI?PnRc3bb8L_c2RMHmoa4@%DAbeQ@rWKk=(+=_8UGxr(r+^s&IYs2
z4;`v;GX6J~g$qWnBma4dHi&@^^(f?YdNJ=lzDdf+n!55)&E{D#eoi0zk+`pPB1(lp
z)~sICQ$ylW$Nb#wD>*>GDP&%!<NauTvql5u?g$m{+4=I;1I|w$N&Py@o=?V-a`K7@
zYakaRuBqH<Nc_RCW}%!w2M2*pxh@oT@}8lDqXMI~(7W(fiTqTL&&VS8aq#)n`iQ|Y
z3qc4Q(O?*p8&S464SM<$Aar~_Or90jAAPc(KSHItc3)7Td4I9?kHf~#?Y1YoAyxjt
zPk?3ES7T^<@b7^^!lsMhi}J0G@jt(!b+gRAqr|C~^Lp-t-^p;L2qC5sed8|(KS;?|
z+~2Dwvg{hIMe3^zu0&mx1|?ja8X!5^A9=4-JhrmV#{0sN&9D4>?N@Q%qK|g3*=^t5
zytT$Smx?;8r--~5&cj5W``okeAsct@zI;L3lF|O(G2@WK{Wn$ZzG{S;-iD3M{cAT3
zUtT)106*E<Y5j+;HE4xUp}`yKhsV(FvD%K;@G|zd8yM*{bgpo3mjvQR&_IM4b$ng5
z%ul69B>lms-~PA}&=#1SgBq={J)aMvLA>*^Z$#43Zx;9YQRkcmAzH!1)Fqc#h&GH0
zDc-9ZQE$$Z=|ZTHvG>CqpMRQd6i|J8`{as^5)T@pmC1<Guvox^tLDT<84;R{`k`-T
z;<1-e(frR;c65iS3E;x0nfw|C^`P9K9U9`7SV8??;JE|(s1&Lk+YCy91+*Od;Zp+C
zKBpqm*)^^ZZ%vIhh=p<q1s1}a2%>k;{L5vZ0^U3E>s8o689Wq4i0&jCO7%50p7^M@
zAt<A6s<C4J^|D!>+}nsAuz*oNafPTd_|l_yw&hW%zsVBV^jH%%_fmy1>T<o)liYHJ
z>HBd09K;{^cld*A2uOA$xJ{3Etweqbn)+6A29qB}_zRd?Tmy?lBF@yZyJ{;!U4C*k
zeH|80CJTXw)G6_61hM&a6q@>iWb1GKCh<)GD|^pg_`P4z8p<pMGKR}AJ)`tT;Q3P)
z0TlK(UJhXGXdE0K1nOdg17uc;TSE_~dcgl&a=UZI_?hz)v$w`j-JwJ{0KFA_LPSb3
z;>gd}vLcJX!=~Xrr%%rrwhEyBVn^VLO2aLV6klxHYyHIhk0DB%-1+d-%N*#<LyJr0
zWUofi<9Zk%x~(=XT4&LYJd?_nsQCD6;g?c&T=JEZHyVh_;V6^4ny^*Pu2#R}4H}xa
zO`9AJUeA-<?_JHvBlozySN*)}9@pT$qq^n@r$%Has0r(4snhhr9;Q`k&Wc`%zl&!D
z%;Fl|(a%Ef#^zhTqKFJ7o;)hb^;FW&X~L<|3eo+H>)Die>{IQupD3{>r;e>@!t{k;
z=@Tma==~n6S1qF*0wtiTuSWkYm50w{4xe1x8vL-7t=_TkX^{S-qH~NKGEPH%V&9p~
zY?$gL9{zl25c!HF4{wLiUkJs_A#boAiKDkeb5`>75D+Ir#c}{gb#U%TwDwMR@6ZSy
zP*Ea3KU~RE)i)=3kF><gvj-J(=`vlr`=;tt7~^hUsGf!k`|_$JDxW?gZTHQOCnNf!
zb#bV%p#g^U)ve2oR*yfti`QS^-tZVu#9g$EXw9eCrd38k4Q;JFkT8)_abXT&s2?W;
zJH5l9kfsbVXlAiMpij!NA3P<%!F^Lve#{m8W&y;FA@wB$UV}(_yrv)T(02Bfx@5qt
z6?J<VYHpT*H>q<>iP8OmBD<)m<)hkHu_`nZj9x}UJPkHG7On5n1nyr$?=@`=4VZOw
znJ$UplbakgBT+j-a^j-Iy}}w)`g4C>9abMP=56Bu_x{i!cSY?HTRJ4WSuo3>4>dG&
z@$#)jP{RUe<_o2|7o{)=%M+pdIXqk^&EmUclZ5ZH&MhC7H0z$>LAE60@<Eeon$R0i
zbnwP405b$qS(GO<(L}nRz)6eeHJSFTGMa!WjZI^+pTd?-d3pYEO?anLQN~7pUL0h&
z#8XxFHQEy5d?U6EeiQxbC2r&BLa;TH+hCLw<8d~nFl`0zc6VgYG)4)%ZpZTXfYl3O
zrp`h>aA7etsk}5EFNfmEOq}s(?#TdXr(S*&uIN=Eg4Hr@1#2=q0P{pM+i3mY63Ocp
z(Y)H_bNL`}R|qKicOJ~8;$jhZ?U(AEP4*P++0Wx8>zI2pg<Xga8_)j0Kct>(Bkx*?
zx0CB0*u0InMrHnen4~fRd$JiUYi1QiK{w+Xt#+e%^h_yw8GH@*lX=soafdc8FajCk
ztL+l4x73I=4<^iI_;W@iyAZ!h3B-A-ca?r!Hlppb+K<YuL>+0WChQg7KvN71!A+Q^
zXZt<u+18i~C`GpAKvoIf5am9ph6KN96`ggL5Gj8W`cdHIKsegp2>md;Pb$#9md5>>
zMEbcerSx*tJkrQe>>poT=-;jVJMZpeLkXWEdU{9lVwgY<<2d_FB!d05YzLt|;9mbB
z^!NeIqaeiVlW6XZ%SeWGWKC-3Fnw~NY)T+!yKJuhA3t(4rimZ`&3{FXH|aMI-a0$+
zYvkEQ%#gkvZL*68pc6z{H@Zu-!X2dWm+B<lBsSzD2O^hWvv1Ue68V+|7@hVsSN6@k
zkdA{VE$;l+e(mS`?`R$dngC&HU&c4lne3W3Z0~anX~;t30x7?o{urW7{tp4V6xb<T
zqN2I-=E9+N-H(?q%%3IhRchC>qm`fUiD<_nO`&<?!7byd({aPq?3j@*MD~1G>BA(~
zjf|v`D<9=-Qa&DsxJ7A1$8<=}gve{(qp9A_2=24|;J|BC%tt27P-;1Wt@GLMGquDH
zVH|!ckOKh5J@ezw%w9+6b=T;_dbL_#SB5;C@jQP7N7aW~XS7b1+_ySuT5NY+*0h2`
zyQ4#~Zd_TY-?RhzQJqq>IH{rQH@`ujB9yR7Y^d6N>^!1`8Id04<txhRm<HwSz~VKN
zMUZ{lyLs8KkS>u{QCpEZadUp<#91PLdm}-RE@h=vmsXTBjK5E=51X7fSSD}wgAZA-
ze9G2Q*AsVdY44pzhk&1Iv81;^;GR^=Lzk`uyF?>oGB(DMAs932l$VcBWBQ>8k_K-j
zy*a1L3IQYIVmE9bC<K2nUk`vl<U%<paBM?viG8ieo`{sAX@9FuaM-XF%zRXp>5e&t
zb7481ueYT`gYC4dZ|+}-zvOyH$IbaZ@>lIf<JiGpK^+Ent4fXhi_^|_IrrXc)^9E*
zB84*>t4hBoYCc{XvNQY=8<48r<zA?y7%e4K#G9w=&YLKPeI-0>7je9%SMRtF_<mqi
zsZ-g}xjZ0gEE?@p0Ni0qFtVL<_bir1D;Hu>&>;U3WSY9MLN<!zYKyCT3T=nh$IOp9
z<{kMR4=^uCxf;%WefH2;p+)OAN{V(jpZpv<FpY#u*4Dhpj?_m&Mj}z^8FjpoPr8vy
zWnep*RzICs&;a$7`Z<U=gYO}`T^@%2D)mFaNCS~<qS$6?#GU6YK9lvzHZ$q;^kFCR
z#VB`!=paql%=-9Vwed?Ss)duuUu8C3t)I&{4oJZkJF&)AP$?&EGrS`Y^F2l2v@tc1
zGy16u;T@14;{HWEq&TVU<7RA)Po0@uW$UEQe1Icb2B(=MljtBdF#SCFtR}37fp2D(
z)pZD*Ecq<==*BGpsKh8~5KbLD#BBdcSdsC-tn-yaa?#u3cQ<aghcx@uf2#Q=<Wb6k
zO2_SJ+|S6v(n0x-%|2-A66QF;<U%XvFQBK0hZg*uLixS^QsVIk=Q-auKwHDW9jD%H
zd3axq<)p*DZ~mga%fR1>Z%aVUqBX~H>wC*iN+b4`LAuZ)(_<d2+tTzLmnT;M1lK7s
z>WjQ*EFw=r`}=U-bRcu15Hl1t&TZ1{NS`git3xscHiy{_P7#Z?!z{Mt|E*^XrT?FL
z?l$BKL@CnPF=QEq@Y}N0!_P_+W&-~ZmLT6oN<V@$fqPshLyIX*s_@o5dA_ftRHh`R
zJ2O9$8&L{`H7C@mcsPoWdwWe<*E{lBmAAxg=lo&t66|gM4sj<(_RZNPc_x*Bas*lA
z&!`;YHWuPor8vQmq6y)Z;o}zmm2I|mg#-8z+<fD<cUNgvun8mFS@`YCjNrDt$vWvC
zw*t}vd)YUm)mi$~cVn^^Nw=}5&eyv?A|tk%^H|4}M6rxl?=g>FSM>5EaO<M>eBT9U
zU7I(jKwe{kot=@}uCLxa#sVO(;*n?s`J(ij;twewUH!$bO5R68BlrA2PrSu1f6wwi
zE;)DU<?nUrIe)KdAcCC#C-{_Vjsf8gkS6j~I-9>S+~y7IGIb>Vm`UMKb0yIs9S9m6
ztmikI(3gj&F?{^qjxw^Y_rAV%^}A(NMO(V$itY<{;U9yJ_MRIlj<zkJONIz<qAoP2
zvf}0C=W+}2b}FYGO%P_mz0qk`IY7l^E<1ami@Wq+KMlsU!3wd5<l9m9B`2^swsZTo
z-?SlG>w+QLC`6lmW?P9}i{HHPdwf%{q=>nWoi-GqM~XMtfAY3GJ?dsU7qEJmjwsGw
zgH67aVYuXZegehr#HO-IzUGrZtVdkx$&QUba^F6`pDQNw0%nNT^Qc4j1r2}Y&+=L@
zc7Ai{0d9V@J}WHlQIxFbsne}~Z+F*Lb;{}_Ve@=BK-{dMM3wA^F^R6rT3IfOC@a}_
zPk$d;Yn{RsggpE_A%%4!bo8ob7$RG;gbnUT>n`tS+3b^@8TL@;_PxJ!;mgKls8P3&
z(!0147EiBUtn^M8ek}K_>T{V_?aplnbfgd22)D;0y9XNsS)v7)25FSYFSBi(2FUp0
z0eT=^Aqjd{eUBa8dHZ!D!sa>S^W^!IH_K&ReL=dfnloRVj-&L|^Ftb}+fS`Plj05y
zSK^0ui+RfWzsCqVDKKK1!a5%>nS9TWSY5=(7*8i-HZJ)a)Ox^sYaD3b<P!!Nj%w;m
zi9ymddPxq<V<m;W1sr<mNyc_H-W`9f*yZgT&Tkw1vWAKoxU2|We31nC+sRRt&g5}X
z&|@C|NM*A0iC{@;<GpBL-tzX9?hhpOW|{jk!92f?wOC=TA?Y;HnRKe|gG^}`bdluB
zWXDhU0J@|cxwF>KlPy9e-M<Bv6b-tj9y=a*%J(Z|{e(7?@rII^xH_qM=GuLVm74pl
z*?>?}SCEML(T{t~1{FpeZVnsq`G}fU1W62UcMpWqnO2Cv*B)b%)RESl!O(pXt*Ng5
zHv{IwC;!}Y<o7M)Ypp!q;;vHNF&6Aj(hoF5Tae(t36BE2?n@oF@kwu7O!iGsdlxAI
zm66dT24X&E$$xyL;fw0*m0e(sZj^NV7!mp0HQwAU<c%q4cKY(u#QkO$9LAfl$OI`{
zeMhb%N@KE<u}_fmCs};J9gtV=yq1)v2T5-GNC)Qb_&AQpGQ+-8`Vpa<Ul&_AG_SaK
zfBnyF##5>pl;<XwPNe%g4VeV{97+{?s&Q<I4>&b5rc~|elVeCb7b}GglHPskDQuM#
zuc;8BTi6Naw5AybpPprOtzo*hVjOd5Y7OJV;>h9a_9Fw+-x11$MobFy8UYwymhpB<
zAqQ~pz<r+Iee~@2=2=fxAg<9y)DV5%MwW~fyE?|WqTBHoQ{Go4E<IpIXLm|>aqm<h
zz4I*)R61iI%lPotmeeqJJ8ZPA7Bgzi>YO%|vSJq%{9#JvRrac06k_kk{uuibq_x_&
z{wg$YL?VImTvgV$NZPdv;WmMN%zXGh)z&j*JHBtF&iUlCIMQo2I?Fx8o6IhlfJrr`
zWa=KB?Zkr^5zs_TW{YQ?2;FlD_HdfEOS5;)T5V;+SA<}n$#$L#HEKC__OFsXeqRpq
zeIE(YE;E|arGB{GprEry*aN?@V;jEXu|SyUBUbpEIoK<K$oC5YQ{ThTfkQR&FJ5$U
zbAU&B?)i+^c>8lV%H<v%(Q#@f(LV6}Yb!(EGM<S%B*|QEm_$kRAl|-kf+*`;wz$Yk
zeU)rT;+p`DP$de1h2BjDdR-B+2eu*y!V4-b%<DWQdZZa<l}*5%(p4}aN|{n8Yg1P4
z@@@d7_qW7s0K12dF}g>|0O<81EbE=ie;dOe2t(SG=E21QXY7_=chn1l*XHZ!Q#j!W
zLsE_HLVCI5<#)HsKDxP}o-Z|1>}&sjo@xT%Z}FnQu#J0k3qO&Niy+eL%vt3Fq3$~E
zW-Mj>UK#^FRx|spYU-sFX>s{`1o7_qY(};|_zLygii^+md*lgrxWoe`W&R6sJ@sAm
zA}k}5Jl%JvCBisI{@Lrq(1+>-4e1uYhE22=u(GvJtW;_4R>4!vo{BTOCuMrAI%EW8
zy47vKq!=as#yiA)rwX5izaFp6T05$Me{om37*SE=R2pN>z?1)WuTCszbds*Ru6NPp
zML4R0$nX6X6p*_g)I+Dls8qi&yS?pVkQk1O8gIwb<YG&ayrY{oXX`{T)fI+!F%0Pc
zZ}7RFFQbXX1i05}Cr@BTTaJA}{Xq8DK=&FP_3NoK!=7<Nx3t0c`Yv=uFb4~fPwh1P
zfTN{npjfhpeRkf0Nf@%xQg!&+$C#!JBf-;c0@L_4nXu$IDsgX=a7a4u7^>2_oKLoR
zW4@i$IqMjlGmd@d^fmm}@XdP-uQrItw@SwwhI$PSbrXA?tzKT=?74gQ*t|U4JP}Mp
znW#W_op5@ZV^SoER>LxzGqrWdQMywDCPkQnI!2vf;Ehwe&Fvl-xblZ_OW}$k^8@Bs
zL|%?vKNY63{7CNXsOjV$oyV9lOw-Op$o~{Fw$$%Hd&0iJY52n|2m_l*12(TH;{qiG
zD7BAs`<rjmlHRwR_(nJv#L}_bAnp@4i)QJQ69OVXSPD1hYW%X_U;Jv_nnddey;4x<
zWjozT$aT=`U@4v#j*1}?`BxhW!sSb4UDArRd_VF#SzJ(;&+m}m<(*%hKzUeE^?c@8
z-D+j479WI2$3Vn@xcPUSE9TG6#MOF@mUN;w?DWob{U1}6=bsr_KA1b`c_k?**TKYR
zSL7dA|JTtfHi?Yt$#iBrW}orm`kx;98B9mLB0!f4ZmhyQEQspKcQU(`vy^u-7`icM
zcC^l_u2)UVqP(gOx(Yz<?2t*kmIQ>EB^W4b&1d91_a-xj&z0$g{xHgFUcolVjn&je
zqm@5ll`NDKS5CGcmrRX3`wp6N9tRO0pt<IrUd=yuBxd>7DSsQUR!M(#+DYP7ee}gl
zv%UiQzQD(aA{$16(T`eK9=nzkH07x4_4<$brXv{;84*Ac(xs_I=V^t{MH6bIxN~xH
zp6i**#le-^%zo56`CTURE0OCT^CuM~D2!~UG9E~OI?Gbo#-P0>Ay*94H#=X__rdj3
z?E{P5*}OTMIOD4ZyJ!kDkq33*@X%WkmGwhwevMJk(DntR{)@JwB-*R|ZdI{L(pHp5
ztu~0mz72PI-IAF6>e)k{lqV)Y#30x9XlYG9b;SJ1PjzLlMkQQ;!$wk+<XPVpFIyqm
z-(Nj_p5A^ruZ#2Th0SV#B#3v8<rNCQ{A2sQGiJuT{y1*kjIL+J)frp0)^Gdg7f*V5
zUF&!3(x{Pc{r+Ph@~?%bAy{%BK{o1FJ1Op^6BYf0$p5Vd+ekQ2>fje4@?c;HO*UTr
zxZwxgOz&cu(Gccc!QTqjhnpr_A5n9*-z~Bz4f627MwD6~%VfX8*0~wo2*_-Cafr!e
zYsEnmN$4_%5Er<vpWF0C`j-~_5`&Rs_K}Eyl(au&YnBoj9<>FZK~JY5?t>Cj#_XOo
z7B{v~9$uc|Q83||lXB!S_qEac?|00s&l5OFN}cM9tiKitKAyC=T6Z;_x)k}-5(ukL
zs5{|V<sANx>~8rzVsK_pWOY6H-jes%T)4G84JA@Eq1#z9Px>`#j%+(fwhB{BMv$-F
zf#d-OM5RObpLAvMMio&9#Z*?p&=XFSO;Xl1hYsgZzR_x*G*SMzu6SUMeK;g!&QT_1
z;+*Z+&8z-82u3;;^GbM*nN(Weh~Az3<?_vxGbzwKL3v8bXxnTO`Ub-9&I4#ZO84lg
zbjUUdl*fDvXG?k4cGZdh3DY#?0F9#iM@m2!*{h4O29;amI8-leyy=LgfQ1CNnN?du
zh1RU2EaR<@kDzTx=$X_>zoBdXkJ)oeQ8myksH#sZfls`5WM1^_ZkJast?TeS2zJ?T
z!{6IyCQ$NJWigGjI?OQWdlI7|2@|D#E#8s*(%xuey=XCH*Bx{b*)iBq0INr`=BwEd
zDEU;`uKOa*U?%%NMx=--GPlN>w$F%~=%(hYNBI#^_Qd(k`shCnUPRS@ZW$5zDNO_s
zt!2SCX&<58lzsimy+<-FAlbNnH_rLqJfHuL{xZQ^nIklML#)4(P@<)k_nZYk#ND}G
twyQjX8PhxsA(}U(POh)U*kEhxb$4FdX@a6$7iit#3e~ed9Kjj<{{Ym6hN=Jn

delta 13027
zcmaiac{r4R+xMj@WSf+I8%3pTDMhxS1tn`mSu2$-B->cVrGz3TCKP2zLSiCIS+j?%
zVM>-}?E4Hej4{i5`Q7*ZJkPrv$NSH9G=EIy`hL&zvz=#RGS_f2*QZ0fwIt`YbOPV}
zu(A{^bUd6cL})s?q!_R@dOy^PN7p`mCYI(RG$!q2+hKMc`S?@E7n-e%Ff~h{3hTE!
z277<E2&k|1SBsgrfm;^@S(2g%uKLN~8V>Nuiv!Gl&xef$${ugC{T!x$W=J_Bc&qRG
zSA+|`);<d^N%8$Q;#*BF`#m4ga-RddJ<T@XBz(5wX-i`SPNxN|SFZQ}vA~6%zEU7c
zi>M~nDm16!?hbx*{c5yv?b&g#sws(X#q*tZeBN1=ex$(T;c^X9!P9p1lHfxfxK9uL
z#+DSYI-H6+Sn-OOc{uKhc0l0K4m7X6(vG9sKON*7A+J0>Suf(asbe1B@xf2AYqbS2
zh>=`hSTd9dq#_=(1!fsX%xZ5W*eu?7e5Y9nokp+!>lkLm(~}Z>9q<bN3jklSKmbA`
zi|>jR&wC~}2S^%QV>jSAK+b`j|9K`r7)u1OHr%!Skl_GNnZns9uyjXNO4i!9M#r|*
z4jjgB@qvC}Ru=qqjU5@|OkUrL7tzU0UaJ#jk>KlCJn6*Ga<sG|LXz+^LWO>l$N@xs
zzqm)VM2xULi~!d-Kt-+e=2kDyj}!vxGqJ9ocq+uLa_j1oxnLiDCB%|YA)PX74m{Ab
z;yMqA{PV!-3Z2IG@*O{Pd3e~-#j(xXYiHWN@As3=MZdy)l6Bv~N5k(9-&0d{yNaUH
z+OLMK7El3C7?nEct92+}v!lBrPz>w<&c6!?!98a4*#Eg#MLGHHD>nagWResIs2O#b
z)%aO}cgW)PR$2O}nmpV;ri@aMFeMp&$oa*W9?7Q!o0RospB;{6q0Klzwv<yC+gR3&
zT{p!6db<Ftw~_yMS%LqvuEuAxcEkZn|KqVUHw&&yacdZbJHq{QALcsCfdTbj{g>a=
z6=JgS=d$20YRvw2`8Q%WLvu4!&JooVHs4}r$9w;7*$s+|^*TG$*8b4w5r~4z#7SNY
z-4oi&(<ZuHr%G3F4Xaye!tx(c#vB?tbLkQqbCqS%AE|e=j8+eyuam7km0t9sqLc#&
zQC7FR;~2w=`wsS|fF{;Z;20Pbv5)S8rx)J&hFLtN+B57UlF-D8o*&QJ$P&`rVG7*N
zjI{g77NPq=ce*<|eQbQmuBmABo4_{?P^zO&>?zRXnjG6hKV9S%tri$Ld^M?IcwuXf
z2l{jH!FmOkGrip6F*VZJber*x{*R5OO>kXie>=^0_LzQQ5Om6yfVa@|p9gGGA7mS=
zTK@Zt)t`WqcLtFQt1c{oUhY5<uDSaEy>;n=96+IVh85+)0gTTZ%5eZ}1D`Z_c~J8W
z(*CoT{>5(PeCZDtzc$e1V#kWDc;e}mfBogd>iN6AJTQN;($fc(`XT1Ruo7&d`j2$2
z&fJo3QEtT>1L}ipTf{?rA#FuqmVK~%>r|1eY$K+s`Q}g7kW3t~d7u{A0`3YlS4Iy#
zS}OP>6rK#c;`y1wmM|FLnik!sSaQKv`Mk=#<o!(w0(;QBS-i7Y3fvU?Nt+kEe3rai
zMAGdkojY}W)YO1t9c?uK1BFNc?htBGCVw+C-Y-^DMB?)uh0a&qJ`+1$&5ruj%Y#L^
zG)T?FyYavOjgyzrAE;NkK;JNEIy_vL7sKnFZF=MBfv4P}!e>5U40$dBM5Zb^zNt>`
zh+jbLm0IWU&Pcs-<YJy>&&e??g-^mLJf4KylniuXurfHnS<SkG5G>@_asd2XJ@tg4
z<ct3=R6wPp2y%6!a=ul;<8L8{#UA?jJ+6wMN<p7q)=xYi5Wuw-8*i|8p-x8K&pq~X
ziOl9F{K_7%><uWKv=GNIh%`8Xe;<CGT;0SzGVV8pezf^@%e?C0mw!NlYCz>$I~iD~
zLiZ!VcYikX_g&A}gQ~9F^&Gw)dzFcz9*R7DR0+WP@74ng4#b%~wN=V-OX?OCxs9FO
z1(-Nz6V)H3zb`CZa7ilbyxfzVB{>2rG9RGxVO@(K)o9AO<t62*)4^LM&CxH}JL>Sg
zH79cDo`l6=^JhWx5o=s&;Ks{UcUWimL9%8=XT3;Q;^XTM^+*#xX4)FwEsL)z=#}fc
z>$#b4yhQoSs{#^FLVZ9R{D7rMAkJ(enmNGgNzZW_ON$-j5q=q}&$}fYz-0}hn$|3V
z4@$6~5is!kN(cwILEr#=N!Da(wn<pe7I!`SSzM$(8%vZ>0e8v^KRPa`3WN~i>xA5^
z{)QDP_@snBxF5gu3SPC{R-l4&ydgq~p4Lz`^LS>JW_COY`}m7x`VoT!mJqxgec9j3
zH|*Xz!OBE$uYlfztUEVnjTQ_MGImv{&wtTxrWf#nmn!__%s)22%wP>o;^z%sa1jG@
zKF-?XVpr^iJFk3B9kTA#CdPiqe5T(5xM4K}R6)C|MBmuqjriMkr4_3m*VdeRQZVG@
zwmCW}^0Ym}wb6w?97#l{2_n{`8Gl)C10^oA8>ub>5auVNEQxB>B=a;a(}2!>JsCR8
zW#n&*`lsKhU1)@%%t|zqKZ|~GKMUtBH6syL5aHoKS=5i&6>#N0l(fxDSVIvxOUt<W
zNuMEbfeo@G2I^Fy!fkSZnEWvZU})sS4#WY}@yPL2-7)ajNe)20ebwb}YPMS*eJH4=
zDsII5HpTeP{p^u;Nxl;!k?yeBP>5zvnJUd!oysV_UY=RtCORO<Jg#;hZ8Q)6QaHdx
zblGFD$K>s`5ik3L;|v)-X9usSHF*wjr<w&fn6A|&nJ}?kzU*4enh4kGAk-D|;MFB%
z<AL(Rsn0)Ea;x9eKRVgXQ=kuQ=vnQOT3^}C6s&w)M0h&0uio0try2U6LKwKLwt-%r
z6gt=#N_e__Ny_{gc9(XDWv2|dqVX54Ll_K74XO}p_j2<Utr32v=qG>GB};^BCUYNM
zJ!fv08m+>~X7+)?on~Hel&#(d$9Pgd0G=P-MqlO2$r72zW*<I+wUXrplbH}kDrA1;
z05_#E+h?aCK`E|6&L8xHw18BkM*cQUE$bbo*tw9qJ40@>`%0Yk;$w85gP&L?y^~>f
za6^O*-Hs&mkkowT^Vd}$N@_W7%8$A@9v%LrxbI8At&e%~QI7-xbczd-ZaHC5^6p;6
z(7=3Z6h`y4MbtsoasqvA#Smd=i;S(YdrN;i(DQEWT}uShWGV?ccKJJlnk5(qU5A<s
z>?Iw;0Tef&d&RplTXae4Oe_mJWcx^iYgi($^xZ=h6<=z;n1x+a%=8c%NK{n$fd*vK
z897Y`0=>*5zEg8So~6REU$E0D>>?I7PdN}`_qdLCYzzWcmDUcqw?lgcir8xp#A(Qj
z8#+LC&TOdj!`mv<LZzacvS);@F-qnwzZO!l`vr7B#36boMtWg<eJ_MfeF-MdIKX5u
zp?>i{r{?}Y$emv-xY8co;dFhZ(_7C?S<&KZ+U$9|EVv69*QWd=$ex}#-ed7-b=1(0
zy&8bDh6rCi3y!B2xtSAGQrFCC;+DDDN`JHs9tZOLB6R4SsxF{kjG*}jRJVKImAxn@
z5qfO438Df*;3Rt=ecUail+gSs{)pv{`8}7+*{iE|G{i-K78f@>Kej%sn{j<2b2s;7
z@*#nb`bLmevWJdVQ4E$&e+nHf2iQJa<6L^|KOeAoiGw?jC~>(%Bz5ZEk=<>FBE@QE
zTIwY=o%W%NF_l+qWPFTSjffz&`djv`NPTV+iQiO?H0@4|(fB%<^ymY|#`5B(OLOK4
zf}S1n)dy$sCo;J;po%q1obehL3KWqtUi#`?|LFKfJ9+m?Ub2?+v-igGm~TVo*gigS
zkb5Jh#;q7-1D-Iex&7>QZ4FM-25rZ9h6ruQf@jw#P%!6*8toT$X%_x$z3n^lb=`F%
zB4IOqb2`$XbEj}pf88xIt{L2ey7bO=1L0Z1lVSGgI=}&L(_ES`R_?I5`H7l?6x}BF
z(ceCw(J`zqTdEBJ1j#q6wAR+Mgj%t}dt{}Ad`aT}sxrAW5}f%&YK9e3znTS4$%oEF
zN(f$xU54i>g7a8bpCKG7!eii!p5;UjoNg@{$p3A_FudY9aSCb8L|(>t0<#M<a8afs
zh3CoOQjx>f(RK`@I@rJ!ZXA(E{x<Wvmj}{Xq~O?amD`meT}uo&omMHIqDSuPi+B?<
z>ou(dEb2K?)SW3FD8-D$pq&&CuFY#;MQG*fda=QOloEE6cfXpxB@x$*Mh7m)a<5J-
zt*0I59q7K0tfs4Fqn`-!P1Z?sfcCR};KC%RV2HR3VYsxk%uF+4bI-{h#u_+fMuD-<
z$@;Tv-m$N6trlS)no5hVpF_?oaDV{tD1@yOW9l0`t696TeI|#%QavM5k$B^JT|^OD
zMpDxB^DC&10!<EYT5;E+#N+Btc``D}>mK2Lr`GBXHC<|2=mUk&v#?*x3#6rejj7Pr
zP$egJxIT>VxD9;|Echn-9iO6ByzIN|JiSbEcshAH<j#X1Z39?CxoA@Tv*EGwX$SYs
z{(C*reV=tPK_YC#+f{w@ka9rz#0jxm7Y-|CQeQwldJJof(3bYE--DbqA|Mw!ET>E`
z;N+Qq{>kEwG`PAA8IlHV<s1NG3NEB`fTL!X^tCAVS3LA)+t+)*W(^ZBr#~33sHzf>
zPfB!XzN@9CCDH)n%X_vqFd1fR*Onm1Sp9QxK(*+0D!=uyFG(Uv61vA6dhds1!NqIq
zst0^B=hvqlUO}&L@WGF98_!y7pQ$c(QdSEA2}}%f@+|aoT3uK>`zCre_t1J6lr#=t
zq{+4VX8X*g)>YG(s6QNlaFBflYsmqg2UfEDEGLzAwERnFeS`dxP!tK+5aZRX{@G2p
z+wcL0EpSiTCY)G6cp(Yg=~cJ)b#d-p^RL=w`E~9%$|5>Z5N)Zc;drjMOUg|8*>Rmo
z;blnVdTxx1h41Sx*WYkPBnAAz_B{Xg68eRj%;Wm`uy2JfF64@GVg}~0?Wfc~evRLa
ztvSHxuA2>60^7)}A*<sX1F0J;staQi@x6~c&b}xY4`~~PO(}qp4)u}r{uw4DYz`cP
zSjPD{q<9X!nZRudpB!h9GdO@D^uh_-FD$bz27Q$8bYYNi4Bi~bTRp#;$GUIH+G(}x
z1N4Vf!2MNa6?GM1Ue%~#`R6w$QeevP`=J8^T$2VWbZOsXkbv)*Y$q)GmmJ{$Kt#^$
z?}Y(7Jmj?PD-!Z&)4JRWew%sd;J(VYn&j*AZphdTR&__eM?Gr!NORCG&en$|USSkK
z44(ymzA#jCoqT3}pq3Cbt@WIC3VZCjI7@zLc03V26seb*S5^L{^lMmHb%^TpGa@O4
zuM~GbM1%`#EuaL|-Dw1UhJDUFm|b%SvV?+i+K?RC=LE@-@NotgWahR3RZ15ADFDB@
z^EybV0%s3kO`*3vnZ^oh;{aK4kUMYSJ9V4;@RJ%uHvRp0$KX91TkFRu5!d(4HoXoD
zSTCvqHaA*8?vq6l`rUC|ABGbz9wpsIEOP*HCJ)Y&!53+8^3LsBANf9M%O{>kpC{GF
z)*&djy$?>|u_2h$c>xt1<}TA6Hnp9F$e%3~vlS0b=(qp%$IM@s^^0`o2gqe8Jr233
z$?VyB+A1;$e}RrB$E;l#@D8l?nLX?0Gj`kd`SV}MrAo_h5dmBaWsrf8O5VRq@;m`t
zL};-i-6?qHli>|kR31c-cOwm~w)^dpL49BS<nEhL8T3JAaG=fyf@sVhlzFmp7^eJJ
zVny1{QZ9D;e0{83Rxi)b!4|qDDJ{0fx)i3hj$XTgPM?Y#0e|@<Fc6-=5`;dd`FH9c
zf#3%6D9h>CV+yjM<?e8G0`qd|-?nvr?uIS%M%+vAQ>_D4j>^^gRx?pp<g>;%;5OF9
z8g-et9z{I3_xhiZ%CP$c|L}f<V~{nK>2D}!O+-Fo)D=yQdOaJyQ!vxf>gGqC#bFEv
zSEyXrEV`Qsc@X~_k_AfxtZA4U2UyXV94h};Hi0GZNg{5d2<&s-VeE#V&^vxMn{e6O
zdvbh>2kPLr!;uD-&_T$9x){E2JBoNrg+r(MdP*Dz=;V*leF2uTOeT_+9Q6UT3DNgS
zrX1zg$Dp(`{Eer}aC{sf_AXO-s`uh^Bl*O~Ry@B0*V>{DG#1h&&AhGN*#7X_8fyNo
zkE`$dOcUXv>fb=QwL&gT9-cb}J0=<C5pn(Ud@VaiVJuQ>HVeB7R~6!27E;YcE+VGP
zYqU#WV@DJvKo9pSu{;w0U}^v2yQ&v7UTvfKwzn96L&@^xI$<bIxMj1_{Gf>1l;kaW
z;Gn@wVMNq|q1;s>ZkJb`^6Nvd?p~57F5lj~Wmy=^5P-_??bw*~wY~f(8;Kl=a|!=(
zp^$;XoFP-p3J0a$kY~a>x5~CnX!Fb;QLb^QjE?N*KAeD@G=b!lEF?nT4r3%a!1p9P
z`0LaJ2RL62guj8h<1o1Mlcj1|yXM{+o9qr94cp7}ymQu8Jo%{06L$;V-zz6fcw3D4
z&8UgpMznJP8ARwR17Q3yq8KEQ9DI=%>JtBQtsS~Q$6n1mAD-hD7MM4Z+m-U8;%9b8
z`@WQ$^f+K^Z;fT6i;P=z^3Rq%n-uKBupN;L>{TK7fa|O~?2q)()Rd1N!=veA9>JeR
zzk!K|y5ZTfM(cXLyN0(#6}Y1=%P-Fnf^TuQaYn&w4?B?70C|snh6uU7?Lf_n+k)p%
zBp(&zWy%f(wwe{)IF)wWRM!ba%dcPFh$w>4b8XAaEMx0MMOkGsDMV<m((}(6>pSW`
zkY}Q(NWMQII#<(Me`qLntz9@gSf6L<39OFq4u^li-GQ9MDHRG$?EWz|a4aHf_5>jx
z@14b`W1H}PHpt8E{q^%+J2=2juV*-$Q0#I@;lMUw9+1d90RfeQGXzxLZR~hYm;dwJ
zGoW+VxW=7`DfUbkB-ixgbIV@ew?EMYKcgeOi2C^V`2EwGTvUAHilnEUMxW2!fwxcZ
zmcKaXEN{#{k*tfzy~`BsEjO1oszYDwa5x)yyv9sv_(yi=L_UN>{o9~;z$&lswL77W
z{bmvZPID~>?cQ=22e9Y|XAeah7;Wc4u>J^chbCmAo5!sSQqnxAPPL#@^j4GX1_guZ
zcmL1aN6W(6(Xqh;HsaVNbFB6Fs%gVjlY+y!z#UW%`&<Ilrj-bWYGHQ`ni6#>!=k1y
zKs(~k`b+2CJ~(Bmm|YfqgP6maJP_z#eDNA`llfc_?kIuEpU8PlE_+ZP-}U}sCAWOt
zy87+tN91o=2lB*-6%WGY65Wy*99yGa3LakC=-kfEWb;_jv-JfVS#UWrz7Z#kdXd`<
z$<4dJDw&mEE`N>PIx@i4mZMQ~fd_SB^g!YY-_dq2ul4wbq2>FMYwVsoOr#9q>c5iM
zUD$ki`Exk@YgCwxL25DtGu#;@1&p9)Ec^BJHst`}@!M%BPcD-Ox>C{#y8$}F@3VAa
z$m?|xSW)k`BZgj{UQMXq)IrC*cmoA@xxKb=q=_Z1FPq--^7Tw!sx@t<R|=S4m}UjH
zWvbAl9t;P*wrTO4K-@0t=9^*LYx&W>t!Ke4DQ>NTJVtvRJbs)sBPLZGV)Y}|AsF)i
zS?(9g)1;3yk90xB+<w*T<6h|<gDhaH_oCflJLXqf>-VWe{ctJ*Vc|I}r!p7I%NziG
z(q{oDqEa?amF4a~-F(*lIkQAfXBRA=nGegPs5ez(;Im%&s=umk%Ucx6?SPGhMuw$B
z&EQ3*S2r!LwPxPuR(@I1YPx_qgT&_oD5D~rC@R8N$m_xHx=SWA(UgtnR46EE{I{|F
zCxpc^;3A+h*yPMl2)8rnQtL{WzrV3JyaZ~htoHRR0Y1jQMjUs56XH(y=gO<qGK-L@
z{&%9z2dt_Kgd0%A<2q^+UM2p@^vT;)K%5<{2fXG_UOms3bu=;3iid}2`PcwKX8(zw
zl**l*319ec#2f#2tc|7f9X)r4{&J#m7%GaqBlT3+1yT-Xw-<s&L_)m?WsBR9G5jKZ
z-}V)@v7)v+Wa2#&qJ^Y+aEk}Rxa?d2y&IdYbO^M(GDu~Kb>UCyelNP=GLce(oz=Je
zl*OMyicq5yHUBI!)!|sVW4>;VkRtQbffw{RKZ?{hYj#%c3s?1;nXfDmzWL&5ga<^e
zkdj~cIY8(cMn*x&iP#7SaQEEXDPzOz3JfsQ!x@Gk6L$gLsJsceO8n{4-W;rXr9W6E
z(d~x)bAPLzEQen3cu&SOy*#C~rhYvH;CaK&dyDxF6~JrqO5;{_Z>9ZYg4e$AMv>Dg
z^<(<m-n*5e-Yn?gqAGCd-n|}X$<uBJ*7OQ>&QP{Hhr|H{>1qRWRp+bBj|axVR-@)u
zx-F-@+T39-P85qG(vf%<N1QT<R1s#D;ECh&4?JTsy^XVdd{Z(BKU}o;Hz;TN(XDn4
z8^nXf<Eojs-N|R$4?{_-<xLmExr!h$TkE#>4I;Kv!SYY;8?9tgxqhIe8?@xa)59_f
z8&p2aTdExh;(9$fQ$S^;FS^6hC=e-wm1W%Smjg$$kDqT-K|TeA8S)RE&I92;jCBY%
zDbh{gui?I$%1idndz)a7BPJNVJO~I!#T>k54J3xb&HN`lPVIHvbAQ#97Y8Lfc75Z5
zIoImsW*e@kE!=N|6)2_yQ#D7(W@bNw9O;kF4-}gioSC|LmN`!OwvM;T;{E&RABs(u
zcz#pi-xQmS&EBnjPJ`*__jmLj2N=-#hc^#Dv(VdDjAO)%fU^}@aO^*|PsKb0SM_@!
zLADp1^)RPHkgqU_qfH!UKYqbQJqZ~NIq<%LD!n?39)>Q*3R=+prW(hq&$53~ncN^B
zu$=cLW{4+~bUQ|=)%k+N(R`XB6`!t7s}~QyIPh4g-+b@uP2q@T-r62|$)QMn@s@h&
zkN5b8Q4d7xUf%b4_x-8mCupF+7}APmi_NVYI5Ns`_uhnPs0+jA$MgClPn&AigJR+5
zYuUYZUy@DN9{Sx6YR%~r0p&Fo;;5jcCl<=buudMt1Ifpe+E$;*D+)n5cl&2DK3Y$F
zhqGs;aHsa1Y8!rw_XNv4M0S)m$XwDNZN_%A9ej7o96%p<Rvh>J!N=pg)clY26RqRy
zuj;EJTod(sm?sR<@a2<A{SKbsUhy;Brq5UH)+W*D!r#9|<5I0IzJ6qK5ul6<g&!ZF
zosezK%aFEtU>&=14ULSYFbjXp!k!OC$jzw|tK$XLx(<BRMoqb<s_5)!E19%&F+^yZ
zi#)$X{*fS)RAOf=anxj|u+w-85-zQ39{5s(SMB?CzhOfiwtPfLmJo&r;+{?n1eEv9
zXjhe8F2hY=^dc0rnT4HYs?Of<YZU`G+mF><H)^G^4>HdKXi=-{&PMsy&xYJ15?{e`
z;Aai@^%+WL?(f@ApoV-r^iAz+0tfgqikY#8!g%kc*9`RHP_V|!>FBts%x1gXa2s-4
z#va1b<>_ZTzgp?w@3LcelC$7ykZ9c}3!BHq*00LD<7m^k4%}b$%{^2kg92GSBN47%
zC`Ql%uh`kyuG{uIj#|p>G1k&cK6cf5{zBGnul5=TN?rGWeqPK?@yh{P{zruITZM!5
zGVmfGh^=2&UP`toSucE<wf6CS98x24Y)N_-^u?2FYWN~g`>8c&&*0M0ZZOW6Xp$uh
ztjkz&qqhf=atUNDwh0uM<0X;QgYW3FkYQ0+0|_s1UH=YoMZoU_rgaYL7GX)MoO*wU
zfhnV!161W;4W(#n93bA{#DY|Nn7N>|S-Fp1%jq|5TTPIM`W6DQ^fii0vp%fRv_05w
zX3TY}EK2b3#BUP<xeJAhjga;XtyUfS&~wM9%}?NjGMguFfPDp65oQ`wOcIubaGn@%
z?;}Smf<nN9Z8?>;ES~bfV4b|^2swNG+Sg_WCxmV#R9>86Rj`HGhFJnQp>Wmn2|hJv
z4<X|hOHKiZB_ur}u#Fzy8e0G4S8j8id2xgT>M1{+1E5&oMPNhlv?hHqDWIiNY`ke5
zU5ym*bLPoL#oLf+1YF*Th~Ngv$C{l3Ax$qh*ntWr$zh7C9KM^U{JF32g832r?MrFM
z;$~Vv)NyozT?JXrY@2XPZL;Ht+X-U>D2R%?!U$LP%l~tR0|Z@2aY$@_Xoxuf8*GcN
zXFYJ4JbL?lf|#cCRbxOl8c^N7h+QIFmUch)4{2t+F#x~n-SL06q>uR<Quu-kT=S;A
zW&~1)f{&EC%j97EoUs&_rup8p#SYD|{j1A-iCL^6BnI&$i`PJe{<OK@T}z1X*l~wc
zy{J(WV$WZ%o_oFrf39TJJSLec3e{Wr8zL@T738;za_U>6NoZeBn|5Ti$_Ue?fO@0-
zMv8y=fO{n-dg=oCa;HTK3k708p8S*F>G7mfX*s?z1lI9czjAm-9>ii$oa8Z~4hI{x
z@E6=4m)Au7h|H{pyDsx8kb6RNioMe6<Ok9jn+*?6eZY^Nr;2dl{xg35uR%20>96mm
zu%`MJPZMO?`M^hzT{h_j7iE1R*D@4&Y8DECu~fJU_+>>b+=9#jqP70iykBRbwp73F
zEZ_j%RqESG<G1JB=HV@*o~O|8IbM>KIk8xwNb&HwSb$iPD}sjKyMWN1c(#Unl|-16
zd~rdjc8Auo)z+u2@$<}<aEhVy*g;C_SJGEzmklNJ*n`wLXcwRlENI{ODE03dGq@c_
zQ!O8kuOhWrAs3iK8DtaoMJNizUjQM=Ao|}gX5FP{+gtXtBgr^QWqDgBABt4Ot5AGf
zyL2s%+oM8H4yf^P6Ib`2T^fa&e%T$?N7J`$GA<44y11--ZC~N{Uy@Y0yy3dLtIW$3
zerv6L7fw#VqCmyid)9EHUuYmpXrl%4Vv-xb>GrBo*|l#@bAZVGb2Q(Mw|x-^_c6K1
zd8vWEk(#sqnt8$|bh&!Wl%b4m3ei^>)i^tDvHIf06khG+o2EZ{m8-?^%`T#6V#>)(
z?KXsG3P=^@0EmvrQ$=M+e$<XWU*C7v)xenR#xt(qZ2TYQB;mR9xYN9rs$ZDj!RD^P
zNeKRf3>4{a@;x6Gm6eC_Ke+Fxwc?028D%1MddM@8C(&WIyK%0cZ+@{n$|ABmXh`Q5
zTt7Zs(0piZZ{i;bv>&QnX=3l`*VC2CaPT|$9dYtUlp5M^4l0#zC^sT{D0{Ed!e0H?
z=j*|V5%m>`Fzg1XLMZCxc|lJj-f5}WDiZ8!b~68rdt*|<R?l-O3Wc0)pNfr<^7*5h
z5B4`HLlfebu<@tiCvv?)!<Pa*9|b-uwTKdWCj`w>6981%*ulYKP3xSVp#I?<ympcb
zvfQDhQi`EaoL$V7)DyAB0(m2LUf$&X70`;@RZ-v?P^4E95;NMW6$+pu%D8@_d`ZzU
zSqc?HBG(-=_4?Qa7<MO}vf_L>%r!XAvAle#=J};soAZ!orCIUxD=i%0F&6lET++e_
zzk|FIU+yA!F```wB{8)$IVC+ei*5nJXb;AuDQqlhN9tlk8)|&D)s~3aQXL1W(D-}c
zAJ5zVH*tEm{3}@7fs{K6@=;f4XkUxoCG*zo2hZ{30^|vtp-wum7Uugnt>?q{O0A@}
z8_D6tHSGu1zs<sZLPMoun_WbF@l6hMy!W#7Lf)|{vFuJm+1%D5A|CK)Y&|vHG%ZG{
z*p$hfIs9y*pau0TS+7)6Az4?qr1OMKvO6q~g20qJg`_L`d$?QfY5t`MmW2^i4Y^K3
z{3a%EP)c@VK9z#AqP!uaZr@MWc5SfVL;d;Br(W(ZNDUnXZG9?9g33)5cUNp{6Niz+
zkq9?~xSZsO7hJi{qNJ6CN<zbu-syeU8hS@Ba;3z6f;=C?il-2j(OM<`_^*1w7?*~m
znR}gw%WQ339&D`6?#q|-`C#aA;r;XF;H3By2pda#U<FEtqMQjoQW^fxSn=BHgc7JY
z!7^4IcYI>)LXSJPk5>eQ^+xOl+YMb9)HH@Q*^GUW4<vl7?~gQyhn(_s_|*v(55!+#
z4-Qth+72OJ^}GkG^t*t7!8NxygF|#|>t1PT^ia@)H-#Ra7d&`!v}_pya8H1S5f8uK
z8-B8GsIN6IqDCUyHwnIF`I9VZ@CC0<CN%M9e5;h|ly<iS4K5;Hvn-7mILJ(U$sMBD
zH7nC9@1_QH+w_kHHD^wOwucv-9~OC4X*DhI7I|y_th2i0*Ia$p_1Wjlyfsa(B;rbR
z5cz|nS8_;y;Fg{VFLDeTT|B&21vay+nde1Yv#hRZ{3074+k+tW*qFF<N9x}l9_H5;
z<9D}#<Su9}0wDQ9b<CvNrMXSGQXYPn-^k({uE^lV=Z<)6z%;s}FD%1v8N_l!UDs`2
zm*$`2iJjXp+<o)j-dc$uh2h%6q`a8Psz<612DLQ#OaBU0>uM3dGff~1J}y{PR$agE
zNuW}ytp?Ty-;pM>s{7hawClRFqX!|$+5Wq0Iw&UaOc#o{O$7vmcHKewI#*KU5=5CW
zFGT7!NMlXpvvYp+au@xZAO%;Ls1cBGh<ybLSlSZWSPy)s3=yZSq0uKwmwjGPnO%$B
zwEnxYxsQrxtXobQiqX(gO^&5r=4VFl7|;5x?WggSK(ptGF-7{F{JDO~05Ce4LGUr1
z_;Go}wVvqw?MHs@t4H~gim$m05&G_Ra)W`LC6E+ZG-@(6x`LS>t$!05`y@hfPUlR!
zZ|>vTs-A<|36=#ukcF8slulRuI40A3^<;v}lAP7S_b>j+Q`;voMX1!zBw0)6FKdDP
z@1Mg~P5SoDD#2<g@RmE4#}EHG^F%j5+(w9}s$hZ6Hm+%mRB`bXPfzJ|bRM~&_c6o`
zG9Qc&(9nYqA&2%RT4Q);oBA8!EU_{UAn*+3Z~;M`g0mx)U>4g$0`N%jf|Pg=--qKD
z^p&^<r=e}JpYJ?$mjX^YS<L!P&+N%&(0rRPp#asPbHQBa_cGq+R>ebZ`wNaoG0lXI
z2ziR_XHr;_U!jZMmSy{%O;-$SPzH{-2WaZz;rHY8*CWK^fB3|#-`O<TdOAK2tx6m*
z6ff6zu^Pflt=ZJ13d{%1uU6dG=gC?ZHO3~%$bzMGm*Q)vu$J**`*$RM)Y9#pONu4(
zHvk>1V@jzaLF2sQBgJ68{$t;*EH4|%VM#p+I(5ZzKhK<fKU^v2<FAX!CnAN=3;vem
z`WU9DD=Rm=Q!(jb#w$V)e2c#uDUc43^pNB=JA$K=r@(s9a<KS|yCw`6v&$WoK}u->
zyS>Zy3{e8SVy$cIM`dH{U2=h7DYXRRxX`6kG3PU5rMF+nNWKouq%&^VJ<ceM`Ko0c
zA~01MQr*4Cm??%voEQHxMJar$R%hRPc9qiKlyp5)se48Bg|3@@9MU69#Rx=d|1pD~
zh!y$6TTPWD?V@2|^R4OF5xFD6$>+d>*RS1pq!Z;tXwe6*&G*fdj|e+c)MZp6LF=Bs
z?ywyC9g<&xM~=xC<eZqtRt1}h<{{~zHJtAdmz=1oK?G~7i_IKB2Sd8tfJgz*aI0ab
z2GB!vwCs`TU90a+Uo?S*3{>$Fv{?mfIW7{YKk!W02HH`~Jlva0UjC{NK7QriFv^dc
zINJ-YBP|PtrZwZ>;5~Pjdk_8nUDG6FO2IvB6S5{(GgFfSU$>}c;nWOz^7p(tERA@R
zN9jfiJ9-1?6OMV&;jXS>ME8}G(!a7Ze}x3gxkg@)m6o+IK{`UvZ0GDoa)~P}FA6t!
zj2&9OI>(;JO@l{)6|mgf%d$_^yc=StJo2N6yX&w5JWjowzsq8B#Kp1QPZ<<Rz;S@9
z{}K}52NTuzuMoeAaLUvBwuTVS@sNne5$gfZVGfYycR*VA&`kI92TiP$`G{I2*sB;x
zpM$116W%3t51x}$K?8V?Ys@3$)!zD!N`Wu!x--~YOrA+4m}}$E&)+Cli;AyjW%|B-
zm><2uuuG+5NEQW!F_k5US_f#n<25&zN9*|jDnb~wa-kP?b{hSD(QmRf6yZNyb1b<T
zJJ1Vlcs|i{D?~-U$aEF`s%IqJ$Tp}?xUn6#%kmH+5EA6+m-;GDzS6r3c}}9Yj56nq
zU_VmYL4!C?1oV$MLYa;tDm@ucYZ~?Luy0y39eagn)bCb;y~AfP9Vl*26s0h)U>)y=
z6YUS{f*}KQnJOXoH4E-Q@r8^n-348Bft3?cgU%d{5rMY3SmJZ->-cmfSUyA=f?+Mi
z?O}ma_HN2v+_oBXnTrwSoxT6H?rFt^t>Kded5TfeS1T!Q(Qdsb4=AScGC#k*lfxF1
zn@2IZR-6NgPJv8){Gahh;U5>@m{5#EIlvkHZ&`d&(a#v(%yl~@+iNm8JM?u~aC;~K
zd>SF=WOpgHSt|6Yy;9ptSIM;ldNE975)@}1`&5!)Qy0@Q0@GX;=F)CIDRR7hXS*hN
z$;R~8@lT78?f?A2!>&qo>-zYaTdz*H-O}9?7Drof*Tg(wOU8tK>F7axUw#P}c{wB3
zR;&a|C%ZI<5Z?r_Lj4tm{A4?6&lnoZ`dgryWpk$3fZmDVDW^m9YwTqr*Dnt67g|R9
zLrc*4)2t)fl1U%aeZm2P<(q)8V@S|`Red*Vb*m}E`!2%c(<u8R4Kq-{egiFNbvfHx
z-V8aJ`{VBE><5W?C8D#4J=ALNyNz(sxt-?wl+0o*dz0pM5RH)sp+%j`Uu%#Lj7Kz6
zyH*C%HgVlGYnFAmOZAV|e^+$EVlNvaOox19X)gm=Qi8^!yQkrAwA@^U-bR9&Prf%y
zLY`Z4qDN3Gw!BqC-a+}T^El5k15d<#kE4O&^uX{7+3oJ{R2vb;dGujIZNI2l@Ev$t
z2x*%5W%YMW*B`f-*OD-5zQCsZD+GSnTcccvio0~p0jDIRxw86}ZBW-(3wNetb7v{&
zSfd!-v}fM7Min9q*SmOPTP<*{UvD=NA`b`tC1>~IgGQ;Tf1?xNrJ41KUL7=Ve;b0D
z9_!59^v#VcuaB~Lj0|OGp0JPljCyCaUJaFgWrw3fap~nvyL79a1iLgXf}mgEi7w^9
z;M0e)|Fy6K!xBxE!Cz9AQu@!Sm6+{hgQe|iUM}}M56Xq^{`c1Ihk2=#)D-uP(C7i%
zPlv~n?f8(a04~oOua<EcRPxWp-Ys-h0LvP_=GzgeZ{kEz!w?Hy%H|1ue~xT@uz+U4
zqt@;C7rqdd*eosMm+yT2@|@X!vE^kiPYzwBa>BXb!wTeVyx%*>=}pUbPi!70Vpp7<
z4ELI;U_1Id74#CJl?X+gji=5(i(K-;d>{>sX3I;(yitn|C60~y^F{=pLn+fig(YW?
zcb@*UjH0@1A3sHSwOq~@rm@L{p}3x1HZnGFbJ5hmdh@R<(=;@~5z|<!+nRpcS(O~F
zlkzA8+x!T1|M-u0;hnT~nms_OlR-V7W&A36)AL*1^o*_|u5tV)WxZH8I}82}JSnU5
z)3zkJV_jHf@3Z5$X??+)D0p*hv;E#zzoa`|r@AmCs&8`yG}nEDYWim9EQ9c>2&yVk
zeY6$fUaJLSEOWyyzy1C!F*_xOFX=gypN{WhO_kRSkw;9QcWbEJOeDyWs>h|6r;~ZG
zHoe*ELi}d;YA=tP_$>`b>djYy<AUM3gC`q5k1Agqz90O({sd;-ga)nG)kjw?4%rj?
zFQ!%2wYMRw19vVn4I7yH%Ac)?ma#$MDdGaPs1Rs(n`@H~SSV}>zd(_`I#g-oZk~|M
z7D4L5gV<0u&fn~e?AMef7otm-P^-7}palbuYEXwf4~$TF;%UCZ!ln^FPbZEeRcqXM
z4eII>ZuVkJ_;f&Yg1d$rFEs&7G1&ja3M^IPqFQk|s>esDy30wad)*r!4qj-8;ojWC
z0m@iNF6=@aur!^njI#HbG|e$e4ZiUqX8`M>LeFWlUx}YNY*Vd#PsnCSJ@p_{7v~RU
z7<kBWr2C{h|7_|j&Uv7%1y9<6@L8eX2umm10|NvJJ(EIu;@Xl<IknOTwi=^-MJ_Jq
zA>Gyq(qu@GCpY{EMd-I;Z!sc<+bldYmY&972hIf9VXrx#4fgqSC%UO%Lxii3X++-a
zWtu1^9a8l@p>j<I#x$cNdgG|xXNJey*nB3DA36^_RXEJP{#_@EfoYA@t8S<}U?5Gf
znH#9BiRyf1#NIbfUT_+VCKJs1a<mw%W;<1hc3V*Qre$%&#zb2=5Zq^qJ(t|tkTh`o
z)3RX5%+m;^KMv=HG7;7)U;eDeI3572ba8luJ+K_8@7!ckuB8**ZzM5$Z516kZyvq@
z7C6tR9U?L2&7N9PPIntbRT2$l2~X;dj*M_u_L5HZQML90ju0R)*rGEpfX$io>EKKf
zzg?=`X}6$u*JY3GjUgjrE}D3p!G3o+;b(1Oelt!FTi!`{=vM|=)0ih*)S=~~9~TrS
zv15_CI5Flm0@EVdzR~#AjlWSlh0L_u_K!a@`Sgv({l)&Dj{Iw)aAJ;mWWoz0U_bKh
zQ^>;pQ3=(^gmHr`%BG!$q15n9+)3X(U3KiqvKv$TEmQW*A`7NVw{4vs495BWzW~6f
B_Xhv~

diff --git a/javascript/loader.js b/javascript/loader.js
index a280f3cf5..f32c7d294 100644
--- a/javascript/loader.js
+++ b/javascript/loader.js
@@ -34,9 +34,10 @@ async function createSplash() {
     </div>`;
   document.body.insertAdjacentHTML('beforeend', splash);
   await preloadImages();
-  const imgElement = `<div class="splash-img" alt="logo" style="background-image: url(file=html/logo-bg-${dark ? 'dark' : 'light'}.jpg), url(file=html/logo-bg-${num}.jpg); background-blend-mode: ${dark ? 'darken' : 'lighten'}"></div>`;
+  const imgElement = `<div class="splash-img" alt="logo" style="background-image: url(file=html/logo-bg-${dark ? 'dark' : 'light'}.jpg), url(file=html/logo-bg-${num}.jpg); background-blend-mode: ${dark ? 'multiply' : 'lighten'}"></div>`;
   document.getElementById('splash').insertAdjacentHTML('afterbegin', imgElement);
 }
+
 async function removeSplash() {
   const splash = document.getElementById('splash');
   if (splash) splash.remove();
diff --git a/javascript/script.js b/javascript/script.js
index 6ad77aa26..d9e35376f 100644
--- a/javascript/script.js
+++ b/javascript/script.js
@@ -75,7 +75,7 @@ let executedOnLoaded = false;
 
 document.addEventListener('DOMContentLoaded', () => {
   const mutationObserver = new MutationObserver((m) => {
-    if (!executedOnLoaded && gradioApp().querySelector('#txt2img_prompt')) {
+    if (!executedOnLoaded && gradioApp().getElementById('txt2img_prompt')) {
       executedOnLoaded = true;
       executeCallbacks(uiLoadedCallbacks);
     }
diff --git a/javascript/setHints.js b/javascript/setHints.js
index 7398c7a5e..1ce221723 100644
--- a/javascript/setHints.js
+++ b/javascript/setHints.js
@@ -51,7 +51,7 @@ async function setHints() {
   if (localeData.data.length === 0) {
     const res = await fetch('/file=html/locale_en.json');
     const json = await res.json();
-    localeData.data = Object.values(json).flat();
+    localeData.data = Object.values(json).flat().filter((e) => e.hint.length > 0);
     for (const e of localeData.data) e.label = e.label.toLowerCase().trim();
   }
   const elements = [
diff --git a/javascript/style.css b/javascript/style.css
index 29c7b7cb2..78f63f8fa 100644
--- a/javascript/style.css
+++ b/javascript/style.css
@@ -230,7 +230,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
 .extra-networks .description { flex: 3; }
 .extra-networks .tab-nav > button { margin-right: 0; height: 24px; padding: 2px 4px 2px 4px; }
 .extra-networks-tab { padding: 0 !important; }
-.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: max(20%, 120px); padding-top: 0.5em; }
+.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: max(15%, 120px); padding-top: 0.5em; }
 .extra-networks-page { display: flex }
 .extra-networks .custom-button { width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 16px; text-indent: -8px; box-shadow: none; line-break: auto; }
 .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) }
@@ -294,8 +294,8 @@ div.controlnet_main_options { display: grid; grid-template-columns: 1fr 1fr; gri
 /* Workaround for Gradio dropdowns capturing clicks during and after fadeout */
 .gradio-dropdown > label > div > div:first-child:not(.showOptions) ~ ul.options { pointer-events: none; }
 
-.splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 100; display: block; text-align: center; }
-.splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: move 5s infinite alternate; }
+.splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; }
+.splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: color 10s infinite alternate; }
 .loading { color: white; position: absolute; top: 20%; left: 50%; transform: translateX(-50%); }
 .loader { width: 300px; height: 300px; border: var(--spacing-md) solid transparent; border-radius: 50%; border-top: var(--spacing-md) solid var(--primary-600); animation: spin 4s linear infinite; position: relative; }
 .loader::before, .loader::after { content: ""; position: absolute; top: 6px; bottom: 6px; left: 6px; right: 6px; border-radius: 50%; border: var(--spacing-md) solid transparent; }
@@ -311,3 +311,8 @@ div.controlnet_main_options { display: grid; grid-template-columns: 1fr 1fr; gri
   from { transform: rotate(0deg); }
   to { transform: rotate(360deg); }
 }
+
+@keyframes color {
+  from { filter: hue-rotate(0deg) }
+  to { filter: hue-rotate(360deg) }
+}
diff --git a/modules/shared.py b/modules/shared.py
index 8dd49648d..9897ed3fc 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -376,24 +376,26 @@ options_templates.update(options_section(('sd', "Execution & Models"), {
     "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on server start"),
     "sd_model_checkpoint": OptionInfo(default_checkpoint, "Base model", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints),
     "sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
-    "sd_checkpoint_cache": OptionInfo(0, "Number of cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
-    "sd_vae_checkpoint_cache": OptionInfo(0, "Number of cached VAEs", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
     "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list),
     "sd_model_dict": OptionInfo('None', "Use baseline data from a different model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
     "stream_load": OptionInfo(False, "Load models using stream loading method"),
-    "model_reuse_dict": OptionInfo(False, "When loading models attempt to reuse previous model dictionary"),
+    "model_reuse_dict": OptionInfo(False, "When loading models attempt to reuse previous model dictionary", gr.Checkbox, {"visible": False}),
     "prompt_attention": OptionInfo("Full parser", "Prompt attention parser", gr.Radio, lambda: {"choices": ["Full parser", "Compel parser", "A1111 parser", "Fixed attention"] }),
     "prompt_mean_norm": OptionInfo(True, "Prompt attention mean normalization"),
     "comma_padding_backtrack": OptionInfo(20, "Prompt padding for long prompts", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
+    "sd_checkpoint_cache": OptionInfo(0, "Number of cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
+    "sd_vae_checkpoint_cache": OptionInfo(0, "Number of cached VAEs", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}),
     "sd_disable_ckpt": OptionInfo(False, "Disallow usage of models in ckpt format"),
 }))
 
 options_templates.update(options_section(('optimizations', "Optimizations"), {
     "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Cross-attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention() }),
     "cross_attention_options": OptionInfo([], "Cross-attention advanced options", gr.CheckboxGroup, lambda: {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
-    "sub_quad_q_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}),
-    "sub_quad_kv_chunk_size": OptionInfo(512, "Sub-quadratic cross-attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),
-    "sub_quad_chunk_threshold": OptionInfo(80, "Sub-quadratic cross-attention chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}),
+    "sub_quad_sep": OptionInfo("<h2>Sub-quadratic options</h2>", "", gr.HTML),
+    "sub_quad_q_chunk_size": OptionInfo(512, "cross-attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8}),
+    "sub_quad_kv_chunk_size": OptionInfo(512, "cross-attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8}),
+    "sub_quad_chunk_threshold": OptionInfo(80, "cross-attention chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}),
+    "token_merging_sep": OptionInfo("<h2>Token Merging</h2>", "", gr.HTML),
     "token_merging_ratio": OptionInfo(0.0, "Token merging ratio", gr.Slider, {"minimum": 0.0, "maximum": 0.9, "step": 0.1}),
     "token_merging_ratio_img2img": OptionInfo(0.0, "Token merging ratio for img2img", gr.Slider, {"minimum": 0.0, "maximum": 0.9, "step": 0.1}),
     "token_merging_ratio_hr": OptionInfo(0.0, "Token merging ratio for hires pass", gr.Slider, {"minimum": 0.0, "maximum": 0.9, "step": 0.1}),
@@ -529,19 +531,19 @@ options_templates.update(options_section(('saving-paths', "Image Naming & Paths"
 options_templates.update(options_section(('ui', "User Interface"), {
     "gradio_theme": OptionInfo("black-teal", "UI theme", gr.Dropdown, lambda: {"choices": list_themes()}, refresh=refresh_themes),
     "theme_style": OptionInfo("Auto", "Theme mode", gr.Radio, {"choices": ["Auto", "Dark", "Light"]}),
-    "tooltips": OptionInfo("UI Tooltips", "UI tooltips", gr.Radio, {"choices": ["None", "Browser default", "UI tooltips"]}),
-    "return_grid": OptionInfo(True, "Show grid in results for web"),
-    "return_mask": OptionInfo(False, "For inpainting, include the greyscale mask in results for web"),
-    "return_mask_composite": OptionInfo(False, "For inpainting, include masked composite in results for web"),
+    "tooltips": OptionInfo("UI Tooltips", "UI tooltips", gr.Radio, {"choices": ["None", "Browser default", "UI tooltips"], "visible": False}),
+    "return_grid": OptionInfo(True, "Show grid in results"),
+    "return_mask": OptionInfo(False, "For inpainting, include the greyscale mask in results"),
+    "return_mask_composite": OptionInfo(False, "For inpainting, include masked composite in results"),
     "disable_weights_auto_swap": OptionInfo(True, "Do not change selected model when reading generation parameters"),
     "send_seed": OptionInfo(True, "Send seed when sending prompt or image to other interface"),
     "send_size": OptionInfo(True, "Send size when sending prompt or image to another interface"),
     "font": OptionInfo("", "Font for image grids that have text"),
-    "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001}),
-    "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing <extra networks:0.9>", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001}),
-    "keyedit_delimiters": OptionInfo(".,\/!?%^*;:{}=`~()", "Ctrl+up/down word delimiters"), # pylint: disable=anomalous-backslash-in-string
+    "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}),
+    "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing <extra networks:0.9>", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}),
+    "keyedit_delimiters": OptionInfo(".,\/!?%^*;:{}=`~()", "Ctrl+up/down word delimiters", gr.Textbox, { "visible": False }), # pylint: disable=anomalous-backslash-in-string
     "quicksettings_list": OptionInfo(["sd_model_checkpoint"] if backend == Backend.ORIGINAL else ["sd_model_checkpoint", "sd_model_refiner"], "Quicksettings list", ui_components.DropdownMulti, lambda: {"choices": list(opts.data_labels.keys())}),
-    "ui_scripts_reorder": OptionInfo("", "UI scripts order"),
+    "ui_scripts_reorder": OptionInfo("", "UI scripts order", gr.Textbox, { "visible": False }),
 }))
 
 options_templates.update(options_section(('live-preview', "Live Previews"), {
@@ -552,7 +554,7 @@ options_templates.update(options_section(('live-preview', "Live Previews"), {
     "notification_audio_path": OptionInfo("html/notification.mp3","Path to notification sound", component_args=hide_dirs, folder=True),
     "show_progress_every_n_steps": OptionInfo(1, "Live preview display period", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
     "show_progress_type": OptionInfo("Approximate NN", "Live preview method", gr.Radio, {"choices": ["Full VAE", "Approximate NN", "Approximate simple", "TAESD"]}),
-    "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"]}),
+    "live_preview_content": OptionInfo("Combined", "Live preview subject", gr.Radio, {"choices": ["Combined", "Prompt", "Negative prompt"], "visible": False}),
     "live_preview_refresh_period": OptionInfo(500, "Progress update period", gr.Slider, {"minimum": 0, "maximum": 5000, "step": 25}),
     "logmonitor_show": OptionInfo(True, "Show log view"),
     "logmonitor_refresh_period": OptionInfo(5000, "Log view update period", gr.Slider, {"minimum": 0, "maximum": 30000, "step": 25}),
@@ -638,7 +640,7 @@ options_templates.update(options_section(('interrogate', "Interrogate"), {
     "interrogate_clip_num_beams": OptionInfo(1, "Interrogate: num_beams for BLIP", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1}),
     "interrogate_clip_min_length": OptionInfo(32, "Interrogate: minimum description length", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1}),
     "interrogate_clip_max_length": OptionInfo(192, "Interrogate: maximum description length", gr.Slider, {"minimum": 1, "maximum": 256, "step": 1}),
-    "interrogate_clip_dict_limit": OptionInfo(2048, "CLIP: maximum number of lines in text file"),
+    "interrogate_clip_dict_limit": OptionInfo(2048, "CLIP: maximum number of lines in text file", gr.Slider, { "visible": False }),
     "interrogate_clip_skip_categories": OptionInfo(["artists", "movements", "flavors"], "Interrogate: skip categories", gr.CheckboxGroup, lambda: {"choices": modules.interrogate.category_types()}, refresh=modules.interrogate.category_types),
     "interrogate_deepbooru_score_threshold": OptionInfo(0.65, "Interrogate: deepbooru score threshold", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}),
     "deepbooru_sort_alpha": OptionInfo(False, "Interrogate: deepbooru sort alphabetically"),
@@ -652,10 +654,10 @@ options_templates.update(options_section(('extra_networks', "Extra Networks"), {
     "extra_networks_card_cover": OptionInfo("sidebar", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}),
     "extra_networks_height": OptionInfo(53, "UI height (%)", gr.Slider, {"minimum": 10, "maximum": 100, "step": 1}),
     "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}),
-    "extra_networks_card_lazy": OptionInfo(True, "UI card preview lazy loading"),
+    "extra_networks_card_lazy": OptionInfo(True, "UI card preview lazy loading", gr.Checkbox, { "visible": False }),
     "extra_networks_card_size": OptionInfo(160, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}),
     "extra_networks_card_square": OptionInfo(True, "UI disable variable aspect ratio"),
-    "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"]}),
+    "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"], "visible": False}),
     "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox),
     "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all LoRA types", gr.Checkbox),
     "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }),

From cefb5959e33fc04cb679c7f12d78dcb36a93586d Mon Sep 17 00:00:00 2001
From: Aptronymist <108482020+Aptronymist@users.noreply.github.com>
Date: Mon, 18 Sep 2023 21:27:20 -0400
Subject: [PATCH 11/27] Correcting legacy image optimization code

Legacy A1111 code improperly implemented Pillow image optimization options, "quality=" never applied to PNG files (or anything else), only JPEG files, per https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#png-saving and https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#jpeg-saving

This is now corrected, other than turning on the optimize=True, for backup and giggles, compress_level=9 is also set. For JPEG files, I also enabled optimize=True as it supposedly makes more optimized files.
---
 modules/images.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/images.py b/modules/images.py
index 39d9e7246..d8bda5e03 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -456,7 +456,7 @@ def atomically_save_image():
             pnginfo_data = PngImagePlugin.PngInfo()
             for k, v in params.pnginfo.items():
                 pnginfo_data.add_text(k, str(v))
-            image.save(fn, format=image_format, quality=shared.opts.jpeg_quality, pnginfo=pnginfo_data if shared.opts.image_metadata else None)
+            image.save(fn, format=image_format, optimize=True, compress_level=9, pnginfo=pnginfo_data if shared.opts.image_metadata else None)
         elif image_format == 'JPEG':
             if image.mode == 'RGBA':
                 shared.log.warning('Saving RGBA image as JPEG: Alpha channel will be lost')
@@ -464,7 +464,7 @@ def atomically_save_image():
             elif image.mode == 'I;16':
                 image = image.point(lambda p: p * 0.0038910505836576).convert("L")
             exif_bytes = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(exifinfo, encoding="unicode") } })
-            image.save(fn, format=image_format, quality=shared.opts.jpeg_quality, exif=exif_bytes)
+            image.save(fn, format=image_format, optimize=True, quality=shared.opts.jpeg_quality, exif=exif_bytes)
         elif image_format == 'WEBP':
             if image.mode == 'I;16':
                 image = image.point(lambda p: p * 0.0038910505836576).convert("RGB")

From a5973b6d96cdf42daf8ce955895cfe24193536a8 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 22:18:16 -0400
Subject: [PATCH 12/27] update

---
 extensions-builtin/sd-extension-system-info | 2 +-
 javascript/style.css                        | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info
index 83dd4d8f6..b73ffdcc8 160000
--- a/extensions-builtin/sd-extension-system-info
+++ b/extensions-builtin/sd-extension-system-info
@@ -1 +1 @@
-Subproject commit 83dd4d8f65511af720ddb034125fe4ac0a46fb18
+Subproject commit b73ffdcc8d6a9622d3d8074862b109b20726ac73
diff --git a/javascript/style.css b/javascript/style.css
index 78f63f8fa..4c4950fe7 100644
--- a/javascript/style.css
+++ b/javascript/style.css
@@ -1,6 +1,7 @@
 :root, .dark{ --checkbox-label-gap: 0.25em 0.1em; --section-header-text-size: 12pt; --block-background-fill: transparent;}
 a { font-weight: bold; cursor: pointer; }
 h2 { margin-top: 1em !important; font-size: 1.4em !important; }
+table { overflow-x: auto !important; overflow-y: auto !important; }
 div.gradio-container{ max-width: unset !important; padding: 8px !important; }
 div.tabitem { padding: 0 !important; }
 div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em 1em; flex-grow: 1 !important; }

From e6d6ee621a2acc3bd2cf35bf03edce4b2b90e856 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 22:30:36 -0400
Subject: [PATCH 13/27] diffusers lora unload logic

---
 modules/processing_diffusers.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index 9cd74f162..d12e3e670 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -315,7 +315,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         task_specific_kwargs = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": 8 * math.ceil(p.height / 8), "width": 8 * math.ceil(p.width / 8)}
 
     if shared.state.interrupted or shared.state.skipped:
-        unload_diffusers_lora()
+        if lora_state['active']:
+            unload_diffusers_lora()
         return results
 
     if shared.opts.diffusers_move_base and not shared.sd_model.has_accelerate:
@@ -361,11 +362,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
     if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0:
         p.extra_generation_params['Embeddings'] = ', '.join(shared.sd_model.embedding_db.embeddings_used)
 
-    if lora_state['active']:
-        p.extra_generation_params['LoRA method'] = shared.opts.diffusers_lora_loader
-        unload_diffusers_lora()
-
     if shared.state.interrupted or shared.state.skipped:
+        if lora_state['active']:
+            unload_diffusers_lora()
         return results
 
     # optional hires pass
@@ -407,6 +406,10 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 except AssertionError as e:
                     shared.log.info(e)
 
+    if lora_state['active']:
+        p.extra_generation_params['LoRA method'] = shared.opts.diffusers_lora_loader
+        unload_diffusers_lora()
+
     # optional refiner pass or decode
     if is_refiner_enabled:
         if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_refiner and hasattr(shared.sd_model, 'vae'):
@@ -423,6 +426,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             sd_samplers.create_sampler(sampler.name, shared.sd_refiner) # TODO(Patrick): For wrapped pipelines this is currently a no-op
 
         if shared.state.interrupted or shared.state.skipped:
+            if lora_state['active']:
+                unload_diffusers_lora()
             return results
 
         if shared.opts.diffusers_move_refiner and not shared.sd_refiner.has_accelerate:

From 947be2905532399838d836b2ba5ae8d74c0ca2e0 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 18 Sep 2023 23:04:09 -0400
Subject: [PATCH 14/27] fix save

---
 modules/ui_common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/modules/ui_common.py b/modules/ui_common.py
index cac1363a9..fa5569162 100644
--- a/modules/ui_common.py
+++ b/modules/ui_common.py
@@ -84,6 +84,8 @@ def save_files(js_data, images, html_info, index):
             self.infotexts = getattr(self, 'infotexts', [html_info])
             self.infotext = self.infotexts[0] if len(self.infotexts) > 0 else html_info
             self.index_of_first_image = getattr(self, 'index_of_first_image', 0)
+            self.batch_size = 1
+
     try:
         data = json.loads(js_data)
     except Exception:
@@ -93,6 +95,8 @@ def save_files(js_data, images, html_info, index):
     if index > -1 and shared.opts.save_selected_only and (index >= p.index_of_first_image):  # ensures we are looking at a specific non-grid picture, and we have save_selected_only # pylint: disable=no-member
         images = [images[index]]
         start_index = index
+    else:
+        p.batch_size = len(images)
     filenames = []
     fullfns = []
     for image_index, filedata in enumerate(images, start_index):

From c33bbc83e71a012ac29b8ae2f5ee82c87c4c1c60 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Tue, 19 Sep 2023 11:19:32 +0300
Subject: [PATCH 15/27] Update OpenVINO

---
 installer.py      | 2 +-
 modules/shared.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/installer.py b/installer.py
index ae227723d..9a322b677 100644
--- a/installer.py
+++ b/installer.py
@@ -496,7 +496,7 @@ def check_torch():
     if opts.get('cuda_compile_backend', '') == 'hidet':
         install('hidet', 'hidet')
     if args.use_openvino or opts.get('cuda_compile_backend', '') == 'openvino_fx':
-        install('openvino==2023.1.0.dev20230811', 'openvino')
+        install('openvino==2023.1.0', 'openvino')
         os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX')
     if args.profile:
         print_profile(pr, 'Torch')
diff --git a/modules/shared.py b/modules/shared.py
index 9897ed3fc..f1da73266 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -443,7 +443,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
     "diffusers_attention_slicing": OptionInfo(True if devices.backend == "ipex" else False, "Enable attention slicing"),
     "diffusers_model_load_variant": OptionInfo("default", "Diffusers model loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}),
     "diffusers_vae_load_variant": OptionInfo("default", "Diffusers VAE loading variant", gr.Radio, lambda: {"choices": ['default', 'fp32', 'fp16']}),
-    "diffusers_lora_loader": OptionInfo("sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}),
+    "diffusers_lora_loader": OptionInfo("diffusers" if cmd_opts.use_openvino else "sequential apply", "Diffusers LoRA loading variant", gr.Radio, lambda: {"choices": ['diffusers', 'sequential apply', 'merge and apply']}),
     "diffusers_force_zeros": OptionInfo(True, "Force zeros for prompts when empty"),
     "diffusers_aesthetics_score": OptionInfo(False, "Require aesthetics score"),
 }))

From 4d9ccbc035912d5747feaef8eeedd7f0b73ee566 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Tue, 19 Sep 2023 11:13:38 -0400
Subject: [PATCH 16/27] add batch info to metadata

---
 html/locale_en.json     |  2 +-
 modules/images.py       | 28 ++++++++++++++++++++--------
 modules/processing.py   | 19 ++++++++++---------
 modules/sd_hijack.py    | 15 ++++-----------
 modules/sd_models.py    |  4 ++--
 modules/shared.py       |  5 +++--
 modules/shared_items.py |  2 +-
 modules/ui_common.py    | 24 +++++++++++++-----------
 modules/ui_tempdir.py   |  2 +-
 9 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/html/locale_en.json b/html/locale_en.json
index 1436a6786..f17742039 100644
--- a/html/locale_en.json
+++ b/html/locale_en.json
@@ -356,7 +356,7 @@
   {"id":"","label":"Enable splitting of hires batch processing","localized":"","hint":"Reduces VRAM usage when using hires fix on batches of images"},
   {"id":"","label":"Load models using stream loading method","localized":"","hint":"When loading models attempt stream loading optimized for slow or network storage"},
   {"id":"","label":"When loading models attempt to reuse previous model dictionary","localized":"","hint":""},
-  {"id":"","label":"Disable cross-attention layer optimization","localized":"","hint":"Disable the all cross-attention optimization. May result in higher VRAM usage and longer generation times"},
+  {"id":"","label":"Disabled","localized":"","hint":""},
   {"id":"","label":"xFormers","localized":"","hint":"Memory optimization. Non-Deterministic (different results each time)"},
   {"id":"","label":"Scaled-Dot-Product","localized":"","hint":"Memory optimization. Non-Deterministic unless SDP memory attention is disabled."},
   {"id":"","label":"Doggettx's","localized":"","hint":""},
diff --git a/modules/images.py b/modules/images.py
index 7f29983bb..98b44c4d0 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -291,7 +291,8 @@ def sanitize_filename_part(text, replace_spaces=True):
 
 class FilenameGenerator:
     replacements = {
-        'batch_number': lambda self: NOTHING if self.index <= 1 else self.index,
+        'batch_number': lambda self: self.batch_number,
+        'iter_number': lambda self: self.iter_number,
         'cfg': lambda self: self.p and self.p.cfg_scale,
         'clip_skip': lambda self: self.p and self.p.clip_skip,
         'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'),
@@ -320,12 +321,17 @@ class FilenameGenerator:
     }
     default_time_format = '%Y%m%d%H%M%S'
 
-    def __init__(self, p, seed, prompt, image, index = 0):
+    def __init__(self, p, seed, prompt, image, grid=False):
         self.p = p
         self.seed = seed
         self.prompt = prompt
         self.image = image
-        self.index = index if self.p is None or self.p.batch_size == 1 else self.p.batch_index + 1
+        if not grid:
+            self.batch_number = NOTHING if self.p is None or getattr(self.p, 'batch_size', 1) == 1 else (self.p.batch_index + 1 if hasattr(self.p, 'batch_index') else NOTHING)
+            self.iter_number = NOTHING if self.p is None or getattr(self.p, 'n_iter', 1) == 1 else (self.p.iteration + 1 if hasattr(self.p, 'iteration') else NOTHING)
+        else:
+            self.batch_number = NOTHING
+            self.iter_number = NOTHING
 
     def hasprompt(self, *args):
         lower = self.prompt.lower()
@@ -449,7 +455,7 @@ def atomically_save_image():
             image_format = 'JPEG'
         if shared.opts.image_watermark_enabled:
             image = set_watermark(image, shared.opts.image_watermark)
-        shared.log.debug(f'Saving: image={fn} type={image_format} size={image.width}x{image.height}')
+        shared.log.debug(f'Saving: image="{fn}" type={image_format} size={image.width}x{image.height}')
         # actual save
         exifinfo = (exifinfo or "") if shared.opts.image_metadata else ""
         if image_format == 'PNG':
@@ -489,8 +495,14 @@ def atomically_save_image():
         with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file:
             file.write(exifinfo)
         if shared.opts.save_log_fn != '' and len(exifinfo) > 0:
-            entry = { 'filename': filename, 'time': datetime.datetime.now().isoformat(), 'info': exifinfo }
-            shared.writefile(entry, os.path.join(paths.data_path, shared.opts.save_log_fn), mode='a+')
+            fn = os.path.join(paths.data_path, shared.opts.save_log_fn)
+            entries = shared.readfile(fn)
+            idx = len(list(entries))
+            if idx == 0:
+                entries = []
+            entry = { 'id': idx, 'filename': filename, 'time': datetime.datetime.now().isoformat(), 'info': exifinfo }
+            entries.append(entry)
+            shared.writefile(entries, fn, mode='w')
         save_queue.task_done()
 
 
@@ -499,7 +511,7 @@ save_thread = threading.Thread(target=atomically_save_image, daemon=True)
 save_thread.start()
 
 
-def save_image(image, path, basename, seed=None, prompt=None, extension='jpg', info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix="", save_to_dirs=None, index=0):
+def save_image(image, path, basename, seed=None, prompt=None, extension='jpg', info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix="", save_to_dirs=None):
     """Save an image.
     Args:
         image (`PIL.Image`):
@@ -537,7 +549,7 @@ def save_image(image, path, basename, seed=None, prompt=None, extension='jpg', i
         return None, None
     if path is None or len(path) == 0: # set default path to avoid errors when functions are triggered manually or via api and param is not set
         path = shared.opts.outdir_save
-    namegen = FilenameGenerator(p, seed, prompt, image, index)
+    namegen = FilenameGenerator(p, seed, prompt, image, grid=grid)
     if save_to_dirs is None:
         save_to_dirs = (grid and shared.opts.grid_save_to_dirs) or (not grid and shared.opts.save_to_dirs and not no_prompt)
     if save_to_dirs:
diff --git a/modules/processing.py b/modules/processing.py
index 772bbafba..90c4038af 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -472,6 +472,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su
         "CFG scale": p.cfg_scale,
         "Size": f"{p.width}x{p.height}",
         "Batch": f'{p.n_iter}x{p.batch_size}' if p.n_iter > 1 or p.batch_size > 1 else None,
+        "Index": f'{p.iteration + 1}x{index + 1}' if (p.n_iter > 1 or p.batch_size > 1) and index >= 0 else None,
         "Parser": shared.opts.prompt_attention,
         "Model": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_model.sd_checkpoint_info.model_name) else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''),
         "Model hash": getattr(p, 'sd_model_hash', None if (not shared.opts.add_model_hash_to_info) or (not shared.sd_model.sd_model_hash) else shared.sd_model.sd_model_hash),
@@ -798,7 +799,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 p.scripts.postprocess_batch_list(p, batch_params, batch_number=n)
                 x_samples_ddim = batch_params.images
 
-            def infotext(index=0): # pylint: disable=function-redefined # noqa: F811
+            def infotext(index): # pylint: disable=function-redefined # noqa: F811
                 return create_infotext(p, p.prompts, p.seeds, p.subseeds, index=index, all_negative_prompts=p.negative_prompts)
 
             for i, x_sample in enumerate(x_samples_ddim):
@@ -816,7 +817,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                         p.restore_faces = False
                         info = infotext(i)
                         p.restore_faces = orig
-                        images.save_image(Image.fromarray(x_sample), path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-face-restoration")
+                        images.save_image(Image.fromarray(x_sample), path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-face-restore")
                     p.ops.append('face')
                     x_sample = modules.face_restoration.restore_faces(x_sample)
                     image = Image.fromarray(x_sample)
@@ -831,7 +832,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                         info = infotext(i)
                         p.color_corrections = orig
                         image_without_cc = apply_overlay(image, p.paste_to, i, p.overlay_images)
-                        images.save_image(image_without_cc, path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-color-correction")
+                        images.save_image(image_without_cc, path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-color-correct")
                     p.ops.append('color')
                     image = apply_color_correction(p.color_corrections[i], image)
                 image = apply_overlay(image, p.paste_to, i, p.overlay_images)
@@ -840,7 +841,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 image.info["parameters"] = text
                 output_images.append(image)
                 if shared.opts.samples_save and not p.do_not_save_samples:
-                    images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=text, p=p)
+                    images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=text, p=p) # main save image
                 if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]):
                     image_mask = p.mask_for_overlay.convert('RGB')
                     image_mask_composite = Image.composite(image.convert('RGBA').convert('RGBa'), Image.new('RGBa', image.size), images.resize_image(3, p.mask_for_overlay, image.width, image.height).convert('L')).convert('RGBA')
@@ -865,13 +866,13 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             if images.check_grid_size(output_images):
                 grid = images.image_grid(output_images, p.batch_size)
                 if shared.opts.return_grid:
-                    text = infotext()
+                    text = infotext(-1)
                     infotexts.insert(0, text)
                     grid.info["parameters"] = text
                     output_images.insert(0, grid)
                     index_of_first_image = 1
                 if shared.opts.grid_save:
-                    images.save_image(grid, p.outpath_grids, "grid", p.all_seeds[0], p.all_prompts[0], shared.opts.grid_format, info=infotext(), short_filename=not shared.opts.grid_extended_filename, p=p, grid=True)
+                    images.save_image(grid, p.outpath_grids, "", p.all_seeds[0], p.all_prompts[0], shared.opts.grid_format, info=infotext(-1), short_filename=not shared.opts.grid_extended_filename, p=p, grid=True, suffix="-grid") # main save grid
 
     if not p.disable_extra_networks and extra_network_data:
         modules.extra_networks.deactivate(p, extra_network_data)
@@ -880,7 +881,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
         p,
         images_list=output_images,
         seed=p.all_seeds[0],
-        info=infotext(),
+        info=infotext(0),
         comments="\n".join(comments),
         subseed=p.all_subseeds[0],
         index_of_first_image=index_of_first_image,
@@ -987,7 +988,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             info = create_infotext(self, self.all_prompts, self.all_seeds, self.all_subseeds, [], iteration=self.iteration, position_in_batch=index)
             self.extra_generation_params = orig1
             self.restore_faces = orig2
-            images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], shared.opts.samples_format, info=info, suffix="-before-hires", index=index+1)
+            images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], shared.opts.samples_format, info=info, suffix="-before-hires")
 
         if shared.backend == shared.Backend.DIFFUSERS:
             modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE)
@@ -1140,7 +1141,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
             self.init_img_width = img.width # pylint: disable=attribute-defined-outside-init
             self.init_img_height = img.height # pylint: disable=attribute-defined-outside-init
             if shared.opts.save_init_img:
-                images.save_image(img, path=shared.opts.outdir_init_images, basename=None, forced_filename=self.init_img_hash, save_to_dirs=False)
+                images.save_image(img, path=shared.opts.outdir_init_images, basename=None, forced_filename=self.init_img_hash, save_to_dirs=False, suffix="-init-image")
             image = images.flatten(img, shared.opts.img2img_background_color)
             if crop_region is None and self.resize_mode != 4:
                 image = images.resize_image(self.resize_mode, image, self.width, self.height)
diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 5cd407e19..3f465a600 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -37,45 +37,38 @@ def apply_optimizations():
     can_use_sdp = hasattr(torch.nn.functional, "scaled_dot_product_attention") and callable(torch.nn.functional.scaled_dot_product_attention)
     if devices.device == torch.device("cpu"):
         if opts.cross_attention_optimization == "Scaled-Dot-Product":
-            shared.log.warning("Scaled dot product cross attention is not available on CPU")
+            shared.log.warning("Cross-attention: Scaled dot product is not available on CPU")
             can_use_sdp = False
         if opts.cross_attention_optimization == "xFormers":
-            shared.log.warning("xFormers cross attention is not available on CPU")
+            shared.log.warning("Cross-attention: xFormers is not available on CPU")
             shared.xformers_available = False
 
-    if opts.cross_attention_optimization == "Disable cross-attention layer optimization":
-        shared.log.warning("Cross-attention optimization disabled")
+    shared.log.info(f"Cross-attention: optimization={opts.cross_attention_optimization} options={opts.cross_attention_options}")
+    if opts.cross_attention_optimization == "Disabled":
         optimization_method = 'none'
     if can_use_sdp and opts.cross_attention_optimization == "Scaled-Dot-Product" and 'SDP disable memory attention' in opts.cross_attention_options:
-        shared.log.info("Applying scaled dot product cross attention optimization (without memory efficient attention)")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.scaled_dot_product_no_mem_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.sdp_no_mem_attnblock_forward
         optimization_method = 'sdp-no-mem'
     elif can_use_sdp and opts.cross_attention_optimization == "Scaled-Dot-Product":
-        shared.log.info("Applying scaled dot product cross attention optimization")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.scaled_dot_product_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.sdp_attnblock_forward
         optimization_method = 'sdp'
     if shared.xformers_available and opts.cross_attention_optimization == "xFormers":
-        shared.log.info("Applying xformers cross attention optimization")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.xformers_attnblock_forward
         optimization_method = 'xformers'
     if opts.cross_attention_optimization == "Sub-quadratic":
-        shared.log.info("Applying sub-quadratic cross attention optimization")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.sub_quad_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.sub_quad_attnblock_forward
         optimization_method = 'sub-quadratic'
     if opts.cross_attention_optimization == "Split attention":
-        shared.log.info("Applying split attention optimization")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
         optimization_method = 'v1'
     if opts.cross_attention_optimization == "InvokeAI's":
-        shared.log.info("Applying InvokeAI cross attention optimization")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_invokeAI
         optimization_method = 'invokeai'
     if opts.cross_attention_optimization == "Doggettx's":
-        shared.log.info("Applying Doggettx cross attention optimization")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
         optimization_method = 'doggettx'
diff --git a/modules/sd_models.py b/modules/sd_models.py
index 5278015e6..d77134238 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -261,7 +261,7 @@ def select_checkpoint(op='model'):
         return None
     checkpoint_info = get_closet_checkpoint_match(model_checkpoint)
     if checkpoint_info is not None:
-        shared.log.debug(f'Select checkpoint: {op} {checkpoint_info.title if checkpoint_info is not None else None}')
+        shared.log.debug(f'Select checkpoint: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"')
         return checkpoint_info
     if len(checkpoints_list) == 0 and not shared.cmd_opts.no_download:
         shared.log.error("Cannot run without a checkpoint")
@@ -275,7 +275,7 @@ def select_checkpoint(op='model'):
             shared.log.info("Selecting first available checkpoint")
         # shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}")
         shared.opts.data['sd_model_checkpoint'] = checkpoint_info.title
-    shared.log.debug(f'Select checkpoint: {checkpoint_info.title if checkpoint_info is not None else None}')
+    shared.log.debug(f'Select checkpoint: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"')
     return checkpoint_info
 
 
diff --git a/modules/shared.py b/modules/shared.py
index f1da73266..d69f0ff3b 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -340,7 +340,7 @@ def readfile(filename, silent=False):
     return data
 
 
-def writefile(data, filename, mode='w'):
+def writefile(data, filename, mode='w', silent=False):
 
     def default(obj):
         log.error(f"Saving: {filename} not a valid object: {obj}")
@@ -350,7 +350,8 @@ def writefile(data, filename, mode='w'):
         with fasteners.InterProcessLock(f"{filename}.lock"):
             # skipkeys=True, ensure_ascii=True, check_circular=True, allow_nan=True
             output = json.dumps(data, indent=2, default=default)
-            log.debug(f'Saving: {filename} len={len(output)}')
+            if not silent:
+                log.debug(f'Saving: {filename} len={len(output)}')
             with open(filename, mode, encoding="utf8") as file:
                 file.write(output)
     except Exception as e:
diff --git a/modules/shared_items.py b/modules/shared_items.py
index 8bb05b064..061fe4929 100644
--- a/modules/shared_items.py
+++ b/modules/shared_items.py
@@ -20,7 +20,7 @@ def refresh_vae_list():
 
 def list_crossattention():
     return [
-        "Disable cross-attention layer optimization",
+        "Disabled",
         "xFormers",
         "Scaled-Dot-Product",
         "Doggettx's",
diff --git a/modules/ui_common.py b/modules/ui_common.py
index fa5569162..8d1bb48fd 100644
--- a/modules/ui_common.py
+++ b/modules/ui_common.py
@@ -75,17 +75,21 @@ def save_files(js_data, images, html_info, index):
     class PObject: # pylint: disable=too-few-public-methods
         def __init__(self, d=None):
             if d is not None:
-                for key, value in d.items():
-                    setattr(self, key, value)
-            self.seed = getattr(self, 'seed', None) or getattr(self, 'Seed', None)
+                for k, v in d.items():
+                    setattr(self, k, v)
             self.prompt = getattr(self, 'prompt', None) or getattr(self, 'Prompt', None)
-            self.all_seeds = getattr(self, 'all_seeds', [self.seed])
             self.all_prompts = getattr(self, 'all_prompts', [self.prompt])
+            self.negative_prompt = getattr(self, 'negative_prompt', None)
+            self.all_negative_prompt = getattr(self, 'all_negative_prompts', [self.negative_prompt])
+            self.seed = getattr(self, 'seed', None) or getattr(self, 'Seed', None)
+            self.all_seeds = getattr(self, 'all_seeds', [self.seed])
+            self.subseed = getattr(self, 'subseed', None)
+            self.all_subseeds = getattr(self, 'all_subseeds', [self.subseed])
+            self.width = getattr(self, 'width', None)
+            self.height = getattr(self, 'height', None)
+            self.index_of_first_image = getattr(self, 'index_of_first_image', 0)
             self.infotexts = getattr(self, 'infotexts', [html_info])
             self.infotext = self.infotexts[0] if len(self.infotexts) > 0 else html_info
-            self.index_of_first_image = getattr(self, 'index_of_first_image', 0)
-            self.batch_size = 1
-
     try:
         data = json.loads(js_data)
     except Exception:
@@ -95,8 +99,6 @@ def save_files(js_data, images, html_info, index):
     if index > -1 and shared.opts.save_selected_only and (index >= p.index_of_first_image):  # ensures we are looking at a specific non-grid picture, and we have save_selected_only # pylint: disable=no-member
         images = [images[index]]
         start_index = index
-    else:
-        p.batch_size = len(images)
     filenames = []
     fullfns = []
     for image_index, filedata in enumerate(images, start_index):
@@ -114,12 +116,12 @@ def save_files(js_data, images, html_info, index):
             fullfns.append(fullfn)
             destination = shared.opts.outdir_save
             if shared.opts.use_save_to_dirs_for_ui:
-                namegen = modules.images.FilenameGenerator(p, seed=p.all_seeds[i], prompt=p.all_prompts[i], image=None, index=image_index)  # pylint: disable=no-member
+                namegen = modules.images.FilenameGenerator(p, seed=p.all_seeds[i], prompt=p.all_prompts[i], image=None)  # pylint: disable=no-member
                 dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]").lstrip(' ').rstrip('\\ /')
                 destination = os.path.join(destination, dirname)
                 os.makedirs(destination, exist_ok = True)
             shutil.copy(fullfn, destination)
-            shared.log.info(f"Copying image: {fullfn} -> {destination}")
+            shared.log.info(f'Copying image: file="{fullfn}" folder="{destination}"')
             tgt_filename = os.path.join(destination, os.path.basename(fullfn))
             modules.script_callbacks.image_save_btn_callback(tgt_filename)
         else:
diff --git a/modules/ui_tempdir.py b/modules/ui_tempdir.py
index 945a004b6..38b4b5923 100644
--- a/modules/ui_tempdir.py
+++ b/modules/ui_tempdir.py
@@ -61,7 +61,7 @@ def pil_to_temp_file(self, img, dir: str, format="png") -> str: # pylint: disabl
     file_obj = tempfile.NamedTemporaryFile(delete=False, suffix=".png", dir=dir)
     img.save(file_obj, pnginfo=(metadata if use_metadata else None))
     name = file_obj.name
-    shared.log.debug(f'Saving temp image: {name}')
+    shared.log.debug(f'Saving temp: image="{name}"')
     return name
 
 

From df377d6d279ac5845caeeffa96ab81258b61e88a Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Tue, 19 Sep 2023 12:55:32 -0400
Subject: [PATCH 17/27] safe handling of accelerate

---
 modules/processing_diffusers.py | 11 +++++++----
 modules/sd_models.py            | 18 +++++++++---------
 modules/sd_vae.py               |  4 ++--
 3 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index d12e3e670..5362fa31a 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -65,7 +65,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
 
     def full_vae_decode(latents, model):
         t0 = time.time()
-        if shared.opts.diffusers_move_unet and not model.has_accelerate:
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
             shared.log.debug('Moving to CPU: model=UNet')
             unet_device = model.unet.device
             model.unet.to(devices.cpu)
@@ -80,7 +80,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype)
 
         decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0]
-        if shared.opts.diffusers_move_unet and not model.has_accelerate:
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
             model.unet.to(unet_device)
         t1 = time.time()
         shared.log.debug(f'VAE decode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={upcast} images={latents.shape[0]} latents={latents.shape} time={round(t1-t0, 3)}s')
@@ -88,7 +88,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
 
     def full_vae_encode(image, model):
         shared.log.debug(f'VAE encode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={model.vae.config.get("force_upcast", None)}')
-        if shared.opts.diffusers_move_unet and not model.has_accelerate:
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
             shared.log.debug('Moving to CPU: model=UNet')
             unet_device = model.unet.device
             model.unet.to(devices.cpu)
@@ -96,7 +96,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload:
             model.vae.to(devices.device)
         encoded = model.vae.encode(image.to(model.vae.device, model.vae.dtype))
-        if shared.opts.diffusers_move_unet and not model.has_accelerate:
+        if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False):
             model.unet.to(unet_device)
         return encoded
 
@@ -358,6 +358,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         output = shared.sd_model(**base_args) # pylint: disable=not-callable
     except AssertionError as e:
         shared.log.info(e)
+    except ValueError as e:
+        shared.state.interrupted = True
+        shared.log.error(e)
 
     if hasattr(shared.sd_model, 'embedding_db') and len(shared.sd_model.embedding_db.embeddings_used) > 0:
         p.extra_generation_params['Embeddings'] = ', '.join(shared.sd_model.embedding_db.embeddings_used)
diff --git a/modules/sd_models.py b/modules/sd_models.py
index d77134238..c58a22932 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -896,7 +896,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
 
         base_sent_to_cpu=False
         if (shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none') or shared.opts.ipex_optimize:
-            if op == 'refiner' and not sd_model.has_accelerate:
+            if op == 'refiner' and not getattr(sd_model, 'has_accelerate', False):
                 gpu_vram = memory_stats().get('gpu', {})
                 free_vram = gpu_vram.get('total', 0) - gpu_vram.get('used', 0)
                 refiner_enough_vram = free_vram >= 7 if "StableDiffusionXL" in sd_model.__class__.__name__ else 3
@@ -917,7 +917,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                     devices.torch_gc(force=True)
                     sd_model.to(devices.device)
                     base_sent_to_cpu=True
-            elif not sd_model.has_accelerate:
+            elif not getattr(sd_model, 'has_accelerate', False):
                 sd_model.to(devices.device)
 
             compile_diffusers(sd_model)
@@ -931,10 +931,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
         shared.opts.data["sd_checkpoint_hash"] = checkpoint_info.sha256
         if hasattr(sd_model, "set_progress_bar_config"):
             sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining}', ncols=80, colour='#327fba')
-        if op == 'refiner' and shared.opts.diffusers_move_refiner and not sd_model.has_accelerate:
+        if op == 'refiner' and shared.opts.diffusers_move_refiner and not getattr(sd_model, 'has_accelerate', False):
             shared.log.debug('Moving refiner model to CPU')
             sd_model.to(devices.cpu)
-        elif not sd_model.has_accelerate: # In offload modes, accelerate will move models around
+        elif not getattr(sd_model, 'has_accelerate', False): # In offload modes, accelerate will move models around
             sd_model.to(devices.device)
         if op == 'refiner' and base_sent_to_cpu:
             shared.log.debug('Moving base model back to GPU')
@@ -1067,6 +1067,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None,
             shared.log.info(f'LDM: {line.strip()}')
     shared.log.debug(f"Model created from config: {checkpoint_config}")
     sd_model.used_config = checkpoint_config
+    sd_model.has_accelerate = False
     timer.record("create")
     ok = load_model_weights(sd_model, checkpoint_info, state_dict, timer)
     if not ok:
@@ -1090,7 +1091,6 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None,
     sd_hijack.model_hijack.hijack(sd_model)
     timer.record("hijack")
     sd_model.eval()
-    sd_model.has_accelerate = False
     if op == 'refiner':
         model_data.sd_refiner = sd_model
     else:
@@ -1126,12 +1126,12 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model')
         current_checkpoint_info = getattr(sd_model, 'sd_checkpoint_info', None)
         if current_checkpoint_info is not None and checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename:
             return
-        if not sd_model.has_accelerate:
+        if not getattr(sd_model, 'has_accelerate', False):
             if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
                 lowvram.send_everything_to_cpu()
             else:
                 sd_model.to(devices.cpu)
-        if (reuse_dict or shared.opts.model_reuse_dict) and not sd_model.has_accelerate:
+        if (reuse_dict or shared.opts.model_reuse_dict) and not getattr(sd_model, 'has_accelerate', False):
             shared.log.info('Reusing previous model dictionary')
             sd_hijack.model_hijack.undo_hijack(sd_model)
         else:
@@ -1164,7 +1164,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model')
         timer.record("hijack")
         script_callbacks.model_loaded_callback(sd_model)
         timer.record("callbacks")
-        if sd_model is not None and not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram and not sd_model.has_accelerate:
+        if sd_model is not None and not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram and not getattr(sd_model, 'has_accelerate', False):
             sd_model.to(devices.device)
             timer.record("device")
     shared.log.info(f"Weights loaded in {timer.summary()}")
@@ -1172,7 +1172,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model')
 
 def disable_offload(sd_model):
     from accelerate.hooks import remove_hook_from_module
-    if not sd_model.has_accelerate:
+    if not getattr(sd_model, 'has_accelerate', False):
         return
     for _name, model in sd_model.components.items():
         if not isinstance(model, torch.nn.Module):
diff --git a/modules/sd_vae.py b/modules/sd_vae.py
index 0360d83b3..173f54337 100644
--- a/modules/sd_vae.py
+++ b/modules/sd_vae.py
@@ -243,7 +243,7 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified):
         vae_source = "function-argument"
     if loaded_vae_file == vae_file:
         return
-    if not sd_model.has_accelerate:
+    if not getattr(sd_model, 'has_accelerate', False):
         if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
             lowvram.send_everything_to_cpu()
         else:
@@ -265,6 +265,6 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified):
                 if vae is not None:
                     sd_model.vae = vae
 
-    if not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram and not sd_model.has_accelerate:
+    if not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram and not getattr(sd_model, 'has_accelerate', False):
         sd_model.to(devices.device)
     return sd_model

From f82e05c5f1fbf483146049f0e4e79327b2cf304b Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Tue, 19 Sep 2023 13:05:42 -0400
Subject: [PATCH 18/27] second part of has_accelerate cleanup

---
 modules/processing_diffusers.py | 8 ++++----
 modules/sd_models.py            | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index 5362fa31a..9d8605d75 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -319,7 +319,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             unload_diffusers_lora()
         return results
 
-    if shared.opts.diffusers_move_base and not shared.sd_model.has_accelerate:
+    if shared.opts.diffusers_move_base and not hasattr(shared.sd_model, 'has_accelerate', False):
         shared.sd_model.to(devices.device)
 
     is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING)
@@ -417,7 +417,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
     if is_refiner_enabled:
         if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_refiner and hasattr(shared.sd_model, 'vae'):
             save_intermediate(latents=output.images, suffix="-before-refiner")
-        if shared.opts.diffusers_move_base and not shared.sd_model.has_accelerate:
+        if shared.opts.diffusers_move_base and not hasattr(shared.sd_model, 'has_accelerate', False):
             shared.log.debug('Moving to CPU: model=base')
             shared.sd_model.to(devices.cpu)
             devices.torch_gc()
@@ -433,7 +433,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 unload_diffusers_lora()
             return results
 
-        if shared.opts.diffusers_move_refiner and not shared.sd_refiner.has_accelerate:
+        if shared.opts.diffusers_move_refiner and not hasattr(shared.sd_refiner, 'has_accelerate', False):
             shared.sd_refiner.to(devices.device)
         refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__)
         p.ops.append('refine')
@@ -469,7 +469,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 for refiner_image in refiner_images:
                     results.append(refiner_image)
 
-        if shared.opts.diffusers_move_refiner and not shared.sd_refiner.has_accelerate:
+        if shared.opts.diffusers_move_refiner and not hasattr(shared.sd_refiner, 'has_accelerate', False):
             shared.log.debug('Moving to CPU: model=refiner')
             shared.sd_refiner.to(devices.cpu)
             devices.torch_gc()
diff --git a/modules/sd_models.py b/modules/sd_models.py
index c58a22932..4ace52368 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -965,10 +965,10 @@ class DiffusersTaskType(Enum):
     INPAINTING = 3
 
 def set_diffuser_pipe(pipe, new_pipe_type):
-    sd_checkpoint_info = pipe.sd_checkpoint_info if hasattr(pipe, "sd_checkpoint_info") else None
-    sd_model_checkpoint = pipe.sd_model_checkpoint if hasattr(pipe, "sd_model_checkpoint") else None
-    sd_model_hash = pipe.sd_model_hash if hasattr(pipe, "sd_model_hash") else None
-    has_accelerate = pipe.has_accelerate if hasattr(pipe, "has_accelerate") else None
+    sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None)
+    sd_model_checkpoint = getattr(pipe, "sd_model_checkpoint", None)
+    sd_model_hash = getattr(pipe, "sd_model_hash", None)
+    has_accelerate = getattr(pipe, "has_accelerate", None)
 
     if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE:
         new_pipe = diffusers.AutoPipelineForText2Image.from_pipe(pipe)

From d73f5f6ff6315c37d585ca9379ad86f3d55360b5 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Tue, 19 Sep 2023 15:14:58 -0400
Subject: [PATCH 19/27] allow zero denoising

---
 modules/processing.py | 3 +++
 modules/ui.py         | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/modules/processing.py b/modules/processing.py
index 90c4038af..f325e6000 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -965,6 +965,9 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None':
             self.is_hr_pass = False
             return
+        if self.denoising_strength == 0:
+            self.is_hr_pass = False
+            return
         self.is_hr_pass = True
         if not shared.state.processing_has_refined_job_count:
             if shared.state.job_count == -1:
diff --git a/modules/ui.py b/modules/ui.py
index 8da53b22a..898da57d1 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -392,7 +392,7 @@ def create_ui(startup_timer = None):
                     with FormGroup():
                         with FormRow(elem_id="sampler_selection_txt2img_alt_row1"):
                             latent_index = gr.Dropdown(label='Secondary sampler', elem_id="txt2img_sampling_alt", choices=[x.name for x in modules.sd_samplers.samplers], value='Default', type="index")
-                            denoising_strength = gr.Slider(minimum=0.05, maximum=1.0, step=0.01, label='Denoising strength', value=0.5, elem_id="txt2img_denoising_strength")
+                            denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoising strength', value=0.5, elem_id="txt2img_denoising_strength")
                         with FormRow(elem_id="txt2img_hires_finalres", variant="compact"):
                             hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False)
                         with FormRow(elem_id="txt2img_hires_fix_row1", variant="compact"):
@@ -678,7 +678,7 @@ def create_ui(startup_timer = None):
 
                 with FormGroup(visible=show_denoise.value, elem_id=f"{tab}_denoise_group") as denoise_group:
                     with FormRow():
-                        denoising_strength = gr.Slider(minimum=0.05, maximum=1.0, step=0.01, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength")
+                        denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength")
                         refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start")
 
                 with FormGroup(visible=show_advanced.value, elem_id=f"{tab}_advanced_group") as advanced_group:

From 5411a0fd8f6c6418bdf82f78ecd61b0b6817f971 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Tue, 19 Sep 2023 20:35:19 -0400
Subject: [PATCH 20/27] fix

---
 modules/processing_diffusers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py
index 9d8605d75..56444d4ec 100644
--- a/modules/processing_diffusers.py
+++ b/modules/processing_diffusers.py
@@ -319,7 +319,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             unload_diffusers_lora()
         return results
 
-    if shared.opts.diffusers_move_base and not hasattr(shared.sd_model, 'has_accelerate', False):
+    if shared.opts.diffusers_move_base and not getattr(shared.sd_model, 'has_accelerate', False):
         shared.sd_model.to(devices.device)
 
     is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING)
@@ -417,7 +417,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
     if is_refiner_enabled:
         if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_refiner and hasattr(shared.sd_model, 'vae'):
             save_intermediate(latents=output.images, suffix="-before-refiner")
-        if shared.opts.diffusers_move_base and not hasattr(shared.sd_model, 'has_accelerate', False):
+        if shared.opts.diffusers_move_base and not getattr(shared.sd_model, 'has_accelerate', False):
             shared.log.debug('Moving to CPU: model=base')
             shared.sd_model.to(devices.cpu)
             devices.torch_gc()
@@ -433,7 +433,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 unload_diffusers_lora()
             return results
 
-        if shared.opts.diffusers_move_refiner and not hasattr(shared.sd_refiner, 'has_accelerate', False):
+        if shared.opts.diffusers_move_refiner and not getattr(shared.sd_refiner, 'has_accelerate', False):
             shared.sd_refiner.to(devices.device)
         refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__)
         p.ops.append('refine')
@@ -469,7 +469,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 for refiner_image in refiner_images:
                     results.append(refiner_image)
 
-        if shared.opts.diffusers_move_refiner and not hasattr(shared.sd_refiner, 'has_accelerate', False):
+        if shared.opts.diffusers_move_refiner and not getattr(shared.sd_refiner, 'has_accelerate', False):
             shared.log.debug('Moving to CPU: model=refiner')
             shared.sd_refiner.to(devices.cpu)
             devices.torch_gc()

From 16b1752042fa5d50e789deee86bd337175a9ff3b Mon Sep 17 00:00:00 2001
From: Nuullll <vfirst218@gmail.com>
Date: Wed, 20 Sep 2023 20:16:56 +0800
Subject: [PATCH 21/27] Fix ipex init

getDeviceIdListForCard is renamed since https://github.com/intel/intel-extension-for-pytorch/commit/835b41fd5c8b6facf9efee8312f20699850ee592
---
 modules/intel/ipex/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py
index bd2e8e142..03cee82c7 100644
--- a/modules/intel/ipex/__init__.py
+++ b/modules/intel/ipex/__init__.py
@@ -17,7 +17,11 @@ def ipex_init(): # pylint: disable=too-many-statements
         torch.cuda.device = torch.xpu.device
         torch.cuda.device_count = torch.xpu.device_count
         torch.cuda.device_of = torch.xpu.device_of
-        torch.cuda.getDeviceIdListForCard = torch.xpu.getDeviceIdListForCard
+        # getDeviceIdListForCard is renamed since https://github.com/intel/intel-extension-for-pytorch/commit/835b41fd5c8b6facf9efee8312f20699850ee592
+        if hasattr(torch.xpu, 'getDeviceIdListForCard'):
+            torch.cuda.getDeviceIdListForCard = torch.xpu.getDeviceIdListForCard
+        else:
+            torch.cuda.getDeviceIdListForCard = torch.xpu.get_device_id_list_per_card
         torch.cuda.get_device_name = torch.xpu.get_device_name
         torch.cuda.get_device_properties = torch.xpu.get_device_properties
         torch.cuda.init = torch.xpu.init

From 180f8c9c93d8e03f053602058acef8a1caee3538 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Wed, 27 Sep 2023 20:09:59 +0300
Subject: [PATCH 22/27] Update OpenVINO

---
 README.md                          |  6 +++---
 installer.py                       |  2 +-
 modules/intel/openvino/__init__.py | 20 ++++++++++++++++++++
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index bc1f07692..5926a6470 100644
--- a/README.md
+++ b/README.md
@@ -69,10 +69,10 @@ Additional models will be added as they become available and there is public int
 - *nVidia* GPUs using **CUDA** libraries on both *Windows and Linux*
 - *AMD* GPUs using **ROCm** libraries on *Linux*.  
   Support will be extended to *Windows* once AMD releases ROCm for Windows
-- Any GPU compatibile with *DirectX* on *Windows* using **DirectML** libraries.  
-  This includes support for AMD GPUs that are not supported by native ROCm libraries
 - *Intel Arc* GPUs using **OneAPI** with *IPEX XPU* libraries on both *Windows and Linux*
-- *Intel* GPUs using **OpenVINO** libraries on both *Windows and Linux*
+- Any GPU compatible with *DirectX* on *Windows* using **DirectML** libraries.  
+  This includes support for AMD GPUs that are not supported by native ROCm libraries
+- Any GPU compatible with **OpenVINO** libraries on both *Windows and Linux*
 - *Apple M1/M2* on *OSX* using built-in support in Torch with **MPS** optimizations
 
 ## Install & Run
diff --git a/installer.py b/installer.py
index 9a322b677..d7350654f 100644
--- a/installer.py
+++ b/installer.py
@@ -430,7 +430,7 @@ def check_torch():
     elif allow_openvino and args.use_openvino:
         #Remove this after 2.1.0 releases
         log.info('Using OpenVINO')
-        torch_command = os.environ.get('TORCH_COMMAND', '--pre torch==2.1.0.dev20230726+cpu torchvision==0.16.0.dev20230726+cpu -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html')
+        torch_command = os.environ.get('TORCH_COMMAND', '--pre torch==2.1.0.dev20230820+cpu torchvision==0.16.0.dev20230820+cpu -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html')
     else:
         machine = platform.machine()
         if sys.platform == 'darwin':
diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py
index 460062a7b..30ac588d4 100644
--- a/modules/intel/openvino/__init__.py
+++ b/modules/intel/openvino/__init__.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import torch
 from openvino.frontend import FrontEndManager
 from openvino.frontend.pytorch.fx_decoder import TorchFXPythonDecoder
@@ -13,6 +14,25 @@ from hashlib import sha256
 import functools
 from modules import shared, devices
 
+def BUILD_MAP_UNPACK(self, inst):
+        items = self.popn(inst.argval)
+        # ensure everything is a dict
+        items = [BuiltinVariable(dict).call_function(self, [x], {}) for x in items] # noqa: F821
+        result = dict()
+        for x in items:
+            assert isinstance(x, ConstDictVariable) # noqa: F821
+        result.update(x.items)
+        self.push(
+            ConstDictVariable( # noqa: F821
+                result,
+                dict,
+                mutable_local=MutableLocal(), # noqa: F821
+                **VariableTracker.propagate(items), # noqa: F821
+            )
+        )
+tmp_torch = sys.modules["torch"]
+tmp_torch.BUILD_MAP_UNPACK_WITH_CALL = BUILD_MAP_UNPACK
+
 compiled_cache = {}
 max_openvino_partitions = 0
 partitioned_modules = {}

From 49d181b8a3a24a73400c4f78afe8f1b922bf7b80 Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Thu, 28 Sep 2023 00:26:53 +0300
Subject: [PATCH 23/27] Remove OpenVINO device warning

---
 extensions-builtin/sd-dynamic-thresholding | 1 +
 modules/intel/openvino/__init__.py         | 4 ----
 2 files changed, 1 insertion(+), 4 deletions(-)
 create mode 160000 extensions-builtin/sd-dynamic-thresholding

diff --git a/extensions-builtin/sd-dynamic-thresholding b/extensions-builtin/sd-dynamic-thresholding
new file mode 160000
index 000000000..55ca687fb
--- /dev/null
+++ b/extensions-builtin/sd-dynamic-thresholding
@@ -0,0 +1 @@
+Subproject commit 55ca687fb8f23044666414b7478fd52a8ec85a55
diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py
index 30ac588d4..698c162be 100644
--- a/modules/intel/openvino/__init__.py
+++ b/modules/intel/openvino/__init__.py
@@ -85,10 +85,6 @@ def get_device():
         shared.log.warning("OpenVINO: No compatible GPU detected!")
     os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device)
     shared.log.debug(f"OpenVINO Device: {device}")
-    if shared.opts.cuda_compile_errors and device not in core.available_devices:
-        shared.log.error(f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices")
-    assert device in core.available_devices, f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices"
-
     return device
 
 def cache_root_path():

From ccae1ef02986f00f096cf43098cc5e993a68079c Mon Sep 17 00:00:00 2001
From: Disty0 <semihgulec2005@gmail.com>
Date: Thu, 28 Sep 2023 00:48:45 +0300
Subject: [PATCH 24/27] Fix git submodule

---
 extensions-builtin/sd-dynamic-thresholding | 1 -
 1 file changed, 1 deletion(-)
 delete mode 160000 extensions-builtin/sd-dynamic-thresholding

diff --git a/extensions-builtin/sd-dynamic-thresholding b/extensions-builtin/sd-dynamic-thresholding
deleted file mode 160000
index 55ca687fb..000000000
--- a/extensions-builtin/sd-dynamic-thresholding
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 55ca687fb8f23044666414b7478fd52a8ec85a55

From 01c466632d299c087a9033f1846558e94383d132 Mon Sep 17 00:00:00 2001
From: vladmandic <vladmandic@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:08:55 +0000
Subject: [PATCH 25/27] =?UTF-8?q?Deploying=20to=20master=20from=20@=20vlad?=
 =?UTF-8?q?mandic/automatic@ccae1ef02986f00f096cf43098cc5e993a68079c=20?=
 =?UTF-8?q?=F0=9F=9A=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5926a6470..19318a2eb 100644
--- a/README.md
+++ b/README.md
@@ -175,7 +175,7 @@ General goals:
 ### **Sponsors**
 
 <div align="center">
-<!-- sponsors --><a href="https://github.com/Tillerz"><img src="https://github.com/Tillerz.png" width="60px" alt="Tillerz" /></a><a href="https://github.com/allangrant"><img src="https://github.com/allangrant.png" width="60px" alt="Allan Grant" /></a><a href="https://github.com/demib72"><img src="https://github.com/demib72.png" width="60px" alt="Michael Harris" /></a><a href="https://github.com/BrentOzar"><img src="https://github.com/BrentOzar.png" width="60px" alt="Brent Ozar" /></a><a href="https://github.com/AimwiseTonix"><img src="https://github.com/AimwiseTonix.png" width="60px" alt="ToniX" /></a><a href="https://github.com/inktomi"><img src="https://github.com/inktomi.png" width="60px" alt="Matthew Runo" /></a><a href="https://github.com/HELLO-WORLD-SAS"><img src="https://github.com/HELLO-WORLD-SAS.png" width="60px" alt="HELLO WORLD SAS" /></a><a href="https://github.com/4joeknight4"><img src="https://github.com/4joeknight4.png" width="60px" alt="" /></a><a href="https://github.com/SaladTechnologies"><img src="https://github.com/SaladTechnologies.png" width="60px" alt="Salad Technologies" /></a><a href="https://github.com/gymdreams8"><img src="https://github.com/gymdreams8.png" width="60px" alt="Gym Dreams • GymDreams8" /></a><!-- sponsors -->
+<!-- sponsors --><a href="https://github.com/allangrant"><img src="https://github.com/allangrant.png" width="60px" alt="Allan Grant" /></a><a href="https://github.com/demib72"><img src="https://github.com/demib72.png" width="60px" alt="Michael Harris" /></a><a href="https://github.com/BrentOzar"><img src="https://github.com/BrentOzar.png" width="60px" alt="Brent Ozar" /></a><a href="https://github.com/AimwiseTonix"><img src="https://github.com/AimwiseTonix.png" width="60px" alt="ToniX" /></a><a href="https://github.com/inktomi"><img src="https://github.com/inktomi.png" width="60px" alt="Matthew Runo" /></a><a href="https://github.com/HELLO-WORLD-SAS"><img src="https://github.com/HELLO-WORLD-SAS.png" width="60px" alt="HELLO WORLD SAS" /></a><a href="https://github.com/4joeknight4"><img src="https://github.com/4joeknight4.png" width="60px" alt="" /></a><a href="https://github.com/SaladTechnologies"><img src="https://github.com/SaladTechnologies.png" width="60px" alt="Salad Technologies" /></a><a href="https://github.com/gymdreams8"><img src="https://github.com/gymdreams8.png" width="60px" alt="Gym Dreams • GymDreams8" /></a><!-- sponsors -->
 </div>
 
 <br>

From 272a50fd61bf04d6cafde71d502f32c3fdc7c2df Mon Sep 17 00:00:00 2001
From: QuantumSoul <jeuxcooletgratuit.simdif@gmail.com>
Date: Sun, 1 Oct 2023 16:14:36 +0200
Subject: [PATCH 26/27] Create invokeai.css

---
 javascript/invokeai.css | 311 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 311 insertions(+)
 create mode 100644 javascript/invokeai.css

diff --git a/javascript/invokeai.css b/javascript/invokeai.css
new file mode 100644
index 000000000..8357b2b33
--- /dev/null
+++ b/javascript/invokeai.css
@@ -0,0 +1,311 @@
+/* generic html tags */
+:root, .light, .dark {
+  --font: 'system-ui', 'ui-sans-serif', 'system-ui', "Roboto", sans-serif;
+  --font-mono: 'ui-monospace', 'Consolas', monospace;
+  --font-size: 16px;
+  --left-column: 490px;
+  --primary-400: #566176; 
+  --primary-500: #483a90; 
+  --primary-700: #6b7994; 
+  --primary-800: #5b49b3; 
+  --highlight-color: var(--primary-500);
+  --inactive-color: var(--primary--800);
+  --body-text-color: var(--neutral-100);
+  --body-text-color-subdued: var(--neutral-300);
+  --background-color: #2b303b; 
+  --background-fill-primary: var(--input-background-fill);
+  --input-padding: 8px;
+  --input-background-fill: #15181e;
+  --input-shadow: none;
+  --button-secondary-text-color: white;
+  --button-secondary-background-fill: var(--primary-400);
+  --button-secondary-background-fill-hover: var(--primary-700);
+  --block-title-text-color: var(--neutral-300);
+  --radius-sm: 1px;
+  --radius-lg: 6px;
+  --spacing-md: 4px;
+  --spacing-xxl: 8px;
+  --line-sm: 1.2em;
+  --line-md: 1.4em;
+}
+
+html { font-size: var(--font-size); }
+body, button, input, select, textarea { font-family: var(--font);}
+button { font-size: 1.2rem; max-width: 400px; }
+img { background-color: var(--background-color); }
+input[type=range] { height: var(--line-sm); appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; }
+input[type=range]::-webkit-slider-runnable-track, input[type=range]::-moz-range-track { width: 100%; height: 6px; cursor: pointer; background: var(--primary-400); border-radius: var(--radius-lg); border: 0px solid #222222; }
+input[type=range]::-webkit-slider-thumb, input[type=range]::-moz-range-thumb { border: 0px solid #000000; height: var(--line-sm); width: 8px; border-radius: var(--radius-lg); background: white; cursor: pointer; appearance: none; margin-top: 0px; }
+input[type="range"]::-moz-range-progress {  background-color: var(--primary-500);  height: 6px;  border-radius: var(--radius-lg); 
+}
+::-webkit-scrollbar-track { background: #333333; }
+::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; }
+div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; margin-bottom: 6px; }
+div.compact { gap: 1em; }
+
+/* gradio style classes */
+fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; }
+.border-2 { border-width: 0; }
+.border-b-2 { border-bottom-width: 2px; border-color: var(--highlight-color) !important; padding-bottom: 2px; margin-bottom: 8px; }
+.bg-white { color: lightyellow; background-color: var(--inactive-color); }
+.gr-box { border-radius: var(--radius-sm) !important; background-color: #111111 !important; box-shadow: 2px 2px 3px #111111; border-width: 0; padding: 4px; margin: 12px 0px 12px 0px }
+.gr-button { font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.8rem; min-width: 32px; min-height: 32px; padding: 3px; margin: 3px; }
+.gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: var(--radius-lg); box-shadow: 2px 2px 3px #111111; }
+.gr-check-radio:checked { background-color: var(--highlight-color); }
+.gr-compact { background-color: var(--background-color); }
+.gr-form { border-width: 0; }
+.gr-input { background-color: #333333 !important; padding: 4px; margin: 4px; }
+.gr-input-label { color: lightyellow; border-width: 0; background: transparent; padding: 2px !important; }
+.gr-panel { background-color: var(--background-color); }
+.eta-bar { display: none !important }
+svg.feather.feather-image, .feather .feather-image { display: none }
+.gap-2 { padding-top: 8px; }
+.gr-box > div > div > input.gr-text-input { right: 0; width: 4em; padding: 0; top: -12px; border: none; max-height: 20px; }
+.output-html { line-height: 1.2rem; overflow-x: hidden; }
+.output-html > div { margin-bottom: 8px; }
+.overflow-hidden .flex .flex-col .relative col .gap-4 { min-width: var(--left-column); max-width: var(--left-column); } /* this is a problematic one */
+.p-2 { padding: 0; }
+.px-4 { padding-lefT: 1rem; padding-right: 1rem; }
+.py-6 { padding-bottom: 0; }
+.tabs { background-color: var(--background-color); }
+.block.token-counter span { background-color: var(--input-background-fill) !important; box-shadow: 2px 2px 2px #111; border: none !important; font-size: 0.8rem; }
+.tab-nav { zoom: 120%; margin-top: 10px; margin-bottom: 10px; border-bottom: 2px solid var(--highlight-color) !important; padding-bottom: 2px; }
+div.tab-nav button.selected {background-color: var(--button-primary-background-fill);}
+#settings div.tab-nav button.selected {background-color: var(--background-color); color: var(--primary-800); font-weight: bold;}
+.label-wrap { background-color: #363c4a; padding: 16px 8px 8px 8px; border-radius: var(--radius-lg);}
+.gradio-slider input[type="number"] { width: 4.5em; font-size: 0.8rem; height: 20px; }
+.gradio-button.tool { border: none; box-shadow: none; border-radius: var(--radius-lg);}
+button.selected {background: var(--button-primary-background-fill);}
+.center.boundedheight.flex {background-color: var(--input-background-fill);}
+.compact {border-radius: var(--border-radius-lg);}
+#logMonitorData {background-color: var(--input-background-fill);}
+#tab_extensions table td, #tab_extensions table th, #tab_config table td, #tab_config table th { border: none; padding: 0.5em; }
+#tab_extensions table, #tab_config table { width: 96vw }
+#tab_extensions table thead, #tab_config table thead { background-color: var(--neutral-700); }
+#tab_extensions table, #tab_config table { background-color: #222222; }
+
+/* automatic style classes */
+.progressDiv { border-radius: var(--radius-sm) !important; position: fixed; top: 44px; right: 26px; max-width: 262px; height: 48px; z-index: 99; box-shadow: var(--button-shadow); }
+.progressDiv .progress { border-radius: var(--radius-lg) !important; background: var(--highlight-color); line-height: 3rem; height: 48px; }
+.gallery-item { box-shadow: none !important; }
+.performance { color: #888; }
+.extra-networks { border-left: 2px solid var(--highlight-color) !important; padding-left: 4px; }
+.image-buttons { gap: 10px !important; justify-content: center; }
+.image-buttons > button { max-width: 160px; }
+.tooltip { background: var(--primary-800); color: white; border: none; border-radius: var(--radius-lg) }
+#system_row > button, #settings_row > button, #config_row > button { max-width: 190px; }
+
+/* gradio elements overrides */
+#div.gradio-container { overflow-x: hidden; }
+#img2img_label_copy_to_img2img { font-weight: normal; }
+#txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt { background-color: var(--background-color); box-shadow: 4px 4px 4px 0px #333333  !important; }
+#txt2img_prompt > label > textarea, #txt2img_neg_prompt > label > textarea, #img2img_prompt > label > textarea, #img2img_neg_prompt > label > textarea { font-size: 1.1rem; }
+#img2img_settings { min-width: calc(2 * var(--left-column)); max-width: calc(2 * var(--left-column)); background-color: #111111; padding-top: 16px; }
+#interrogate, #deepbooru { margin: 0 0px 10px 0px; max-width: 80px; max-height: 80px; font-weight: normal; font-size: 0.95em; }
+#quicksettings .gr-button-tool { font-size: 1.6rem; box-shadow: none; margin-top: -2px; height: 2.4em; }
+#quicksettings button {padding: 0 0.5em 0.1em 0.5em;}
+#open_folder_extras, #footer, #style_pos_col, #style_neg_col, #roll_col, #extras_upscaler_2, #extras_upscaler_2_visibility, #txt2img_seed_resize_from_w, #txt2img_seed_resize_from_h { display: none; }
+#save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; background-color: #111111; }
+#script_list { padding: 4px; margin-top: 20px; margin-bottom: 20px; }
+#settings > div.flex-wrap { width: 15em; }
+#txt2img_cfg_scale { min-width: 200px; }
+#txt2img_checkboxes, #img2img_checkboxes { background-color: transparent; }
+#txt2img_checkboxes, #img2img_checkboxes { margin-bottom: 0.2em; }
+#txt2img_actions_column, #img2img_actions_column { flex-flow: wrap; justify-content: space-between; }
+#txt2img_enqueue_wrapper, #img2img_enqueue_wrapper { min-width: unset; width: 48%; }
+#txt2img_generate_box, #img2img_generate_box { min-width: unset; width: 48%; }
+
+#extras_upscale { margin-top: 10px }
+#txt2img_progress_row > div { min-width: var(--left-column); max-width: var(--left-column); }
+#txt2img_results, #img2img_results, #extras_results { background-color: var(--background-color); padding: 0; }
+#txt2img_settings { min-width: var(--left-column); max-width: var(--left-column); background-color: #111111; padding-top: 16px; }
+#pnginfo_html2_info { margin-top: -18px; background-color: var(--input-background-fill); padding: var(--input-padding) }
+#txt2img_tools, #img2img_tools { margin-top: -4px; margin-bottom: -4px; }
+#txt2img_styles_row, #img2img_styles_row { margin-top: -6px; z-index: 200; }
+
+/* custom elements overrides */
+#steps-animation, #controlnet { border-width: 0; }
+
+/* based on gradio built-in dark theme */
+:root, .light, .dark {
+  --body-background-fill: var(--background-color);
+  --color-accent-soft: var(--neutral-700);
+  --background-fill-secondary: none;
+  --border-color-accent: var(--background-color);
+  --border-color-primary: var(--background-color);
+  --link-text-color-active: var(--primary-500);
+  --link-text-color: var(--secondary-500);
+  --link-text-color-hover: var(--secondary-400);
+  --link-text-color-visited: var(--secondary-600);
+  --shadow-spread: 1px;
+  --block-background-fill: None;
+  --block-border-color: var(--border-color-primary);
+  --block_border_width: None;
+  --block-info-text-color: var(--body-text-color-subdued);
+  --block-label-background-fill: var(--background-fill-secondary);
+  --block-label-border-color: var(--border-color-primary);
+  --block_label_border_width: None;
+  --block-label-text-color: var(--neutral-200);
+  --block_shadow: None;
+  --block_title_background_fill: None;
+  --block_title_border_color: None;
+  --block_title_border_width: None;
+  --panel-background-fill: var(--background-fill-secondary);
+  --panel-border-color: var(--border-color-primary);
+  --panel_border_width: None;
+  --checkbox-background-color: var(--primary-400);
+  --checkbox-background-color-focus: var(--primary-700);
+  --checkbox-background-color-hover: var(--primary-700);
+  --checkbox-background-color-selected: var(--primary-500);
+  --checkbox-border-color: transparent;
+  --checkbox-border-color-focus: var(--primary-800);
+  --checkbox-border-color-hover: var(--primary-800);
+  --checkbox-border-color-selected: var(--primary-800);
+  --checkbox-border-width: var(--input-border-width);
+  --checkbox-label-background-fill: None;
+  --checkbox-label-background-fill-hover: None;
+  --checkbox-label-background-fill-selected: var(--checkbox-label-background-fill);
+  --checkbox-label-border-color: var(--border-color-primary);
+  --checkbox-label-border-color-hover: var(--checkbox-label-border-color);
+  --checkbox-label-border-width: var(--input-border-width);
+  --checkbox-label-text-color: var(--body-text-color);
+  --checkbox-label-text-color-selected: var(--checkbox-label-text-color);
+  --error-background-fill: var(--background-fill-primary);
+  --error-border-color: var(--border-color-primary);
+  --error-text-color: #ef4444;
+  --input-background-fill-focus: var(--secondary-600);
+  --input-background-fill-hover: var(--input-background-fill);
+  --input-border-color: var(--background-color);
+  --input-border-color-focus: var(--primary-800);
+  --input-placeholder-color: var(--neutral-500);
+  --input-shadow-focus: None;
+  --loader_color: None;
+  --slider_color: None;
+  --stat-background-fill: linear-gradient(to right, var(--primary-400), var(--primary-800));
+  --table-border-color: var(--neutral-700);
+  --table-even-background-fill: #222222;
+  --table-odd-background-fill: #333333;
+  --table-row-focus: var(--color-accent-soft);
+  --button-border-width: var(--input-border-width);
+  --button-cancel-background-fill: linear-gradient(to bottom right, #dc2626, #b91c1c);
+  --button-cancel-background-fill-hover: linear-gradient(to bottom right, #dc2626, #dc2626);
+  --button-cancel-border-color: #dc2626;
+  --button-cancel-border-color-hover: var(--button-cancel-border-color);
+  --button-cancel-text-color: white;
+  --button-cancel-text-color-hover: var(--button-cancel-text-color);
+  --button-primary-background-fill: var(--primary-500);
+  --button-primary-background-fill-hover: var(--primary-800);
+  --button-primary-border-color: var(--primary-500);
+  --button-primary-border-color-hover: var(--button-primary-border-color);
+  --button-primary-text-color: white;
+  --button-primary-text-color-hover: var(--button-primary-text-color);
+  --button-secondary-border-color: var(--neutral-600);
+  --button-secondary-border-color-hover: var(--button-secondary-border-color);
+  --button-secondary-text-color-hover: var(--button-secondary-text-color);
+  --secondary-50: #eff6ff;
+  --secondary-100: #dbeafe;
+  --secondary-200: #bfdbfe;
+  --secondary-300: #93c5fd;
+  --secondary-400: #60a5fa;
+  --secondary-500: #3b82f6;
+  --secondary-600: #2563eb;
+  --secondary-700: #1d4ed8;
+  --secondary-800: #1e40af;
+  --secondary-900: #1e3a8a;
+  --secondary-950: #1d3660;
+  --neutral-50: #f0f0f0;
+  --neutral-100: #e0e0e0;
+  --neutral-200: #d0d0d0;
+  --neutral-300: #b0b0b0;
+  --neutral-400: #909090;
+  --neutral-500: #707070;
+  --neutral-600: #606060;
+  --neutral-700: #404040;
+  --neutral-800: #333333;
+  --neutral-900: #111827;
+  --neutral-950: #0b0f19;
+  --spacing-xxs: 1px;
+  --spacing-xs: 2px;
+  --spacing-sm: 4px;
+  --spacing-lg: 8px;
+  --spacing-xl: 10px;
+  --radius-xxs: 0;
+  --radius-xs: 0;
+  --radius-md: 0;
+  --radius-xl: 0;
+  --radius-xxl: 0;
+  --text-xxs: 9px;
+  --text-xs: 10px;
+  --text-sm: 12px;
+  --text-md: 14px;
+  --text-lg: 16px;
+  --text-xl: 22px;
+  --text-xxl: 26px;
+  --body-text-size: var(--text-md);
+  --body-text-weight: 400;
+  --embed-radius: var(--radius-lg);
+  --color-accent: var(--primary-500);
+  --shadow-drop: 0;
+  --shadow-drop-lg: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);
+  --shadow-inset: rgba(0,0,0,0.05) 0px 2px 4px 0px inset;
+  --block-border-width: 1px;
+  --block-info-text-size: var(--text-sm);
+  --block-info-text-weight: 400;
+  --block-label-border-width: 1px;
+  --block-label-margin: 0;
+  --block-label-padding: var(--spacing-sm) var(--spacing-lg);
+  --block-label-radius: calc(var(--radius-lg) - 1px) 0 calc(var(--radius-lg) - 1px) 0;
+  --block-label-right-radius: 0 calc(var(--radius-lg) - 1px) 0 calc(var(--radius-lg) - 1px);
+  --block-label-text-size: var(--text-sm);
+  --block-label-text-weight: 400;
+  --block-padding: var(--spacing-xl) calc(var(--spacing-xl) + 2px);
+  --block-radius: var(--radius-lg);
+  --block-shadow: var(--shadow-drop);
+  --block-title-background-fill: none;
+  --block-title-border-color: none;
+  --block-title-border-width: 0;
+  --block-title-padding: 0;
+  --block-title-radius: none;
+  --block-title-text-size: var(--text-md);
+  --block-title-text-weight: 400;
+  --container-radius: var(--radius-lg);
+  --form-gap-width: 1px;
+  --layout-gap: var(--spacing-xxl);
+  --panel-border-width: 0;
+  --section-header-text-size: var(--text-md);
+  --section-header-text-weight: 400;
+  --checkbox-border-radius: var(--radius-sm);
+  --checkbox-label-gap: 2px;
+  --checkbox-label-padding: var(--spacing-md);
+  --checkbox-label-shadow: var(--shadow-drop);
+  --checkbox-label-text-size: var(--text-md);
+  --checkbox-label-text-weight: 400;
+  --checkbox-check: url("data:image/svg+xml,%3csvg viewBox='0 0 16 16' fill='white' xmlns='http://www.w3.org/2000/svg'%3e%3cpath d='M12.207 4.793a1 1 0 010 1.414l-5 5a1 1 0 01-1.414 0l-2-2a1 1 0 011.414-1.414L6.5 9.086l4.293-4.293a1 1 0 011.414 0z'/%3e%3c/svg%3e");
+  --radio-circle: url("data:image/svg+xml,%3csvg viewBox='0 0 16 16' fill='white' xmlns='http://www.w3.org/2000/svg'%3e%3ccircle cx='8' cy='8' r='3'/%3e%3c/svg%3e");
+  --checkbox-shadow: var(--input-shadow);
+  --error-border-width: 1px;
+  --input-border-width: 1px;
+  --input-radius: var(--radius-lg);
+  --input-text-size: var(--text-md);
+  --input-text-weight: 400;
+  --loader-color: var(--color-accent);
+  --prose-text-size: var(--text-md);
+  --prose-text-weight: 400;
+  --prose-header-text-weight: 600;
+  --slider-color: ;
+  --table-radius: var(--radius-lg);
+  --button-large-padding: 2px 10px;
+  --button-large-radius: var(--radius-lg);
+  --button-large-text-size: var(--text-lg);
+  --button-large-text-weight: 400;
+  --button-shadow: none;
+  --button-shadow-active: none;
+  --button-shadow-hover: none;
+  --button-small-padding: var(--spacing-sm) calc(2 * var(--spacing-sm));
+  --button-small-radius: var(--radius-lg);
+  --button-small-text-size: var(--text-md);
+  --button-small-text-weight: 400;
+  --button-transition: none;
+  --size-9: 64px;
+  --size-14: 64px;
+}

From 7b6e3344da55626564f0b9be968cb3e38af03024 Mon Sep 17 00:00:00 2001
From: QuantumSoul <jeuxcooletgratuit.simdif@gmail.com>
Date: Sun, 1 Oct 2023 17:10:58 +0200
Subject: [PATCH 27/27] Update invokeai.css

---
 javascript/invokeai.css | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/javascript/invokeai.css b/javascript/invokeai.css
index 8357b2b33..363aff97b 100644
--- a/javascript/invokeai.css
+++ b/javascript/invokeai.css
@@ -4,6 +4,9 @@
   --font-mono: 'ui-monospace', 'Consolas', monospace;
   --font-size: 16px;
   --left-column: 490px;
+  --primary-100: #2b303b;
+  --primary-200: #15181e;
+  --primary-300: #0a0c0e;
   --primary-400: #566176; 
   --primary-500: #483a90; 
   --primary-700: #6b7994; 
@@ -12,10 +15,10 @@
   --inactive-color: var(--primary--800);
   --body-text-color: var(--neutral-100);
   --body-text-color-subdued: var(--neutral-300);
-  --background-color: #2b303b; 
+  --background-color: var(--primary-100); 
   --background-fill-primary: var(--input-background-fill);
   --input-padding: 8px;
-  --input-background-fill: #15181e;
+  --input-background-fill: var(--primary-200);
   --input-shadow: none;
   --button-secondary-text-color: white;
   --button-secondary-background-fill: var(--primary-400);
@@ -36,8 +39,7 @@ img { background-color: var(--background-color); }
 input[type=range] { height: var(--line-sm); appearance: none; margin-top: 0; min-width: 160px; background-color: var(--background-color); width: 100%; background: transparent; }
 input[type=range]::-webkit-slider-runnable-track, input[type=range]::-moz-range-track { width: 100%; height: 6px; cursor: pointer; background: var(--primary-400); border-radius: var(--radius-lg); border: 0px solid #222222; }
 input[type=range]::-webkit-slider-thumb, input[type=range]::-moz-range-thumb { border: 0px solid #000000; height: var(--line-sm); width: 8px; border-radius: var(--radius-lg); background: white; cursor: pointer; appearance: none; margin-top: 0px; }
-input[type="range"]::-moz-range-progress {  background-color: var(--primary-500);  height: 6px;  border-radius: var(--radius-lg); 
-}
+input[type=range]::-moz-range-progress {  background-color: var(--primary-500);  height: 6px;  border-radius: var(--radius-lg); }
 ::-webkit-scrollbar-track { background: #333333; }
 ::-webkit-scrollbar-thumb { background-color: var(--highlight-color); border-radius: var(--radius-lg); border-width: 0; box-shadow: 2px 2px 3px #111111; }
 div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; margin-bottom: 6px; }
@@ -79,10 +81,10 @@ button.selected {background: var(--button-primary-background-fill);}
 .center.boundedheight.flex {background-color: var(--input-background-fill);}
 .compact {border-radius: var(--border-radius-lg);}
 #logMonitorData {background-color: var(--input-background-fill);}
-#tab_extensions table td, #tab_extensions table th, #tab_config table td, #tab_config table th { border: none; padding: 0.5em; }
-#tab_extensions table, #tab_config table { width: 96vw }
-#tab_extensions table thead, #tab_config table thead { background-color: var(--neutral-700); }
-#tab_extensions table, #tab_config table { background-color: #222222; }
+#tab_extensions table td, #tab_extensions table th, #tab_config table td, #tab_config table th { border: none; padding: 0.5em; background-color: var(--primary-200); }
+#tab_extensions table, #tab_config table { width: 96vw; }
+#tab_extensions table input[type=checkbox] {appearance: none; border-radius: 0px;}
+#tab_extensions button:hover { background-color: var(--button-secondary-background-fill-hover);}
 
 /* automatic style classes */
 .progressDiv { border-radius: var(--radius-sm) !important; position: fixed; top: 44px; right: 26px; max-width: 262px; height: 48px; z-index: 99; box-shadow: var(--button-shadow); }
@@ -183,8 +185,8 @@ button.selected {background: var(--button-primary-background-fill);}
   --slider_color: None;
   --stat-background-fill: linear-gradient(to right, var(--primary-400), var(--primary-800));
   --table-border-color: var(--neutral-700);
-  --table-even-background-fill: #222222;
-  --table-odd-background-fill: #333333;
+  --table-even-background-fill: var(--primary-300);
+  --table-odd-background-fill: var(--primary-200);
   --table-row-focus: var(--color-accent-soft);
   --button-border-width: var(--input-border-width);
   --button-cancel-background-fill: linear-gradient(to bottom right, #dc2626, #b91c1c);