From dc79cec19cbc06ed25f62a74c5fea81d02ebdc61 Mon Sep 17 00:00:00 2001 From: vladmandic Date: Fri, 3 Nov 2023 07:12:28 +0000 Subject: [PATCH 01/43] =?UTF-8?q?Deploying=20to=20master=20from=20@=20vlad?= =?UTF-8?q?mandic/automatic@2336ffc0eef3265b49688e73ef70f8cbafee1202=20?= =?UTF-8?q?=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9e136c0e..721955890 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ General goals: ### **Sponsors**
-Allan GrantMichael HarrisBrent OzarToniXMatthew RunoHELLO WORLD SASSalad TechnologiesGym Dreams • GymDreams8a.v.mantzaris +Allan GrantMichael HarrisBrent OzarToniXMatthew RunoHELLO WORLD SASSalad Technologiesa.v.mantzaris

From 361903660f291fba681166ed2c2f0a29acc46ec8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 6 Nov 2023 17:50:29 -0500 Subject: [PATCH 02/43] log cleanup --- installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.py b/installer.py index 01a48ad8e..adb57121f 100644 --- a/installer.py +++ b/installer.py @@ -201,7 +201,7 @@ def installed(package, friendly: str = None, reload = False, quiet = False): def uninstall(package): - if installed(package, package): + if installed(package, package, quiet=True): log.warning(f'Uninstalling: {package}') pip(f"uninstall {package} --yes --quiet", ignore=True, quiet=True) From a2b50b6922e9666e46be2d4a95d7ff4b4990f86f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 6 Nov 2023 17:51:00 -0500 Subject: [PATCH 03/43] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41055f528..b4896751f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Another pretty big release, this time with focus on new models (3 new model type Plus quite a few fixes Also, [Wiki](https://github.com/vladmandic/automatic/wiki) has been updated with new content, so check it out! -Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IntelArc](https://github.com/vladmandic/automatic/wiki/Intel-ARC), [DirectML](https://github.com/vladmandic/automatic/wiki/DirectML), [ONNX/Olive>](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime) +Some highlights: [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IntelArc](https://github.com/vladmandic/automatic/wiki/Intel-ARC), [DirectML](https://github.com/vladmandic/automatic/wiki/DirectML), [ONNX/Olive](https://github.com/vladmandic/automatic/wiki/ONNX-Olive) - **Diffusers** - since now **SD.Next** supports **12** different model types, we've added reference model for each type in From e53db4e259904f5cac71381f5921e1c74fcf8072 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Nov 2023 08:00:44 -0500 Subject: [PATCH 04/43] use ThreadPoolExecutor for extra networks --- .../Lora/ui_extra_networks_lora.py | 12 ++-- extensions-builtin/sd-webui-agent-scheduler | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- modules/modelloader.py | 62 ++++++++-------- modules/ui_extra_networks.py | 12 ++-- modules/ui_extra_networks_checkpoints.py | 59 ++++++++------- modules/ui_extra_networks_styles.py | 66 +++++++++-------- .../ui_extra_networks_textual_inversion.py | 72 +++++++++++-------- 8 files changed, 165 insertions(+), 122 deletions(-) diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index 249f84890..7df667acb 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -1,5 +1,6 @@ import os import json +import concurrent import network import networks from modules import shared, ui_extra_networks @@ -8,6 +9,7 @@ from modules import shared, ui_extra_networks class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): def __init__(self): super().__init__('Lora') + self.list_time = 0 def refresh(self): networks.list_available_networks() @@ -74,10 +76,12 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): return None def list_items(self): - for _index, name in enumerate(networks.available_networks): - item = self.create_item(name) - if item is not None: - yield item + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks} + for future in concurrent.futures.as_completed(future_items): + item = future.result() + if item is not None: + yield item def allowed_directories_for_previews(self): return [shared.cmd_opts.lora_dir, shared.cmd_opts.lyco_dir] diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 99b2cafbc..02da7abf4 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 99b2cafbc2b4a2fc93ffcabd56b0ff915396d1f1 +Subproject commit 02da7abf4be093499c32c0f758621838968b2212 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index fce6775a6..05ef0b1cd 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit fce6775a6dddef52ecd658259e909687d9dedf72 +Subproject commit 05ef0b1cd1374cf285dd8d5ccd7db9997549893c diff --git a/modules/modelloader.py b/modules/modelloader.py index 4d4858cec..f0fd4241e 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -334,23 +334,23 @@ def load_reference(name: str): return True -modelloader_directories = {} +cache_folders = {} cache_last = 0 cache_time = 1 -def directory_has_changed(dir:str, *, recursive:bool=True) -> bool: # pylint: disable=redefined-builtin +def directory_updated(path:str, *, recursive:bool=True) -> bool: # pylint: disable=redefined-builtin try: - dir = os.path.abspath(dir) - if dir not in modelloader_directories: + path = os.path.abspath(path) + if path not in cache_folders: return True if cache_last > (time.time() - cache_time): return False - if not (os.path.exists(dir) and os.path.isdir(dir) and os.path.getmtime(dir) == modelloader_directories[dir][0]): + if not (os.path.exists(path) and os.path.isdir(path) and os.path.getmtime(path) == cache_folders[path][0]): return True if recursive: - for _dir in modelloader_directories: - if _dir.startswith(dir) and _dir != dir and not (os.path.exists(_dir) and os.path.isdir(_dir) and os.path.getmtime(_dir) == modelloader_directories[_dir][0]): + for folder in cache_folders: + if folder.startswith(path) and folder != path and not (os.path.exists(folder) and os.path.isdir(folder) and os.path.getmtime(folder) == cache_folders[folder][0]): return True except Exception as e: shared.log.error(f"Filesystem Error: {e.__class__.__name__}({e})") @@ -358,44 +358,48 @@ def directory_has_changed(dir:str, *, recursive:bool=True) -> bool: # pylint: di return False -def directory_directories(dir:str, *, recursive:bool=True) -> dict[str,tuple[float,list[str]]]: # pylint: disable=redefined-builtin - dir = os.path.abspath(dir) - if directory_has_changed(dir, recursive=recursive): - for _dir in list(modelloader_directories): - if os.path.exists(_dir) or os.path.isdir(_dir): +def directory_list(path:str, *, recursive:bool=True) -> dict[str,tuple[float,list[str]]]: # pylint: disable=redefined-builtin + path = os.path.abspath(path) + res = {} + if not os.path.exists(path): + return res + if directory_updated(path, recursive=recursive): + for folder in list(cache_folders): + del cache_folders[folder] + if os.path.exists(folder) or os.path.isdir(folder): + continue + for folder, files in walk(path, lambda e, path: shared.log.debug(f"FS walk error: {e} {path}")): + if not os.path.exists(folder): continue - del modelloader_directories[_dir] - for _dir, _files in walk(dir, lambda e, path: shared.log.debug(f"FS walk error: {e} {path}")): try: - mtime = os.path.getmtime(_dir) - if _dir not in modelloader_directories or mtime != modelloader_directories[_dir][0]: - modelloader_directories[_dir] = (mtime, [os.path.join(_dir, fn) for fn in _files]) + mtime = os.path.getmtime(folder) + if folder not in cache_folders or mtime != cache_folders[folder][0]: + cache_folders[folder] = (mtime, [os.path.join(folder, fn) for fn in files]) except Exception as e: shared.log.error(f"Filesystem Error: {e.__class__.__name__}({e})") - del modelloader_directories[_dir] - res = {} - for _dir in modelloader_directories: - if _dir == dir or (recursive and _dir.startswith(dir)): - res[_dir] = modelloader_directories[_dir] + del cache_folders[folder] + for folder in cache_folders: + if folder == path or (recursive and folder.startswith(path)): + res[folder] = cache_folders[folder] if not recursive: break return res -def directory_mtime(dir:str, *, recursive:bool=True) -> float: # pylint: disable=redefined-builtin - return float(max(0, *[mtime for mtime, _ in directory_directories(dir, recursive=recursive).values()])) +def directory_mtime(path:str, *, recursive:bool=True) -> float: # pylint: disable=redefined-builtin + return float(max(0, *[mtime for mtime, _ in directory_list(path, recursive=recursive).values()])) def directories_file_paths(directories:dict) -> list[str]: return sum([dat[1] for dat in directories.values()],[]) -def unique_directories(directories:list[str], *, recursive:bool=True) -> list[str]: +def directories_unique(directories:list[str], *, recursive:bool=True) -> list[str]: '''Ensure no empty, or duplicates''' - directories = { os.path.abspath(dir): True for dir in directories if dir }.keys() + directories = { os.path.abspath(path): True for path in directories if path }.keys() if recursive: '''If we are going recursive, then directories that are children of other directories are redundant''' - directories = [dir for dir in directories if not any(_dir != dir and dir.startswith(os.path.join(_dir,'')) for _dir in directories)] + directories = [path for path in directories if not any(d != path and path.startswith(os.path.join(d,'')) for d in directories)] return directories @@ -404,7 +408,7 @@ def unique_paths(paths:list[str]) -> list[str]: def directory_files(*directories:list[str], recursive:bool=True) -> list[str]: - return unique_paths(sum([[*directories_file_paths(directory_directories(dir, recursive=recursive))] for dir in unique_directories(directories, recursive=recursive)],[])) + return unique_paths(sum([[*directories_file_paths(directory_list(d, recursive=recursive))] for d in directories_unique(directories, recursive=recursive)],[])) def extension_filter(ext_filter=None, ext_blacklist=None): @@ -485,7 +489,7 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None @param ext_filter: An optional list of filename extensions to filter by @return: A list of paths containing the desired model(s) """ - places = unique_directories([model_path, command_path]) + places = directories_unique([model_path, command_path]) output = [] try: output:list = [*filter(extension_filter(ext_filter, ext_blacklist), directory_files(*places))] diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index ff48c82ac..0d0445971 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -222,6 +222,8 @@ class ExtraNetworksPage: self.items = [] shared.log.error(f'Extra networks error listing items: class={self.__class__.__name__} tab={tabname} {e}') for item in self.items: + if item is None: + continue self.metadata[item["name"]] = item.get("metadata", {}) t1 = time.time() debug(f'EN create-items: page={self.name} items={len(self.items)} time={t1-t0:.2f}') @@ -237,7 +239,7 @@ class ExtraNetworksPage: return f"
Extra network page not ready
Click refresh to try again
" subdirs = {} allowed_folders = [os.path.abspath(x) for x in self.allowed_directories_for_previews()] - for parentdir, dirs in {d: modelloader.directory_directories(d) for d in allowed_folders}.items(): + for parentdir, dirs in {d: modelloader.directory_list(d) for d in allowed_folders}.items(): for tgt in dirs.keys(): if shared.backend == shared.Backend.DIFFUSERS: if os.path.join(paths.models_path, 'Reference') in tgt: @@ -539,9 +541,11 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): refresh_time = time.time() threads = [] for page in get_pages(): - # page.create_items(ui.tabname) - threads.append(threading.Thread(target=page.create_items, args=[ui.tabname])) - threads[-1].start() + if os.environ.get('SD_EN_DEBUG', None) is not None: + threads.append(threading.Thread(target=page.create_items, args=[ui.tabname])) + threads[-1].start() + else: + page.create_items(ui.tabname) for thread in threads: thread.join() for page in get_pages(): diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index f31604079..5dae382b7 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -1,8 +1,10 @@ +import os import html import json -import os +import concurrent from modules import shared, ui_extra_networks, sd_models, paths + reference_dir = os.path.join(paths.models_path, 'Reference') class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): @@ -36,31 +38,38 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "description": v.get('desc', ''), } + def create_item(self, name): + record = None + try: + checkpoint: sd_models.CheckpointInfo = sd_models.checkpoints_list.get(name) + exists = os.path.exists(checkpoint.filename) + record = { + "type": 'Model', + "name": checkpoint.name, + "title": checkpoint.title, + "filename": checkpoint.filename, + "hash": checkpoint.shorthash, + "search_term": self.search_terms_from_path(checkpoint.title), + "preview": self.find_preview(checkpoint.filename), + "local_preview": f"{os.path.splitext(checkpoint.filename)[0]}.{shared.opts.samples_format}", + "metadata": checkpoint.metadata, + "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"', + "mtime": os.path.getmtime(checkpoint.filename) if exists else 0, + "size": os.path.getsize(checkpoint.filename) if exists else 0, + } + record["info"] = self.find_info(checkpoint.filename) + record["description"] = self.find_description(checkpoint.filename, record["info"]) + except Exception as e: + shared.log.debug(f"Extra networks error: type=model file={name} {e}") + return record + def list_items(self): - checkpoint: sd_models.CheckpointInfo - checkpoints = sd_models.checkpoints_list.copy() - for name, checkpoint in checkpoints.items(): - try: - exists = os.path.exists(checkpoint.filename) - record = { - "type": 'Model', - "name": checkpoint.name, - "title": checkpoint.title, - "filename": checkpoint.filename, - "hash": checkpoint.shorthash, - "search_term": self.search_terms_from_path(checkpoint.title), - "preview": self.find_preview(checkpoint.filename), - "local_preview": f"{os.path.splitext(checkpoint.filename)[0]}.{shared.opts.samples_format}", - "metadata": checkpoint.metadata, - "onclick": '"' + html.escape(f"""return selectCheckpoint({json.dumps(name)})""") + '"', - "mtime": os.path.getmtime(checkpoint.filename) if exists else 0, - "size": os.path.getsize(checkpoint.filename) if exists else 0, - } - record["info"] = self.find_info(checkpoint.filename) - record["description"] = self.find_description(checkpoint.filename, record["info"]) - yield record - except Exception as e: - shared.log.debug(f"Extra networks error: type=model file={name} {e}") + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + future_items = {executor.submit(self.create_item, cp): cp for cp in list(sd_models.checkpoints_list.copy())} + for future in concurrent.futures.as_completed(future_items): + item = future.result() + if item is not None: + yield item for record in self.list_reference(): yield record diff --git a/modules/ui_extra_networks_styles.py b/modules/ui_extra_networks_styles.py index 49a34affb..12f5f0a53 100644 --- a/modules/ui_extra_networks_styles.py +++ b/modules/ui_extra_networks_styles.py @@ -1,6 +1,7 @@ import os import html import json +import concurrent from modules import shared, extra_networks, ui_extra_networks, styles @@ -62,35 +63,44 @@ class ExtraNetworksPageStyles(ui_extra_networks.ExtraNetworksPage): } return item - def list_items(self): - for k, style in shared.prompt_styles.styles.items(): - try: - fn = os.path.splitext(getattr(style, 'filename', ''))[0] - name = getattr(style, 'name', '') - if name == '': - continue - txt = f'Prompt: {getattr(style, "prompt", "")}' - if len(getattr(style, 'negative_prompt', '')) > 0: - txt += f'\nNegative: {style.negative_prompt}' - yield { - "type": 'Style', - "name": name, - "title": k, - "filename": style.filename, - "search_term": f'{txt} {self.search_terms_from_path(name)}', - "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else self.find_preview(fn), - "description": style.description if getattr(style, 'description', None) is not None and len(style.description) > 0 else txt, - "prompt": getattr(style, 'prompt', ''), - "negative": getattr(style, 'negative_prompt', ''), - "extra": getattr(style, 'extra', ''), - "local_preview": f"{fn}.{shared.opts.samples_format}", - "onclick": '"' + html.escape(f"""return selectStyle({json.dumps(name)})""") + '"', - "mtime": getattr(style, 'mtime', 0), - "size": os.path.getsize(style.filename), - } - except Exception as e: - shared.log.debug(f"Extra networks error: type=style file={k} {e}") + def create_item(self, k): + item = None + try: + style = shared.prompt_styles.styles.get(k) + fn = os.path.splitext(getattr(style, 'filename', ''))[0] + name = getattr(style, 'name', '') + if name == '': + return item + txt = f'Prompt: {getattr(style, "prompt", "")}' + if len(getattr(style, 'negative_prompt', '')) > 0: + txt += f'\nNegative: {style.negative_prompt}' + item = { + "type": 'Style', + "name": name, + "title": k, + "filename": style.filename, + "search_term": f'{txt} {self.search_terms_from_path(name)}', + "preview": style.preview if getattr(style, 'preview', None) is not None and style.preview.startswith('data:') else self.find_preview(fn), + "description": style.description if getattr(style, 'description', None) is not None and len(style.description) > 0 else txt, + "prompt": getattr(style, 'prompt', ''), + "negative": getattr(style, 'negative_prompt', ''), + "extra": getattr(style, 'extra', ''), + "local_preview": f"{fn}.{shared.opts.samples_format}", + "onclick": '"' + html.escape(f"""return selectStyle({json.dumps(name)})""") + '"', + "mtime": getattr(style, 'mtime', 0), + "size": os.path.getsize(style.filename), + } + except Exception as e: + shared.log.debug(f"Extra networks error: type=style file={k} {e}") + return item + def list_items(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + future_items = {executor.submit(self.create_item, style): style for style in list(shared.prompt_styles.styles)} + for future in concurrent.futures.as_completed(future_items): + item = future.result() + if item is not None: + yield item def allowed_directories_for_previews(self): return [v for v in [shared.opts.styles_dir] if v is not None] + ['html'] diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py index 004e4cecc..f67031029 100644 --- a/modules/ui_extra_networks_textual_inversion.py +++ b/modules/ui_extra_networks_textual_inversion.py @@ -1,5 +1,6 @@ import json import os +import concurrent from modules import shared, sd_hijack, sd_models, ui_extra_networks from modules.textual_inversion.textual_inversion import Embedding @@ -8,6 +9,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): def __init__(self): super().__init__('Embedding') self.allow_negative_prompt = True + self.embeddings = [] def refresh(self): if sd_models.model_data.sd_model is None: @@ -17,51 +19,61 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): elif hasattr(sd_models.model_data.sd_model, 'embedding_db'): sd_models.model_data.sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True) + def create_item(self, embedding: Embedding): + record = None + try: + path, _ext = os.path.splitext(embedding.filename) + tags = {} + if embedding.tag is not None: + tags[embedding.tag]=1 + name = os.path.splitext(embedding.basename)[0] + record = { + "type": 'Embedding', + "name": name, + "filename": embedding.filename, + "preview": self.find_preview(embedding.filename), + "search_term": self.search_terms_from_path(name), + "prompt": json.dumps(f" {os.path.splitext(embedding.name)[0]}"), + "local_preview": f"{path}.{shared.opts.samples_format}", + "tags": tags, + "mtime": os.path.getmtime(embedding.filename), + "size": os.path.getsize(embedding.filename), + } + record["info"] = self.find_info(embedding.filename) + record["description"] = self.find_description(embedding.filename, record["info"]) + except Exception as e: + shared.log.debug(f"Extra networks error: type=embedding file={embedding.filename} {e}") + return record + def list_items(self): + def list_folder(folder): for filename in os.listdir(folder): fn = os.path.join(folder, filename) if os.path.isfile(fn) and (fn.lower().endswith(".pt") or fn.lower().endswith(".safetensors")): embedding = Embedding(vec=0, name=os.path.basename(fn), filename=fn) embedding.filename = fn - embeddings.append(embedding) + self.embeddings.append(embedding) elif os.path.isdir(fn) and not fn.startswith('.'): list_folder(fn) if sd_models.model_data.sd_model is None: - embeddings = [] + self.embeddings = [] list_folder(shared.opts.embeddings_dir) elif shared.backend == shared.Backend.ORIGINAL: - embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values()) + self.embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values()) elif hasattr(sd_models.model_data.sd_model, 'embedding_db'): - embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values()) + self.embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values()) else: - embeddings = [] - embeddings = sorted(embeddings, key=lambda emb: emb.filename) - for embedding in embeddings: - try: - path, _ext = os.path.splitext(embedding.filename) - tags = {} - if embedding.tag is not None: - tags[embedding.tag]=1 - name = os.path.splitext(embedding.basename)[0] - record = { - "type": 'Embedding', - "name": name, - "filename": embedding.filename, - "preview": self.find_preview(embedding.filename), - "search_term": self.search_terms_from_path(name), - "prompt": json.dumps(f" {os.path.splitext(embedding.name)[0]}"), - "local_preview": f"{path}.{shared.opts.samples_format}", - "tags": tags, - "mtime": os.path.getmtime(embedding.filename), - "size": os.path.getsize(embedding.filename), - } - record["info"] = self.find_info(embedding.filename) - record["description"] = self.find_description(embedding.filename, record["info"]) - yield record - except Exception as e: - shared.log.debug(f"Extra networks error: type=embedding file={embedding.filename} {e}") + self.embeddings = [] + self.embeddings = sorted(self.embeddings, key=lambda emb: emb.filename) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + future_items = {executor.submit(self.create_item, net): net for net in self.embeddings} + for future in concurrent.futures.as_completed(future_items): + item = future.result() + if item is not None: + yield item def allowed_directories_for_previews(self): return list(sd_hijack.model_hijack.embedding_db.embedding_dirs) From 1f068feb448801714fceb14625fa5d41d123b513 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Nov 2023 09:35:54 -0500 Subject: [PATCH 05/43] fix dpm sde --- javascript/sdnext.css | 3 +-- modules/processing_diffusers.py | 6 ++++++ modules/sd_samplers.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 61aebf947..f245dbe0e 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -106,11 +106,10 @@ div#extras_scale_to_tab div.form{ flex-direction: row; } #quicksettings > button { padding: 0 1em 0 0; align-self: end; margin-bottom: var(--text-sm); } #settings { display: flex; gap: var(--layout-gap); } #settings div { border: none; gap: 0; margin: 0 0 var(--layout-gap) 0px; padding: 0; } -#settings .gr-group { max-width: 70em; } #settings > div.tab-content { flex: 10 0 75%; display: grid; } #settings > div.tab-content > div { border: none; padding: 0; } #settings > div.tab-content > div > div > div > div > div { flex-direction: unset; } -#settings > div.tab-nav { display: grid; grid-template-columns: repeat(auto-fill, .5em minmax(10em, 1fr)); flex: 1 0 auto; width: 12em; align-self: flex-start; gap: var(--spacing-lg); } +#settings > div.tab-nav { display: grid; grid-template-columns: repeat(auto-fill, .5em minmax(10em, 1fr)); flex: 1 0 auto; width: 12em; align-self: flex-start; gap: var(--spacing-xxl); } #settings > div.tab-nav button { display: block; border: none; text-align: left; white-space: initial; padding: 0; } #settings > div.tab-nav > #settings_show_all_pages { padding: var(--size-2) var(--size-4); } #settings .block.gradio-checkbox { margin: 0; width: auto; } diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 3dedcb85b..0c7c16f2b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -263,6 +263,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args['negative_pooled_prompt_embeds'] = negative_pooled else: args['negative_prompt'] = negative_prompts + if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'): + model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values + model.scheduler.noise_sampler_seed = seeds[0] # some schedulers have internal noise generator and do not use pipeline generator + if 'noise_sampler_seed' in possible: + args['noise_sampler_seed'] = seeds[0] if 'guidance_scale' in possible: args['guidance_scale'] = p.cfg_scale if 'generator' in possible: @@ -294,6 +299,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clean.pop('callback', None) clean.pop('callback_steps', None) clean.pop('callback_on_step_end', None) + clean.pop('callback_on_step_end_tensor_inputs', None) if 'latents' in clean: clean['latents'] = clean['latents'].shape if 'image' in clean: diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index db9ad937f..62f0c8a73 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -53,14 +53,14 @@ def create_sampler(name, model): sampler.config = config sampler.initialize(p=None) sampler.name = name - shared.log.debug(f'Sampler: sampler={sampler.name} config={sampler.config.options}') + shared.log.debug(f'Sampler: sampler="{sampler.name}" config={sampler.config.options}') return sampler elif shared.backend == shared.Backend.DIFFUSERS: sampler = config.constructor(model) if not hasattr(model, 'scheduler_config'): model.scheduler_config = sampler.sampler.config.copy() model.scheduler = sampler.sampler - shared.log.debug(f'Sampler: sampler={sampler.name} config={sampler.config}') + shared.log.debug(f'Sampler: sampler="{sampler.name}" config={sampler.config}') return sampler.sampler else: return None From ca7cd437af6fae2a717d83dc51fafb4230fe8a3b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Nov 2023 09:37:21 -0500 Subject: [PATCH 06/43] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4896751f..a1ab38d3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log for SD.Next +## Update for 2023-11-07 + +- **Scheduler**: Fix DPM SDE +- **Extra networks**: Use multi-threading for 5x load speedup + ## Update for 2023-11-06 Another pretty big release, this time with focus on new models (3 new model types), new backends and optimizations From 56cf80b8e0c6f897b3346ad9549ec82dd32116c8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Nov 2023 10:12:21 -0500 Subject: [PATCH 07/43] fix inpaint --- CHANGELOG.md | 7 +++++-- javascript/black-teal.css | 2 +- modules/processing.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1ab38d3e..9767f1d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,11 @@ ## Update for 2023-11-07 -- **Scheduler**: Fix DPM SDE -- **Extra networks**: Use multi-threading for 5x load speedup +- **Extra networks** + - Use multi-threading for 5x load speedup +- **Diffusers** + - Fix DPM SDE scheduler + - Fix inpaint ## Update for 2023-11-06 diff --git a/javascript/black-teal.css b/javascript/black-teal.css index 70e1807ac..b2cd0b984 100644 --- a/javascript/black-teal.css +++ b/javascript/black-teal.css @@ -38,7 +38,7 @@ html { font-size: var(--font-size); font-family: var(--font); } body, button, input, select, textarea { font-family: var(--font); } button { font-size: 1.2rem; max-width: 400px; } img { background-color: var(--background-color); } -input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: 100% !important; +input[type=range] { height: var(--line-sm) !important; appearance: none !important; margin-top: 0 !important; min-width: max(4em, 100%) !important; background-color: var(--background-color) !important; width: 100% !important; background: transparent !important; } input[type=range]::-webkit-slider-runnable-track { width: 100% !important; height: var(--line-sm) !important; cursor: pointer !important; box-shadow: 2px 2px 3px #111111 !important; background: var(--input-background-fill) !important; border-radius: var(--radius-lg) !important; border: 0px solid #222222 !important; } diff --git a/modules/processing.py b/modules/processing.py index b0213db25..3e64bff9a 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -1185,7 +1185,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None: shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING) - self.sd_model.dtype = self.sd_model.unet.dtype + # self.sd_model.dtype = self.sd_model.unet.dtype elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None: shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE) From af30c1e27c0f1e8990d48402e8f4d1ad4b875ecf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Nov 2023 13:15:45 -0500 Subject: [PATCH 08/43] en error handler --- modules/ui_extra_networks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 0d0445971..0584b4f71 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -267,7 +267,8 @@ class ExtraNetworksPage: self.create_items(tabname) self.create_xyz_grid() htmls = [] - self.items.sort(key=lambda x: x["mtime"], reverse=True) + if len(self.items) > 0 and self.items[0].get('mtime', None) is not None: + self.items.sort(key=lambda x: x["mtime"], reverse=True) for item in self.items: htmls.append(self.create_html(item, tabname)) self.html += ''.join(htmls) From 924fae3d8428e6d6c0e2835d26df9ed7db780b0a Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 8 Nov 2023 17:30:24 +0900 Subject: [PATCH 09/43] DirectML LCMScheduler fix --- modules/dml/hijack/diffusers.py | 104 ++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/modules/dml/hijack/diffusers.py b/modules/dml/hijack/diffusers.py index 7ad66336f..881ac44fd 100644 --- a/modules/dml/hijack/diffusers.py +++ b/modules/dml/hijack/diffusers.py @@ -1,5 +1,8 @@ import torch import diffusers +import diffusers.utils.torch_utils +from typing import Optional, Union, Tuple + def PNDMScheduler__get_prev_sample(self, sample: torch.FloatTensor, timestep, prev_timestep, model_output): # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf @@ -45,8 +48,10 @@ def PNDMScheduler__get_prev_sample(self, sample: torch.FloatTensor, timestep, pr return prev_sample + diffusers.PNDMScheduler._get_prev_sample = PNDMScheduler__get_prev_sample # pylint: disable=protected-access + def UniPCMultistepScheduler_multistep_uni_p_bh_update( self: diffusers.UniPCMultistepScheduler, model_output: torch.FloatTensor, @@ -153,4 +158,103 @@ def UniPCMultistepScheduler_multistep_uni_p_bh_update( x_t = x_t.to(x.dtype) return x_t + diffusers.UniPCMultistepScheduler.multistep_uni_p_bh_update = UniPCMultistepScheduler_multistep_uni_p_bh_update + + +def LCMScheduler_step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[diffusers.schedulers.scheduling_lcm.LCMSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] or `tuple`. + Returns: + [`~schedulers.scheduling_utils.LCMSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # 1. get previous step value + prev_step_index = self.step_index + 1 + if prev_step_index < len(self.timesteps): + prev_timestep = self.timesteps[prev_step_index] + else: + prev_timestep = timestep + + # 2. compute alphas, betas + sample.__str__() + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + # 3. Get scalings for boundary conditions + c_skip, c_out = self.get_scalings_for_boundary_condition_discrete(timestep) + + # 4. Compute the predicted original sample x_0 based on the model parameterization + if self.config.prediction_type == "epsilon": # noise-prediction + predicted_original_sample = (sample - beta_prod_t.sqrt() * model_output) / alpha_prod_t.sqrt() + elif self.config.prediction_type == "sample": # x-prediction + predicted_original_sample = model_output + elif self.config.prediction_type == "v_prediction": # v-prediction + predicted_original_sample = alpha_prod_t.sqrt() * sample - beta_prod_t.sqrt() * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for `LCMScheduler`." + ) + + # 5. Clip or threshold "predicted x_0" + if self.config.thresholding: + predicted_original_sample = self._threshold_sample(predicted_original_sample) + elif self.config.clip_sample: + predicted_original_sample = predicted_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 6. Denoise model output using boundary conditions + denoised = c_out * predicted_original_sample + c_skip * sample + + # 7. Sample and inject noise z ~ N(0, I) for MultiStep Inference + # Noise is not used for one-step sampling. + if len(self.timesteps) > 1: + noise = diffusers.utils.torch_utils.randn_tensor(model_output.shape, generator=generator, device=model_output.device) + prev_sample = alpha_prod_t_prev.sqrt() * denoised + beta_prod_t_prev.sqrt() * noise + else: + prev_sample = denoised + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample, denoised) + + return diffusers.schedulers.scheduling_lcm.LCMSchedulerOutput(prev_sample=prev_sample, denoised=denoised) + + +diffusers.LCMScheduler.step = LCMScheduler_step From 00562a50841676a03bdc2876f0802c52a8aa5bd1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 07:38:45 -0500 Subject: [PATCH 10/43] cleanup model paths --- extensions-builtin/sd-webui-agent-scheduler | 2 +- modules/deepbooru.py | 2 +- modules/ui_interrogate.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 02da7abf4..dcb085cf8 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 02da7abf4be093499c32c0f758621838968b2212 +Subproject commit dcb085cf814ffca53a9682a7e9af0a9026530ddc diff --git a/modules/deepbooru.py b/modules/deepbooru.py index de50853d6..46d2c3ed5 100644 --- a/modules/deepbooru.py +++ b/modules/deepbooru.py @@ -18,7 +18,7 @@ class DeepDanbooru: return files = modelloader.load_models( - model_path=os.path.join(paths.models_path, "torch_deepdanbooru"), + model_path=os.path.join(paths.models_path, "DeepDanbooru"), model_url='https://github.com/AUTOMATIC1111/TorchDeepDanbooru/releases/download/v1/model-resnet_custom_v3.pt', ext_filter=[".pt"], download_name='model-resnet_custom_v3.pt', diff --git a/modules/ui_interrogate.py b/modules/ui_interrogate.py index 3adc5b72c..b9bb074ea 100644 --- a/modules/ui_interrogate.py +++ b/modules/ui_interrogate.py @@ -35,7 +35,7 @@ class BatchWriter: def load(clip_model_name): global ci # pylint: disable=global-statement if ci is None: - config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'clip-interrogator'), clip_model_name=clip_model_name, quiet=True) + config = Config(device=devices.get_optimal_device(), cache_path=os.path.join(paths.models_path, 'Interrogator'), clip_model_name=clip_model_name, quiet=True) if low_vram: config.apply_low_vram_defaults() shared.log.info(f'Interrogate load: config={config}') From a03d7ce99c566397e3c7b8978b911564badef907 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 07:41:42 -0500 Subject: [PATCH 11/43] fix manual save --- modules/ui_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ui_common.py b/modules/ui_common.py index c5d1fe62e..f27589a8b 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -93,6 +93,7 @@ def save_files(js_data, images, html_info, index): self.index_of_first_image = getattr(self, 'index_of_first_image', 0) self.infotexts = getattr(self, 'infotexts', [html_info]) self.infotext = self.infotexts[0] if len(self.infotexts) > 0 else html_info + self.outpath_grids = None try: data = json.loads(js_data) except Exception: From 8bcb14d0ac0deb1b440786402160df4f94ad3ebd Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 08:35:13 -0500 Subject: [PATCH 12/43] fix adetailer with controlnet --- modules/processing.py | 30 ++++++++++++++++++++++++++++++ modules/styles.py | 3 ++- requirements.txt | 2 +- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 3e64bff9a..1ee141fbc 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -7,6 +7,7 @@ import random import warnings from contextlib import nullcontext from typing import Any, Dict, List +from dataclasses import dataclass, field import torch import numpy as np import cv2 @@ -115,6 +116,7 @@ def txt2img_image_conditioning(sd_model, x, width, height): return x.new_zeros(x.shape[0], 5, 1, 1, dtype=x.dtype, device=x.device) +@dataclass(repr=False) class StableDiffusionProcessing: """ The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing @@ -203,11 +205,39 @@ class StableDiffusionProcessing: self.all_hr_negative_prompts = [] self.comments = {} self.is_api = False + self.scripts_value: modules.scripts.ScriptRunner = field(default=None, init=False) + self.script_args_value: list = field(default=None, init=False) + self.scripts_setup_complete: bool = field(default=False, init=False) + @property def sd_model(self): return shared.sd_model + @property + def scripts(self): + return self.scripts_value + + @scripts.setter + def scripts(self, value): + self.scripts_value = value + if self.scripts_value and self.script_args_value and not self.scripts_setup_complete: + self.setup_scripts() + + @property + def script_args(self): + return self.script_args_value + + @script_args.setter + def script_args(self, value): + self.script_args_value = value + if self.scripts_value and self.script_args_value and not self.scripts_setup_complete: + self.setup_scripts() + + def setup_scripts(self): + self.scripts_setup_complete = True + self.scripts.setup_scrips(self, is_ui=not self.is_api) + def comment(self, text): self.comments[text] = 1 diff --git a/modules/styles.py b/modules/styles.py index 053cfd6fb..9b63a6d01 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -5,7 +5,6 @@ import os import csv import json from installer import log -from modules import paths class Style(): @@ -66,6 +65,8 @@ def apply_styles_to_extra(p, style: Style): class StyleDatabase: def __init__(self, opts): + from modules import paths + self.no_style = Style("None") self.styles = {} self.path = opts.styles_dir diff --git a/requirements.txt b/requirements.txt index f833a42e1..f34360881 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ requests==2.31.0 tqdm==4.66.1 accelerate==0.20.3 opencv-python-headless==4.7.0.72 -diffusers==0.22.0 +diffusers==0.22.3 einops==0.4.1 gradio==3.43.2 huggingface_hub==0.18.0 From a68f0bcbe48ea2f31fe86a5e33d9988d37892c3a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 11:33:40 -0500 Subject: [PATCH 13/43] rework prompt paste --- .../stable-diffusion-webui-rembg | 2 +- modules/generation_parameters_copypaste.py | 69 +++++++------------ modules/processing.py | 15 ++-- modules/sd_models.py | 2 +- modules/ui.py | 7 +- 5 files changed, 38 insertions(+), 57 deletions(-) diff --git a/extensions-builtin/stable-diffusion-webui-rembg b/extensions-builtin/stable-diffusion-webui-rembg index 7f5772962..d5cd87bd4 160000 --- a/extensions-builtin/stable-diffusion-webui-rembg +++ b/extensions-builtin/stable-diffusion-webui-rembg @@ -1 +1 @@ -Subproject commit 7f57729626503837a70ad9eed92313bc36db7bf3 +Subproject commit d5cd87bd434f1d82403ef740e0ab727afaf9dc96 diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index e2cc4f92f..fb54cfb9e 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -3,20 +3,20 @@ import io import os import re import json - from PIL import Image import gradio as gr from modules.paths import data_path from modules import shared, ui_tempdir, script_callbacks, images + re_param_code = r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)' re_param = re.compile(re_param_code) re_imagesize = re.compile(r"^(\d+)x(\d+)$") re_hypernet_hash = re.compile("\(([0-9a-f]+)\)$") # pylint: disable=anomalous-backslash-in-string type_of_gr_update = type(gr.update()) - paste_fields = {} registered_param_bindings = [] +debug = shared.log.info if os.environ.get('SD_PASTE_DEBUG', None) is not None else lambda *args, **kwargs: None class ParamBinding: @@ -203,37 +203,19 @@ def find_hypernetwork_key(hypernet_name, hypernet_hash=None): def parse_generation_parameters(x: str): - """parses generation parameters string, the one you see in text field under the picture in UI: -``` -girl with an artist's beret, determined, blue eyes, desert scene, computer monitors, heavy makeup, by Alphonse Mucha and Charlie Bowater, ((eyeshadow)), (coquettish), detailed, intricate -Negative prompt: ugly, fat, obese, chubby, (((deformed))), [blurry], bad anatomy, disfigured, poorly drawn face, mutation, mutated, (extra_limb), (ugly), (poorly drawn hands), messy drawing -Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model hash: 45dee52b -``` - - returns a dict with field values - """ - if x is None: - return {} res = {} - prompt = "" - negative_prompt = "" - done_with_prompt = False - *lines, lastline = x.strip().split("\n") - if len(re_param.findall(lastline)) < 3: - lines.append(lastline) - lastline = '' - for line in lines: - line = line.strip() - if line.startswith("Negative prompt:"): - done_with_prompt = True - line = line[16:].strip() - if done_with_prompt: - negative_prompt += ("" if negative_prompt == "" else "\n") + line - else: - prompt += ("" if prompt == "" else "\n") + line - res["Prompt"] = prompt - res["Negative prompt"] = negative_prompt - for k, v in re_param.findall(lastline): + if x is None: + return res + remaining = x.strip() + if len(remaining) == 0: + return res + remaining = x[7:] if x.startswith('Prompt: ') else x + res["Prompt"], remaining = remaining.split(' Negative prompt: ', maxsplit=1) if ' Negative prompt: ' in remaining else (remaining, '') + res["Negative prompt"], remaining = remaining.split(' Steps: ', maxsplit=1) if ' Steps: ' in remaining else (remaining, None) + if remaining is None: + return res + remaining = f'Steps: {remaining}' + for k, v in re_param.findall(remaining): try: if v[0] == '"' and v[-1] == '"': v = unquote(v) @@ -245,16 +227,9 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model res[k] = v except Exception: pass - - # Missing CLIP skip means it was set to 1 (the default) - if "Clip skip" not in res: - res["Clip skip"] = "1" - hypernet = res.get("Hypernet", None) - if hypernet is not None: - res["Prompt"] += f"""""" - if "Hires resize-1" not in res: - res["Hires resize-1"] = 0 - res["Hires resize-2"] = 0 + res["Full quality"] = res.get('VAE', None) != 'TAESD' + for k, v in res.items(): + debug(f"Parse prompt: '{k}'={v}") return res @@ -328,16 +303,16 @@ def create_override_settings_dict(text_pairs): def connect_paste(button, local_paste_fields, input_comp, override_settings_component, tabname): def paste_func(prompt): - if prompt is not None and 'Negative prompt' not in prompt and 'Steps' not in prompt: - prompt = None - if not prompt and not shared.cmd_opts.hide_ui_dir_config: + if prompt is None or len(prompt.strip()) == 0 and not shared.cmd_opts.hide_ui_dir_config: filename = os.path.join(data_path, "params.txt") if os.path.exists(filename): with open(filename, "r", encoding="utf8") as file: prompt = file.read() + shared.log.debug(f'Paste prompt last: {prompt}') else: prompt = '' - shared.log.debug(f'Paste prompt: {prompt}') + else: + shared.log.debug(f'Paste prompt current: {prompt}') params = parse_generation_parameters(prompt) script_callbacks.infotext_pasted_callback(prompt, params) res = [] @@ -346,6 +321,8 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp v = key(params) else: v = params.get(key, None) + if v is not None: + debug(f"Parse apply: '{key}'={v}") if v is None: res.append(gr.update()) elif isinstance(v, type_of_gr_update): diff --git a/modules/processing.py b/modules/processing.py index 1ee141fbc..d51905d01 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -588,7 +588,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["Denoising strength"] = p.denoising_strength args["Latent sampler"] = p.latent_sampler args["Image CFG scale"] = p.image_cfg_scale - args["CFG rescale"] = p.diffusers_guidance_rescale if shared.backend == shared.Backend.DIFFUSERS else None + args["CFG rescale"] = p.diffusers_guidance_rescale if 'refine' in p.ops: args["Second pass"] = p.enable_hr args["Refiner"] = None if (not shared.opts.add_model_name_to_info) or (not shared.sd_refiner) or (not shared.sd_refiner.sd_checkpoint_info.model_name) else shared.sd_refiner.sd_checkpoint_info.model_name.replace(',', '').replace(':', '') @@ -597,7 +597,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args['Refiner start'] = p.refiner_start args["Hires steps"] = p.hr_second_pass_steps args["Latent sampler"] = p.latent_sampler - args["CFG rescale"] = p.diffusers_guidance_rescale if shared.backend == shared.Backend.DIFFUSERS else None + args["CFG rescale"] = p.diffusers_guidance_rescale if 'img2img' in p.ops or 'inpaint' in p.ops: args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}" args["Init image hash"] = getattr(p, 'init_img_hash', None) @@ -756,7 +756,13 @@ def process_images(p: StableDiffusionProcessing) -> Processed: return res -def validate_sample(sample): +def validate_sample(tensor): + if tensor.dtype == torch.bfloat16: # numpy does not support bf16 + tensor = tensor.to(torch.float16) + if shared.backend == shared.Backend.ORIGINAL: + sample = 255.0 * np.moveaxis(tensor.cpu().numpy(), 0, 2) + else: + sample = 255. * tensor with warnings.catch_warnings(record=True) as w: cast = sample.astype(np.uint8) if len(w) > 0: @@ -914,7 +920,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: image = x_sample x_sample = np.array(x_sample) else: - x_sample = 255. * (np.moveaxis(x_sample.cpu().numpy(), 0, 2) if shared.backend == shared.Backend.ORIGINAL else x_sample) x_sample = validate_sample(x_sample) image = Image.fromarray(x_sample) if p.restore_faces: @@ -1118,7 +1123,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae), self.full_quality) decoded_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0) for i, x_sample in enumerate(decoded_samples): - x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) x_sample = validate_sample(x_sample) image = Image.fromarray(x_sample) bak_extra_generation_params, bak_restore_faces = self.extra_generation_params, self.restore_faces @@ -1134,7 +1138,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): decoded_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0) batch_images = [] for _i, x_sample in enumerate(decoded_samples): - x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) x_sample = validate_sample(x_sample) image = Image.fromarray(x_sample) image = images.resize_image(1, image, target_width, target_height, upscaler_name=self.hr_upscaler) diff --git a/modules/sd_models.py b/modules/sd_models.py index c627cea7b..ddea17a01 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -268,7 +268,7 @@ def select_checkpoint(op='model'): shared.log.info(f'Select: {op}="{checkpoint_info.title if checkpoint_info is not None else None}"') return checkpoint_info if len(checkpoints_list) == 0 and not shared.cmd_opts.no_download: - shared.log.error("Cannot generate without a checkpoint") + shared.log.warning("Cannot generate without a checkpoint") shared.log.info("Set system paths to use existing folders in a different location") shared.log.info("Or use --ckpt to force using existing checkpoint") return None diff --git a/modules/ui.py b/modules/ui.py index 262507000..ec75ee3a3 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -513,7 +513,7 @@ def create_ui(startup_timer = None): txt2img_paste_fields = [ (txt2img_prompt, "Prompt"), (txt2img_negative_prompt, "Negative prompt"), - # (txt2img_prompt_styles, "Styles"), + (txt2img_prompt_styles, "Styles"), (steps, "Steps"), (seed, "Seed"), (sampler_index, "Sampler"), @@ -530,8 +530,8 @@ def create_ui(startup_timer = None): (refiner_start, "Refiner start"), (full_quality, "Full quality"), (restore_faces, "Face restoration"), - (batch_size, "Batch size"), - (batch_count, "Batch count"), + (batch_count, "Batch-1"), + (batch_size, "Batch-2"), (seed_resize_from_w, "Seed resize from-1"), (seed_resize_from_h, "Seed resize from-2"), (enable_hr, "Second pass"), @@ -548,6 +548,7 @@ def create_ui(startup_timer = None): (tiling, "Tiling"), (refiner_negative, "Negative2"), (refiner_prompt, "Prompt2"), + # TODO restore params complete list *modules.scripts.scripts_txt2img.infotext_fields ] parameters_copypaste.add_paste_fields("txt2img", None, txt2img_paste_fields, override_settings) From f8ae261fc30ab18495c6dcfbf09d25f8308edbc0 Mon Sep 17 00:00:00 2001 From: Redacted Date: Wed, 8 Nov 2023 10:49:50 -0600 Subject: [PATCH 14/43] SD-Upscaler fix -changed upscaler list from radio input to dropdown. --- scripts/sd_upscale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sd_upscale.py b/scripts/sd_upscale.py index 0d35e262b..8e22febbe 100644 --- a/scripts/sd_upscale.py +++ b/scripts/sd_upscale.py @@ -18,7 +18,7 @@ class Script(scripts.Script): info = gr.HTML("

Will upscale the image by the selected scale factor; use width and height sliders to set tile size

") overlap = gr.Slider(minimum=0, maximum=256, step=16, label='Tile overlap', value=64, elem_id=self.elem_id("overlap")) scale_factor = gr.Slider(minimum=1.0, maximum=4.0, step=0.05, label='Scale Factor', value=2.0, elem_id=self.elem_id("scale_factor")) - upscaler_index = gr.Radio(label='Upscaler', choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name, type="index", elem_id=self.elem_id("upscaler_index")) + upscaler_index = gr.Dropdown(label='Upscaler', choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name, type="index", elem_id=self.elem_id("upscaler_index")) return [info, overlap, upscaler_index, scale_factor] From 55d5c09f09cc62d1a05da2ba73a8b803aa172f1a Mon Sep 17 00:00:00 2001 From: Redacted Date: Wed, 8 Nov 2023 10:56:20 -0600 Subject: [PATCH 15/43] Fix dropdown width -added max-width: fit-content; to dropdown in sdnext.css to prevent the dropdown taking up the entire width of the panel for no reason. --- javascript/sdnext.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/sdnext.css b/javascript/sdnext.css index f245dbe0e..d0cc5cb98 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -30,7 +30,7 @@ textarea { overflow-y: auto !important; } .gradio-column { min-width: min(160px, 100%) !important; } .gradio-container { max-width: unset !important; padding: var(--block-label-padding) !important; } .gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; } -.gradio-dropdown { margin-right: var(--spacing-sm) !important; } +.gradio-dropdown { margin-right: var(--spacing-sm) !important; max-width: fit-content; } .gradio-dropdown ul.options { z-index: 1000; min-width: fit-content; max-height: 33vh !important; white-space: nowrap; } .gradio-dropdown ul.options li.item { padding: var(--spacing-xs); } .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--primary-500); } From a5f6e884e7a7a51026a50bed5c4ce47cbef856e0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 12:00:51 -0500 Subject: [PATCH 16/43] cleanup codeformer --- CHANGELOG.md | 8 +- .../codeformer/facelib/detection/__init__.py | 43 +-- .../scripts/download_pretrained_models.py | 48 ++- .../download_pretrained_models_from_gdrive.py | 30 +- .../codeformer/web-demos/hugging_face/app.py | 280 ------------------ .../codeformer/web-demos/replicate/cog.yaml | 30 -- .../codeformer/web-demos/replicate/predict.py | 189 ------------ .../codeformer/weights/CodeFormer/.gitkeep | 0 repositories/codeformer/weights/README.md | 3 - .../codeformer/weights/facelib/.gitkeep | 0 10 files changed, 40 insertions(+), 591 deletions(-) delete mode 100644 repositories/codeformer/web-demos/hugging_face/app.py delete mode 100644 repositories/codeformer/web-demos/replicate/cog.yaml delete mode 100644 repositories/codeformer/web-demos/replicate/predict.py delete mode 100644 repositories/codeformer/weights/CodeFormer/.gitkeep delete mode 100644 repositories/codeformer/weights/README.md delete mode 100644 repositories/codeformer/weights/facelib/.gitkeep diff --git a/CHANGELOG.md b/CHANGELOG.md index 9767f1d7d..1e88ec824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,18 @@ # Change Log for SD.Next -## Update for 2023-11-07 +## Update for 2023-11-08 - **Extra networks** - Use multi-threading for 5x load speedup +- **General**: + - Reworked parser when pasting previously generated images/prompts - **Diffusers** - Fix DPM SDE scheduler +- **Fixes** - Fix inpaint + - More uniform models paths + - Improve extension compatibility + - Improve BF16 support ## Update for 2023-11-06 diff --git a/repositories/codeformer/facelib/detection/__init__.py b/repositories/codeformer/facelib/detection/__init__.py index 5d1f8fc21..1c021d410 100644 --- a/repositories/codeformer/facelib/detection/__init__.py +++ b/repositories/codeformer/facelib/detection/__init__.py @@ -1,14 +1,16 @@ import os +from copy import deepcopy import torch from torch import nn -from copy import deepcopy - from facelib.utils import load_file_from_url from facelib.utils import download_pretrained_models from facelib.detection.yolov5face.models.common import Conv - from .retinaface.retinaface import RetinaFace from .yolov5face.face_detector import YoloDetector +from modules import paths + + +model_dir = os.path.join(paths.models_path, 'Codeformer') def init_detection_model(model_name, half=False, device='cuda'): @@ -32,7 +34,7 @@ def init_retinaface_model(model_name, half=False, device='cuda'): else: raise NotImplementedError(f'{model_name} is not implemented.') - model_path = load_file_from_url(url=model_url, model_dir='weights/facelib', progress=True, file_name=None) + model_path = load_file_from_url(url=model_url, model_dir=model_dir, progress=True, file_name=None) load_net = torch.load(model_path, map_location=lambda storage, loc: storage) # remove unnecessary 'module.' for k, v in deepcopy(load_net).items(): @@ -55,8 +57,8 @@ def init_yolov5face_model(model_name, device='cuda'): model_url = 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/yolov5n-face.pth' else: raise NotImplementedError(f'{model_name} is not implemented.') - - model_path = load_file_from_url(url=model_url, model_dir='weights/facelib', progress=True, file_name=None) + + model_path = load_file_from_url(url=model_url, model_dir=model_dir, progress=True, file_name=None) load_net = torch.load(model_path, map_location=lambda storage, loc: storage) model.detector.load_state_dict(load_net, strict=True) model.detector.eval() @@ -69,32 +71,3 @@ def init_yolov5face_model(model_name, device='cuda'): m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility return model - - -# Download from Google Drive -# def init_yolov5face_model(model_name, device='cuda'): -# if model_name == 'YOLOv5l': -# model = YoloDetector(config_name='facelib/detection/yolov5face/models/yolov5l.yaml', device=device) -# f_id = {'yolov5l-face.pth': '131578zMA6B2x8VQHyHfa6GEPtulMCNzV'} -# elif model_name == 'YOLOv5n': -# model = YoloDetector(config_name='facelib/detection/yolov5face/models/yolov5n.yaml', device=device) -# f_id = {'yolov5n-face.pth': '1fhcpFvWZqghpGXjYPIne2sw1Fy4yhw6o'} -# else: -# raise NotImplementedError(f'{model_name} is not implemented.') - -# model_path = os.path.join('weights/facelib', list(f_id.keys())[0]) -# if not os.path.exists(model_path): -# download_pretrained_models(file_ids=f_id, save_path_root='weights/facelib') - -# load_net = torch.load(model_path, map_location=lambda storage, loc: storage) -# model.detector.load_state_dict(load_net, strict=True) -# model.detector.eval() -# model.detector = model.detector.to(device).float() - -# for m in model.detector.modules(): -# if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]: -# m.inplace = True # pytorch 1.7.0 compatibility -# elif isinstance(m, Conv): -# m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility - -# return model \ No newline at end of file diff --git a/repositories/codeformer/scripts/download_pretrained_models.py b/repositories/codeformer/scripts/download_pretrained_models.py index daa6e8ca1..16ff6e170 100644 --- a/repositories/codeformer/scripts/download_pretrained_models.py +++ b/repositories/codeformer/scripts/download_pretrained_models.py @@ -1,40 +1,26 @@ -import argparse import os -from os import path as osp - +from modules import paths from basicsr.utils.download_util import load_file_from_url -def download_pretrained_models(method, file_urls): - save_path_root = f'./weights/{method}' - os.makedirs(save_path_root, exist_ok=True) +urls = { + 'CodeFormer': { + 'codeformer.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth' + }, + 'facelib': { + # 'yolov5l-face.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/yolov5l-face.pth', + 'detection_Resnet50_Final.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth', + 'parsing_parsenet.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth' + } +} + +def download_pretrained_models(file_urls): + model_dir = os.path.join(paths.models_path, 'Codeformer') for file_name, file_url in file_urls.items(): - save_path = load_file_from_url(url=file_url, model_dir=save_path_root, progress=True, file_name=file_name) + load_file_from_url(url=file_url, model_dir=model_dir, progress=True, file_name=file_name) if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument( - 'method', - type=str, - help=("Options: 'CodeFormer' 'facelib'. Set to 'all' to download all the models.")) - args = parser.parse_args() - - file_urls = { - 'CodeFormer': { - 'codeformer.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth' - }, - 'facelib': { - # 'yolov5l-face.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/yolov5l-face.pth', - 'detection_Resnet50_Final.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth', - 'parsing_parsenet.pth': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth' - } - } - - if args.method == 'all': - for method in file_urls.keys(): - download_pretrained_models(method, file_urls[method]) - else: - download_pretrained_models(args.method, file_urls[args.method]) \ No newline at end of file + for method in urls.keys(): + download_pretrained_models(urls[method]) diff --git a/repositories/codeformer/scripts/download_pretrained_models_from_gdrive.py b/repositories/codeformer/scripts/download_pretrained_models_from_gdrive.py index 7df5be6fc..5a5c6bd44 100644 --- a/repositories/codeformer/scripts/download_pretrained_models_from_gdrive.py +++ b/repositories/codeformer/scripts/download_pretrained_models_from_gdrive.py @@ -1,18 +1,16 @@ -import argparse import os +from modules import paths from os import path as osp - -# from basicsr.utils.download_util import download_file_from_google_drive import gdown -def download_pretrained_models(method, file_ids): - save_path_root = f'./weights/{method}' - os.makedirs(save_path_root, exist_ok=True) +model_dir = os.path.join(paths.models_path, 'Codeformer') + +def download_pretrained_models(file_ids): for file_name, file_id in file_ids.items(): file_url = 'https://drive.google.com/uc?id='+file_id - save_path = osp.abspath(osp.join(save_path_root, file_name)) + save_path = osp.abspath(osp.join(model_dir, file_name)) if osp.exists(save_path): user_response = input(f'{file_name} already exist. Do you want to cover it? Y/N\n') if user_response.lower() == 'y': @@ -29,21 +27,13 @@ def download_pretrained_models(method, file_ids): # download_file_from_google_drive(file_id, save_path) if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument( - 'method', - type=str, - help=("Options: 'CodeFormer' 'facelib'. Set to 'all' to download all the models.")) - args = parser.parse_args() - # file name: file id # 'dlib': { # 'mmod_human_face_detector-4cb19393.dat': '1qD-OqY8M6j4PWUP_FtqfwUPFPRMu6ubX', # 'shape_predictor_5_face_landmarks-c4b1e980.dat': '1vF3WBUApw4662v9Pw6wke3uk1qxnmLdg', # 'shape_predictor_68_face_landmarks-fbdc2cb8.dat': '1tJyIVdCHaU6IDMDx86BZCxLGZfsWB8yq' # } - file_ids = { + urls = { 'CodeFormer': { 'codeformer.pth': '1v_E_vZvP-dQPF55Kc5SRCjaKTQXDz-JB' }, @@ -52,9 +42,5 @@ if __name__ == '__main__': 'parsing_parsenet.pth': '16pkohyZZ8ViHGBk3QtVqxLZKzdo466bK' } } - - if args.method == 'all': - for method in file_ids.keys(): - download_pretrained_models(method, file_ids[method]) - else: - download_pretrained_models(args.method, file_ids[args.method]) \ No newline at end of file + for method in urls.keys(): + download_pretrained_models(urls[method]) diff --git a/repositories/codeformer/web-demos/hugging_face/app.py b/repositories/codeformer/web-demos/hugging_face/app.py deleted file mode 100644 index 7da0fc947..000000000 --- a/repositories/codeformer/web-demos/hugging_face/app.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -This file is used for deploying hugging face demo: -https://huggingface.co/spaces/sczhou/CodeFormer -""" - -import sys -sys.path.append('CodeFormer') -import os -import cv2 -import torch -import torch.nn.functional as F -import gradio as gr - -from torchvision.transforms.functional import normalize - -from basicsr.utils import imwrite, img2tensor, tensor2img -from basicsr.utils.download_util import load_file_from_url -from facelib.utils.face_restoration_helper import FaceRestoreHelper -from facelib.utils.misc import is_gray -from basicsr.archs.rrdbnet_arch import RRDBNet -from basicsr.utils.realesrgan_utils import RealESRGANer - -from basicsr.utils.registry import ARCH_REGISTRY - - -os.system("pip freeze") - -pretrain_model_url = { - 'codeformer': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth', - 'detection': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth', - 'parsing': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth', - 'realesrgan': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/RealESRGAN_x2plus.pth' -} -# download weights -if not os.path.exists('CodeFormer/weights/CodeFormer/codeformer.pth'): - load_file_from_url(url=pretrain_model_url['codeformer'], model_dir='CodeFormer/weights/CodeFormer', progress=True, file_name=None) -if not os.path.exists('CodeFormer/weights/facelib/detection_Resnet50_Final.pth'): - load_file_from_url(url=pretrain_model_url['detection'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None) -if not os.path.exists('CodeFormer/weights/facelib/parsing_parsenet.pth'): - load_file_from_url(url=pretrain_model_url['parsing'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None) -if not os.path.exists('CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth'): - load_file_from_url(url=pretrain_model_url['realesrgan'], model_dir='CodeFormer/weights/realesrgan', progress=True, file_name=None) - -# download images -torch.hub.download_url_to_file( - 'https://replicate.com/api/models/sczhou/codeformer/files/fa3fe3d1-76b0-4ca8-ac0d-0a925cb0ff54/06.png', - '01.png') -torch.hub.download_url_to_file( - 'https://replicate.com/api/models/sczhou/codeformer/files/a1daba8e-af14-4b00-86a4-69cec9619b53/04.jpg', - '02.jpg') -torch.hub.download_url_to_file( - 'https://replicate.com/api/models/sczhou/codeformer/files/542d64f9-1712-4de7-85f7-3863009a7c3d/03.jpg', - '03.jpg') -torch.hub.download_url_to_file( - 'https://replicate.com/api/models/sczhou/codeformer/files/a11098b0-a18a-4c02-a19a-9a7045d68426/010.jpg', - '04.jpg') -torch.hub.download_url_to_file( - 'https://replicate.com/api/models/sczhou/codeformer/files/7cf19c2c-e0cf-4712-9af8-cf5bdbb8d0ee/012.jpg', - '05.jpg') - -def imread(img_path): - img = cv2.imread(img_path) - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - return img - -# set enhancer with RealESRGAN -def set_realesrgan(): - half = True if torch.cuda.is_available() else False - model = RRDBNet( - num_in_ch=3, - num_out_ch=3, - num_feat=64, - num_block=23, - num_grow_ch=32, - scale=2, - ) - upsampler = RealESRGANer( - scale=2, - model_path="CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth", - model=model, - tile=400, - tile_pad=40, - pre_pad=0, - half=half, - ) - return upsampler - -upsampler = set_realesrgan() -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -codeformer_net = ARCH_REGISTRY.get("CodeFormer")( - dim_embd=512, - codebook_size=1024, - n_head=8, - n_layers=9, - connect_list=["32", "64", "128", "256"], -).to(device) -ckpt_path = "CodeFormer/weights/CodeFormer/codeformer.pth" -checkpoint = torch.load(ckpt_path)["params_ema"] -codeformer_net.load_state_dict(checkpoint) -codeformer_net.eval() - -os.makedirs('output', exist_ok=True) - -def inference(image, background_enhance, face_upsample, upscale, codeformer_fidelity): - """Run a single prediction on the model""" - try: # global try - # take the default setting for the demo - has_aligned = False - only_center_face = False - draw_box = False - detection_model = "retinaface_resnet50" - print('Inp:', image, background_enhance, face_upsample, upscale, codeformer_fidelity) - - img = cv2.imread(str(image), cv2.IMREAD_COLOR) - print('\timage size:', img.shape) - - upscale = int(upscale) # convert type to int - if upscale > 4: # avoid memory exceeded due to too large upscale - upscale = 4 - if upscale > 2 and max(img.shape[:2])>1000: # avoid memory exceeded due to too large img resolution - upscale = 2 - if max(img.shape[:2]) > 1500: # avoid memory exceeded due to too large img resolution - upscale = 1 - background_enhance = False - face_upsample = False - - face_helper = FaceRestoreHelper( - upscale, - face_size=512, - crop_ratio=(1, 1), - det_model=detection_model, - save_ext="png", - use_parse=True, - device=device, - ) - bg_upsampler = upsampler if background_enhance else None - face_upsampler = upsampler if face_upsample else None - - if has_aligned: - # the input faces are already cropped and aligned - img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR) - face_helper.is_gray = is_gray(img, threshold=5) - if face_helper.is_gray: - print('\tgrayscale input: True') - face_helper.cropped_faces = [img] - else: - face_helper.read_image(img) - # get face landmarks for each face - num_det_faces = face_helper.get_face_landmarks_5( - only_center_face=only_center_face, resize=640, eye_dist_threshold=5 - ) - print(f'\tdetect {num_det_faces} faces') - # align and warp each face - face_helper.align_warp_face() - - # face restoration for each cropped face - for idx, cropped_face in enumerate(face_helper.cropped_faces): - # prepare data - cropped_face_t = img2tensor( - cropped_face / 255.0, bgr2rgb=True, float32=True - ) - normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True) - cropped_face_t = cropped_face_t.unsqueeze(0).to(device) - - try: - with torch.no_grad(): - output = codeformer_net( - cropped_face_t, w=codeformer_fidelity, adain=True - )[0] - restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1)) - del output - torch.cuda.empty_cache() - except RuntimeError as error: - print(f"Failed inference for CodeFormer: {error}") - restored_face = tensor2img( - cropped_face_t, rgb2bgr=True, min_max=(-1, 1) - ) - - restored_face = restored_face.astype("uint8") - face_helper.add_restored_face(restored_face) - - # paste_back - if not has_aligned: - # upsample the background - if bg_upsampler is not None: - # Now only support RealESRGAN for upsampling background - bg_img = bg_upsampler.enhance(img, outscale=upscale)[0] - else: - bg_img = None - face_helper.get_inverse_affine(None) - # paste each restored face to the input image - if face_upsample and face_upsampler is not None: - restored_img = face_helper.paste_faces_to_input_image( - upsample_img=bg_img, - draw_box=draw_box, - face_upsampler=face_upsampler, - ) - else: - restored_img = face_helper.paste_faces_to_input_image( - upsample_img=bg_img, draw_box=draw_box - ) - - # save restored img - save_path = f'output/out.png' - imwrite(restored_img, str(save_path)) - - restored_img = cv2.cvtColor(restored_img, cv2.COLOR_BGR2RGB) - return restored_img, save_path - except Exception as error: - print('Global exception', error) - return None, None - - -title = "CodeFormer: Robust Face Restoration and Enhancement Network" -description = r"""
CodeFormer logo
-Official Gradio demo for Towards Robust Blind Face Restoration with Codebook Lookup Transformer (NeurIPS 2022).
-🔥 CodeFormer is a robust face restoration algorithm for old photos or AI-generated faces.
-🤗 Try CodeFormer for improved stable-diffusion generation!
-""" -article = r""" -If CodeFormer is helpful, please help to ⭐ the Github Repo. Thanks! -[![GitHub Stars](https://img.shields.io/github/stars/sczhou/CodeFormer?style=social)](https://github.com/sczhou/CodeFormer) - ---- - -📝 **Citation** - -If our work is useful for your research, please consider citing: -```bibtex -@inproceedings{zhou2022codeformer, - author = {Zhou, Shangchen and Chan, Kelvin C.K. and Li, Chongyi and Loy, Chen Change}, - title = {Towards Robust Blind Face Restoration with Codebook Lookup TransFormer}, - booktitle = {NeurIPS}, - year = {2022} -} -``` - -📋 **License** - -This project is licensed under S-Lab License 1.0. -Redistribution and use for non-commercial purposes should follow this license. - -📧 **Contact** - -If you have any questions, please feel free to reach me out at shangchenzhou@gmail.com. - -
- 🤗 Find Me: - Twitter Follow - Github Follow -
- -
visitors
-""" - -demo = gr.Interface( - inference, [ - gr.inputs.Image(type="filepath", label="Input"), - gr.inputs.Checkbox(default=True, label="Background_Enhance"), - gr.inputs.Checkbox(default=True, label="Face_Upsample"), - gr.inputs.Number(default=2, label="Rescaling_Factor (up to 4)"), - gr.Slider(0, 1, value=0.5, step=0.01, label='Codeformer_Fidelity (0 for better quality, 1 for better identity)') - ], [ - gr.outputs.Image(type="numpy", label="Output"), - gr.outputs.File(label="Download the output") - ], - title=title, - description=description, - article=article, - examples=[ - ['01.png', True, True, 2, 0.7], - ['02.jpg', True, True, 2, 0.7], - ['03.jpg', True, True, 2, 0.7], - ['04.jpg', True, True, 2, 0.1], - ['05.jpg', True, True, 2, 0.1] - ] - ) - -demo.queue(concurrency_count=2) -demo.launch() \ No newline at end of file diff --git a/repositories/codeformer/web-demos/replicate/cog.yaml b/repositories/codeformer/web-demos/replicate/cog.yaml deleted file mode 100644 index 3f4589690..000000000 --- a/repositories/codeformer/web-demos/replicate/cog.yaml +++ /dev/null @@ -1,30 +0,0 @@ -""" -This file is used for deploying replicate demo: -https://replicate.com/sczhou/codeformer -""" - -build: - gpu: true - cuda: "11.3" - python_version: "3.8" - system_packages: - - "libgl1-mesa-glx" - - "libglib2.0-0" - python_packages: - - "ipython==8.4.0" - - "future==0.18.2" - - "lmdb==1.3.0" - - "scikit-image==0.19.3" - - "torch==1.11.0 --extra-index-url=https://download.pytorch.org/whl/cu113" - - "torchvision==0.12.0 --extra-index-url=https://download.pytorch.org/whl/cu113" - - "scipy==1.9.0" - - "gdown==4.5.1" - - "pyyaml==6.0" - - "tb-nightly==2.11.0a20220906" - - "tqdm==4.64.1" - - "yapf==0.32.0" - - "lpips==0.1.4" - - "Pillow==9.2.0" - - "opencv-python==4.6.0.66" - -predict: "predict.py:Predictor" diff --git a/repositories/codeformer/web-demos/replicate/predict.py b/repositories/codeformer/web-demos/replicate/predict.py deleted file mode 100644 index 61935e9e7..000000000 --- a/repositories/codeformer/web-demos/replicate/predict.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -This file is used for deploying replicate demo: -https://replicate.com/sczhou/codeformer -running: cog predict -i image=@inputs/whole_imgs/04.jpg -i codeformer_fidelity=0.5 -i upscale=2 -push: cog push r8.im/sczhou/codeformer -""" - -import tempfile -import cv2 -import torch -from torchvision.transforms.functional import normalize -try: - from cog import BasePredictor, Input, Path -except Exception: - print('please install cog package') - -from basicsr.utils import imwrite, img2tensor, tensor2img -from basicsr.archs.rrdbnet_arch import RRDBNet -from basicsr.utils.realesrgan_utils import RealESRGANer -from basicsr.utils.registry import ARCH_REGISTRY -from facelib.utils.face_restoration_helper import FaceRestoreHelper - - -class Predictor(BasePredictor): - def setup(self): - """Load the model into memory to make running multiple predictions efficient""" - self.device = "cuda:0" - self.upsampler = set_realesrgan() - self.net = ARCH_REGISTRY.get("CodeFormer")( - dim_embd=512, - codebook_size=1024, - n_head=8, - n_layers=9, - connect_list=["32", "64", "128", "256"], - ).to(self.device) - ckpt_path = "weights/CodeFormer/codeformer.pth" - checkpoint = torch.load(ckpt_path)[ - "params_ema" - ] # update file permission if cannot load - self.net.load_state_dict(checkpoint) - self.net.eval() - - def predict( - self, - image: Path = Input(description="Input image"), - codeformer_fidelity: float = Input( - default=0.5, - ge=0, - le=1, - description="Balance the quality (lower number) and fidelity (higher number).", - ), - background_enhance: bool = Input( - description="Enhance background image with Real-ESRGAN", default=True - ), - face_upsample: bool = Input( - description="Upsample restored faces for high-resolution AI-created images", - default=True, - ), - upscale: int = Input( - description="The final upsampling scale of the image", - default=2, - ), - ) -> Path: - """Run a single prediction on the model""" - - # take the default setting for the demo - has_aligned = False - only_center_face = False - draw_box = False - detection_model = "retinaface_resnet50" - - self.face_helper = FaceRestoreHelper( - upscale, - face_size=512, - crop_ratio=(1, 1), - det_model=detection_model, - save_ext="png", - use_parse=True, - device=self.device, - ) - - bg_upsampler = self.upsampler if background_enhance else None - face_upsampler = self.upsampler if face_upsample else None - - img = cv2.imread(str(image), cv2.IMREAD_COLOR) - - if has_aligned: - # the input faces are already cropped and aligned - img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR) - self.face_helper.cropped_faces = [img] - else: - self.face_helper.read_image(img) - # get face landmarks for each face - num_det_faces = self.face_helper.get_face_landmarks_5( - only_center_face=only_center_face, resize=640, eye_dist_threshold=5 - ) - print(f"\tdetect {num_det_faces} faces") - # align and warp each face - self.face_helper.align_warp_face() - - # face restoration for each cropped face - for idx, cropped_face in enumerate(self.face_helper.cropped_faces): - # prepare data - cropped_face_t = img2tensor( - cropped_face / 255.0, bgr2rgb=True, float32=True - ) - normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True) - cropped_face_t = cropped_face_t.unsqueeze(0).to(self.device) - - try: - with torch.no_grad(): - output = self.net( - cropped_face_t, w=codeformer_fidelity, adain=True - )[0] - restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1)) - del output - torch.cuda.empty_cache() - except Exception as error: - print(f"\tFailed inference for CodeFormer: {error}") - restored_face = tensor2img( - cropped_face_t, rgb2bgr=True, min_max=(-1, 1) - ) - - restored_face = restored_face.astype("uint8") - self.face_helper.add_restored_face(restored_face) - - # paste_back - if not has_aligned: - # upsample the background - if bg_upsampler is not None: - # Now only support RealESRGAN for upsampling background - bg_img = bg_upsampler.enhance(img, outscale=upscale)[0] - else: - bg_img = None - self.face_helper.get_inverse_affine(None) - # paste each restored face to the input image - if face_upsample and face_upsampler is not None: - restored_img = self.face_helper.paste_faces_to_input_image( - upsample_img=bg_img, - draw_box=draw_box, - face_upsampler=face_upsampler, - ) - else: - restored_img = self.face_helper.paste_faces_to_input_image( - upsample_img=bg_img, draw_box=draw_box - ) - - # save restored img - out_path = Path(tempfile.mkdtemp()) / 'output.png' - imwrite(restored_img, str(out_path)) - - return out_path - - -def imread(img_path): - img = cv2.imread(img_path) - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - return img - - -def set_realesrgan(): - if not torch.cuda.is_available(): # CPU - import warnings - - warnings.warn( - "The unoptimized RealESRGAN is slow on CPU. We do not use it. " - "If you really want to use it, please modify the corresponding codes.", - category=RuntimeWarning, - ) - upsampler = None - else: - model = RRDBNet( - num_in_ch=3, - num_out_ch=3, - num_feat=64, - num_block=23, - num_grow_ch=32, - scale=2, - ) - upsampler = RealESRGANer( - scale=2, - model_path="./weights/realesrgan/RealESRGAN_x2plus.pth", - model=model, - tile=400, - tile_pad=40, - pre_pad=0, - half=True, - ) - return upsampler diff --git a/repositories/codeformer/weights/CodeFormer/.gitkeep b/repositories/codeformer/weights/CodeFormer/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/repositories/codeformer/weights/README.md b/repositories/codeformer/weights/README.md deleted file mode 100644 index 67ad334bd..000000000 --- a/repositories/codeformer/weights/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Weights - -Put the downloaded pre-trained models to this folder. \ No newline at end of file diff --git a/repositories/codeformer/weights/facelib/.gitkeep b/repositories/codeformer/weights/facelib/.gitkeep deleted file mode 100644 index e69de29bb..000000000 From 28365d6b691e2304c383f6106cdaf0d5b0b17ffc Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 12:09:09 -0500 Subject: [PATCH 17/43] fix forced filename --- modules/images.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/images.py b/modules/images.py index 2c4ecefe4..072604409 100644 --- a/modules/images.py +++ b/modules/images.py @@ -559,16 +559,17 @@ def save_image(image, path, basename = '', seed=None, prompt=None, extension=sha if shared.opts.save_to_dirs: dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]") path = os.path.join(path, dirname) + file_decoration = '' if forced_filename is None: - if short_filename or seed is None: - file_decoration = "" if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0: file_decoration = shared.opts.samples_filename_pattern else: file_decoration = "[seq]-[prompt_words]" file_decoration = namegen.apply(file_decoration) file_decoration += suffix - filename = os.path.join(path, f"{file_decoration}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{file_decoration}.{extension}") + filename = os.path.join(path, f"{file_decoration}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{file_decoration}.{extension}") + else: + filename = forced_filename pnginfo = existing_info or {} if info is not None: pnginfo[pnginfo_section_name] = info From 3c9c9cf471ec16c57f2e08caaf3bbab067c66c65 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Nov 2023 12:21:36 -0500 Subject: [PATCH 18/43] fix paste parser --- modules/generation_parameters_copypaste.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index fb54cfb9e..1ec23abc6 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -206,12 +206,14 @@ def parse_generation_parameters(x: str): res = {} if x is None: return res - remaining = x.strip() + remaining = x.replace('\n', ' ').strip() if len(remaining) == 0: return res remaining = x[7:] if x.startswith('Prompt: ') else x - res["Prompt"], remaining = remaining.split(' Negative prompt: ', maxsplit=1) if ' Negative prompt: ' in remaining else (remaining, '') - res["Negative prompt"], remaining = remaining.split(' Steps: ', maxsplit=1) if ' Steps: ' in remaining else (remaining, None) + prompt, remaining = remaining.split('Negative prompt: ', maxsplit=1) if 'Negative prompt: ' in remaining else (remaining, '') + res["Prompt"] = prompt.strip() + negative, remaining = remaining.split('Steps: ', maxsplit=1) if 'Steps: ' in remaining else (remaining, None) + res["Negative prompt"] = negative.strip() if remaining is None: return res remaining = f'Steps: {remaining}' From 6b3f3e3892976fb271d58dbdc5d19fad2a26fe8f Mon Sep 17 00:00:00 2001 From: Redacted Date: Wed, 8 Nov 2023 12:01:04 -0600 Subject: [PATCH 19/43] Update sdnext.css -changed min-width from fit-content to 160px and added max-width: fit-content --- javascript/sdnext.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/sdnext.css b/javascript/sdnext.css index d0cc5cb98..10a2d6089 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -30,7 +30,7 @@ textarea { overflow-y: auto !important; } .gradio-column { min-width: min(160px, 100%) !important; } .gradio-container { max-width: unset !important; padding: var(--block-label-padding) !important; } .gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; } -.gradio-dropdown { margin-right: var(--spacing-sm) !important; max-width: fit-content; } +.gradio-dropdown { margin-right: var(--spacing-sm) !important; min-width:160px; width:fit-content } .gradio-dropdown ul.options { z-index: 1000; min-width: fit-content; max-height: 33vh !important; white-space: nowrap; } .gradio-dropdown ul.options li.item { padding: var(--spacing-xs); } .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--primary-500); } From 05b5b0f6747b49a50131afaaba35354f502b8960 Mon Sep 17 00:00:00 2001 From: Redacted Date: Wed, 8 Nov 2023 12:02:05 -0600 Subject: [PATCH 20/43] Update sdnext.css fixed typo --- javascript/sdnext.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 10a2d6089..94652f34f 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -30,7 +30,7 @@ textarea { overflow-y: auto !important; } .gradio-column { min-width: min(160px, 100%) !important; } .gradio-container { max-width: unset !important; padding: var(--block-label-padding) !important; } .gradio-container .prose a, .gradio-container .prose a:visited{ color: unset; text-decoration: none; } -.gradio-dropdown { margin-right: var(--spacing-sm) !important; min-width:160px; width:fit-content } +.gradio-dropdown { margin-right: var(--spacing-sm) !important; min-width:160px; max-width:fit-content } .gradio-dropdown ul.options { z-index: 1000; min-width: fit-content; max-height: 33vh !important; white-space: nowrap; } .gradio-dropdown ul.options li.item { padding: var(--spacing-xs); } .gradio-dropdown ul.options li.item:not(:has(.hide)) { background-color: var(--primary-500); } From f6dd495eb385ce221ee699b1945c0e44fb20d679 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 09:04:50 -0500 Subject: [PATCH 21/43] multiple fixes --- CHANGELOG.md | 6 +- html/locale_en.json | 2 +- modules/generation_parameters_copypaste.py | 30 +++--- modules/images.py | 13 ++- modules/img2img.py | 5 + modules/processing.py | 6 +- modules/shared_items.py | 12 ++- modules/styles.py | 29 +++++- modules/ui.py | 111 +++++++++++---------- modules/ui_common.py | 2 +- modules/ui_extra_networks.py | 2 +- modules/ui_extra_networks_checkpoints.py | 4 +- wiki | 2 +- 13 files changed, 129 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e88ec824..82d6297dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,15 @@ - **Extra networks** - Use multi-threading for 5x load speedup - **General**: - - Reworked parser when pasting previously generated images/prompts + - Reworked parser when pasting previously generated images/prompts + includes all `txt2img`, `img2img` and `override` params - **Diffusers** - Fix DPM SDE scheduler + - Add additional pipeline types for manual model loads when loading from `safetensors` - **Fixes** - Fix inpaint + - Fix manual grid image save + - Fix img2img init image save - More uniform models paths - Improve extension compatibility - Improve BF16 support diff --git a/html/locale_en.json b/html/locale_en.json index f0c84af87..65f239e10 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -61,7 +61,7 @@ {"id":"","label":"Skip","localized":"","hint":"Stop processing current job and continue processing"}, {"id":"","label":"Interrupt","localized":"","hint":"Interrupt current processing job"}, {"id":"","label":"Pause","localized":"","hint":"Pause processing"}, - {"id":"","label":"Restore","localized":"","hint":"Restore parameters from last known generated image"}, + {"id":"","label":"Restore","localized":"","hint":"Restore parameters from current prompt or last known generated image"}, {"id":"","label":"Clear","localized":"","hint":"Clear prompts"}, {"id":"","label":"Networks","localized":"","hint":"Open extra network interface"}, {"id":"","label":"Interrogate\nCLIP","localized":"","hint":"Run interrogate using CLIP model"}, diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index 1ec23abc6..e742e32fc 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -210,14 +210,15 @@ def parse_generation_parameters(x: str): if len(remaining) == 0: return res remaining = x[7:] if x.startswith('Prompt: ') else x - prompt, remaining = remaining.split('Negative prompt: ', maxsplit=1) if 'Negative prompt: ' in remaining else (remaining, '') + remaining = x[11:] if x.startswith('parameters: ') else x + prompt, remaining = remaining.strip().split('Negative prompt: ', maxsplit=1) if 'Negative prompt: ' in remaining else (remaining, '') res["Prompt"] = prompt.strip() - negative, remaining = remaining.split('Steps: ', maxsplit=1) if 'Steps: ' in remaining else (remaining, None) + negative, remaining = remaining.strip().split('Steps: ', maxsplit=1) if 'Steps: ' in remaining else (remaining, None) res["Negative prompt"] = negative.strip() if remaining is None: return res remaining = f'Steps: {remaining}' - for k, v in re_param.findall(remaining): + for k, v in re_param.findall(remaining.strip()): try: if v[0] == '"' and v[-1] == '"': v = unquote(v) @@ -230,8 +231,7 @@ def parse_generation_parameters(x: str): except Exception: pass res["Full quality"] = res.get('VAE', None) != 'TAESD' - for k, v in res.items(): - debug(f"Parse prompt: '{k}'={v}") + debug(f"Parse prompt: {res}") return res @@ -239,7 +239,7 @@ settings_map = {} infotext_to_setting_name_mapping = [ - ('Backed', 'sd_backend'), + ('Backend', 'sd_backend'), ('Model hash', 'sd_model_checkpoint'), ('Refiner', 'sd_model_refiner'), ('VAE', 'sd_vae'), @@ -282,13 +282,6 @@ infotext_to_setting_name_mapping = [ def create_override_settings_dict(text_pairs): - """creates processing's override_settings parameters from gradio's multiselect - Example input: - ['Clip skip: 2', 'Model hash: e6e99610c4', 'ENSD: 31337'] - - Example output: - {'CLIP_stop_at_last_layers': 2, 'sd_model_checkpoint': 'e6e99610c4', 'eta_noise_seed_delta': 31337} - """ res = {} params = {} for pair in text_pairs: @@ -310,25 +303,25 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp if os.path.exists(filename): with open(filename, "r", encoding="utf8") as file: prompt = file.read() - shared.log.debug(f'Paste prompt last: {prompt}') + shared.log.debug(f'Paste prompt: type="params" prompt="{prompt}"') else: prompt = '' else: - shared.log.debug(f'Paste prompt current: {prompt}') + shared.log.debug(f'Paste prompt: type="current" prompt="{prompt}"') params = parse_generation_parameters(prompt) script_callbacks.infotext_pasted_callback(prompt, params) res = [] + applied = {} for output, key in local_paste_fields: if callable(key): v = key(params) else: v = params.get(key, None) - if v is not None: - debug(f"Parse apply: '{key}'={v}") if v is None: res.append(gr.update()) elif isinstance(v, type_of_gr_update): res.append(v) + applied[key] = v else: try: valtype = type(output.value) @@ -337,8 +330,10 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp else: val = valtype(v) res.append(gr.update(value=val)) + applied[key] = val except Exception: res.append(gr.update()) + debug(f"Parse apply: {applied}") return res if override_settings_component is not None: @@ -359,6 +354,7 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp continue vals[param_name] = v vals_pairs = [f"{k}: {v}" for k, v in vals.items()] + shared.log.debug(f'Settings overrides: {vals_pairs}') return gr.Dropdown.update(value=vals_pairs, choices=vals_pairs, visible=len(vals_pairs) > 0) local_paste_fields = local_paste_fields + [(override_settings_component, paste_settings)] diff --git a/modules/images.py b/modules/images.py index 072604409..cbb60b5ff 100644 --- a/modules/images.py +++ b/modules/images.py @@ -547,7 +547,7 @@ save_thread = threading.Thread(target=atomically_save_image, daemon=True) save_thread.start() -def save_image(image, path, basename = '', seed=None, prompt=None, extension=shared.opts.samples_format, info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix="", save_to_dirs=None): # pylint: disable=unused-argument +def save_image(image, path, basename='', seed=None, prompt=None, extension=shared.opts.samples_format, info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix='', save_to_dirs=None): # pylint: disable=unused-argument if image is None: shared.log.warning('Image is none') return None, None @@ -556,27 +556,30 @@ def save_image(image, path, basename = '', seed=None, prompt=None, extension=sha if path is None or len(path) == 0: # set default path to avoid errors when functions are triggered manually or via api and param is not set path = shared.opts.outdir_save namegen = FilenameGenerator(p, seed, prompt, image, grid=grid) + suffix = suffix if suffix is not None else '' + basename = basename if basename is not None else '' if shared.opts.save_to_dirs: dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]") path = os.path.join(path, dirname) - file_decoration = '' if forced_filename is None: if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0: file_decoration = shared.opts.samples_filename_pattern else: file_decoration = "[seq]-[prompt_words]" file_decoration = namegen.apply(file_decoration) - file_decoration += suffix + file_decoration += suffix if suffix is not None else '' filename = os.path.join(path, f"{file_decoration}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{file_decoration}.{extension}") else: - filename = forced_filename + forced_filename += suffix if suffix is not None else '' + filename = os.path.join(path, f"{forced_filename}.{extension}") if basename == '' else os.path.join(path, f"{basename}-{forced_filename}.{extension}") pnginfo = existing_info or {} if info is not None: pnginfo[pnginfo_section_name] = info params = script_callbacks.ImageSaveParams(image, p, filename, pnginfo) params.filename = namegen.sanitize(filename) dirname = os.path.dirname(params.filename) - os.makedirs(dirname, exist_ok=True) + if dirname is not None and len(dirname) > 0: + os.makedirs(dirname, exist_ok=True) # sequence if shared.opts.save_images_add_number or '[seq]' in params.filename: if '[seq]' not in params.filename: diff --git a/modules/img2img.py b/modules/img2img.py index b05254434..0152cf9d5 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -211,6 +211,11 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s p.extra_generation_params['Resize mode'] = resize_mode if mask: p.extra_generation_params["Mask blur"] = mask_blur + p.extra_generation_params["Mask alpha"] = mask_alpha + p.extra_generation_params["Mask invert"] = inpainting_mask_invert + p.extra_generation_params["Mask content"] = inpainting_fill + p.extra_generation_params["Mask area"] = inpaint_full_res + p.extra_generation_params["Mask padding"] = inpaint_full_res_padding p.is_batch = mode == 5 if p.is_batch: process_batch(p, img2img_batch_files, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, args) diff --git a/modules/processing.py b/modules/processing.py index d51905d01..5e7832c5f 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -166,11 +166,6 @@ class StableDiffusionProcessing: self.disable_extra_networks = False self.token_merging_ratio = 0 self.token_merging_ratio_hr = 0 - if not seed_enable_extras: - self.subseed = -1 - self.subseed_strength = 0 - self.seed_resize_from_h = 0 - self.seed_resize_from_w = 0 self.scripts = None self.script_args = script_args or [] self.per_script_args = {} @@ -603,6 +598,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["Init image hash"] = getattr(p, 'init_img_hash', None) args["Mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None args['Resize mode'] = getattr(p, 'resize_mode', None) + args['Resize scale'] = getattr(p, 'scale_by', None) args["Mask blur"] = p.mask_blur if getattr(p, 'mask', None) is not None and getattr(p, 'mask_blur', 0) > 0 else None args["Denoising strength"] = getattr(p, 'denoising_strength', None) if 'face' in p.ops: diff --git a/modules/shared_items.py b/modules/shared_items.py index 85fba87a3..49387e56e 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -27,7 +27,7 @@ def list_crossattention(): def get_pipelines(): import diffusers from installer import log - pipelines = { + pipelines = { # note: not all pipelines can be used manually as they require prior pipeline next to decoder pipeline 'Autodetect': None, 'Stable Diffusion': getattr(diffusers, 'StableDiffusionPipeline', None), 'Stable Diffusion Img2Img': getattr(diffusers, 'StableDiffusionImg2ImgPipeline', None), @@ -37,9 +37,15 @@ def get_pipelines(): 'Stable Diffusion XL Img2Img': getattr(diffusers, 'StableDiffusionXLImg2ImgPipeline', None), 'Stable Diffusion XL Inpaint': getattr(diffusers, 'StableDiffusionXLInpaintPipeline', None), 'Stable Diffusion XL Instruct': getattr(diffusers, 'StableDiffusionXLInstructPix2PixPipeline', None), + 'Latent Consistency Model': getattr(diffusers, 'LatentConsistencyModelPipeline', None), + 'PixArt Alpha': getattr(diffusers, 'PixArtAlphaPipeline', None), + 'UniDiffuser': getattr(diffusers, 'UniDiffuserPipeline', None), + 'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None), + 'Kandinsky 2.1': getattr(diffusers, 'KandinskyPipeline', None), + 'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22Pipeline', None), + 'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None), 'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None), - # 'Test': getattr(diffusers, 'TestPipeline', None), - # 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E', 'Kandinsky V1 Img2Img', 'Kandinsky V2 Img2Img', 'DeepFloyd IF Img2Img', 'Shap-E Img2Img', + # Segmind SSD-1B, Segmind Tiny } for k, v in pipelines.items(): if k != 'Autodetect' and v is None: diff --git a/modules/styles.py b/modules/styles.py index 9b63a6d01..427e76ea9 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -136,18 +136,33 @@ class StyleDatabase: return found[0] if len(found) > 0 else self.no_style def get_style_prompts(self, styles): + if styles is None or not isinstance(styles, list): + log.error(f'Invalid styles: {styles}') + return [] return [self.find_style(x).prompt for x in styles] def get_negative_style_prompts(self, styles): + if styles is None or not isinstance(styles, list): + log.error(f'Invalid styles: {styles}') + return [] return [self.find_style(x).negative_prompt for x in styles] def apply_styles_to_prompt(self, prompt, styles): + if styles is None or not isinstance(styles, list): + log.error(f'Invalid styles: {styles}') + return prompt return apply_styles_to_prompt(prompt, [self.find_style(x).prompt for x in styles]) def apply_negative_styles_to_prompt(self, prompt, styles): + if styles is None or not isinstance(styles, list): + log.error(f'Invalid styles: {styles}') + return prompt return apply_styles_to_prompt(prompt, [self.find_style(x).negative_prompt for x in styles]) def apply_styles_to_extra(self, p): + if p.styles is None or not isinstance(p.styles, list): + log.error(f'Invalid styles: {p.styles}') + return for style in p.styles: s = self.find_style(style) apply_styles_to_extra(p, s) @@ -173,19 +188,25 @@ class StyleDatabase: log.error(f'Failed to save style: name={name} file={path} error={e}') count = len(list(self.styles)) if count > 0: - log.debug(f'Saved styles: {path} {count}') + log.debug(f'Saved styles: folder="{path}" items={count}') def load_csv(self, legacy_file): if not os.path.isfile(legacy_file): return with open(legacy_file, "r", encoding="utf-8-sig", newline='') as file: reader = csv.DictReader(file, skipinitialspace=True) + num = 0 for row in reader: try: - self.styles[row["name"]] = Style(row["name"], row["prompt"] if "prompt" in row else row["text"], row.get("negative_prompt", "")) + name = row["name"] + prompt = row["prompt"] if "prompt" in row else row["text"] + negative = row.get("negative_prompt", "") if "negative_prompt" in row else row.get("negative", "") + self.styles[name] = Style(name, desc=name, prompt=prompt, negative_prompt=negative, extra="") + log.debug(f'Migrated style: {self.styles[name].__dict__}') + num += 1 except Exception: - log.error(f'Styles error: file={legacy_file} row={row}') - log.debug(f'Load legacy styles: file={legacy_file} items={len(self.styles.keys())}') + log.error(f'Styles error: file="{legacy_file}" row={row}') + log.info(f'Load legacy styles: file="{legacy_file}" loaded={num} created={len(list(self.styles))}') """ def save_csv(self, path: str) -> None: diff --git a/modules/ui.py b/modules/ui.py index ec75ee3a3..ee490a87d 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -511,44 +511,48 @@ def create_ui(startup_timer = None): txt_prompt_img.change(fn=modules.images.image_data, inputs=[txt_prompt_img], outputs=[txt2img_prompt, txt_prompt_img]) txt2img_paste_fields = [ + # prompt (txt2img_prompt, "Prompt"), (txt2img_negative_prompt, "Negative prompt"), - (txt2img_prompt_styles, "Styles"), - (steps, "Steps"), - (seed, "Seed"), - (sampler_index, "Sampler"), - (cfg_scale, "CFG scale"), + # main (width, "Size-1"), (height, "Size-2"), - (subseed, "Variation seed"), - (subseed_strength, "Variation strength"), - (clip_skip, "Clip skip"), - (latent_index, "Latent sampler"), - (latent_index, "Secondary sampler"), - (denoising_strength, "Denoising strength"), - (refiner_steps, "Refiner steps"), - (refiner_start, "Refiner start"), - (full_quality, "Full quality"), - (restore_faces, "Face restoration"), + # sampler + (sampler_index, "Sampler"), + (steps, "Steps"), + # batch (batch_count, "Batch-1"), (batch_size, "Batch-2"), - (seed_resize_from_w, "Seed resize from-1"), - (seed_resize_from_h, "Seed resize from-2"), + # seed + (seed, "Seed"), + (subseed, "Variation seed"), + (subseed_strength, "Variation strength"), + # advanced + (cfg_scale, "CFG scale"), + (clip_skip, "Clip skip"), + (image_cfg_scale, "Image CFG scale"), + (diffusers_guidance_rescale, "CFG rescale"), + (full_quality, "Full quality"), + (restore_faces, "Face restoration"), + (tiling, "Tiling"), + # second pass (enable_hr, "Second pass"), - (hr_force, "Hires force"), - (hr_scale, "Hires upscale"), + (latent_index, "Latent sampler"), + (denoising_strength, "Denoising strength"), (hr_upscaler, "Hires upscaler"), + (hr_force, "Hires force"), (hr_second_pass_steps, "Hires steps"), + (hr_scale, "Hires upscale"), (hr_resize_x, "Hires resize-1"), (hr_resize_y, "Hires resize-2"), - (diffusers_guidance_rescale, "CFG rescale"), - (image_cfg_scale, "Image CFG scale"), - (refiner_steps, "Refiner steps"), + # refiner (refiner_start, "Refiner start"), - (tiling, "Tiling"), - (refiner_negative, "Negative2"), + (refiner_steps, "Refiner steps"), (refiner_prompt, "Prompt2"), - # TODO restore params complete list + (refiner_negative, "Negative2"), + # hidden + (seed_resize_from_w, "Seed resize from-1"), + (seed_resize_from_h, "Seed resize from-2"), *modules.scripts.scripts_txt2img.infotext_fields ] parameters_copypaste.add_paste_fields("txt2img", None, txt2img_paste_fields, override_settings) @@ -839,46 +843,45 @@ def create_ui(startup_timer = None): ui_extra_networks.setup_ui(extra_networks_ui_img2img, img2img_gallery) img2img_paste_fields = [ + # prompt (img2img_prompt, "Prompt"), (img2img_negative_prompt, "Negative prompt"), - # (img2img_prompt_styles, "Styles"), - (steps, "Steps"), - (seed, "Seed"), + # sampler (sampler_index, "Sampler"), - (cfg_scale, "CFG scale"), + (steps, "Steps"), + # resize + (resize_mode, "Resize mode"), (width, "Size-1"), (height, "Size-2"), + (scale_by, "Resize scale"), + # batch + (batch_count, "Batch-1"), + (batch_size, "Batch-2"), + # seed + (seed, "Seed"), (subseed, "Variation seed"), (subseed_strength, "Variation strength"), - (full_quality, "Full quality"), - (clip_skip, "Clip skip"), - (latent_index, "Latent sampler"), - (latent_index, "Secondary sampler"), + # denoise (denoising_strength, "Denoising strength"), + (refiner_start, "Refiner start"), + # advanced + (cfg_scale, "CFG scale"), + (image_cfg_scale, "Image CFG scale"), + (clip_skip, "Clip skip"), + (diffusers_guidance_rescale, "CFG rescale"), + (full_quality, "Full quality"), (restore_faces, "Face restoration"), - (batch_size, "Batch size"), - (batch_count, "Batch count"), + (tiling, "Tiling"), + # inpaint + (mask_blur, "Mask blur"), + (mask_alpha, "Mask alpha"), + (inpainting_mask_invert, "Mask invert"), + (inpainting_fill, "Masked content"), + (inpaint_full_res, "Mask area"), + (inpaint_full_res_padding, "Masked padding"), + # hidden (seed_resize_from_w, "Seed resize from-1"), (seed_resize_from_h, "Seed resize from-2"), - (resize_mode, "Resize mode"), - (image_cfg_scale, "Image CFG scale"), - (diffusers_guidance_rescale, "CFG rescale"), - (tiling, "Tiling"), - (mask_blur, "Mask blur"), - # TODO scale_by add to paste fields - (scale_by, "UNKNOWN"), - # from txt2img - (hr_force, "Hires force"), - (hr_scale, "Hires upscale"), - (hr_upscaler, "Hires upscaler"), - (hr_second_pass_steps, "Hires steps"), - (hr_second_pass_steps, "Hires steps"), - (hr_resize_x, "Hires resize-1"), - (hr_resize_y, "Hires resize-2"), - (refiner_steps, "Refiner steps"), - (refiner_start, "Refiner start"), - (refiner_prompt, "Prompt2"), - (refiner_negative, "Negative2"), *modules.scripts.scripts_img2img.infotext_fields ] parameters_copypaste.add_paste_fields("img2img", init_img, img2img_paste_fields, override_settings) diff --git a/modules/ui_common.py b/modules/ui_common.py index f27589a8b..9206c6c59 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -93,7 +93,7 @@ def save_files(js_data, images, html_info, index): self.index_of_first_image = getattr(self, 'index_of_first_image', 0) self.infotexts = getattr(self, 'infotexts', [html_info]) self.infotext = self.infotexts[0] if len(self.infotexts) > 0 else html_info - self.outpath_grids = None + self.outpath_grids = shared.opts.outdir_grids or shared.opts.outdir_txt2img_grids try: data = json.loads(js_data) except Exception: diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 0584b4f71..ba84f9040 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -321,7 +321,7 @@ class ExtraNetworksPage: return 'html/card-no-preview.png' if shared.opts.diffusers_dir in path: path = os.path.relpath(path, shared.opts.diffusers_dir) - ref = os.path.join(paths.models_path, 'Reference') + ref = os.path.join('models', 'Reference') fn = os.path.join(ref, path.replace('models--', '').replace('\\', '/').split('/')[0]) files = listdir(ref) else: diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 5dae382b7..98aac49af 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -2,10 +2,10 @@ import os import html import json import concurrent -from modules import shared, ui_extra_networks, sd_models, paths +from modules import shared, ui_extra_networks, sd_models -reference_dir = os.path.join(paths.models_path, 'Reference') +reference_dir = os.path.join('models', 'Reference') class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): def __init__(self): diff --git a/wiki b/wiki index e999774e3..c0b5cb267 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit e999774e3096ceb89a264548fdfaaa76d891c0df +Subproject commit c0b5cb2672f7b0ac0add0a321f22bc0e6b738d78 From 7cbd2bc9b5db01315a36d750f5516580e911b25a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 09:28:44 -0500 Subject: [PATCH 22/43] fix list view --- javascript/extraNetworks.js | 1 + modules/ui_extra_networks.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 06c0cf819..8415d2d11 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -40,6 +40,7 @@ const setENState = (state) => { // methods function showCardDetails(event) { + console.log('showCardDetails', event) const tabname = getENActiveTab(); const btn = gradioApp().getElementById(`${tabname}_extra_details_btn`); btn.click(); diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index ba84f9040..4815e8210 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -40,9 +40,11 @@ card_full = ''' ''' card_list = '''
- 🛈  -
{title}
  -
+
+ 🛈  +
{title}
  +
+
''' From 294af698e51d4cd6dc75147aa47f21db76975b45 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 12:27:07 -0500 Subject: [PATCH 23/43] modify base/hires/refiner steps calculations --- extensions-builtin/sd-webui-agent-scheduler | 2 +- javascript/ui.js | 2 +- modules/deepbooru.py | 5 +-- modules/images.py | 2 ++ modules/interrogate.py | 5 +-- modules/processing_diffusers.py | 38 +++++++++++---------- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index dcb085cf8..8970f485b 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit dcb085cf814ffca53a9682a7e9af0a9026530ddc +Subproject commit 8970f485b767929cb36b5fee0df1d98840023f2a diff --git a/javascript/ui.js b/javascript/ui.js index d15ac24de..daffbe352 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -170,7 +170,7 @@ function submit_postprocessing(...args) { return args; } -const submit = submit_txt2img; +window.submit = submit_txt2img; function modelmerger(...args) { const id = randomId(); diff --git a/modules/deepbooru.py b/modules/deepbooru.py index 46d2c3ed5..24f970a7d 100644 --- a/modules/deepbooru.py +++ b/modules/deepbooru.py @@ -16,9 +16,10 @@ class DeepDanbooru: def load(self): if self.model is not None: return - + model_path = os.path.join(paths.models_path, "DeepDanbooru") + shared.log.debug(f'Loading interrogate model: type=DeepDanbooru folder={model_path}') files = modelloader.load_models( - model_path=os.path.join(paths.models_path, "DeepDanbooru"), + model_path=model_path, model_url='https://github.com/AUTOMATIC1111/TorchDeepDanbooru/releases/download/v1/model-resnet_custom_v3.pt', ext_filter=[".pt"], download_name='model-resnet_custom_v3.pt', diff --git a/modules/images.py b/modules/images.py index cbb60b5ff..28046e159 100644 --- a/modules/images.py +++ b/modules/images.py @@ -532,6 +532,8 @@ def atomically_save_image(): file.write(exifinfo) if shared.opts.save_log_fn != '' and len(exifinfo) > 0: fn = os.path.join(paths.data_path, shared.opts.save_log_fn) + if not fn.endswith('.json'): + fn += '.json' entries = shared.readfile(fn) idx = len(list(entries)) if idx == 0: diff --git a/modules/interrogate.py b/modules/interrogate.py index e2c6f9577..76685dae9 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -87,9 +87,10 @@ class InterrogateModels: def load_blip_model(self): self.create_fake_fairscale() import models.blip # pylint: disable=no-name-in-module - + model_path = os.path.join(paths.models_path, "BLIP") + shared.log.debug(f'Loading interrogate model: type=BLIP folder={model_path}') files = modelloader.load_models( - model_path=os.path.join(paths.models_path, "BLIP"), + model_path=model_path, model_url='https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth', ext_filter=[".pth"], download_name='model_base_caption_capfilt_large.pth', diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 0c7c16f2b..54564f822 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -377,29 +377,32 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if shared.opts.diffusers_move_base and not getattr(shared.sd_model, 'has_accelerate', False): shared.sd_model.to(devices.device) - is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or - sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING) + is_img2img = bool(sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or sd_models.get_diffusers_task(shared.sd_model) == sd_models.DiffusersTaskType.INPAINTING) use_refiner_start = bool(is_refiner_enabled and not p.is_hr_pass and not is_img2img and p.refiner_start > 0 and p.refiner_start < 1) use_denoise_start = bool(is_img2img and p.refiner_start > 0 and p.refiner_start < 1) def calculate_base_steps(): + steps = p.steps if use_refiner_start: - return int(p.steps // p.refiner_start + 1) if shared.sd_model_type == 'sdxl' else p.steps - elif use_denoise_start and shared.sd_model_type == 'sdxl': - return int(p.steps // (1 - p.refiner_start)) - elif is_img2img: - return int(p.steps // p.denoising_strength + 1) - else: - return p.steps + steps = p.steps // (1.0 - p.refiner_start) if shared.sd_model_type == 'sdxl' else p.steps + if os.environ.get('SD_STEPS_DEBUG', None) is not None: + shared.log.debug(f'Steps: type=base input={p.steps} output={steps} refiner={use_refiner_start}') + return int(steps) + + def calculate_hires_steps(): + steps = p.hr_second_pass_steps * p.denoising_strength if p.hr_second_pass_steps > 0 else p.steps * p.denoising_strength + if os.environ.get('SD_STEPS_DEBUG', None) is not None: + shared.log.debug(f'Steps: type=hires input={p.hr_second_pass_steps} output={steps} denoise={p.denoising_strength}') + return int(steps) def calculate_refiner_steps(): - refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__) - if p.refiner_start > 0 and p.refiner_start < 1 and refiner_is_sdxl: - refiner_steps = int(p.refiner_steps // (1 - p.refiner_start)) + if p.refiner_start > 0 and p.refiner_start < 1: + steps = p.refiner_steps // p.refiner_start if p.refiner_steps > 0 else p.steps // p.refiner_start else: - refiner_steps = int(p.refiner_steps // p.denoising_strength + 1) if refiner_is_sdxl else p.refiner_steps - p.refiner_steps = min(99, refiner_steps) - return p.refiner_steps + steps = p.denoising_strength * p.refiner_steps if p.refiner_steps > 0 else p.denoising_strength * p.steps + if os.environ.get('SD_STEPS_DEBUG', None) is not None: + shared.log.debug(f'Steps: type=refiner input={p.refiner_steps} output={steps} start={p.refiner_start} denoise={p.denoising_strength}') + return int(steps) # pipeline type is set earlier in processing, but check for sanity if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: # reset pipeline @@ -465,7 +468,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, - num_inference_steps=int(p.hr_second_pass_steps // p.denoising_strength + 1), + num_inference_steps=calculate_hires_steps(), eta=shared.opts.scheduler_eta, guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, guidance_rescale=p.diffusers_guidance_rescale, @@ -517,12 +520,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro image = vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil') p.extra_generation_params['Noise level'] = noise_level output_type = 'np' - calculate_refiner_steps() refiner_args = set_pipeline_args( model=shared.sd_refiner, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i], negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts[i], - num_inference_steps=p.refiner_steps, + num_inference_steps=calculate_refiner_steps(), eta=shared.opts.scheduler_eta, # strength=p.denoising_strength, noise_level=noise_level, # StableDiffusionUpscalePipeline only From 290970e536d837ab8d63a1ea811c671cdc4a7b9c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 12:57:46 -0500 Subject: [PATCH 24/43] safe move offloads --- modules/processing_diffusers.py | 12 ++++++------ modules/sd_models.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 54564f822..f401a7b0a 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -89,12 +89,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def full_vae_decode(latents, model): t0 = time.time() - if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False): + if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'): shared.log.debug('Moving to CPU: model=UNet') unet_device = model.unet.device model.unet.to(devices.cpu) devices.torch_gc() - if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload: + if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload and hasattr(model, 'vae'): model.vae.to(devices.device) latents.to(model.vae.device) @@ -104,7 +104,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0] - if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False): + if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'): model.unet.to(unet_device) t1 = time.time() shared.log.debug(f'VAE decode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={upcast} images={latents.shape[0]} latents={latents.shape} time={round(t1-t0, 3)}') @@ -112,15 +112,15 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def full_vae_encode(image, model): shared.log.debug(f'VAE encode: name={sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "baked"} dtype={model.vae.dtype} upcast={model.vae.config.get("force_upcast", None)}') - if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False): + if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'): shared.log.debug('Moving to CPU: model=UNet') unet_device = model.unet.device model.unet.to(devices.cpu) devices.torch_gc() - if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload: + if not shared.cmd_opts.lowvram and not shared.opts.diffusers_seq_cpu_offload and hasattr(model, 'vae'): model.vae.to(devices.device) encoded = model.vae.encode(image.to(model.vae.device, model.vae.dtype)).latent_dist.sample() - if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False): + if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and hasattr(model, 'unet'): model.unet.to(unet_device) return encoded diff --git a/modules/sd_models.py b/modules/sd_models.py index ddea17a01..b9d673e5b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -777,7 +777,7 @@ def set_diffuser_options(sd_model, vae, op: str): shared.log.debug(f'Setting {op} VAE: name={sd_vae.loaded_vae_file} upcast={sd_model.vae.config.get("force_upcast", None)}') if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'): sd_model.enable_xformers_memory_efficient_attention() - if shared.opts.opt_channelslast: + if shared.opts.opt_channelslast and hasattr(sd_model, 'unet'): shared.log.debug(f'Setting {op}: enable channels last') sd_model.unet.to(memory_format=torch.channels_last) From 0cd8e454e26c1298986b3c807290ff9e032f976a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 13:02:23 -0500 Subject: [PATCH 25/43] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d6297dc..a3dad88e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,10 @@ - Reworked parser when pasting previously generated images/prompts includes all `txt2img`, `img2img` and `override` params - **Diffusers** - - Fix DPM SDE scheduler - Add additional pipeline types for manual model loads when loading from `safetensors` + - Updated logic for calculating steps when using base/hires/refiner workflows + - Safe model offloading for non-standard models + - Fix DPM SDE scheduler - **Fixes** - Fix inpaint - Fix manual grid image save From 0b6005570275ff53db74d3add8946b93b7314a15 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 16:26:20 -0500 Subject: [PATCH 26/43] add brackets --- modules/processing_diffusers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index f401a7b0a..a2d650ca7 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -384,22 +384,22 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def calculate_base_steps(): steps = p.steps if use_refiner_start: - steps = p.steps // (1.0 - p.refiner_start) if shared.sd_model_type == 'sdxl' else p.steps + steps = (p.steps // (1.0 - p.refiner_start)) if shared.sd_model_type == 'sdxl' else p.steps if os.environ.get('SD_STEPS_DEBUG', None) is not None: shared.log.debug(f'Steps: type=base input={p.steps} output={steps} refiner={use_refiner_start}') return int(steps) def calculate_hires_steps(): - steps = p.hr_second_pass_steps * p.denoising_strength if p.hr_second_pass_steps > 0 else p.steps * p.denoising_strength + steps = (p.hr_second_pass_steps * p.denoising_strength) if p.hr_second_pass_steps > 0 else (p.steps * p.denoising_strength) if os.environ.get('SD_STEPS_DEBUG', None) is not None: shared.log.debug(f'Steps: type=hires input={p.hr_second_pass_steps} output={steps} denoise={p.denoising_strength}') return int(steps) def calculate_refiner_steps(): if p.refiner_start > 0 and p.refiner_start < 1: - steps = p.refiner_steps // p.refiner_start if p.refiner_steps > 0 else p.steps // p.refiner_start + steps = (p.refiner_steps // p.refiner_start) if p.refiner_steps > 0 else (p.steps // p.refiner_start) else: - steps = p.denoising_strength * p.refiner_steps if p.refiner_steps > 0 else p.denoising_strength * p.steps + steps = (p.denoising_strength * p.refiner_steps) if p.refiner_steps > 0 else (p.denoising_strength * p.steps) if os.environ.get('SD_STEPS_DEBUG', None) is not None: shared.log.debug(f'Steps: type=refiner input={p.refiner_steps} output={steps} start={p.refiner_start} denoise={p.denoising_strength}') return int(steps) From fd3f971b407eaa7247404fc25191d773e3bb5cb7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 17:34:17 -0500 Subject: [PATCH 27/43] update steps --- html/reference.json | 7 ++++++- .../Reference/latent-consistency--lcm-sdxl.jpg | Bin 0 -> 63331 bytes modules/processing_diffusers.py | 15 ++++++++------- modules/sd_samplers_diffusers.py | 3 +++ requirements.txt | 2 +- 5 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 models/Reference/latent-consistency--lcm-sdxl.jpg diff --git a/html/reference.json b/html/reference.json index 8c6d752d2..49f85b189 100644 --- a/html/reference.json +++ b/html/reference.json @@ -24,7 +24,12 @@ "desc": "Segmind's Tiny-SD offers a compact, efficient, and distilled version of Realistic Vision 4.0 and is up to 80% faster than SD1.5", "preview": "segmind--tiny-sd.jpg" }, - "LCM Dreamshaper 7": { + "LCM SD-XL": { + "path": "latent-consistency/lcm-sdxl", + "desc": "Latent Consistencey Models enable swift inference with minimal steps on any pre-trained LDMs, including Stable Diffusion. By distilling classifier-free guidance into the model's input, LCM can generate high-quality images in very short inference time. LCM can generate quality images in as few as 3-4 steps, making it blazingly fast.", + "preview": "latent-consistency--lcm-sdxl.jpg" + }, + "LCM SD-1.5 Dreamshaper 7": { "path": "SimianLuo/LCM_Dreamshaper_v7", "desc": "Latent Consistencey Models enable swift inference with minimal steps on any pre-trained LDMs, including Stable Diffusion. By distilling classifier-free guidance into the model's input, LCM can generate high-quality images in very short inference time. LCM can generate quality images in as few as 3-4 steps, making it blazingly fast.", "preview": "simianluo--lcm_dreamshaper_v7.jpg" diff --git a/models/Reference/latent-consistency--lcm-sdxl.jpg b/models/Reference/latent-consistency--lcm-sdxl.jpg new file mode 100644 index 0000000000000000000000000000000000000000..34e6b239476e4a9000e7ee756cc41dac251e9de2 GIT binary patch literal 63331 zcmbTdbx>SQ^fovIOVHrXBxr)$;F3V_0KpxC%ODwa@FWD63GNWwA-GGB8Qk67ZE)w~ z{q63zRr}ZOp4;c%uG`(W&eL68_tbe#Kg~R?0A4G|$jbmwP*4D0pD)1ED!^PCYGVli zC@BG0006*C02&G<0R6d!@~q-0RR6m!gYpi5`akbqJU4O!p6>v>XH|aI|K#hl{x{}- zp5IJt99%dYEL`4zIJf{$^8hIT>Wdfum1jYFF6b}O(b3S*u`n?)UgBWk;9z56W8>lx zzQ)BPz{AFVP5PREh?sp_b3jln!9Q}Dt{+pKnQ&3(!+lYbr@)Z{LbBCJOfEOsJs4vh^ z|FinJx6gAs0F3~h@GbX83?fw%OgacL&yT3gm-Lbq|47tEPZ)SjoqoQ;A|)fIpk!oX ze)pb*55zAZC?qWPNm@o$PF_J>LsLszM_13x+``hz+Q!z|#nsIn>f!1CDFpaEpO~DQo|&CnU0dJS+}hsR z-9w(9onKsDUEkdPhYRHy=l>=D2eAJKE`n!VFVN6X(J=qRh4RAvSy2hl(BE=n5Pnp} zG=UJ&@%(s6EE$zq@$VHqui6QTsnaMHDFfdsBl16J|AXxR9k8GOzmWZJVE;F+IRFkS z%Jbr(5&$Fs_qP=$Gy@h-fXO5D*0^#|xRx(bOfn@N!zIq|76ssUpVX z+q8h%0FoN+Sd*G3lI;;fY_ohfW#GQ7Fuyrmjwb+D zJaa=4Hj{~vXtIL5gQQZ5>axt7aSZtb(tE4Ip;ncdT{>dY-r4oP1--bAuRX&g+}s|JiZ z)^ut}I&TZOXkaqnT~B8n@@8HWm8R7ngYjguyNd^mxzc|}wXFA{>Qd0)e*$zAu`P<> zTaMS@94;(8fCIdeM;cZi%QzVRAV2RX2z(_!A~yKFU;q$QU}e;)en;la}=19 z8`++}XHBj>(EFhK7p2JU)beA@MIm3s+>C*RHl*8WnzG~)LUfOs-@;1lMwT4QN`dShgIH1amLlVI+*+7DQ9H!uHYwqYzj~u(6T|l<9SSS6-Bid9E~JfG6t(F76B0cy zB}c~=gG%2+b?!a9Kj6(fAyJ$cvZAM+Yq7cC6uR4H<8qo|-N~2ny^t)Q>;q44rx%RU z9~0g@0bF7-+NU88X{*OSC0>@E+Y{d8PpNewaF`@OgT0c93K)Wmsp-Nfil535C25iB ztIaXcsF z&gf_q2rqNRe{G#WdU?qof6Q1kpXHP8@%CRs`e*9Gel*SbJGi>PVuBLpKvTRp@Y-GB z1lHLGT-uFV{76LZvTOAza`kXs`i#+z`b3ubd$Z4~USm)Q-!PT6IPqBZApf?^q)vy! zT3ZwU>tRy<9!AZZIey1g)^XJEDCv@Jzb&EV??BA_YNSkN(fmH8LuQtdjXh0kac&=6 zrJA>p`*-JNti1?Kng&1L7>s0HFq!YY~a-ygAU!XvkeOl`j9 zUtQGK&-|lTzPmhDN;w7mCw7GjAf{ewyoqEG8)rm!quSq5@S;tXX$OGbdS& z(-I@z?0x)44vc^IS?KcK8-;lH57gjw@zTXqLX1BF zN;MqVn?J4qarcD%f|C2cM4m1mlAaYxaCnCML0!&wHPSAm`EJw4vki(8ntu+EH zrNWU{twFci!|9aBt>O|S1*y}J z(-m%?W>17j^32?r8BAJ^Z$T}zv`xW9*Db4SJEzikvl=IzEoSd7Lbf&hL$)R6yBXk4 z;KbhsvK$l?5M8)RVMK!Zh)b#U!|7z#+}`JsY%$@fP%WCMgE^zN_D*e-oIOLlW8WG` zibJQGY}Kyk)iLo=Zy83X?&*y1-=0zjHJ@SYpe`l2e)5@!3bFUBTj6-YP164F{if-_ zZjV%7CbUgsSgE9Cs4b#UwuAx?dj+p%8+Z~;9P!upilQv?=XUM(x7*Gm);7`U3>blT z*YTj_gwZHXS7V`(qHoQ=-Ze&736Rl`Lz2i=d$7CA@Vb!)WDO8)=JqlBlf^(|S}y#r ze#B&lVksc!Xe-%TC_$>qUf)WljO@;Nsc3EM!0ibzFKXrbt#Us_9krAaKXL!KrXbXx zbrsVHJX9n0jeJ(ODyCaxM!R8laP3_rmc)zZN_$uP)7*#K31z?c zN7pdDSJhX)H}9+^PtiGi*eshOYhVsShl;2K>Q0Q#&Pp-;CYq%hE0#HdkxNV0`n$ZI zJ^V50lf#AvJae5d9e86jz3A~|-xE zV)W=-c8v8Wt=)Y=$)&-Uo)<-1T6pTpA~hzxS0&JyW5Hc@$X>0$!3I65&;>|qbkU5S z#cCcFvXVIgh~O6(4ifVtWFNe8v9FB|NFK8*b;y&(-n!$bzakq6H~6 zE7Q}I29uH=1Ig{&e=xz|8nb8fJ@jy^SkrDag&QXbI(TX{xV@*ZXtCy-^qBgbC-kD( zZM%!Itki|QB1c+%!?S*H)RsC8>4zOZ46wn^EV!<5J{F&s~(W7}xMU&(V5gl~dz;vL7zp($&!f#NtbM*D7iwBUS#Lh5*5=lH{gM-tI#Aggd2 zp^mlt5_vS7@eUN{Ul460;ocJGH(5;Ez+kl-dY!T6cnGpzm**3?k(!sQY?_ikR_D1hrkW?xVFGp`W zUILWhPpxW&LMX&1f4sC4{5m8G&?Fk|TNKtq7374hX3mR^!?m52lLHp6)Rtcw2_>1c z)MIXF0v;LelF#JMvCoJ|BHDLDN0na@^zv3tiZh5H9KW$IauWIpQ4PIyLIJw?_@z00 zxpuh8isfUQ57MFzO^3o}2lvhXq#)D>T{v^%)Hc^U_`dSy;RCrhh=xIZ_;`zmcPQjU zhq8ReUpAw6Txwgzr`VIhi8ZeXCSOv`f7|#P4pcSRx4xoM)_Bxt$X@WSI&j==WosTK zTykR1i`KN!sH)untTW&nm}6}%D>f**380QuUq10?Ko`bAHo#md+FYG;gmXS4M|RS>jB-EHdXDwZ=LFW;G}Ttdq%!zp+xKj`>JrpYt7Y&Aa$9+|Oee-niJ zHv6%B^?X2dhd7k|nQ*TrtB^=V)5g*1`GEfWN97xplOIN6o5Pf;7^q`mf%&6P0K6xF zHTEKzM$OMV?TNMV%UY8_>3T=*5DDn_8HV$1lC#pdE~KS2TP3S^NwaCPyCekcyyz{x zzKJAa44L5A(mAF)C9Y@rOC1wnlInyiOTN{_^>!Bp!F)S}BA4t>!9~ zs@1nvwh_*Qi$mH&Y!yp?2Kb_r$ZNkiZ-EuRIVs~4x0%DTjbylpRdpyPkj-{L{|(-a zP+k=CE<9`%RDW{OU0n+a=5U69SyxkR9;xrN1MZ0)a`|Y~{|SrbP8Bi;=eCY$x!wIq zfBadZWtDeLyP7${S-ndS@v_~>L;mc41dT$+JKSg)?j(X;loYSVvWgKDK!H|klJJbz zy0F*0Y#B`!>&fM~D;4;TywziftKJQy!d%q5LRQ`$Z^$3^fX+i;(++}WzYcj6UPp@Y zL0-lnFgs3XrBJrwY+{TG90WjWxGKV`mIkS)4^3Iqr>sD?ylx29Xu&Io_F7#QJEe)%tQ$|iN?T(1m##9B4EiljToH&U@$dt2wsk2w`L zExC5wa7fzv#&zcgsCT+=Oi1$icrERS>9AtI?If0$0~>Nt&JttHeTa@+N)U|_5L34I z>xb*p?Dtc3TYj1w?%A<9%tW^an>^=mBYz5P78iAS5lez54-LxAH8%owH!=$d3ZX^YQ`8T zF+5h-FQoGYuY^})W^5!998sB}P@wPTJQYaro~1wwT-U1KQq$ihZ6QYu}0~lEw_W*6?VG+ zitU3th4p&v{F@+p2P*7?VPu+dl4&#N7iGhptz;1?BEo8BUo6W5NXpVd|GuO?fZI-% z1H6OBq>WS*hT~{7cHKq%@CO=wlc;!BqDI^c*-aLKLxQSfFsVvzcK#eAQg^}Ok2<#j zEuF`{p-^mD?)>+PU6k|v*s=~>?Pk8hTCE`~v!H-a^$&AqZ+(1rCJ(d_M5`^N>l$4S z2HE_&Gr;tgF+|6$W8nEX#BGn1OCfay|FMOyDWVb|2TGR# zobxoqTLR;<$idrn(mFXXPwv#V^yyq=tGvyx4_A+alD+JkLvkHsoC!`*7jCsTfiQ8d zy;i}Avb<)Y=+#OMV4Um+bWW&EhP?NOhf+J2Cjhy^BMo*zv3_t)^OD=`U)1^)XJeXO zVcN_RLn`pQ+;m-Wt(ULv2hA5uk7jesgzO@V*MQx{$FFO;H)1|BwZ!(l;ZVSXq_tp@ z?6hJ&#DO>o{j^+NJ`#l`X`Wy=hrg50@tduMJ<2>uws7*8nvdn9oOt&r zCs+QZ8f6(|eJDKQ#RB(@jHM7*FPqt!7POaPIwUGY+sra+-i3w zpHzZV1UV{g*AzaPkXU5F%5FEfSe)L_D%hJMcbcZtxe@F}OR zo+UvY`VCHe-yYJ-1Imr!bFo+D_4dsYe7#$2vrfbV zf{)C~i>W$J?EniLk2x%;$>r8I{Zv4%erN4d^Mdz#V)+~ie48rf-pB4I z0xCfRbhFtAk;~}plX5@sBFHW6pTIX2GsQXaO)tS^vu$~Cg85ztH~p@HP5`{cXMeHW zY@9C0jyrH$WStyQqf!laYDwOKC4=z&-`f_MNY_)ZU+ZYMEb6zviRvtgnDqn>mCuWB z`F5mFi5ofwo|S2Ny~!7Kpm>n>!4}KGDLYD$fl^cYjwbc}5(Wwku8@1Tlv`_7rOTpe zHw^#LH*nw*4!qpW$W<#E@GXlk1&SJZbJxV-MBV4UTyY=t(#p+|b>v^J6zaXH<@j(s zSFLFto-4y?2ushqb|_lX8cqy9e*iD|{mpN+WT&DI$KezmzZ|3ALt1?o_#XcD`-lB# zp>pSh4tMrfAucy{mvc=?u(9HKIaqNSB7pO!4BpI%Z`{Esl<85%f}c!em|Q19yB7bs zF;w7JsQJrN-d&?zs}+$O23An`ca$dpd;13oRF!?~g2$qAFIrM>20Z+P?=qMq=k|4- zWQ}tlgRuMpoTR0?bm~{VGWZvuq$-Tw^NkX!A^-}lTB3N@X)nd_TxumZv%qj)-k z{aW5Z%>IjVXJq{paEP%bEZ~tQke8xDKOaFKkC-YnP2ccQsX!_r{O?mdk_>s@zHA%5 zXFb^QCBJt6Qj?Zb^%EUd^cz5XeUVp@hH?6s-^H&dHWb5mg~7fi2(*FAtnvHbPLCVC zyLQgfLqKpVG{fr|gQWh!aUH4E_MBAEZh^p?as3{RXbkYo#c=?)l*?!4A>2_}Ged$d z>D1_sL0U6#xnWKyeO#8v54Iq*yJ;n_6$&;pM-B=~3$z0Ss2MC{sBEK}>Pg#uQgsQN zXl5if@~vXMA!o(usuZXID%_NK)H?I@g;(Nb7Q2d+;xwp?my*H9??2c0xPOYK5@LdF zM~cfphV`!JS5y$-a`Ywr4?}SJcWAucnIcF%XG5LduDj@IjfRP8-Gj1hS{E}gzQXiL zZD#KEw!GpzhB`fmvCYQz)K5xrgX;+EWd1^tbX4j`(75lY5lQqIW;~tZ`ysKWDYY=Y zRM7iReHyiW{Tv&b zYt<-&q3vWT(!I70C9b`l?}c8VeO zFPv45={5i+spZ%7yR$}b{Ls5gW(aJgA>BlzHyiS%or0-RjDOHqHBw2$*o3@z)mWHL zQ`~Y7JZfh*@eWP%U6q()<`x<4E7W82>EiyJoV0?CkA*i6`Ng)LV@=_#y)DQYSBYH1 zaWpqi!-r9c=Go2s$@)jltsIwrFM5$Lt>EI_Elc(-At8xel~s0jSJ-QPX`y*5(*3Z& z^AL=!N1c0H4_AH%zBOqgdpm0X1v9#j; zCo>a^C%>=W$7*m+<95T=jGbi@5Zdnn+bL3|6F`#z{UB8W! zg)4$qw=rU|N>cF$qQ;6s+x*Gf<@eqg+*}j$k)0+gfG`a(PEU%!mc;ybtu$qw>BeF z(lW{t@5QO-t(NxGL8&lp(`q#qF#zAm7UrM4wI(-8Tw!*rRVVzWDZMUR?o_|L|D0s( zTKyAuaN0DRL=XwH#+L_ixQsa=GsUj{?XS_9dJj2jpHazaDZ>bCAijRJseF{mpE67Y zki0~cH@D~0-Y{jfriv(5Fz~)uFw3VQ#NQo-+M{+{@A-5$#@DniF7fvyTgH$DFRI zn8i1U?wQP71Zx$M0aB*0Y?&gzK3d!CJMaEnYsj_x9orJ3K-#&m@ORQKz4Ba~F8sZ&An4|iRMtGXI!e9uIjl&XI0zbiQ zU&iwW)yPK9AF34$C((s_ZZoy;Dvlb%$pQ_f)5oQLt@p`vF4SE)&v<5Z)NcwfAEh-S zq*7`opFM;Vm=nFA4IP!KiktI&Ny*(1g`)zr+)f%)LMz#NrAk^ z_{{e$y2Tc&rJTxY;^PdzE|w&>ca4M!ylrbIII#5fhy8>h)LPakvzQdL>e)c8bg0BhVYYN-u3;i zf*f{J4<_`$6dI9}lU;DZJPo@m&?bB75;i1-YQ`%*dba&hQ}_bNrlQ!0{1rd7ekVyd z>NjlIK(aN}`li)JvfB~DOYm6w`|w^`xhX>0RhXjYkW8)FxsJf)MyJe=B}Vq7azCQc zhwCFyI|iNiyjx+*2dh~ms-`q_e)v-HGMH;<7Vil1Ol~Ekh zJ@{59?@G)6xejYFE>PM{tM$cb%`2RE_cL9^nT9hm3FR6+?z?_B2tM(WDu!Wh+h!8T zt2>=!N|b*P;%?_?mk{M{6K~*G6RC|PQT#+b%@aNO_9iVSWY|d%6oVPc&Qu@C$=4${ zY$PoUiiq{de%_s0PdfA4ePGr9z7JFCi=z{ z6WN=2g(N^eYh_wg2-*Mtf?d`n@!X$5SV#FBe;AQgRpI&+fwha86PGsM7~i`(`3b*CbvF@` zxs+S+TYXnC?#60RWOn?9VtZz~K!4{6fOF3&+-yWslm5ktp_yIi34rK*)X43-FjWJ~ zb*IZhkcx^`QQ65d-#Xu_x&wQ(%M85L#_cSaRYpq=vw^Nzb(-Y&fRbdGQ>ohWt2G{P zIiC5-+1=?=YUJizkGs_Z*7<|}dR*5?HN-HtxNzf}#OKl8*Uj@YC%9&C%Hsr%McNxo z%R7-mirZ4pz+5EV){Qr!&E_9PkHkCr`^UI%F+q}ycnFbsgF<+{I?8b03oEgtg*tgoG4d%j%q2BoFA{r13NR>Fq?g z6UFDzC=T3Ut-Y#l>d2?JS((ne*EmV#Kclb*SnJ1r6`syku1ZkvE0m3Ker9JZfiT?F zlpVhPhFbU$O7C6!WqBf-Bhxr%wyUK){WlpgY4`U`5Ct*S+fpR!hZ>|+08?Wft+OJq zPgby}T<{_BkxLI6)}gvvH-?dlTkouQZ zYeM>7PP=|i_yJ6oIaEqv>;2dF6tAQ7I_7hXRCBU`=sQ6^(zRqRPKSO&5r6-&Du&qO z1bgKeriUkhEZYK;3N=66+w8Xv_8Mz-J_XoM( z?VGmO4`CJ-#c)Rw$Ki7aD z;sWt8Y9kscdh647GneV53D={#os-dbXK7Vz$5@NGn;P~S^)Y@E>E5!BVEswms{2$| zw(V}-*Pl8`MHOJ@tYtYm+L3OFu3giN)!oFYM>DgQbaPdr=Vt+RSB#jsxq5XOBsBwU zy2T2Ddsf=&ZQKclihlBch$DZD>jsgRX}GH!ES)irxRUcd?+j?5soUbBEN(1|MB{L) z)Dyt_5AcL#OUM#HR97aV%^R^eD5UPr)&o_&9>36sn4ff~_Kg;|Rfzt$F--}RifqCS zsngIpaJS4UQyZ&@2q9q%m#}Ws6noWs&mtEW%@in*y!WmnZQujy!g=X2s_qlO7TVVy zJD@65?{`J7hPY9mjXn>-491Kq8hnu&_N~H~1t3vo8yh3G*j0z)blTv2ojT?;5nTLs z>c(nN(cz_C)KuPjYd~%|H7`UqDH_w{V^U!rJZj*S=yx{#QXxamd&G+$DRW9)5Wetj zwB*cue)A8WnsBX|1mdB`sojB>eO;t0@AIrH5I#PH{sAAaY>y$QId16$bSow(x)?ZP014kjVEA+upKXN<75c-|L#0tl>KG(;w&9w~Or> z$Mz1XtjaRpNj$=D+q&!(a8Tn3!Jd#gsRn+HlNz7ng66ZMqCyI3g!kVmMt27&x4e-$)gU-%-?gjjH ze6H-axGa{D6rU8r`uSgEJ#!iBY%aUvZ?E(31ni=A_%oW@8t(Geq{fIHW@uDQj=*FB z%$Y@@Ic12l?6u(#y@Xj&X3MQH9yKM&m4iSY&uFg)4DeVDez2ZZ0DE9uUN!&JRDVDu z;~bY@JK_wKo-qB*`?67w{1*RUI&*ID@;QvMH!aTCXq?z?@XtZ)FWwd~_*OgVx@sMs z>Hr*?GF0|)^0@aA(F#+_vYg1xwi@LZh6m*Qtr|T%aA~q8Mvok0RF*&)^tJ#ai>2=j z#vh9IE|meNYo?eE{(Lt@F4A}8rAe1(*OTT9DSy>bsec#~Fe8d}K`7LxMtWV%Fm9Y@ zsztWC&_^|YG7eTV9Py?u@&kvH*j%f6)%9rZg8iq~rNl=b;RoAELlBdW*O8BSI;X@w zTA!55qOq{9xAHhAabbpsn7HFqmS%&yg)G))rTEVnct2k$g)2fC7n+sU>D5`+3Y=m zFY-D4Oyk)8Ggznn#}tfZ$tIEO-(s%ipPA`7LtBva(oUGpok75WLwI+{LTABwNsnXt z+V}uCgALRg5#o7@B(VCRz6DR4t#Y zNJoZ<=7!F9+uIBnmn2z*w4EATS5Z3&$@wz}Q`kf#HH|oI5CC`R$%8iB*t%YtN17;o zevtaqlehrkJi-Bzi8N1MQ{P7Yr?r4Nos2ww4IO%k)p6zP+;)E|iVu&^3w=);R}Ssa zkzsIJZ@?qQw>`l%?k4bHtD_8P#?FZC+h*`CoXPKmaPzN+($)u^Ds;a!izU^&sSV@Bn4y-O>wb!km*PMU^f?j&Vwq+JTKx ze3^9Uu}kTOJe(tb?-4X-z!YpnbIy#-5;KhBPhW@7dzk(FvsUTK&Z&DcFX{;kaxA4$ z=8-`nyh}Bk!*3EJ=%X0uESG!s{dM-&?ZrEg9JI{3+JU+#_i*dYgW&DG!Fd`E@VSn~ zK-im(BQdcm4f|F)U0eG`t-v?R}eASD`7CM)H0Ay+=^j7?-SI;UuyQIqwZD4s#vkem(lLP`az4g{C)qXZ61LN zOO(8!IjIS(vWfSfPZ+0z9MSf%J^Z@*1k1}~+THvJA8DI=$-Uw}bJ53id%3K;xliYM*n-MT?XmcN1|@k}#CBIA0miqk7rmkEJ!I>@=^JLSo6g+J5;h6EYT z6w(KXhhF1WVBI0LhqD@ zvpa+?qBMHac98-*rSMqmkx}bIW~=4ywKmMAm3@(MBIS&l`?pZB{NF9>n?VXgVFUj(w+u3vwJT5|(J;hG3P)Aef?WWge_3AgIH%C743)`e<5!zsS!yk?i5-F7Q^O>`Tk9qXUaGDZ) zy(;@B0FWEr{GC_hH~Y|WyRdvqtMtEKA*MQUilGYQw_kYVaP_j4uEgU{Fk-0PkQ_o^ z-g^rbGPcyam_&Ya7sTu4v}m2UGccLuyFyhL&0Si&mf$Mo`RZ(}RbQHQ?~ zlc|6Ahv`}`t_4;^Bm!HB0)DbA*r6epX>5daLOv>G&)n1J&qEzmYX8?JS(B+m4=8yG%G$fXg!BuNZ& zq2n**U?Ns$!>C9LxbH*72JU6R#$et-7qw>;n?rMq@XKLoKugo3|VBC1pXt@ zE*HyTYeVn?n;eGj!VzPO=epQltQoGk!27|um%E}ucS8564`d+$TVow<>7z-k;UVR8k69Cz??zVIx9*DtW80 z-DD)Y65&JyDX^&}lO}fXFqP>!5TX$)?rvMiZ*y_bMy120CX`nl*&@c1+$MsLsoxkyMaWUQ{t`>WbI zLHV1f+=;#ZbGlo%BW?!Aafe>qSlNg8QPGR%poYqMc^O}M=(r-r+B%eC#xK9cc2a}_ zPNF_>UBl-We4A%lcaq-3YIXDw`T^x$+GiEB%d5PyTlg!6!EX%xWYtUU(UdkD+8}tc zY`a9b5sRtHFUGO;6^feoYKyXM3>jms&285K1-4jZ`2Ou??Ig+?(^6@|brmbZ_y@d& z#(d7#b=@cP(vycS!Tq&jqKNkU_|~9Nmy~jfPyP0ghUo-1Q%ygV>0MRIQHK%jSL$99 zagnSYepQYg#xI(+m9lUh*2JaZg>UPHfCm`~=jGN_C+TU~U6hB9lPuwU(d($RVIx}i zfzHE^(I?BR|1wnTvCd}}3t zxMta(A?+@`^LQ}5QOIxfn`9CZKU4$92+%N={>_q63xo*g$JvIK@_u1u=z+&@XtR0X z>J)3ur{Afp)RxUp0Ju2INlVK($wbMv_rq=={2oC<>eKJEsTWwhCYRqyZ0$fN#4NP< z1ehvA)j;O`DCm#!c_FyInTSizPH<#51-zD>4t>SpWV*Fn={ zt-qg@2dyLqi9GRDIx^>OF_qlK5!d+n>@6u$ct)>UO%9W{{H#<+Kq-tR2u5X%+tA}| z%}%NQVp^+0tIObP`pbPA&aXAgRQQQMMy4mpzUe}SJmd z+(8q#^s`(_NH^Zw#0O>4*qybB-Jm?U{PWcqp_8+}Vv4H1B1M9l$GC`_sm<@l+b@4y za~5l`b+n~USsp>`#dJ&F+Z#qIt?vQZ zt;)A*?oGVeFlT(J-h#}Oe%M2Hx6_`Z=}ZF~;cO(`a?`zSqB?_rEi^AKsIM`hMlloO z+JM0{7jQ=$y}iY38vf`i^FDG+IC7%obBtXBgTIZVFAHvm zQuZGbepH#~BNa_|6NYjmrV-FNp+ZYHCr*83?OJJ`i4YRN#+0Kv)8e67X+?KygtJ?F zWK{Pfa{tz{3>qjMrZDbuJ%L}<*>JsgIK1P@jwzcaksVA9Zas&o)&oL;;9OI3=Gyx?MPZKHL+v%TeV{=%KQT9bug$ns!5CKv z=Q8*ExU0d(&a8Mc6z96;NNnh#^(l#xYnkuYIZ(wm(#SD*J|#vj+leYHOp&FVSBL&FFHR8^8NZ!`ey8v#Y0fpZDo4%@csbe|OM~(+bT+5@cIdhD3DU zb;s_B(@)!tnf$DU$O+P>`Q<_lgeM;9V!VUb{bWq#ho|aUl<_lnI^yIw!%jdK{XMOd zEOYn0FA@FYAhEL8+FJ+*526&Ja4c0sxaqlPx<`6N`=(;_T##Kye_?^%0N`PtYJ5ytBm+mR4=*g>csWxlEfH2nlECr3~s*_>ft_l?_Cvmw*74<0*^Ow2Z%77*%`f4iBI;XJ*o61|@+bs2y472<3w)j3s|w{zma z9gQj{%>76|m+Rk2%P?Ke=&$fsH%qE!OO5p1a8pLJuRn<2$!*7Bg03MwKLwi5aa&dg z9bdMs!{reF(&m10S?)9DOZ1VkEvbADYsVi$#O-vQd8Ak?XcILE=8M)r?PRE9B z%0%vYIAHiX@>SvYwa$C_VeO9G5a{X;6!9O{*4jw$5Korr91Sd#U5cY6)hyz@UUrPV zWLcP=pS~4~|0`E_5?jfv$uz>@^Ju5;U7lH@zvu0f32ie@=lZ*pU2R5gd#iV~BAOiC z9RElbJ*3_5dytiSdt&1xTanMJ&)aVDd=`yW`xP4}pVYA(Xy>-=pnu&jUSGtIQ3N3x zs4BW7E3iozIK55gs?;8}8$WlpEBTj-1q41N zn(gJU$`X~)6>{kuc4auac6nW6>ru5<>J8!O?v z_t|B)A0>Z$V_B0QV86&?Yxbsk!zvi+{?C^UX@3sbt;u~++9 zR)>oYIH_DbXDGx8?!TvQ(=l7wr;tkOuSM5uDupYK}$Jhc8sN)Ty-lx$hSt^)3K zd`5EW;ge_Tkp;ah<8BsIX#$AVb3f3%uk$dU)cUFkT?^_Tsu{`woue>IrSX(bkhAP1qB~A`=Esju#ako>? zR!DW3%rM$tVmXl>i*e50xxx*9JnbWsD4K3h&ghaqG*swkQ{(LGrPmk=UmD}le1B-R zPvhK`fg3Nw+#!t9BIkHekH4B$gG`(5Vomw0F+}X%t(GsMlk%@>r^OMTj&er10F{-} zNrlvQynXgTtzpJwIHe2`Z!xD!^|EfGH{ua_Q2&3vs7UlqB@2gJ_D=Vz*do7W%+34a z=&{3b?ZiM`v}mUs{ul4^`}-#p4CY}I;;)TAi7tv?ip(4Ml}J^7TEC9Ty!AJ9OUgHz z?QD}z9h&Ww75S>J33Rehaucmp!B8mMb+)tc{;H5lBDgaB_qz6dsY@cKXu!`4IWhv& zdY<%OKu5hv4WA)hRhU_qy@^E?^hn^)!ax#}t~EoGgFv8irWtYMz%vv)F3EgHYh7$dm2okjg1*rsvf*|lYhk~lWZqQ~4d{1y|x zgY^{Yob870__FGW2dzQq<>0+dF60}glL?>I`G+OO<7D&7(T(n_ zWzx@j%ocVfA8htFq_TGdGxz9{otrdp$cM%SpS-DhBj1l_z&~5bqYufmIf&cNP1u$- z=`9_-KoY&cRirKYAi7R%0!@+{ywU#xyg)<01Ax@|eFPG4DJVsJx7(+E9D^roTy~_!p(Bmuq!z z!lA{N6)5a{q5C=fE0^ubb#oa|4t`-@S;yggg@#cInhtAFJz&EZbGNC!TAXRSn}z?X zwCYrQBYH7gvob#!K(Zk6)uJcn;=J=sn{=u3Wl83_WfvrRRNrIhuLx?%dm!?7jEv&3 zejTz!svrs|GkT|UR>RL%hL$B}#i3P3X z?{6BbmBA$R1F5MzJ9!n2zNsXQxAz`iGtNn?O*t8B%c*ke3yo6fSXs)(;^3E>Wf&00 zo}=9M&vRXmf|lRIJ|7y*Q zT(B^v@g}ouq60PQEzt=M?x*y+!y%#uTHepqCgaAqg61I|71&1)PzmB^iX zceSq0lf$~*#lj+9r;#8T`K!phMdQtDQ@B52vypoTeiRCcDMkCbg`qWlkD^<~8fKpu zXORq;;YWJ#Zwq*LQq&nPH4Cpf+mQ@s3;3FzdaG!W%TKJ&d0!A6V%N-e@*yYXMLdz~ zUPJK%;-;P8dl)oJ23y@)Me`>q(k1LYs+^_Ec4U&2trwx!ccCnjS3h?XQXNq_?%yZWcws%APas zJu4Hy-ZHe0#8oKqubp^FjqMZ=vUNNr`y_TI`pwm zdN$6Z2`p`f9^d}Dxlw%?o2fmS;<~rP{{R=>w5@Bh;x-xD=WBi*)$E#=i4TWlju{zc zgtyBuAHq8ydYJz0*^-&_7l}Ru_|sQ~WNj`Nmv#Y3#!uI^eOsq^+ey|JYsj%acx51d zHL|H~6C$M*Q&xPBrGCpl5ZSjjhZu~EhL-67{VVNRJZEnVBHCG3EHIDR!2Y#<*SW{_ zepY;mWBV}p>1PaJ(`@oK;herX>wqii``?HfUXe6V>Qb}@P;oYJxXuaBZlZ-YYN^E^ zJQUdsUd}~qM|FY9spua#81<>V zy9vfU8S~QoKk=`F;Y-bDLA!-EPSV{2I$Dl!q026}d{PP$(W{7kqSowh7wl1;0q zMIIX7On<7nQTLSK?38@N1y1ZOTB7$x4!YV$$o?>}eLqfk=5r+0w+21OxUSRohVX^Y zg?v?ar`)E=TG=Mxf~}nL2;hT(gWkHSrnfk2K4o~GRj2rhO*L7gCnQ%kw+C+8>QOw| zJDtCW^@p0_i9EL^u(X-}%^ONiFe2+q)jM zBq}4Ami`?6bddS4u*UghAoGEWQS+Q*IX!7y)25+u`?wVV<&I4~ZW24lnU{Ja0$0}^ zsMVSIlIxM1f3n)18YqE(_nYQFrYf!d%$K)P&nGP-7C((s97rQj+?sCkxfrOqUPYTc z{izuJ)j2;(O+HB0aBef?b$)Pmjz}MtDaUeb(j5$Y<1Bg#AUKjVt8LEBADoh?^!eWdIgjsM(zJ3T%{k&vU{`d1kr zjUM42{4b z^{yI~)V4Q`4oxJFA@T2z;)>!+MVB1l^H|>zei7;ynw*z0ZW%e^x$h}%RO!M8Egq<5 zz7AU}+N@akKJQLtj&$68aa6C==62m3mxlEPf@Ip+^EjsH`frtW7$ZDm=}kKWvp#G6 zqOIls0EDsRL6AjuKOb~`+Vp-}ZZZxADCbuVFOVGjVIS zb7{FxA26(?Q67yrG06v9)Z_wxPt^WWE_-5n7 z`hDfwmCCNvVt6t+004h6`qtFreOTu!D<((c?-S3eYLeSn&-QC=I4n^Z@Wc?mCmxs; z&rPO{1{WgYgo0>9MV$nq3V7+__1j$Hj=}A zYjHMqn_b!2j;wor74n_Oh?43?j3<`tM%QD`K9$oM9L_SnhqPrYB$e+aN;;1U+*&>2OYYa>QmDQV z7OX8r?Dw~XZ;;I^7-3KEwm1XWb+0_}AH~Qmbcu8?3(C=1mBrSiic(v~1NcGZQ`0=0 z_N(@7>Yjvi#Va*-eHr7wk6Jgv-wyu(V`z5^u6UM6_Oy+I6a@8W$6R(n$4s8p@cq`P z_U-4~Gh5r1+*$tf{y);L(DgBV*S(KZhr~BU?zKDn^CiseURfr8nnou$≀uUIpSW z8C+_Yim)?9X&(xwy-SBM{iBZhpF-LA!DF_T;NC5~Y<^fHJh!#`B4DEs$)=3FI zhoy1e2bts4;+|5=3b$YWy>rS63h4A{)N^{DVryP3SuO2S-*HbX2E6gSO&#T}s#uXM zxc&~q70Z5Zhi;HtoCl9}CbRJnx_4l+DNVkot!DgWjh9V|Rm_)2;DEh3>s=J$rJ>JK zq@$_oKeCRh_PU0zbR+;xAVM?OAO5=L{u11u3h1_>qi8N2vgGg>k&{tbb}EkE_fzOk z9e9FEm}XRwRY2g1@xK-L=j|J#-UD|zu2mN{bh#*=U-1WCm&4lhyQMODfJmgD%vTlh zO8Dy9lw`@gs*%9OdQ@npz2nKmVkH;umZSK9ZM7&h3uy|+8L!~TxTlWgX$~A zBlsh(M+}m#rFEzIX}zI)ZRZ`x+T=L@0Cbwq>T3G4Pwfk>Vs?}G*vWFqZL8V9!M__zi0M<3fc-BDYl*(MAWLWi9;8FRIh*N7t(L zXR1l@IxR{QWp#9yHq6;bZ$h-tHh3Yv=_5UTI3t{oDAqL14^p#^T_aJr)o&pmyl*Uo z3J2bHz#oXKSa`&bu2g5Yct_)89u~B@(>41yZ805gOm^-OkcQ9NBT@eVEdb=N3d8G~ z@C#j1RkJo%P^_>Y^-_U}BhLj@`t3cz_2Q{fh<##D)cZ5V@U`@J66iPQSkv{3C^aUE z%WAaI7s0rQlY<`PhC|3XQ;rRM>F~$KPaoQR7jJ(wQA4`_07a0wnn-RoZkpXl#^jYj z8-KotdYXw*Nt(kECXZA2UE^y{4EVwem7Q)hE7WUUM%ZJ1(%56Ss3V6{>Vww2H%-?a zyi5heUo^z3Pa6%R%~7;Y7J^3?wTh1#qZ`ie)t9OUy^bEuW zKAhJdbdj7^_Pyf^Saf&!z}mPUm2^{@W)&$)JEPj<&oy;Cz8?V;8o`#Wy1tWITO$SEQFXYHX+YDfL zVt=3pv^-05{hOlL>1mEv&zID*bo~eGR_$#Lc>B`S=A+Kid(^iP{heT6<7C|b0FP?S z(&}3g{ne&4m}TRNtAV|#NV<21?5tCnXJd6aACiOm)q$xYu(8E zj2|Rppm^KG_jb0{_EE^~_Jf|iE0*yU*PY@^Uzd8r8jOCll(|-?qMAOl{hIX&@3goh zn77NvHEZ^L*R<&DZDJ^5X6SL$*KHn2^*pRxTGb=zc|0oyp=zbj5=U`b5NeVAjW#gp zUaV=x$Ir@AzNeLV!@zLcF_jQ{S8b{(yvbp>b`>&;B}v5amC|IjyvnA1IImTc!|3xD zC^Ph~XAZVDjIMmU@jpb8^IVCgiyUxqUd!Xn3OS}j<%VVUt{RI|)uXnjpLi!jNNizG zFP!>UZ{dq07ErD`VAnS(+s$KVPqP*fS_pd!emrEX;V||(9{x;HM7ZNDA5yD#`Pq*`|y;DzVf77)q-zGIXLhs2f*0fgL z7iWD;J8cFnLhB0_)>GVe&2io|_{9#B=4sk2LVL5r&fO#b025tQsnwTF3@TNjPvUx> zhw&4{AMs4-z8!n3eLGA>-!evahh@eKbII?|r?q_hXQk>MD!BVa^~94BbMqcQD(zJH zcV`5vdq?g@=Uc0I>%-n1k7k<#YWHLKj}e3Gnv4DrSS>*NPS_zT8_Z%p@2xoUb-Oc{ zwYHW=39NX!{{X}op70Io$7Co7MIP9$lUB3QbomTPc?)eH!gHGFPAg<^$}@iy&Key< z%w-PEA3XOJuX&~0O>ZQyI+Cr%J?ifkW6L$I%bI=FgD8*9Q_rp|0xMCH7!~{~Wfc0F zLad{sM|WpyY0GUpaq26ZvX0^d`=RH@{>dMxtR)xR>Gq0CW2Z7etM>w^^{iy?&{p5W zsA9gn9&l5STIH!|^g5~DT3qyh48+zJVHI|&uP3>#yG`)@zM%+;-BKwrfWv?*f=`+1 zdXcB0!|RGQWh(njWw3o~yt()Zt=-x^-L|`Ly?NOzM1G*v#!5X!D7Vnh&~9u$vrVnS znTE_A{lEJ4!uX@Zo+t2)!~L(zxQ{>UEy*8_4ONu2xh`C)%^BV_(#^fp(pp@|(GU4$ zBYoe-xgB@KjT~?1NW+W}!xyt@PVTp@nW-t9_w&9LCw;ej>Qm)SZ~{7q=YNk%LBD-etS#A-$Bg(E^^T$PepO z6(yU5LdV{qOQlb06cyov$V@t?6n(tuGGmKatA0l-Gh)n zI;CPCG3gfmF4Hu7TbOQRzq^nUgik8Z8}c)N$-@E%IUiot<#Suyd1wUE7Rx(fk5YO5 zwRSVKuesb_c%ts|7D#O1g58+0lH*|{lgav@YT%?dDvCVT%AURd06Lyj#b?y*EY6=E z=G0`G?Q@X~Gl-(k&;n2CTsp~c7?CSy(45rrzJ)7wJzr0UPS&mDOEU(Zu?^l>X3rk% zNj1i3I>BiXB?x348Gx>NMseAlaOAa)mru94(EMFttXM|*wz!VvQCn$kj0wT)NF9CZ z#M+g@s`-VS4*9IzNo;f~QDx5#SY27`c8PDkSgt1ukYhP4+$xU8BnsTnbqk*l&!+0| zg_7ponc{XC^5iN)xc5>+4{$0u+>=F09J<8ryngoDZmn;k+RkK_%gWPPlw=uERJ#xM ze8!P~8qWB6;mN#5;z(w=mEp3};y2fKU)+rj<|O*D!i@JdG12o1O3Rc!H%gLt`7KqI zg`I>@w$qXr79<~Wn#l1*$?*oQscN{~^Ulwn;1j_7v0V|ChZRWcW0R3YYD$b6zYdz$ zM_?S9&QfK@3#nT{zMjJK&PE{DOKafyVT?N=+l=QNS1i5b(Wc`Sc0sRr((2;DqcX4r zW36-kD77%LjB(z&X(tz{=i(}J=#Dp1Z#qUK=O&pl65Q8oZLJS1ZH}7Pw-LmhSmL+* zHyYZr>gB|bJ22XtA@n-ExJvqL?m6&N10^*0CfRl{HlE|zRjZQml*k^MYmJ_ z^8(+Xp*?OXUFb>V%dhIu#xlj2BKg-D60$BW(xi#ZiV$s(HyZCaH3(79_V+wGMi*;+?2W9pjBY>Lx!LWXA2>t#AIY z9>bcN-rb#%K;@4VtXzp@OnGCS$1H!PB(iRrPj8fcDzqe$*?wZjw=|Z-vt!nx;MI$& z4+qwkGF^g>l^sd*Ixh>`i_5g#_fwNu+I9KUby1X#lTgu#XSV}#Pm^nqCA$ZG4f9*)n1M)H5Ua8|R2l?_(DyJsA zny|L#(BW%eL&Pk!wYb$p@-A0?IrJ6KUBE0Zoi^~f&3V#S*!p}rR=Ybd3HZ4*+l7iL zHZryWUNKmo3H%W!iM7q{%&;ru{&Yg==sM0RU7uk5GuN)Q`xu&c$i^}Sb>9zsBWa^) zaHL7M3Obti=|)_-pDS9VB<&NVhQ>dX9!*&J7H%6ju7#n)Vuq<;w(*t&lUkR`vRdVb z(=?{ljUm2=mTH=B+AlLA{I%LHl(&pY99NrW+3CVbElxsj1^Jf$0BW=1MHwmUU7)j{ zP2~`}dI}sP)eG85Jq}+()T6$S>~nmRuLE}ShqDXy1C zL|kR$$l57>;BB(U)7(`B*R=gY7q+~)7I&yHcN`jVoLbZrCw&pX{9E{)XQJt|!0eD- zMh`8)2k_>&FN>cDd_Af|b9-^GPSPB=&pz+wYnDz|-$T-;7+c}g`CH;o#Vegc;iPRr zg7b*gvQ2Ow5wv|L!}k{kE0&*WAs0V*=DI1uoV?7cI130-M03u0 z=BsG_6V*IJa9Y;c-boMM3&0+g-5ArSvOKv;tygy~1542Dtk=xAm2-@-CcQ7izXLTr z=>_U}kw`&qa7I0=Wj=1A_G&vVP8!$5QbUV49rLA>0adD2K30Zgqup2&AL|E zCA6!FBW`eg!K#d@-`^soEJrW6eCerpg{0IRIG*M%L0KOJoObl)yZ-`t z7V3>7GXRl+lb@wxd^GW0mY=92Uco3~&_;L$xaUt>ozbYJq3f2qhO4I8YCave-(_v6 zq8s$u156`3qmMmygO)hzNc5=F!~;^)9_bN~3Ip~pB#_8d=e1^A$cZrY#Zf$~srA>we~h!Ws-*I8xEz}C4-eiA3r)3W zI5OsD&!=iGMb+$&xOC4P4M{?7`FO#vp1cL|95tG&xGtxTdsIr>ml>qbrlwT9w>=3W zxi1R%h+iuF-*|K;yz0^BN3DgaW~^|YF46wUG@F+!de>p&eK=ge&IV2^!JXpleV!Jg zsI@rn0{l?2@D1`Lu@+Kt+*gx$w%&ErnO7^$1v)cvmCr*FLKB6V_x}KZ{xMtlve^Zp zDlyW$U-oZzn zH7_mE+fG#B@X7th!(SEk4Q}67p2JPm{@!;L5k{qt(}Q1HXx|R}Bcb0i=-N_Ea87*5 zK4bMH)1;vd^*5bbv(h>vUXKW>Bz^ecgf_cipz=fO9;ykkH5 zzRFJ@N=i?$PnGL)_Gl1X2%W#9u}QTU_5I)8?=paDe%ws1Kf^=_h$^P=!*L7G<6YN?CV<~M1LoUD!#`e6YHFu3e#S9YzNeqr+1yWV9Jxp` zLb2y3t}9nuytK293tM*Dq~*5u&tN|~N?g~nCRF21d(TrxL7F=Tm(E0aGu z;Xkt4$7vKfb(cBDOL55S{&7(1vuNQiyoP)`@RTX5T+GPA2{x!X><3K!Nv~4ywu^6L z*u8DEp_S_yXT0*spya&sWkGt6YYHtx=Mg5l{ zknRdH8`8O|!8CNyl$5M|nek)8=16W9Sd>W_`Qzw0* z!lKiHJ}z$$E{m^U$ATh;f0v%1oPk^3Hi4wQdzi8dHhN$mO7>~ha*9m(y3`dp zxkqCHINwybm3HIHJd;eb)C0+K=ytD|3BdFm*4JqYoO#}+vsw+Yx0f2pv;aq8?rV2T zwTAaTYm7H=I&stMQtEP@ijOwNC)jP_1@2Ht*uXt59oS}KiUu3&rdN>j8~)GcoO#MjK(Ep{{Wt0Mk_^C_c4_S%T|vfy7+IT ztfp8sGdy_8j+p+n=<|3Ed&`ho$iWZX%wrip*0fdS+~k%P9gaHN!+Lg;r}?vOxJCJd zWP&r=y9V{o@%?b7f}>LXc*e7fnKt{0~U&p zB5h1(=B)j)iGInmz$CS6Jwc1aeok?U_04C(Hu{vGxhk@*K=&1K#HnaU)#Vp-dG?Q~ zTK%BSHMtHBaGecnoeTRmQj*8dW*sTH#iFB4-*eJD38`Ca3oMgj7}Q~Z3Z>y&?Ke-d z*(8AubM>w|lI)Ji)6-UZ%y-&bM+$%R}$ zF}SbJeW`vKz_v>2%m~eTxLI9Yqr%2U&qJoZ){U5Q1`a9@6W&|HZ!4S=gId5KmVhz;0BG}1)D+xaO&}QFzX!mNbV+XoK=mVbONuGEFAndxanUdV=EX&R`e-6*edo8bLdm$|}M~nFpYx zYbhv}EmT|S^Evy{FpPd2pXo*C?&E?H{42cuPyYa06v@5F84Kd4MG%pmc&U1ku+R*c z$?3&PtI!U;g*z0iI-Rs56*PsXLP*rfw0fRuYgqR&Zj>c)>?P7LV8r_mrAcr%TomiD z0ZovuGT95tqpnR{wYK}Gr!_2BI!#XLudWp%I7roF3OxX=9|>r`Y@0FyMsv+c3r&u% z;lG7$Zl;P&t|RO9uUznbj9OiaHWViqG;*s3vS(M}T^f5c56nQTRvsp{jfP!tIO|;1 zBYlp#PnDg7)<0^u*#7`A#Ydv*iKuNL?%iHRT5(S7`dlShJ&zypw}yghMC`!N2iCny z#-1M&N9F|2Jwl4F$xRPVtjMkTrv{KqUhB1d; zcUPZ?j-1H!aI(IVFZh33QErkuw3c6+ko@!d~t@X?xVh@@r0^u>^of6 zw4ETEJ18NEaq@%!bgT_8;Y5BanWxn*q!v9+*4oYOp2Dswa@8UJ=AM(4*;;QB>Jmqk zx{J!)fSKp>uSwJVFW~G@YYxo>^V<0ma1g9pSLJp;=}^2vTO%s4gkU&L>N zK0Jw^R!vc4wS;b1$s<0sYOJlMg?(O}w`bBb{7};5-tByd$T=D7U!HpIy{>q6?S9NCa1knXEspWCW z9mR7>qh_(RU%Fsf=x+`5CRGPwe532@Tldy)Zza5pTWgXJJqKFPTb(hg+OcOvh8t_a zEXblD#FitDzvEV#&O4|Q1NT=2PfFQCm)=J_q~7PI+DNUYvvIc{onvShGMzj_Ao&zv zfA#CS61;5BBBWZ8O8)>^`!sTs!jE5C&GBTxZDj{yuhO!S*-k1)Bd++7R^DjKfxSG>T_yjsxui^G@EYL+W->bCKMFdX9|zI?awwxJpg)1NOqVEWYBLG3w}p2yPD z>e_rWpD|-*JOzyQuM^PxUuA6|GEUCx&(7JXeBmBdqCLjb$(gbs5h3Su3|Es}d{(oP zP_j;Ai8?P#)HOK@T#HuD)5MlSd#M#<%vcxzv}rwd*7RC}YI7?_ zHe4qpRC`G+gz2enaJr|%d)+2&(m80)Agb52S;-!)V)nNnWDXdPMIPQ;0-%w@d_A** zE3Yl4!6O)NS}wHHqUy#(I11d=$5(SXc0GH;?$)Uyj-b?j5W9-|Of8(ZdiE)8q4U_i z883+)q_J}C+NgMrM*9qk*%%Z$jy(?@*&ZL_$yQ6r6dbDcs(vJGw^Cy)OIH-DIUG}) zlR3j;Hxt^eTL-wfRAGhBKT4iw(2r?lsaNd2PIJlIyAEBK-0I-oG<7n zdw-v$S(-p2C-AF{*|#x)cy}|LQFrq>1J<`!T}@)IsgW=VBC-3q%{K9l*5`t2c(YWw zv@+*&B+PNt=A>3#*~c}huEa9_?l(W)J88O;ZIL7HqbKTXL#;4Jzn^fC_hJP)-eGGrvVRyx3-vru8ofo4Mb@ClI9=EuK~$r1I+~M9HL-5t zM3RsQ;M8sM;~eoxYRTr)V#IK=ub#u?&d_~LX?TbWJAFS+0}T{jWWVn}IQ#}F>`J_} zdl{R9F+Itq#=CJyxZNXGAKluc4!?NSSR`f}N99!Ht)Q)`V$^xg%)^p-sqAf;fjz23 z$>x|tw{0FP{%ESS#j$GiMBn{*JX9IzbJ9tvS? zJRo<)dDZFPRC{B=+jX+@fBQ#+uOCYYmuLN)wnUt0;w4_=&K$a?{ksymRm)* z3Npn`D@$I|p6bogFk6bLN18S>ruij%pEFtNUuV^(k3TkPz970N~>zf z%AOr5b2HWT&xn&upd+)4R#&H$6325p9Sh357u=|%$jQ`&JNV%n)_!>Z94W)ki!bd z?T|{055v}+Yd(XX88vAiA^2zZdGW7`Lu}LD%H11jn(O5sfbaNM=%$t7_ma%W%5p%& zeTnT_`&VO?VX3EPe3Rf0*;HA|zHAa*+=j%L3^>pCPB?m>uNpJA1XH9f2IEswo zG@k}o>2a)ak1^D8-lx{On@=KX3059v?=0i7^r($Eu7*>oC%ZY@%{8rO7bzKaR4cW6 zS5o$|*|SEdSnMo#;N;cvq9ZAEI9YCn*iw9`VYzqz058I}X3++zn59wsvA7&$4E;T- zd383Ft*bNqKdf5o8nfx}PO?gX2^Cu`qz;wOd_M4v?}mJFuU<52R_0yNvW^1hkEU~5 zrO?_|RIeAQ>hKn~I}u4y!8NL2k(MM__-;#jRjm&)_uTS7h?7ihE?09#*~iOWXNTPN7&-Q?3B~n0<0SM) z%DySIHw`38j0&8;wRK)R)5VRo%<%sJuaUuJ^%a~SD-`2&d3CNPnZpgL4iDkptX^7` zgLmDLx~mH4a^^hQE>48jC8n56hbj-<&N2-o5eq9g?cZ>axB-W~MVX{uQya#99JbW1 zCzc_y;YM2>4;9<#x(U9VKz?>+1*H0Qtz%Ks-HupzR@T~|IZU(Z_p-*}v(y^;Pg?j_ zX7^A{4Z6$~O1B;Jf?jxG&qd2_+eGWS{Y#+|E37hyOV=Dnj-_%~w|tt2b3AK>(= z_VnaW)$1e75=i9vlW~!{?KRrB!`U>ugmu9Of~U0}(l2Id8Z{ThbJ}NyC)A|#CEQ~l z!rd#-bo&>A7ea7>eJeLtTOA6HhBQ78TZWD$3X!h_fz5Y%Mb@8c;b&Y)jP~ z80yOUWsuR&mu~r003F@yBU#nH%c+KivkZqA&q~VfT9v0mvzx=8W0`K?bG?S_FG|h5 z@l=u9z`*U=0+8MKs8^6|YuoC9R1?GJn&R~j5h=^$4ac<#FKZDwsO)q;ExHl3ZpQ*q zT&}gM4O2;kA^qfRG0ksIn_HYUX03F3zry*Xg>%bBc&Ppq+^lvniB3AZP`)8AtJ&M+$1ixqC|Pq5;aA)Y-$=Z#{;yyjvu0j{fA({{|tS0k-?EuL#_9(^2p zeAA9d+BmL|=)t2dMh1AJoh^vx?s3N4dCX4%Mr&IB$H}*M9b{~M0Ij1XV?J(%MY_Dr z6d?T4HaB}yCTF#k9UBb2E20{^vG1ZPc%tB4YSU~#B4RYhOaY3caUk5zdw)9Vl$@Q- zsl`E9p>ZBzmA7#kh0;VN@jUJUbSaWxAUaSAC*SpYgCc1HMTROjQQAMfV{S8mxs$z+4-d;{u!+c zO}z@Tx0@?D{v%BHpsk5mk1bd8sPpeaLQxH)89unFS$8M{_(n}T32MzPJ_fw=oE@tY z84F{kDo{VO{4CfwlKNe~`xle?;ABqQ;l zZ0 z0PQVe?&SXf%Sl;>ukQsPHU{tv=Yp$RXBzso2-oFWXPGB|i2l_N`+a+1s zt-S``2(9p#&2&+V>T*`)`zDMyWo0ziZ!HPvYqavun?623ZG1qp#J!GjU2n#%J_)Vl zk+58f@^Lim`HxEvB(y$vxYk|`J&PPM^{h==LvXIB2?}_wJ#2c>lI(Wg7tQ^=WdPnt zt3CmRZ7mU)Y&g#~&n;2Vq|&*U@o!xk9*yLKha$YkaE_UTNV?SE!vW|p%R^F$)Xjc)cERqmd3c`lt?209M z52oJLlPT1>*i6w{>T%uhBVfNSd-twxYsqyRTt$-oDic?^(61$9YV%%rd`)$1s?0WS zRtJx2NNt^OpprnXz90|O3a8w-Nkv$P%+)n$XS#KeTSN{x8es;ON@k&h|%TB?OyE%N3*%Kl52Qighsi@_(%EnWv%-gZ) zo)xoOjW*U@lr6(;7=_HO!81c8%pXppT zi@b3TiDuGyq9s1na;l=WA{1Pnhh3^_T79jovRvFFD)YC1f1P|G@wfJiHZj~mZ8?nc z3kKxZH`Zl0Xh!F6)YKO=l&Adb>XQr zuEm>zTd_T=B^s+)GU4nb?2ppV4EW;TUXCkXS;CfPUU<(+@_&Q(7dlM#v&z_#Whhkg zS0^L<`_TzWy;+S&LwKJ19hoRr5EQoGn>=j?C-kYbT}Cq*g@^AH?L1dhV|Ss=Ed;l) z@?kO$o!Pg38LpjdV_SbTFB^ESg-O`v_KMQJvyz zG?FWb%k2)lH>GBH<5P`%xYk@VbtAB-ld?J~DK)9X>vy&qCYZMF`=nvFJ=(ax8ehS@ zv$k3>ocdP;=Prkr2P0Mlje;Uo#JVhOf&vz3I&B2pAfn2w3 zozqcK(BO3_tu(zp_8Z`)>Nx_YM>*-o7K z8+}UJSmbO?8mW)hBZK-^l3V!dX4H#s+N}K7Y_RgR(dSdd$D5S;1H*Q=2^zF&T0#Lv zBZlo%ydUDxHH;BJL3LFN-)I%)RD_Z3LRUwmSw_}yubs3xJDVNqx7rf>PDBLx(T&HB z=DAT^x75qkmMe)F)qZ3I{c3X}0INV$zkg@lO3EFhOosI}p**%c_G`tG@NZp;P5^4o z)%67z$Ux(gc&?c2b58fv#nrXHCsUf)TjkC$E1tc!GiuV^IV-&JU2%f3$10Lcx2)?@ zeWeN$(+38*{Zimt-2e`I8rC$?gzCQL-IdgrR;~Q153C%Y~ z??OGdFH^YI^;eoJc^S6lc|dEClTm4{l(TLi)WVl3GONwh*tpaU#8}Dt)(YMpMsuDk zUQ|S?s~wJx-)Ohk3C=}o_yDIeZ>FFt zf(3LhbsimBy6Vi$OH$ny&Zeqs7RDJH(zz-=W5}sdw`6md>L%PsbYKTz)2^ylv7kB#=!`hBrCARx`C%A9Pi#$i$Lou&2#(FJp()qje^=^(%16 zE9361cfx%vZ4-#Rkc^8F%Oj(wXIK~Y|hWEC`D}Avn zpDtavQ|Vh8UYPGOkfu@~T>JY|zhgNo>df1Tm91j8PKw*u`n6w!NogA-a(SRhYjZjn zPC8dlX`(YP+@v?A=1s1i*ApTEjmH()==vtxfXuks3H>QEG-dsp%)BZ-;^9~5E3(pj zHX2REbDFf$LrIkMofmSBGQ`)UXr39|z=gm9rUl#Ns=&`3Mc3Hl*$5vt_8g$~u045tQSK?R5P&*jQiKU=-wZ-&^#!X(`J1ZUy zYxomHYke}pbWFs^Gsk*P&Db^E>2+Hd(@nj@46g(5HQQ?58J73M@k_Ctc82*^J8?>J zW zsH~Z_w6Q|g#jvZilj~Y3F5{AvwmN-p#ZcMXJH*&n*C6})SI-|2z9UOxZFcg=BZQnP zspOneb8^uY8eF=w2jeHkSxi?EdA}&n@qbGBQ{p#?=GJDG9p!w(mcir-+MXTMcau4* z;~_3%yO+d*)64PCs=Q8Nkn0ud+NT;V7)_ZPXP= zzyOu)>BrR8E-p^;J8w>s_fBR{1-*`(^4mh7#xh3S5PrVZ>Dq;%T_$MaAxkMxU#<^d z#)(pmv@?z-@<`(J8<%TzazA*-<|o%RRbhJ@!2k{O9!*4V9T}{YWvV?#?HIEgj(%Qi zhOzM_%4$Z|Pe4Mq^RB2>Pu)2xP?yEf)6^#Ox8=5aRef?rnj9Bc+>W%Tb*;xwvPTu- zeMa3bhy1lE#wxC@r@?O{$kKVKj!t{m9Hiyc^r2*Vua2jV;?{XWYz1+Qb*6aHMaGkK z6^;qQ*F2K3-%Z-c@a=0&^Q`3?aEmD z@?;V{Ysjfbsy>pPXw4q?Y_!{K2r*;yB&*7wC5S+pHJ?_YPy zF}Jrp>(QloYjiWm)jf5Zg{ASVPcJac{I6hQjw{ntKH0*?s`{E@bg7&Ls5)v%IAvCX)T83W8Xd! z_;)t8`_~;CrFuVtd@C)zge?~vK&g$+DwDc8xh($8W;ns9WXc52j6;$+`GVsy{m>rvifM%pW;EYTdz^?;Fj;FF)itm`Um?PhS;04jcU z*#_Rm+Z-MD-f`PJQ}3|NkzKK8L`<8w@~V9~G`Bx-&#A?29Su{nG2Ob8+|tT~n%Qb} zT&>lc(v;;+F-&4@`9^;24VDT!3SdEVGf!~rMmfM9fN%v(6hOZtj;4m+Q7?5^j1Vvl zNeo$#5;M(3(9cD;(VFsBaJb1Py(>xZ_|M(Iq~&oORDKx-F~xdhItKR(w~A;E1`iCp zHf3>-Fy#FOdVHD*k$_d;aYAD}*4M(vaQSQ2th_QvA1^drg}WRzkA*lKV!dY1!()t( z(u<17;cR>=1Z0%YI2G#9=s^N5&wiBdCmzQerTA4wK3eqK4Hu!?NNF<*OVEDL2i@z% zcXs-C+M{g)v5}_epx_m*$4%4J_UTH?sYDunm4ymN7166CICnUL< z;U5ZZ^*<1Js`mY@{Ft{LuFHTc)7i$A9C~h<7S~_A$hFGXSsign{3VYP&HaOJ-ekUW zmBDQK*9Gx2#8;YKh_-?swCWO!lZ=EFn~La-<55FNA3pq0zMAvIUS8jogVSr?vjc`lX+SwJk-Zj4Xr83d7Lw03ToHT>k)s{4cC&?H;6& zwX`@MUj#Aqf5VF8sV94**QaPJGxSfxelE0I2>kiN$ipOV;ColZn&-t$OT(H(vt5~% zXh-ma^J9a7>r>AsqC;*Hcb16z^54XeYYUM1L?jM+pI^$pB=|M^RcTs=sScZcbZuFP z2-AF_nD*wXJlFS;jH%Ij(~@V|^Sn^pl5oaGBzsp8apP+#;SxtE-bYYsoRZulPEye9 zbxVsxktR^4Ra_SDTt&X1nuKCcGq1f*H#Vr+G`Vzay{cLlM&q7jbBgHY5ED&tvuBDIidi!sZvINDwpCF+jyTd$CgMJXf^3k z(mb*}996mb;78$oU86&p4^A^&riXpyuEx&gIHfv8(XP*={2#6n5L=Ew9nE?7hxLN+ zY~6yGKaF|yDIUE#>f4^(Cx{FYys^6==)C%RR|R9@19=)GfZ;j{wRW4g_w1&c`Iyk zwQ|$Qc4xg*X&z1h;8de09;8=NZqGD%W9;tiIp`}yn;P;lLa17jTbzqol?=pm6}@w2KxNxoDu$fLB{Vs1m9r&@4brJi z7tUdMIO;22OC7?WIMl9VtbZ#3--^)tDcyjya|O$u4AV{bh%DnPnAOCOk`fCYATx^C=oTkLo1)WoyVzl$6J zU937dOKkTvx@;zy&0A?8x840Kv$E3%A1xrdGHkSA$ID%O79nskK+KD694Q=EOC6|M zdXDC7dSaIQzSX~NYx8n5RFG^|x6@S@?$*-6Jtz`=$`RRxY+|(*9G;X|1hci+?8((U=NQU^hxTtAu!Iitg@zH*OxTAIhm^eb4+)w<`>rPEtk+%O2xCGSmS(4XJ?{B_Ih+^w^;e*VV%C`-n@(Ug7Nj& zg>(kCvzuFsUxG<%ZUW|-lKJ;1D=g`Wl2 zUy9w~d9QBR`IAiSN~t|^eQPPFeI3raO?6*+x$tx0WtF#xzqO>6N#|zBc@9l|PvD=0 z`ZtC2sOP%5jLfT%8E$j*tlT9n(bB3@i?U{xn`a!bBF+_K#|698diJevC7<2ctIGDQ zrS5K8e9p-pZ{p7qK7-@p18lZyW*O{oL9ZqF$>UPGkcney3IoXRp4GuA%1-b-YBb}? zX_;RdJ}+9?+T1iDaf~cTd3MKK`d7@~6E&NMR+3282zMQz9J3nhqla}Rc13JOIuWvG z9pc{=&8SEVkDb0%MfLTrtKkp9i|>dxgIK=*07KDZVqW4-2F2xw{$HJY3Y=*zcRqTi zCRmADI~iUI@Feg?nvLX449Dci_3!krx%?~e#vd7fV&2`@{hLX*hjPYa`HUm(5-unqk6xN%ZKf=F-W^V`TRyT+(%#Q`cjCf-b?qY-e>@9r}uV~R}w(E3NZG^U) z9^J4QGZ0F!9*8hLxU6bybW*6UntGhRhvA!TT5?u6HzIs zG?2?5fGnrAg-!0QawCaJ!j&8X+<+G(u|C-9MPX)ZDyhCy?0EkG#9xP2ULn(O!AG~Z zNYsnDF|gyQ&!ERq*1dyF((Qaz;XPeqkId5T1Ho!=7ws&+D=_p^Au2tWHA+2<{j$BJ zd=27Xg0E)ZXo0_R8kcs#9OwT4)z7_sed51`aqC)L)ue=``Bm;bqlWny7dyYYT$Sn6 zA4<~o640OAN=umVkA}Yxq+2+2eNJg2op8NFUCQNY&BqYVj2?`o-NJNa*^LS!nS)${Xe5n)#c-{{R)WeN)42 zr(PkB_7%dmPFZo^-m;Y!PvJUf<6~9cN3r<3$BS7lic)uwL55bSK9mBN*K+OJ@~IEk6e}R6$F_qbNE(@lQDRxJr7RtCyUY@CMGSC z4@&Z%80v;Qbya+@_p6N;CUexp#t7f>M~fQv#uSaVHwxKYHJDtG%p+{3&M-9sE zaL*c~th6m^y2ZuR1{7>NHU?`>!$n;}1d2iBNk7RPjy_%3$mgbb+mBhZ@8ZKCPgOH<8Z(x!qb11@dA z$x+kmUZ14+MoXKG*HN&ullM}PKh(*d57Q$R9L2R`=N(f|eILY60Y+WtBO@Go*U%px zdUGVxt1iKl2W%S6ZSDVI^n@cP8 z6sg)S$2D(xd&i93Lp80$kW7VHPAk3DG}v`(RhlHt!ZsXNd|lD!X&s)2J>gqcHv<6k z-n}2dx+T2ZR&)x>$TgFCnn%d%bUit5w7B8h^Dj8Aqr&<@hf-DpF`g)QdJ@_?e+1}~ zLW6SRyQ^3uwzUkS9CK8oWO7uQo2fyz+=6k%Wq5;GRgHi>DkRb8RGMhu{8Gj}>%Swl zV|as76Hff`Tr!tKic)>R{Py(Z}Q_C)d^|2yG>l;5& z(yuD2+eSOLCZ;#NnlqEu#TnyoAy*jVCap|&XY#JCk*=08e$R#np&ZruAZ^_$RFIUp zj0x=W**x=G4QgaO4Ab_y*&0Qh)yAF~NypN<4NAyFv5azh*1XQwFGG=SKWlJ1R_j}g z9w~cHhH_}-E-X5nb*_#}5Wv-{o?}w#jxOs@+Mw<0T|TK`s*x)F?xwn9EiTJbXAIV5 ziQ^T~{fB7eRdbqi9Kh55WfV2PUY%_o>S`9p;3n@6ufR7t;D}5sw7)@cJgY%j+6z3RTU(0?lcC(w(*U_ zin9uYT-7MM0}#bh{o_JiSoa?aZBl9Kr2Q(DfSwiIj0&*?XK3P!0QmD$q<++_I|;OH zCZ=e5QD7<@VAH3>YHDSm(rF*BS2_XWxPX_po8KFTF2I8$~Ezd-nHE*R+`$3Ta9q4Q} zF*U7F#Lo;1(~Q-t{W(!uQa$UIdYi!WjaN;Y7Lw*tS)@56)}M?n)=ds_TggB&Sakxq zrA_X2MoB$Skp3k6Ki2$p<3_r;lHSKpi0oU7gMh1D=9OhPjJ#C}-1*OA9HZr9+diK4 zoNp&(sk3S-_C15)Pl2TH?}4oC^vHzmFZ{e?0l(V%S9*_k_86@+;pD7wgp4miTf0SB z8AkGVk;cj4jY2(W+}%jhMIHc%F~L6d+3B`-UKxhwU0P+8KqXy|1lKpS=67dLwsBmj z@=uFiEQ`a?fic)wc?r(nYVe=hGse>Ddd1pA%N%OwDh+uRqf(__SnPY)XvQ$+XOR3< z@wAsWk~|WZe4UG)^~uTbr^OyEllw~a$KlTp?gRTvRDUU0{{XPRdCBwwy$U#PRjGx1 zVo#L}@ZW&0zAX5<8+K-gNwH5l>f;FAgm%+nZGTbJq!&va z%moeOa3V$>RwX%N4<@^!y_&RH;Z>;`^G-;m;eUqKYu(#7b%WrtY?DjA6mV|S; zll!h<#{U2lCQ?!NR;OR$IjuY& zqlUGKX19bXmTU|=anSk>-%9hJj2j)wH+{icTIu{EX6hi;HuE82K&#l!hbK=X_$M;*^Uoq0cl zbpv(c%jhP)xBD&i#q?svIky&+MhJ8C&dl`psO5ICqO^VF&sEbkCb`ml8R5+)B9_{u z3~nyu#w{&iQINUn<<2{d=e{b2jrN;eW5Qo#m8L;sol!y-4Jk}$U#BO8RPyybWOQCK}l(ZiS$)MWBeSR6P zQVA|+5^kxF< zjc;^ScX_za(zma?8#bk6Z{v(bbsG)p4<1__41GBKs<~n>Yf!~@RcIkZhm&g$+jw%f~WCVN%JcCkBf zN$bXcPW9=K_%jC%4{KA6l{S!s64#-VByYgf|~% zx!n(tcX=%T0K1Vga9LD^>&^f)IHoRV9U0DDcs}369}Q>R4Zu;lFw5BW{6%_ChoM_F zhAVi~>6aU#b^hyf$LEocdT~i;5|etLp8hQG9;2b_Gr{B$AU`SrjIU8&S$t{md>V(1 zBDa-Ata4i`+uSJX%bnT4?zf!t%G zu5(GAT3(cFxCBzf{@5vV7a_RsOH>k1mqG1`F^(zGMip>5tgMXb#7k>7l0rb@wJhTD?Ix3<#swuCYhz8_ zk%Op2WRpFq*DTACoYPZ#3RjCUq)@CzDQ>=1+S|kk334mjD3V~LaaL9_3dxSOTH8`wnG(ht zVnE=Up4A4~-1JDvKb1}FWhQ;CA>ktF0_ng?AreQ^dOvdYWsviLKQ4si7N3y=W^FDudRZtH%@%ri8Zf?k23-##SB2 zt5ZmqVcRgGra%o?A+g6+BTVBIy6RdT96jpd#qxpp)c~-mKEkZS2H{a~LPE7_D{?sk zmCLak^gXI;i1tKD;-{Gx%5vKFq4s#+oOP~e$C{9W-d=rcno~E8Jr6PXh2o)Xu0!F(zpYxaC)j#J85MH8R#%2~ z#TIJPbH!-McDcW=U|EoZk|~p4CDM!xip|F5#k0&jLE-C-VgU=eNVCZGtzQ-1T_VVq zq$`d`dghmAbVp50{{RU14#P{kOu{6X^SiM3sXQfht2NRrpf*NX@sGllIJKZkea}LI z+^J|Sz!GiYh~QNB3vk*rgfEc9W6d3FM-o(leorKj4+MvtdS#9%4ls&pA6(}B;WeI>4V64v54VVQ-NjbfW4f-VUFbNSa!yPj`4 z>7nM(>fSQ(z4e{en`>$I>-(W^8r>@!drMSYk^tw7k8(+_E91AtO-9SbdLM=~IR%VD zG&g=pz*KCNh>_53`H$A2CwsPPq^$Hki{lrJU%_?|X#(aMr6D5oA;Kb(GQ@NB@7lf_ z_{Z^id}U*7t#a|qy3}XOR?8UGLvA1J^#lFdzj^4kGLDS*Z8kXkMW}dEUlIu}uCJw* zHZ6~naal}bJe(>4$4ob<74mnDJ~?XsANY}Sx;4l4rKXxb&0}z6I}|f3%In8gY=U}l zI|^RStw+tF^%svmIa%v^KAWpaBzCrllT8dt<+jF*L}o@k9DqJi{9e`ZKZ?F2-0C_l z!dk%%l1(E?J&P{ojhK?p7Jr12PCF8NS43i()P;Dvqttv8@hbAy###s2Rw%$8XnD5? z;{*~3!ReJJ{jLW}@SPvWR@RnA8DhAWe1eY|+Es@}&$b8UPuZiR+FwKTN5g*>ir%iJ zZxY@?bpqj96^TfL@2{E#a4=+F6BaC#c z`CaZ-jai>je$gH{xbato{5rPRMQt@Gr;>ZFf-JHZWACt#2m4jwDe)6Q@dlDK8_hyR z)0Xq?mdIvblOz2@H}`ss{S6UT+)fwR>h!y&@cy{E;sYF7m6x2-$h%B-!~}^?p=_R< zJR(`*Ds>@q7V2&6HT5Vn&zE#43_r{Gv?bVY{pN1TabT-vbkKXXwj?V-v`_1 z^GS6WD3>uSLwOG(Jdc?`{tS+u$BM=Awc2VHwvTFF-qdbbwz&-go=h$Mf(;^)BnylAqO({525>^#JI{oQvrE197WHI+& z+YGcw$1YgqzNfuvp>rNC<@=Uju+^m`V%kP@`!{OhKBJlmv zY7@!k-P?ZgV$a@UbA#=g!dHtmPU!oK#r_?!*R(ifmNmOENV0Fo%(y%sYV*$x{9Ij1 z#9ly3K_Uf~2I{Ow@}<(`scT}l#Def?vhEx0Bb#b754tmo)A)sP75Bs0q>kLo#xips zQ&87bYZ-J$l=v?4Xzni~mObdQZd2)0{u~< zVcbRzIQOlG)6uN+p4GjyIOTJS)HeA}8nS#*JeE^P6y<#@o=L8!B#tM2BU6jeoOaVc`j z)-6eJ%Lzq1RZ{-|FRd3h({i&fnIPT5(oLREtr{&%TU}Y3sbX0P909q+EenJ9iLK(UDK_1cDO^Z-BaE6Rg{F~I40BrYvsnA6rqddZ=>&2z zj&a_L6gq2-+*L}g5;OON+GnI;%Di!(Dn(x;VfN3(6*`>xJDhONaG-UWRG zne;`Qc>KlybQLwcWn(6kV{o{!uIVbpyg6!i4NEb^NFLR0G~iVL)Qs_0B7nDQkq);& zO=#KJvF%RdLIwFi#cJE><-dli;^nzU8-CT$+i6(jid@EuCQLSGBZ}$a)57zLo?^L7 zh-}%gF`JK56{RC8`&MpDwF?zi5gPQSU0y1w=Ap>ZMH#x)z*~05 zrDAxCQ$@Fbo3&=-X&9b6@i$iT?dIp6Yl8TtuYA*>Z1QWGOxYvFyi0s;ZyzVQtbI~p zzDF(15f`b89-%keSj0%$*y5dM8pf>r6Pno=^*)0572+F7XxifK{{VR*`Bt;Pf8q^EUnj|r?vB+CB`4VJd?BE;Y;P_b z<$t<>bQOC_^DJ!0$s~`iXh|2$`itUCSsvb7IGc2gyH!`UI>%2D8=ILxGcx?3_N?W1 zp>b;WI9)r!5a=@t(Sq@YCA$)ReT8UzLh%Kro1)EcF~-#9G&pHkkGu4)e`gr#j*fXf zIvKhSu`h|FpH!Abx6|R6#;T)-z#I?8xg7^r{>Sh`YFF$n?brQOKXps7B;(Yaf%s7D zmD!V)wu!altx`>MO?Yi4@?w-Yc2m$~j8_rjFA!TrZEFYmi>Zps<*4eJ$sWL(#y6<- zCuV%<@yEqlH-q(!b~77&sggB&ry#fm2KeI#p(};YUzqk4;Xf9n)ckqzOIS&*CY@ur zVycgXRsQ=oKu_t?w7NPY3#+q=)-_4|VXrm(R_kd9AS5Vru>ims?_H&z!@WZK&7-gu zu|~grqmZ{gnW^@PXmv)F)`eRi2)3bOc3Ue>Sx1@Q747~M@T}e*@bp)=OA~|(yeZG8 zKZSG3rMjIFqMO+Bo3DYw>dxL)Of!sfI@i-0PM>dar^7pevm2L+!|M`S8!8I=qvi|G zfH#_znt4`mtEzpxx>dFWG<5tw_L;h5+@hnAfS;)nVy9YYWky|csPBQe2F%ntlF{Q5Gx1 zRz1Kz)n3_KRC4`I-$@@nM|-QIB##oY+l~R|y|Ym83t5(gz}vg-Tz(XLW!O@|$*s>2 zn)_Cm6)yTfycWON_Yuzr*{8OJZjur?Xvyo2*&NdL z&ifg>GVXlxJTEj^-5X$a;=PAhg%<7IApsZ+s2p(zyQs3B?>5I?Z(v!v64kp4FYXRgk0Iv+Ey*I`lfG zn)VlaXSR|suOD?|{{YsnCHP;jNu%iT-CVZIo2#A3r>-;8@~DbYT9vGxhk2}gAh^(V z>1Wf68(Eujl^rYJ^jQ+~Nw=0n5=P6AeK?`TZf(saqdcF&b~9MYV!Beqp0((@w};>t z0`4Uscg1BGxVxcA+^=on)syE#gH~eHBZe=P@K>mRelC}`c^Gn z*%;kJR&44jwz^G}V}^XlYdc%iKGl4|PwwOS6nhHj>B@x;X}2au&UTC*Dght}#QIiJ ziKMpJS>)Y<4>eJy?T#vK(7{6Gd#2s>qD9@7?@`OxNpxi?7v$)3S7TA-+)C#O67p+k z$*l)C#wz_x#*@o(-Ec8gnr+eGdgi(m00q^ zRUN8%icunuPgTbiIl4v9YFF+%4_H-eHtSSXByOXb7NTl1A`W=2QvU#7SmPshG4D#_ zW31FXK#P;QypLD$X3fKY>r0rUxYWE%iiGd46xKXn z@*kgy$;hORuf-lL0vRy%<9ervjngpS$JVoxHAf%u6I?Akgq-A>;(TYReWv;F+$pG0 zxIZJB)h=1219#r-S=WryYRMY%ss(ElVJ*$5)UCSYR*s(vw0V$lDuBSfzM9@e5fU8n zTQ~Yj$uiHk02-*UmdC3674gQUqo`T4fyiUrQ~nv~ce<=l#<2kgMq7?WYM(1PC1dJ; zhyFOZ@pS6|Z;)}tY5XhjEw_hOIig}&xgxYzkqS#=*YsT}u(2rH>P2p8Iat<0Li$o> zw>53lT3dj}b9T)`4ab=wcKL>Q0+Au*&2X(NWFY9Hy=q9oY@LdQ{MmSe68`r#*<5~KAmAC zkxH(antqL(S$} z3+Q9=rJC~MM>0LEAnhmB-}sNy^RHvmeh|T?>Az^Vxmy@9lKxA{Dg1x>_0Oz&o5C#T z{2QiU$qZ55Si`4UG5KLt<-O1U0A9Pl3wRe+)vi+N*iSW_L@Ld2%Nm?|bNs3r>UuQ& z-5JnpdR$MT!>21Q9BMlid9PtzB}?=S$G%oh7ziS&+eRCzubHyM|y#PAfZH z(KMN7!nSsH*789;x=9VL7Fo&pcXSHAywp~jOZaEQHrDgrE!>I?&BG1@NCyMx2d{Ey zQMREbrJ^qQuJZ3x)U5Qq3N&k*wpiq8Iana?fIFY%SvLMAk5F4n&ks6BbRW#Pj!s$l z!CZQE{3@k)zQT_*ZkOVA8vc(RosOTT=`8MHn`H5*%SnX!N7S(&n6A&lIvt(8%-W!4 zGU{KwSma&8-eA18Pr2ZGdkWjx^KG$@w{-6JW>%bZ%@;|E^6ixt&h9ZQWMjeloC*Ydun3F<`dw4b8A=?4RYq|#UBjr>~#4f(xX`2W4p7r7dF#^ z$~Sp?gJ+IacQC==;}xg6)s9%oq#M-mzw9Wkyiwunt!+)a>Kc3bR>gLR?`OtbI&BzAB#j(XeC;9weC{z-j!jh_+HywdqHJkHly}iDC8eIrq<;9@CnCGm2Ue~ zUpvFz7lhg)X}Wius-&wdrG8>T=N&&$RHG6dS;?F>hi`GMHm%~TxZ4t2$q$q0gN0wf z41H_XJVB*J;Qs)J)A(}c7~5QX2%2<2GyBC?W>7lt#1rZbN?g~lbMo6$=g$^sqwP>$ zSrFFu3G;!-RuugP*8a_#OZz;A(q(C2kd6xHk@?nYHXD+Y)aSI>Q%Tf}cQLa;FH?>y z(X6~bX{zY#)9r#xlsW7{#ad=^jrKUm{=uwWPoz%1U9+|#xF30Q(zh&pA1J(xtr^cYp4twebh%PXpg0mLSQ~-3jz> zZ+g*rv!6qN)1bJUO-L@FXrzV2kur~Z_WuA3_*EM0Y1)#iW9GM=M-A+K>GH6=dT4iF z4t!Z1zK?5fb~$7Kt5+>?qsk1t+Fo8gzDR|-X1I$s zoXry>9r)zcqc54c;$IENrAaK-^B0Y|+D|o|d#f>p#pI)C9Wzz(DMhoEiuruva&el= z)OD*dazsq36y((2+li)n4v#v>tAkuchl`<&)nk8Zu6D)hdlsDv%)EemRtJTAM3WT- zvQJ7m)Q8HqI(v-?CPVUmmD6dh6~tpW>00vL6J&XQt>ED%;pOMvyMmD1CbCo|Y0+%KAY{F5RdmMZjnm{qs)%#67pqfTu#9*3^MRFx6 zC9#`xA`8Ze2r9g0p^gI2BOa`Ht8t35>R|@ftd~{Io`S29zUoZlzR^{DQ0MzWAdN<0ZCnpZf@TcE(zJ|ONfo3<3OUAU zx6{AA1$%+P{blDnM#39H(h{;)IWl{MU|M_=vope<$lh9jtl<_lkz& z^ZvEuSDqkZ*(`Bc+V73@<$s%u*9m#3l~y^YaoDl0c!}AF4S`%ntEjY+6$XSW*ywF_2DpxQ zEO4t-EEKMmLjCCtWi88%&QDlhGG=lzcV@5ZmgkTM)~}%bPC2)&=Jo1LX^TK59CxS9 zp`R|dIxPy-u8|iR9qX;|HjXAmY-X^OjpG}Zblo1%O$2v=Rw71sbfNvqI-9R?)*8U`4L#j6Ux}*Z~z+kJ5=#jucY7R-u`H|wniGg8g{YgREhex;XQg7 z<5B0M9=k(yug*V*UmvxtCsK@EUA)M6!DGd3fA|ye zqfYTwp&hid?zuSmdYVl>!&NSQU*SJGK#IV-wkN>MX`pa9|gUZ>) zqiV0yzvKBsd;jwnp%%_t@jFue>*O_Zl=l zUHDXQG2_n2KpdQonD(rRuQY#zHg`T8wj)~cO~tvmotX!d3giV$XC(8TrzH2RT{rf$ zI=!x|lV3B=HGL^GReQk;+vx!AZBFruERES?Q_Fq>y?PgiejV!GENJx&Y6p@{>dkX< z1d57J-HAG9j@%EzqBJ?SM*jd=anelYJQv~n?}%Ci%OrOeP)l+MX(Gc2faHwy+KjBZ<*$37+oz{n*$MD9}O}JZYSbU}`zFI~z zApqw&0=-)IQ~tq*)+y~8=5H%%svICy02Y!%SxTCTtvb_|%HupIrQT`!TBObWCsZiFpDevObvlpRH1yX{`wMH7h&GScl?Ppw?H`siND9pDIH< z?->K<9T;)j*RM6s-D;Yf{_e?32>}j(atHWUQmuWC%2-t&i&HD#%%-@ZGJvHwAAb zGq_`^S}y|ZeGGoBWaYh&O!$|s9~{G_#G0GAxJ#^ z(_BA@t|V<%_et>`v^TcWOXb6VZofP%rva76BoEAXHKM6~By)Qg70ncIv-lqS!@A@# zYVpHk_K}z|IPxCg5*1YA<=wlVPC8eA;ZGZBzAyVvhx}V<3>N!DR{E)x=T&t$SCf8D zPCowtQPBE|O>B(hr4{cZ$ovMkl6*+hMz<{5cBN|K-BL6q02wk z-qDr_qS#k3APRti*C1r)*0#I{XC!*&*=K03V*oHX8<)_j>M=?ZY1qg3(V`CA#TS($uVn;;8ZS}`W)ybSL322Wm(^>V8 z3rTzSV-)PW-Z;-2x*GKF1?bvk^0lSAC9JZXh-P5RAwS(UFKS-SKW7Hq;t$~Pj0Gh)sdBhvAOSuxc0&2nBQxw;xap3&QK zuRCe%2<3c1EzRbkG)3`)fB~)t#9l7E)9wYqUz@4TOMYhYG;h2MuC>!jn@l8U@AF)x z=f>?o$F)=p40WWrN9Ii>#QHt>f<_`rMR|se;^88xFO&7E=0VEZJwD>eCdnAA9XG@aFaWhJ9>TD< z-XBRb^47#xX)k=y%8AhPtz*J#^L(9stI)0>j%7Slx@5Q`=YJ4*C35G>>+N2{tY~W{ zaG;9kl?AzzqZWLju6Qk>X?80O)$bbDgv5$Y+Q-=0-o_HcrjL=ePY+9JCR7zG?OxsD z9|ai;5g$)#%B3snb38IyA2iuA6noWTo04GrF3e2 zJ61aRHJ#h=-qprg>RxkTo~EYEM^beIBaynXBb<4JgHx$iH=?-v#j7$IH{|A>$~5M> z!gZ{>x7i+Z+KYzf7l}0@io-arcf~hnYjc`c8A3bunsy_RRrF#QA6i$k6|O_6O!4Kp zz^8bFY&PfIW~Sp&vM^g%1I=vx&});_uRNPsTyd7$PT}e{O;PSH-zOrnbmC=l$El|D z5-w^IZiJeHT5l#CcceCRH&Tt~C!Ez*Q#H_B_pJ!f?DV-e+o0^CjSk(KKAcxOvwEAy zVhu_lu@jC>TGioT$e?6qqV^fi+u4VaaypvbTTmtkZvvZ>1!cMDUIWwCJ8(y(Y4|QM zw(0=q6^xr(n?)m?_?-{>89jw)d_=eNwD~~vG~$9!OCKWmw;yj{IOd<@SWH*1>yuuc zHhGgTd@0jyb#E5i+ugR`7ftrrGN?KR?~3F! z&xqDH7beQeVE$Li+j?YntmPJT#&JEO-&9+kQHKW@u4BWxgnI;!jJI0F=xCkJlHTF2 z8fHZHENedNRAsl4I7Y>8zd~_Q%--gvFNu6bX{D-yN|%rp+vqC?#FnCK%Ue`YJo3r& zvFBpvnwfb8&c8vCTTg3{fFm8jyBeje%$i-kmu`WYCUrru9&0zIgHc#SW2G)kw+@s7+NG*;(99EXLgxuzQ4e<8fJy%iJH8fkca#G0L zqO*vH-jR+nOCMgHtEKTatTg*}iZQX~ks~L8wlI2sI#x?oTW&@-i}mATww9Kk>#ig* zM-*fNj=*itxd%Sn3g>GsLwvRsT6Ck2nL6fe^iGJMNI;`wzuEk8~= zM~C9JSyiu^HHLeVIcDm|-vpYWsa#2M<2wy9DRFML#&X++8J)4uh&IGr*%kqrqHZbqcKTWysT=Tu` zdK4tNd7kH#c%xsQZCW^hj@mf`lK>!)7rPPP+ogHp&nBhf?M%-jHNTpW`Dl4vz;Jo# zg5Lg>=QYgik8<&hQ)j1qIq?H(8sO8_Lz$OoL_a(C#yvXME#ZFb$42YzOSvwittJxR(2}Dui$-tmE^w_H8^}d z;_X{dhW0B>Kkd>e*O0rt4F)Y4X zx;A{mG0(R=d)Jz29wU!i@phLp+$<1z(p@kwjv?DCDf+8^z41gLIXkkaUKG>R?}BNK zu0*$zs!0jh){=arZ6ILhr(Av&pY{`~E&JcBZQ>c^5+ljX@-au1kmIoc?~i(Jxvj;k_!Gh}r^wg8YSP)^ zm?^}JlDfJMI<7e#Mrzz;6rXZoUNdcWJlk53cyGklnqaq@HI=t{WpK#+oDg{*{;IjF zYgYb0@cpgb;THZP)7!`^kDh1`2+n&t9y#Qx8R<`#sx?(qz1b~I?}mSDxzw1!ADar` z%>zHo197-}XZhBbgY0g!AMA}_BWpL2trK5B5TYRC$CC^ z)>93t6YIriYYr!6WvjGGGTE+n#%8*hAUO4=yn)GeJr7IPouos$xvf6qX&7wLaV6CD ztp{B#v*+fx?+)G^gPKL2?WAkpDtz463#IBOa*WoA+hE-GJ3kR^$Io00dV!ALac)ma zzIJBfdX>M47EBG*<#9M_`VTE#l!8Kmi= zeVRUL@dtqf+s~KN+P%+I&|{D`6R)*k=%!}rqvNj-cs5u|{D&3v4~Tpsk~9AA^r&>Z z9MYnEb*XrESlxp!HSXRd@Hop6AL&u-+05z6J0CX5rkGkxsp7qBQ1CzQ5AObAYe+?y zNx1HN9mT*Z6dqXC6}|CdU5;z3L+H0Utv+KLWOPwkdVSe=&UoBv8!YN=X5G(Pp=&W6 zD?65Swvut0tzjYA$flLV>`kgow%qg0Rn-Y-Tj@YV^N$kVvdBkzq2jB78#5lpn1$0X z{H`(wt#V7QEx2R5nuZA3@jkR^bEwiI`9Z~6(Bu8cH4e?_TGYWfH;9M;)f427MinCC-#Dr4VKUw? z8OfueD|65^3leP8jw_(hZ2th!VNJcOot;tG$-2`FvM(J?cJXMUNf+g;mCMlL{{XSd zvH-pDUV=0q-X<5lSrXLH@NR;B%BL9OyKfHZ{{UyR8C+wNQ6+Mghn)OO($?2bn32v6 zTKLTW0O&*x1`RlckBEGE^4`c#W~KOnhP;>SUG$C`o16at3AL>jO-eZRA1NJiwWP)? z$}QVupN)3KSmLOzkI)Ya__M=49aNUnQ;kb~^xi+Ee1YJ#d3jbN@vladBP4m1J;r{H z{44Rkfj*+P_C+T18+JS7^*qo$>*q*7+n$yA*Jt}gYML&w1?Bv&da<9GSobx>Ix>#Op$h4rrfuUWzRqRazXdk| z+mT-wd=>qs{{Z1M@h|oR1rzE_QWGKhB0aKun&kF#>~*TK?tQ7FOeUJ?-{-huLO1&- z{{YuqXNUD|LO4uCpnXaMheVSkE_$Csj@2BcMoFHRB#){_UfKRzG-NB^fN)3ST-D0=4e*VGULWucnnJG>t72{A2dW0kx%!?d_cKd7Sw>hab3Bo%uc2V8{Hk0^=IU?w zYCB&uOVo<1&kRp*zd0Qk?X$ zhFq(b0I1G-5=Y^) zJ?pR3JY{9^Dm&Y|q?S3XE!R%_Exi8#=*uLJm@V=!WRX=e?5Akjz-OrB_BMJQvBgVi zo?)TRJQb~K*Q)n7cCmi$HS=0V+V~0rfzN(G>59wo#r^k!bPa1#yNzMhFE7^0-YDEh z6bh02;~B`>jG&&MyglnU(cb4=u}aA!%>E|pQ|lL2+N7`M*t7W(V2~DI0WoqrDF-<2 zdxKb>A8RdpPuBF^UJIDSx0bM3$!wXFO6AoXVIW|FM>soMuS(W9c*R`8Sh+WDhR%nh zUTL~+k*&)-)^f`Cccbj`D0tym3~K5R0gFhg2|GYSPfS(6jBq}s<6A!rTM=ayfP&fW zqYO9OuiY+vUGn=+0bMXMPI_Yn6N*(GOSW$6nx?+qAJhC(d2+WNA%{>|;kc1LSz}mK z?+7>wyPE?8JC7CC_*X)?@pa9=hOOQfc;`0j3{o>Baxudw-PdRbl6yA*VxKFMxVe;9 zk=pp%#MZI=Jn+V+Wn@3G^)m9aMHb0nnt2J7habbaTX^b7QB-_s;rJx6Yb`Hx>AF-h z>9&DT#+MQ_#!2pqlff(o(T<7_@lva-QWhRulX5)c#nNe-M~XBn>oJ>MN>jdUe2X9$ zP{Xgzci{B-dR5;J_%?qOXge<0BS$0)ERKw;Zp()+>y4~&@5g%NRJ1)AVy!LoJsV8$ zhV5~M#_4pOV$?x8Gn3`Ao$$zmo&oa|{v*A1UK6;FQq$~h%CuLGW!kKFu!TnJ6 zw1~r*L2gmY{m~t)m?;b2iUF*RPCoM{>DmOlLLD>Kc(~(aFlF z!wR0GAN{t2RF&i(DIa)tuQt`ZL2%ID$!evxXA9o0*JKIfKM~&PnyXz{#AgZ^m0wEn zo3GkZ+ruk!Z>edMNp^Fy0|V%DTIww~ML!mNSk-kcPzWw0wdx#3ab8UiiJ!z;#l_)_ zMdwIgYV;X7tS6?%@%Oho**slw3dr)aOju=^&P{W=hJMUQ>59bO$s@DyU&NgwO1PG5 zn=od7;@gw?*O9HymvM@%IvF*6&gaKoDUZb3r=JbF!4T*Lavx|dlh9_6(XBPcBSZ%~ zMP#kPX7;8lv~}JbkS(pn!*R52TEe&08Er+n{HvVijYpx>Sly2>4Dc%k&g`oWYI=g) z^xY2gV*|Rmtt(JJC->pc`T+{qeR|9`PTJpO+e5`;L=ASVe zp7*5qoB|H~*OTc!BHfT1pRH769H-E>9wJGClGWt8cZeToJHJ}KVr+YV+v2#4@-tpt zYvN~KH&CQ29p&Y;R~f*`u71nJYRUs1D_f#Ddywj09g5?2SFg2et@KkaNfhj7D5tsR zy0?Tvvu#d3mEK-PlV=>!w#Ht@GU$BWZ+uob!ER3^S33@@jdAz0is!M|dPbLc3WJ)$ z(rycFApR9P?8c8o)9wVF1!n0tL}Hb!X>+x+U_cc{&U3dU)4j<>%~D{nFz<|3KDT-! zjk)bXN0od;y!$9}4RZeg67@~ZAl*(RzULQtu9e;MS0QH7E+?^Q3%U{Xrf4$lw;xKzRyT=9#GfujN8$$a zr2{?bp-V!Jg9&TZIO8?F;XOaiojB(-Ns1`+&kR|XiAfv*T?U6_Ahl86u{CCmj-F&- zaa{(R2#r-wCZu;6X{7F9p0(8J_A2{Zy*mKNZG4Md5sK5lg`D02$sg|s$9nIJIHeP= z@SdV<=29DvO2pFTF-W9wU9{(8ilh{@J&r4}eF!~HE1l4_FO&j%bgyQd)Rmdy)TH9{ zDeG4cV?3Lq6C?4px*D_M7{1g~1CT2!k?ata_bq%U@jjj59}(N?+JsUlf=%cnW80B~ zTxIkxZ_^+O;*=UkqaS&nWO!Hhh_vwrsj6G}d`+fY+$n2zX2~u`%s#oze0|}+5;lc$ zE+S@Igka0J+OUdI-5W|TPE+k~gZ}^%JVD|g9HoZ0b+*@EKQvaQ)sriL;`@}O?~2_LOvI&g03n!<9mnm6~;v zne1!i--JIKE<8aJ+v`^``6sdCa2~bIIL0eRZwD*teTCrr_u7&~=l5IKhd3aSir}XB zg`-2^_@l793j}*ilaIcna;T%Z&2#9H)U`dqZ#N3A;mU)OsNXU*p**~wfd3~RYFD?Q@CBn%x zW9L}cX<^u$bu_PFv7zFh5Pxj=g7u|O-@zG6Xz_-00FA(B{N}k&Gjzt`B~_Lifblc7 z9L7%Kj6HdAk8xPNUZ#u4^S_3ARgJ-o+S=~bMf%D1S)EvUi=IXR$R4MjK<`_cmHn2R z<2SalgAg>E%dz|6xxn<~^Y2r$Ez0QS{yKPlJW1kk#1o}h0UwnS3Kx48l19%Yv-R{L zM?qS?CX#!L%~s!P+-^mhB#;(Uw>b*KocG77(|5TlRy?1<-V49J@$HVT(AvXmsa{I< z_V*K?GI{o^Mm8_ah{(w&pa;1Ww_4P?exV%xXW5+-Prx)%WE9de2uekPyqELIVZ2pipj2Bisny7dM>f2>HZzl6HK!6 zg_JPFB$40un^cTOqo0|Wvyse6wT1qXX@`)}XP_7t_u#@vKKCDUi_pM`O ztYYtbqtd(=YZjTOL8&+}q){!rF}KP}ZNYQx)ce;5rue#TY44Xr_2!nxqe`o>VCD9lfI?K zNh8rbO|LiG6XeXf8-{-BpGxtKbH*&!f_t0RkgQ6D*8y1inq0iv51k(Gqw4KpZefMD zg4^&-eBY-0P`=Uco9%as(d{8uJ-b%E&D(Q6bLeeT;$)D_A(}Q-Y=y^j?_VctUmq@S zAh||?04n{E)Tx!oDy|j*ZWB0ucaF&F5gX>c(vf`9>Jxjt;wZzK7a0%)M6{X>b z;Y}_VUzNcCV!0=Ccy4t1l9aUr)K#1G^EBW{=M~C!bT}y^EpTeg!xNEL6BdTNn?@=* zN9>qrAf)yr)Z<+Jr3cnCtRNO7loJ3xLk~LRbn(prD-=FQ-wTn#b!Zqi-G~^ z#bwmc^lc{SZ57X0UNFGvQrOQyw$ukq*DG(S6>hxJxYmZgxqKGhL!NP5*1M@~iOD<> zLVDP(4;lE0d|u*Sd)6JJ=OGddzkc&2-m?kd!AZn#lY z4(#l_6{i5o*sUK6ff!71YdJEKXQA1zl87@^A)BW?t1|3#`h44*9MugjMKUPQdPC(s z8%1L3qrE1E;So(*4?%oIHa1=%;*^*pz`i0&_K77K=bFy(gx+=Yu6mm0 zwx-t5`P;{ke)auwYfHx|=V}uu7z^@&UDWnDYJ10$UBY8xan`i8m^{1;bgtMmaY0G8T_G8%ZbWQQXGXEhy=@V>Rc>Hap`bEm6It>N9D!D=JFMxqtu~ z$GW`Ca73r&jd}W>wS=Rgv?T9y-F_SVNLzS~z1+-Eh-05oUM-;Aw3kSMwvMFNG^wMq zH9Mba={_;GnUYAS*pYyEILY*{oAe(VOC7Gj+Y3pBT=TSdu11_q{v+$lUmi^;Xx3QC zU_Nc5cjsOi;qMYjwH==&s|i$*ylKa&?_b-CHluAkimh=G%07CW9uKcI=J5E9K2pz? z$9m72MM{)T=Hh15btoG;=?d_ee!_nsfQvAu2bqlqAX>T9k(_NN<*i?J7sJVg4o zsX7a7!R88MQGx#e*5wqGs=N(5<{}rRZi`Yi%@tYsL?k zae`3@I5^4V<2X346pzKXR`-VGq{#rBV~pd9s|g=iXrb;rA$6+i*U(9>U(Fgt9Lx(@ z!oM?0;Cn!@ZhTX&JW`&tI)qE}_iP^+ucHDD?ZQiJreX>OuO~(uP(sa~=o1yglJ|MQeS?{mzp4tYK%OkMDoSuO9 zB-hM-D)`r^_=?^OSuLT87=MB+fKROolzNcl`X2SJd~>qWyj5#pfnu;yK1}t`bMId+ z>$=n$;@jMf%XXt{dSbNpaWZn1k48;m{{X|jFT7~lc{J;Z9X1nzEG$VM(;nivElb6h z3!~}w+r*PXF}ttJoQmDWEsSG!?0P1<;_XL5wM&SSKPisTGm(sP4S5!?;>*2S)Ilp@ z0q83~XRXaT>RHwNK=Z4GnTnEFN4tTOT&rc9HC(9<_a?g3Ww?)HF*xs=s9o$^U(3_ZckbDkHv5MCh370EPj z2dVA%O0sDqXzf)Xx(%fD?Oc4RGp}@U_Ye^?6H9+F7Y4PCrYxm$bHMFU%ONDyq6S9Y zO*S?i4KX5k+bx=GKYJ8|)X*O?iWlVzOrpZ#J)=3O8cogVK#xK21Tnq;0G|A%x|b*D zD?3fQ8aIP(t^@x5>Aa=&3yx}qCiX^FxvPD`>=w+$xL%~Q3L_lGpV==9bq;d4>xsvqXhIHl?)bTUlLsDnFHR1nFzK&1C7Be$NF zB%4)z=@dejERuENmd+2gP6urBO%iWg-u<2^^4~iQ);6hjvEG$a!VRLW+n^?nt}R=E z&o#?Icv(TsDBq~{-1KV=abdH#`M!2dt6T$WUS2YIsc@dFtqaayA$+c|5w2IEz430W9trEZMwG(K9O zL&&NT*7#-1Tn|c*!taTf>sd*Zu6pj2-?&rlOQW*+DoNl9<&oDEYTBu3fu*&be9kI4 zit2Wn4BH49=Bntk4>5DiWhTozn^_E*^u;cg1cQp7A=uHw1-o-~|7!O>5cc@8#zi9dl0Xn;vW8D{Znk#}&|cqfOZ( z@m(;HgyVC}r(KL{)}ao^JYu_I?s7C!w~+4~(@^Zku4{P7re`H}W_P*`;f4s;J!=b3 zzbME%U{|X~yILM(>2^JbL%j2&QJmK|plZS*nQklDjYTD?;#Q*XdmUD-r;Dgxj&V{$ ze7ATY@!qM$GYIoU@~;wTW;Y}X@AaKFJGkAruRg9Dn^bpFuHyAPI%a6uzad+GAkodd z&4uE;ij-Vg?NX|wWW#b{RCH`H2R_}Zt0`V9nZ=!nofeg-5h7&c6_H~xjzu}eVIX1NVNRTjEFLt&&K51_6&Q@Py=nq2K8@g&cvaQ^@xJXa&6>K*}4n-*4kh9Ym{#BjLMtYyZoXf!NQcxmpKGiG{^r#vw*+=slxnpm;r&?FjQj1zs zIJis>_0-4V*)*#=3z<{}yDVe%J9sDQSl_X6M_Hp>NNjeoIISHX)n>C$7Z@4NE0$76 zTDgaLYvxIu^H;Cq@@Bvorx_B_&fYH7ByvHi@3==etrp}-D;d`8S%zsWTXcuLQ!_=# zhtHalH5nP|Y8RPClyT7m3ez*5h@{g%CH3~j+n<}IXU&VNv-4m9 zn!B=H$t>=M)L+iCbluUz6BCk7Xp*+XM(i2*Ug^GL&1#4Uc0$Uk5dhuWJHD zW!>7PvZ9X$@y?Sa&YO>#Zs{9RC6|v{O8po+Gm?kH=GMd(IKZuKa`Aq|;Z8WE zt7Fk4$UIeV`}r}*sWpOSxGcxzu9%$iJfB*R6A|AvXI@Zm+yUOZBF<@?^s((>Q`|0V zc(VqA-fgZq#Z_FZR`HWDlu_2{dYM*UdR7ID#awg7YrdUhhID$KtD@=$fu1WD!#9Ck z7UglASFcWLNb&1aYoXgoi6d+t4L;sTn$%~2-7Bx5#b%kxYno5m6~0{8L3q)i!2pWF zbfY7t6(=2!AJnw7ZtcfP>^wncjSP$jCyMhb;nG?jojgOjO!J+`%uqPytX*1|%P=2G z@}&lkPBkuS%Xk_%kz3`Ba6Yw!vNH81v2i-1Jr0`BQu3HLO2QXkC<3u?G*k0B!{R}1 z;zs%QgN66O0=T)ZiyUT`vn$#%Y3>Y?GZWUTCS+`Ots;!2CDfUhe>to6ntzzJv|`!L z>LgGW<+<%!qfgtAMgXmSn=|&P!;a0LwRCaYBH$0@R;ih~%*KyRZVuYtx@Cnx>BVd8 zQryNhq~3;1c3~GS*R?Y0^meTIRCYAGmgA*mjd-daV@h*Ke4@GB*{1S<=8_oq0G>rlx#n6sV>9OAJyj*C!SGliza-#1K=O5lqV zA!pndgVL`$jnKu66<&WHYJE!7h8S=&R~9qSR&B0@BF2}MkRN)tVP@-w1E)0)QtWyU zi#14R)-qoMo4)z zzQ!^QFKl-No|T%n1!NrYRco1SYqiK*jE+DxQa0V&lSmD_&0O6OPu)?#tdo(zpfu=P zS^U<>6ceH1yH|WRk1-!I)&?sFMA<7Px{V{AVZWP{)0?>&+d8(O0KX z>R*ab(-6G@sBGKk$m(k$olc*Pqdt`tn-z;a>pe?}YVu91#X&jU)~IZanZC}_cFrl9 zmB^0P2p6y3G?t-zkSyD{s*tf}{A#r5BK*(t;CAa(r(&vTenL!g-xyS=I1SdbW6al( z#!ud04Fft0t8}%sGj7C&Fiw3e7D2>@ImF#Y}6&c7KD=SZ*=0(q5 zD@k0%9~EOCbDL+XJ4#cZD^tg?x@XX-%saU$lZCYS*pe5(Hz`gCwhy)?f#Gn>rrHji){AK3XQQRYN4WBnAKY~ z28X9M~G}G1{hA8qm$WjV@iczsP;-jPX~9 zqS39hB%`O@&ss`1A2NwY#L`>YZeru*HR3)$_=wjV(kTfC-=%FDZJ5fmPK#dgrLwH6 zzdJ@c*U!3_h$NRFIr)#RbgB%)+nt`V;^mr5BVbpY{i%;Ea%))8Q81N9BcQ&y^DY_B zahl7T^Ujr+aJT>u!mE@yg_Q2e>rsZ>xja<}uOnfPYGmBCXpCfup_E5e`c$=8>a34_eEWoAVK_S*rD|uI_PEpxC8RBF2+qXK|5RI%=)D zjPbgms``cLSVU@g#dUJVDCXvJWowIoWNaSwNn>eG0-+}MLv0dC zb+il)YW<#w(*-BJU$T?f&CU_r#JZAOjFL&sbw|VMibqpN)#mqM^@uHwEp1qyYoWcd z5Rx&9&Qy7KHH~gsFyf_;(2^8pkh>hQ?^c#EA0$?Hxobjucp1KI zaZ=mgMyeEYd(zac4?*xJhFe;=h9z9@APS%1-;C{ai|clCw`W@BpEb>6XS+L}6nF(Z zKWw~)IOKs_+UJNZ{83@P7X)Luu6Hd?u3Ie({V`+Ed>bYEo}_{6o|J14i_HcL7{*p1 zo+^6k$%;IiRlIxamy!4fpsN~|n)bH^SXPQlp)yYTm~$Pmu&Y{?ua_Akt#21|GX-S* z=i^bdWJ#Lb#j|$Id6*RKEq+mxS-X_ACzdUXp7kZy-4!TCbdqh&GG)$c(IshSJF2vg zmTIx?JW08R4KmqxF*xFaY+Q_GxZ;u({pNE?j@?ab54;6Wq4`o7nUCEZRJK*kcqSnrpE>g56v^lV?3Rs-{fInsz9;3l(hCaIieo>Q;rB z<|I@X5@BhG&%C%O*#`t3wV!`+mORxYODo^ryozHiSZ6%ebdnY|swCXfz_@u{9r!G9ozQw2!<@k>h?K{{UF#qwxfUGq0_6M8Zcp z$L6O00Lx5OkXpC<%TKjmm{YKh&%?JO(iR|P*kBs1r2WxZGR;psl6~sDj8W?=b}f0G zK8!rY1fRm7@cg-8GhEZq=!U1Ncq-542CjHC{m-RmChT-X^gBp)#=Cmetvq?1I#z1! znrv$6>gygmQ?Blo&di?ps=b8Qb2r8wDu=_iB~BTcPdVvc1@W`Sh8Y?N<=#m3tz!$A z%6z92@jK!Snx*pvkg6{j&3P`RdnMHD%-PAV$Wl?<#niQ#E{VzfYKIM+*Fs7= z7`=B1Gq`&SVQ#fT9k}sK-&20#v8iVPeY9!wZDCQbLquMd*KBouI8DSvp8YCr(WQ~i z$r)E5)}7UfLUJokV>XPgnf0Zc_hzXPVn|!&0;{VKYgo?O7|J@GE}ME)lxK>_w7A>W zyD8LY=c!FJclLL~Zb|K2ygGqc$2I82zK5GEJJ{a7zTulW&0)2}OufxSr_4B0<=kcE z!IM1mNVd--ax0oqZJkM@GDhStTGMSUnX`(`({kuioMXA49lw~}D{{|EJ;gOewjSO+ zO1AcUW6mp}8hZI4cdgY1j9qxPVaX=nEk_uw3ynOM$P~Stkkv?;?A~)^)0*AmY*A($ z%b{ilNaF{L=Bd1HNB|l&5?seqXLc2#P&vkHKTB2@2&a-UT{N39sRd)PwX^f1$rXQ2 zVI8hW&lTv|GRiSZO(OpV3`SMhb;>7>W4dDYs9=+I2$H7PR03a4`+X5?3iIo+Ps zI6GX3w`x_4w@Ssx-;`8ZnmBXKMaxKHuI3eKVa72?s}ou5jzllQ=bDLzMNZ+VKC62f zx^#s6gMsf<-rxA@l0@Fb)@oZVRpZnRyJ3EmOUM?lQq-4^XCjbsS@$l|5a*$!if3=5 zd_cw9O$QVnA~G-a`@Yq+06 z_=0G4DFpBwf-*qnyz{~SEZ7z~eqea!u$2i!*V?IWbT`;&*q)-Kyi{gwjlBrm8x%)D z9E{vZbL&#-QHbO?;<{q)V`cn>{8f(x5{%;UPI+VkBoVl0R1Y3Vy~H& z;M6;UcJY=0^I_;zkW{R99RP z%_d-?e8AJ!Dl0+@Hsj{wnwG?T**u!CI$a|^Sj{e)!B;)222w2C@YPyr_egMZD<)ae z_;iBGM_OlwS9GbLTFRtVu5AaY?EVvtytQTYtKJQVa>d6Ou5I@_5#DJ|VzNT@GW# z-Y9)y{#hBz7HZ|ho8-ynyQ57+7BFVIhggFL7B=YNvA|6 z!0lD^`HXvgE6}GaoE0NutF_Wqz~ZfGmqYCPvy+PNj8(3A?xwaYO{R!*#%o#_4d(n2 z@{Pwnm7<4ZH#-=S#(r)%9GcO*w_AHDBs>Ckk9vm^xv!y0$$ZRY)cS3{<^&qJq9^2A z`w(U6TK2Y2BAg6?*0$wL$J$CKIX0FXCuMa_W`#=iHI(VAY+Ne4?r_G`BXT&cT~YxX zK08;REgg?SGIqN&H26>2H+$5UviWU{*3gZJa=NlQu_u>e-1QZX*T?LxE2+sSoSx0| zT;jYbeH3FRj9s$rZzBdXrhm2639_l|H7isCHVkC6%6%N&eAfbKF!#Sl4xNG`T}T zgW9OrOd)1cYUGT~$s9Cio@%394k|2I`=#=13m?5BV0{Hu)UJ%hM@7d1tq7}(VS%Qv z%}kv|W#c`lx@;~76osI`er$H8M#B_62AQV}q+kP8VVEnZ&orzwZAPWSuU29Jt9lBF zH57Lv%ZM2p@DwPyJ0Z$S3hbN3OC6omR#7(fw~1AV{_z=a*066cKGk(Br2Xb5n?%QJ zvIcLPG>)UKA(mob!k~?S9cURf)WKH*s!YVUtqBr{&&?$RtyPeoeuUu>b%7 literal 0 HcmV?d00001 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index a2d650ca7..bde6efdbc 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -387,22 +387,25 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro steps = (p.steps // (1.0 - p.refiner_start)) if shared.sd_model_type == 'sdxl' else p.steps if os.environ.get('SD_STEPS_DEBUG', None) is not None: shared.log.debug(f'Steps: type=base input={p.steps} output={steps} refiner={use_refiner_start}') - return int(steps) + return max(2, int(steps)) def calculate_hires_steps(): - steps = (p.hr_second_pass_steps * p.denoising_strength) if p.hr_second_pass_steps > 0 else (p.steps * p.denoising_strength) + # denoising strength is applied to steps by diffusers so this is no-op + # steps = (p.hr_second_pass_steps * p.denoising_strength) if p.hr_second_pass_steps > 0 else (p.steps * p.denoising_strength) + steps = p.hr_second_pass_steps if p.hr_second_pass_steps > 0 else p.steps if os.environ.get('SD_STEPS_DEBUG', None) is not None: shared.log.debug(f'Steps: type=hires input={p.hr_second_pass_steps} output={steps} denoise={p.denoising_strength}') - return int(steps) + return max(2, int(steps)) def calculate_refiner_steps(): + # diffusers apply additional math to refiner steps, but we leave numbers as-is without correction if p.refiner_start > 0 and p.refiner_start < 1: - steps = (p.refiner_steps // p.refiner_start) if p.refiner_steps > 0 else (p.steps // p.refiner_start) + steps = ((1 - p.refiner_start) * p.refiner_steps) if p.refiner_steps > 0 else ((1 - p.refiner_start) * p.steps) else: steps = (p.denoising_strength * p.refiner_steps) if p.refiner_steps > 0 else (p.denoising_strength * p.steps) if os.environ.get('SD_STEPS_DEBUG', None) is not None: shared.log.debug(f'Steps: type=refiner input={p.refiner_steps} output={steps} start={p.refiner_start} denoise={p.denoising_strength}') - return int(steps) + return max(2, int(steps)) # pipeline type is set earlier in processing, but check for sanity if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: # reset pipeline @@ -460,8 +463,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) recompile_model(hires=True) update_sampler(shared.sd_model, second_pass=True) - if p.hr_second_pass_steps == 0: - p.hr_second_pass_steps = p.steps hires_args = set_pipeline_args( model=shared.sd_model, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index b613fca99..4891950df 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -17,6 +17,7 @@ try: UniPCMultistepScheduler, LMSDiscreteScheduler, KDPM2AncestralDiscreteScheduler, + LCMScheduler, ) except Exception as e: import diffusers @@ -40,6 +41,7 @@ config = { 'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, + 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, } samplers_data_diffusers = [ @@ -58,6 +60,7 @@ samplers_data_diffusers = [ sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('Heun', LCMScheduler, model), [], {}), ] class DiffusionSampler: diff --git a/requirements.txt b/requirements.txt index f34360881..b62ebcb89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ requests==2.31.0 tqdm==4.66.1 accelerate==0.20.3 opencv-python-headless==4.7.0.72 -diffusers==0.22.3 +diffusers==0.23.0 einops==0.4.1 gradio==3.43.2 huggingface_hub==0.18.0 From 6564e99ccdb45d4edc8390776aecf5003c21f526 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 18:22:24 -0500 Subject: [PATCH 28/43] update pipelines and xyzgrid --- CHANGELOG.md | 18 +++++++++++++----- modules/sd_models.py | 8 ++++++++ modules/sd_samplers_diffusers.py | 6 +++++- scripts/xyz_grid.py | 18 +++++++++++++++--- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3dad88e8..625924d11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,24 @@ ## Update for 2023-11-08 +- **Diffusers** + - **LCM** support for any *SD 1.5* or *SD-XL* model! + - download [lcm-lora-sd15](https://huggingface.co/latent-consistency/lcm-lora-sdv1-5/tree/main) and/or [lcm-lora-sdxl](https://huggingface.co/latent-consistency/lcm-lora-sdxl/tree/main) + - load for favorite *SD 1.5* or *SD-XL* model + - load **lcm lora** + - set **sampler** to **LCM** + - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient + note: LCM scheduler does not support steps higher than 50 + - Add additional pipeline types for manual model loads when loading from `safetensors` + - Updated logic for calculating **steps** when using base/hires/refiner workflows + - Safe model offloading for non-standard models + - Fix **DPM SDE** scheduler - **Extra networks** - Use multi-threading for 5x load speedup - **General**: - Reworked parser when pasting previously generated images/prompts includes all `txt2img`, `img2img` and `override` params -- **Diffusers** - - Add additional pipeline types for manual model loads when loading from `safetensors` - - Updated logic for calculating steps when using base/hires/refiner workflows - - Safe model offloading for non-standard models - - Fix DPM SDE scheduler + - Add refiner options to XYZ Grid - **Fixes** - Fix inpaint - Fix manual grid image save diff --git a/modules/sd_models.py b/modules/sd_models.py index b9d673e5b..0c6db835a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -646,6 +646,14 @@ def detect_pipeline(f: str, op: str = 'model'): guess = 'Stable Diffusion XL Instruct' else: guess = 'Stable Diffusion' + if 'LCM_' in f or 'LCM-' in f: + if shared.backend == shared.Backend.ORIGINAL: + shared.log.warning(f'Model detected as LCM model, but attempting to load using backend=original: {op}={f} size={size} MB') + guess = 'Latent Consistency Model' + if 'PixArt' in f: + if shared.backend == shared.Backend.ORIGINAL: + shared.log.warning(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB') + guess = 'PixArt Alpha' pipeline = shared_items.get_pipelines().get(guess, None) shared.log.info(f'Autodetect: {op}="{guess}" class={pipeline.__name__} file="{f}" size={size}MB') except Exception as e: diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 4891950df..6b1025d65 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -60,7 +60,7 @@ samplers_data_diffusers = [ sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('Heun', LCMScheduler, model), [], {}), + sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), ] class DiffusionSampler: @@ -73,8 +73,10 @@ class DiffusionSampler: return for key, value in config.get('All', {}).items(): # apply global defaults self.config[key] = value + shared.log.debug(f'Sampler: name={name} type=all config={self.config}') for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults self.config[key] = value + shared.log.debug(f'Sampler: name={name} type=scheduler config={self.config}') if hasattr(model.scheduler, 'scheduler_config'): # find model defaults orig_config = model.scheduler.scheduler_config else: @@ -82,9 +84,11 @@ class DiffusionSampler: for key, value in orig_config.items(): # apply model defaults if key in self.config: self.config[key] = value + shared.log.debug(f'Sampler: name={name} type=model config={self.config}') for key, value in kwargs.items(): # apply user args, if any if key in self.config: self.config[key] = value + shared.log.debug(f'Sampler: name={name} type=user config={self.config}') # finally apply user preferences if shared.opts.schedulers_prediction_type != 'default': self.config['prediction_type'] = shared.opts.schedulers_prediction_type diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 5e5c00938..912638777 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -86,6 +86,17 @@ def apply_checkpoint(p, x, xs): p.override_settings['sd_model_checkpoint'] = info.name +def apply_refiner(p, x, xs): + if x == shared.opts.sd_model_refiner: + return + info = sd_models.get_closet_checkpoint_match(x) + if info is None: + shared.log.warning(f"XYZ grid: apply refiner unknown checkpoint: {x}") + else: + sd_models.reload_model_weights(shared.sd_refiner, info) + p.override_settings['sd_model_refiner'] = info.name + + def apply_dict(p, x, xs): if x == shared.opts.sd_model_dict: return @@ -240,11 +251,12 @@ axis_options = [ AxisOption("[Second pass] upscaler", str, apply_field("hr_upscaler"), choices=lambda: [*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]]), AxisOption("[Second pass] sampler", str, apply_latent_sampler, fmt=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]), AxisOption("[Second pass] denoising Strength", float, apply_field("denoising_strength")), - AxisOption("[Second pass] steps", int, apply_field("hr_second_pass_steps")), + AxisOption("[Second pass] hires steps", int, apply_field("hr_second_pass_steps")), AxisOption("[Second pass] CFG scale", float, apply_field("image_cfg_scale")), AxisOption("[Second pass] guidance rescale", float, apply_field("diffusers_guidance_rescale")), - AxisOption("[Second pass] refiner start", float, apply_field("refiner_start")), - AxisOption("[Second pass] refiner start", float, apply_field("refiner_start")), + AxisOption("[Refiner] model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: sorted(sd_models.checkpoints_list)), + AxisOption("[Refiner] refiner start", float, apply_field("refiner_start")), + AxisOption("[Refiner] refiner steps", float, apply_field("refiner_steps")), AxisOption("[TOME] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')), AxisOption("[TOME] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')), AxisOption("[FreeU] 1st stage backbone factor", float, apply_setting('freeu_b1')), From d6206f3f5f226b53779728d6fe283b8944da0bd7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 18:24:20 -0500 Subject: [PATCH 29/43] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 625924d11..f357f0a0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Updated logic for calculating **steps** when using base/hires/refiner workflows - Safe model offloading for non-standard models - Fix **DPM SDE** scheduler + - Update to `diffusers==0.23.0` - **Extra networks** - Use multi-threading for 5x load speedup - **General**: From 898ee57949079829b09404766e457913259aff7a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 10 Nov 2023 02:53:06 +0300 Subject: [PATCH 30/43] Add lcm_convert.py to cli --- cli/lcm_convert.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 cli/lcm_convert.py diff --git a/cli/lcm_convert.py b/cli/lcm_convert.py new file mode 100644 index 000000000..73cefa6fd --- /dev/null +++ b/cli/lcm_convert.py @@ -0,0 +1,42 @@ +import argparse +import torch +from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPipelineForText2Image, LCMScheduler + +parser = argparse.ArgumentParser("lcm_convert") +parser.add_argument("--name", help="Name of the new LCM model", type=str) +parser.add_argument("--model", help="A model to convert", type=str) +parser.add_argument("--huggingface", action="store_true", help="Use Hugging Face models instead of safetensors models") +parser.add_argument("--upload", action="store_true", help="Upload the new LCM model to Hugging Face") +parser.add_argument("--no_save", action="store_true", help="Don't save the new LCM model to local disk") +parser.add_argument("--sdxl", action="store_true", help="Use SDXL models") +parser.add_argument("--ssd_1b", action="store_true", help="Use SSD-1B models") + +args = parser.parse_args() + +if args.huggingface: + pipeline = AutoPipelineForText2Image.from_pretrained(args.model, torch_dtype=torch.float16, variant="fp16") +else: + if args.sdxl or args.ssd_1b: + pipeline = StableDiffusionXLPipeline.from_single_file(args.model) + else: + pipeline = StableDiffusionPipeline.from_single_file(args.model) + +pipeline.scheduler = LCMScheduler.from_config(pipeline.scheduler.config) +if args.sdxl: + pipeline.load_lora_weights("latent-consistency/lcm-lora-sdxl") +elif args.ssd_1b: + pipeline.load_lora_weights("latent-consistency/lcm-lora-ssd-1b") +else: + pipeline.load_lora_weights("latent-consistency/lcm-lora-sdv1-5") +pipeline.fuse_lora() + +#components = pipeline.components +#pipeline = LatentConsistencyModelPipeline(**components) + +pipeline = pipeline.to(dtype=torch.float16) +print(pipeline) + +if not args.no_save: + pipeline.save_pretrained(args.name, variant="fp16") +if args.upload: + pipeline.push_to_hub(args.name, variant="fp16") From baed87285817c8ede31369e13630ba60cd70cdcb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Nov 2023 18:57:07 -0500 Subject: [PATCH 31/43] update changelog --- CHANGELOG.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f357f0a0a..b7f82c758 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,31 +3,33 @@ ## Update for 2023-11-08 - **Diffusers** - - **LCM** support for any *SD 1.5* or *SD-XL* model! - - download [lcm-lora-sd15](https://huggingface.co/latent-consistency/lcm-lora-sdv1-5/tree/main) and/or [lcm-lora-sdxl](https://huggingface.co/latent-consistency/lcm-lora-sdxl/tree/main) - - load for favorite *SD 1.5* or *SD-XL* model - - load **lcm lora** - - set **sampler** to **LCM** + - **LCM** support for any *SD 1.5* or *SD-XL* model! + - download [lcm-lora-sd15](https://huggingface.co/latent-consistency/lcm-lora-sdv1-5/tree/main) and/or [lcm-lora-sdxl](https://huggingface.co/latent-consistency/lcm-lora-sdxl/tree/main) + - load for favorite *SD 1.5* or *SD-XL* model *(original LCM was SD 1.5 only, this is both)* + - load **lcm lora** + - set **sampler** to **LCM** - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient note: LCM scheduler does not support steps higher than 50 + - Add `cli/lcm_convert.py` script to convert any SD 1.5 or SD-XL model to LCM model + by baking in LORA and uploading to Huggingface, thanks @Disty0 - Add additional pipeline types for manual model loads when loading from `safetensors` - Updated logic for calculating **steps** when using base/hires/refiner workflows - - Safe model offloading for non-standard models + - Safe model offloading for non-standard models - Fix **DPM SDE** scheduler - Update to `diffusers==0.23.0` - **Extra networks** - Use multi-threading for 5x load speedup -- **General**: +- **General**: - Reworked parser when pasting previously generated images/prompts includes all `txt2img`, `img2img` and `override` params - Add refiner options to XYZ Grid -- **Fixes** +- **Fixes** - Fix inpaint - - Fix manual grid image save - - Fix img2img init image save - - More uniform models paths - - Improve extension compatibility - - Improve BF16 support + - Fix manual grid image save + - Fix img2img init image save + - More uniform models paths + - Improve extension compatibility + - Improve BF16 support ## Update for 2023-11-06 From 044c31473b0c86c2351d1ff56322337f9d08010b Mon Sep 17 00:00:00 2001 From: Aptronymist <108482020+Aptronymist@users.noreply.github.com> Date: Thu, 9 Nov 2023 21:40:52 -0800 Subject: [PATCH 32/43] Update CHANGELOG.md added cfg for LCM notes --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f82c758..2f9f39f09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ - load **lcm lora** - set **sampler** to **LCM** - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient - note: LCM scheduler does not support steps higher than 50 + note: LCM scheduler does not support steps higher than 50 + - set cfg to 1 or 2 - Add `cli/lcm_convert.py` script to convert any SD 1.5 or SD-XL model to LCM model by baking in LORA and uploading to Huggingface, thanks @Disty0 - Add additional pipeline types for manual model loads when loading from `safetensors` From f775324ea26f8668f679821208db91bed8c39716 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 08:21:59 -0500 Subject: [PATCH 33/43] use ckpt none to skip loading a model --- webui.py | 2 +- wiki | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webui.py b/webui.py index 2e3c76f80..91c0179d6 100644 --- a/webui.py +++ b/webui.py @@ -156,7 +156,7 @@ def initialize(): def load_model(): - if opts.sd_checkpoint_autoload: + if opts.sd_checkpoint_autoload and shared.cmd_opts.ckpt.lower() != 'none': shared.state.begin('load') thread_model = Thread(target=lambda: shared.sd_model) thread_model.start() diff --git a/wiki b/wiki index c0b5cb267..cd040c02e 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit c0b5cb2672f7b0ac0add0a321f22bc0e6b738d78 +Subproject commit cd040c02e4a477135ce08efe4d06672b57456c31 From dc436b1f9de7dde84070e05be35faa97938e1ace Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 08:28:33 -0500 Subject: [PATCH 34/43] fix --- modules/upscaler.py | 16 ++++++++++++++++ webui.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/upscaler.py b/modules/upscaler.py index e60f9a46c..6ad530a8b 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -51,6 +51,19 @@ class Upscaler: except Exception: pass + def find_folder(self, folder, scalers, loaded): + for fn in os.listdir(folder): # from folder + if not fn.endswith('.pth') and not fn.endswith('.pt'): + continue + file_name = os.path.join(folder, fn) + if file_name not in loaded: + model_name = os.path.splitext(fn)[0] + scaler = UpscalerData(name=f'{self.name} {model_name}', path=file_name, upscaler=self) + scaler.custom = True + scalers.append(scaler) + loaded.append(file_name) + modules.shared.log.debug(f'Upscaler type={self.name} folder="{folder}" model="{model_name}" path="{file_name}"') + def find_scalers(self): scalers = [] loaded = [] @@ -66,6 +79,8 @@ class Upscaler: # modules.shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model[0]}" path="{model_path}"') if not os.path.exists(self.user_path): return scalers + self.find_folder(self.user_path, scalers, loaded) + """ for fn in os.listdir(self.user_path): # from folder if not fn.endswith('.pth') and not fn.endswith('.pt'): continue @@ -77,6 +92,7 @@ class Upscaler: scalers.append(scaler) loaded.append(file_name) # modules.shared.log.debug(f'Upscaler type={self.name} folder="{self.user_path}" model="{model_name}" path="{file_name}"') + """ return scalers @abstractmethod diff --git a/webui.py b/webui.py index 91c0179d6..8397aced3 100644 --- a/webui.py +++ b/webui.py @@ -156,7 +156,7 @@ def initialize(): def load_model(): - if opts.sd_checkpoint_autoload and shared.cmd_opts.ckpt.lower() != 'none': + if opts.sd_checkpoint_autoload and (shared.cmd_opts.ckpt is not None and shared.cmd_opts.ckpt.lower() != 'none'): shared.state.begin('load') thread_model = Thread(target=lambda: shared.sd_model) thread_model.start() From 653d253bfecbde9a7f0481928a114c6e1af0b0bf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 08:34:35 -0500 Subject: [PATCH 35/43] allow upscalers in subfolders --- modules/upscaler.py | 8 ++++++-- scripts/postprocessing_upscale.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/upscaler.py b/modules/upscaler.py index 6ad530a8b..52c50ff45 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -53,9 +53,12 @@ class Upscaler: def find_folder(self, folder, scalers, loaded): for fn in os.listdir(folder): # from folder - if not fn.endswith('.pth') and not fn.endswith('.pt'): - continue file_name = os.path.join(folder, fn) + if os.path.isdir(file_name): + self.find_folder(file_name, scalers, loaded) + continue + if not file_name.endswith('.pth') and not file_name.endswith('.pt'): + continue if file_name not in loaded: model_name = os.path.splitext(fn)[0] scaler = UpscalerData(name=f'{self.name} {model_name}', path=file_name, upscaler=self) @@ -63,6 +66,7 @@ class Upscaler: scalers.append(scaler) loaded.append(file_name) modules.shared.log.debug(f'Upscaler type={self.name} folder="{folder}" model="{model_name}" path="{file_name}"') + print(f'Upscaler type={self.name} folder="{folder}" model="{model_name}" path="{file_name}"') def find_scalers(self): scalers = [] diff --git a/scripts/postprocessing_upscale.py b/scripts/postprocessing_upscale.py index 0086a3abc..e97ac9b45 100644 --- a/scripts/postprocessing_upscale.py +++ b/scripts/postprocessing_upscale.py @@ -16,7 +16,7 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing): with FormRow(elem_id="extras_upscale"): with gr.Tabs(elem_id="extras_resize_mode"): with gr.TabItem('Scale by', elem_id="extras_scale_by_tab") as tab_scale_by: - upscaling_resize = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Resize", value=4, elem_id="extras_upscaling_resize") + upscaling_resize = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Resize", value=2.0, elem_id="extras_upscaling_resize") with gr.TabItem('Scale to', elem_id="extras_scale_to_tab") as tab_scale_to: with FormRow(): From 3616fba9bee24a1552df939a174f5f1248e04b85 Mon Sep 17 00:00:00 2001 From: Nuullll Date: Fri, 10 Nov 2023 21:44:09 +0800 Subject: [PATCH 36/43] Fix UnboundLocalError for variable 'updated' --- modules/ui_extensions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py index 1fafce4bf..6a3e1e6b1 100644 --- a/modules/ui_extensions.py +++ b/modules/ui_extensions.py @@ -319,6 +319,7 @@ def create_html(search_text, sort_column): for ext in sorted(extensions_list, key=sort_function, reverse=sort_reverse): installed = get_installed(ext) author = '' + updated = datetime.timestamp(datetime.now()) try: if 'github' in ext['url']: author = ext['url'].split('/')[-2].split(':')[-1] if '/' in ext['url'] else ext['url'].split(':')[1].split('/')[0] From 9778af1bc6e069ea24df190f2a34f3654f35e7e3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 08:46:26 -0500 Subject: [PATCH 37/43] safe check scripts type --- modules/processing.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 5e7832c5f..19655eb36 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -677,7 +677,7 @@ def print_profile(profile, msg: str): def process_images(p: StableDiffusionProcessing) -> Processed: if not hasattr(p.sd_model, 'sd_checkpoint_info'): return None - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): p.scripts.before_process(p) stored_opts = {} for k, v in p.override_settings.copy().items(): @@ -803,7 +803,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.all_subseeds = [int(subseed) + x for x in range(len(p.all_prompts))] if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and shared.backend == shared.Backend.ORIGINAL: modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=False) - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): p.scripts.process(p) infotexts = [] output_images = [] @@ -841,7 +841,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size] p.seeds = p.all_seeds[n * p.batch_size:(n + 1) * p.batch_size] p.subseeds = p.all_subseeds[n * p.batch_size:(n + 1) * p.batch_size] - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) if len(p.prompts) == 0: break @@ -849,7 +849,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if not p.disable_extra_networks: with devices.autocast(): modules.extra_networks.activate(p, extra_network_data) - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) if n == 0: with open(os.path.join(modules.paths.data_path, "params.txt"), "w", encoding="utf8") as file: @@ -898,9 +898,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.cmd_opts.lowvram or shared.cmd_opts.medvram and shared.backend == shared.Backend.ORIGINAL: modules.lowvram.send_everything_to_cpu() devices.torch_gc() - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): p.prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size] p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size] batch_params = modules.scripts.PostprocessBatchListArgs(list(x_samples_ddim)) @@ -928,7 +928,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.ops.append('face') x_sample = modules.face_restoration.restore_faces(x_sample) image = Image.fromarray(x_sample) - if p.scripts is not None: + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner): pp = modules.scripts.PostprocessImageArgs(image) p.scripts.postprocess_image(p, pp) image = pp.image @@ -993,7 +993,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: index_of_first_image=index_of_first_image, infotexts=infotexts, ) - if p.scripts is not None and not (shared.state.interrupted or shared.state.skipped): + if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner) and not (shared.state.interrupted or shared.state.skipped): p.scripts.postprocess(p, res) return res From e2bcbaeed0b5bfc93d0c7585db3b27b8b7581bf5 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 10 Nov 2023 17:12:15 +0300 Subject: [PATCH 38/43] update lcm-convert.py --- CHANGELOG.md | 2 +- cli/{lcm_convert.py => lcm-convert.py} | 25 +++++++++++++++++++------ modules/sd_samplers_diffusers.py | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) rename cli/{lcm_convert.py => lcm-convert.py} (62%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f9f39f09..f9324bc7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient note: LCM scheduler does not support steps higher than 50 - set cfg to 1 or 2 - - Add `cli/lcm_convert.py` script to convert any SD 1.5 or SD-XL model to LCM model + - Add `cli/lcm-convert.py` script to convert any SD 1.5 or SD-XL model to LCM model by baking in LORA and uploading to Huggingface, thanks @Disty0 - Add additional pipeline types for manual model loads when loading from `safetensors` - Updated logic for calculating **steps** when using base/hires/refiner workflows diff --git a/cli/lcm_convert.py b/cli/lcm-convert.py similarity index 62% rename from cli/lcm_convert.py rename to cli/lcm-convert.py index 73cefa6fd..c2d7c266b 100644 --- a/cli/lcm_convert.py +++ b/cli/lcm-convert.py @@ -1,3 +1,4 @@ +import os import argparse import torch from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPipelineForText2Image, LCMScheduler @@ -5,11 +6,13 @@ from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPi parser = argparse.ArgumentParser("lcm_convert") parser.add_argument("--name", help="Name of the new LCM model", type=str) parser.add_argument("--model", help="A model to convert", type=str) +parser.add_argument("--lora-scale", default=1.0, help="Strenght of the LCM", type=float) parser.add_argument("--huggingface", action="store_true", help="Use Hugging Face models instead of safetensors models") parser.add_argument("--upload", action="store_true", help="Upload the new LCM model to Hugging Face") -parser.add_argument("--no_save", action="store_true", help="Don't save the new LCM model to local disk") +parser.add_argument("--no-half", action="store_true", help="Convert the new LCM model to FP32") +parser.add_argument("--no-save", action="store_true", help="Don't save the new LCM model to local disk") parser.add_argument("--sdxl", action="store_true", help="Use SDXL models") -parser.add_argument("--ssd_1b", action="store_true", help="Use SSD-1B models") +parser.add_argument("--ssd-1b", action="store_true", help="Use SSD-1B models") args = parser.parse_args() @@ -28,15 +31,25 @@ elif args.ssd_1b: pipeline.load_lora_weights("latent-consistency/lcm-lora-ssd-1b") else: pipeline.load_lora_weights("latent-consistency/lcm-lora-sdv1-5") -pipeline.fuse_lora() +pipeline.fuse_lora(lora_scale=args.lora_scale) #components = pipeline.components #pipeline = LatentConsistencyModelPipeline(**components) -pipeline = pipeline.to(dtype=torch.float16) +if args.no_half: + pipeline = pipeline.to(dtype=torch.float32) +else: + pipeline = pipeline.to(dtype=torch.float16) print(pipeline) if not args.no_save: - pipeline.save_pretrained(args.name, variant="fp16") + os.makedirs(f"models--local--{args.name}/snapshots") + if args.no_half: + pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}") + else: + pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}", variant="fp16") if args.upload: - pipeline.push_to_hub(args.name, variant="fp16") + if args.no_half: + pipeline.push_to_hub(args.name) + else: + pipeline.push_to_hub(args.name, variant="fp16") diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 6b1025d65..015a55aaa 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -41,7 +41,7 @@ config = { 'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, - 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, + 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, } samplers_data_diffusers = [ From b8a980c7057c90451b8d207e56c91ad6b491d32b Mon Sep 17 00:00:00 2001 From: vladmandic Date: Fri, 10 Nov 2023 14:12:28 +0000 Subject: [PATCH 39/43] =?UTF-8?q?Deploying=20to=20master=20from=20@=20vlad?= =?UTF-8?q?mandic/automatic@f29d3926070d89bd375f7602f8830cd5303fe87d=20?= =?UTF-8?q?=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- cli/{lcm-convert.py => lcm_convert.py} | 25 ++++++------------------- modules/sd_samplers_diffusers.py | 2 +- 3 files changed, 8 insertions(+), 21 deletions(-) rename cli/{lcm-convert.py => lcm_convert.py} (62%) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9324bc7d..2f9f39f09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient note: LCM scheduler does not support steps higher than 50 - set cfg to 1 or 2 - - Add `cli/lcm-convert.py` script to convert any SD 1.5 or SD-XL model to LCM model + - Add `cli/lcm_convert.py` script to convert any SD 1.5 or SD-XL model to LCM model by baking in LORA and uploading to Huggingface, thanks @Disty0 - Add additional pipeline types for manual model loads when loading from `safetensors` - Updated logic for calculating **steps** when using base/hires/refiner workflows diff --git a/cli/lcm-convert.py b/cli/lcm_convert.py similarity index 62% rename from cli/lcm-convert.py rename to cli/lcm_convert.py index c2d7c266b..73cefa6fd 100644 --- a/cli/lcm-convert.py +++ b/cli/lcm_convert.py @@ -1,4 +1,3 @@ -import os import argparse import torch from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPipelineForText2Image, LCMScheduler @@ -6,13 +5,11 @@ from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPi parser = argparse.ArgumentParser("lcm_convert") parser.add_argument("--name", help="Name of the new LCM model", type=str) parser.add_argument("--model", help="A model to convert", type=str) -parser.add_argument("--lora-scale", default=1.0, help="Strenght of the LCM", type=float) parser.add_argument("--huggingface", action="store_true", help="Use Hugging Face models instead of safetensors models") parser.add_argument("--upload", action="store_true", help="Upload the new LCM model to Hugging Face") -parser.add_argument("--no-half", action="store_true", help="Convert the new LCM model to FP32") -parser.add_argument("--no-save", action="store_true", help="Don't save the new LCM model to local disk") +parser.add_argument("--no_save", action="store_true", help="Don't save the new LCM model to local disk") parser.add_argument("--sdxl", action="store_true", help="Use SDXL models") -parser.add_argument("--ssd-1b", action="store_true", help="Use SSD-1B models") +parser.add_argument("--ssd_1b", action="store_true", help="Use SSD-1B models") args = parser.parse_args() @@ -31,25 +28,15 @@ elif args.ssd_1b: pipeline.load_lora_weights("latent-consistency/lcm-lora-ssd-1b") else: pipeline.load_lora_weights("latent-consistency/lcm-lora-sdv1-5") -pipeline.fuse_lora(lora_scale=args.lora_scale) +pipeline.fuse_lora() #components = pipeline.components #pipeline = LatentConsistencyModelPipeline(**components) -if args.no_half: - pipeline = pipeline.to(dtype=torch.float32) -else: - pipeline = pipeline.to(dtype=torch.float16) +pipeline = pipeline.to(dtype=torch.float16) print(pipeline) if not args.no_save: - os.makedirs(f"models--local--{args.name}/snapshots") - if args.no_half: - pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}") - else: - pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}", variant="fp16") + pipeline.save_pretrained(args.name, variant="fp16") if args.upload: - if args.no_half: - pipeline.push_to_hub(args.name) - else: - pipeline.push_to_hub(args.name, variant="fp16") + pipeline.push_to_hub(args.name, variant="fp16") diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 015a55aaa..6b1025d65 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -41,7 +41,7 @@ config = { 'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, - 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, + 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, } samplers_data_diffusers = [ From eda2aa6da209a64782420778fef4dadcb3962e30 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 10 Nov 2023 17:19:50 +0300 Subject: [PATCH 40/43] re-update lcm-convert.py --- CHANGELOG.md | 2 +- cli/{lcm_convert.py => lcm-convert.py} | 25 +++++++++++++++++++------ modules/sd_samplers_diffusers.py | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) rename cli/{lcm_convert.py => lcm-convert.py} (62%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f9f39f09..f9324bc7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - set number of steps to some low number, for SD-XL 6-7 steps is normally sufficient note: LCM scheduler does not support steps higher than 50 - set cfg to 1 or 2 - - Add `cli/lcm_convert.py` script to convert any SD 1.5 or SD-XL model to LCM model + - Add `cli/lcm-convert.py` script to convert any SD 1.5 or SD-XL model to LCM model by baking in LORA and uploading to Huggingface, thanks @Disty0 - Add additional pipeline types for manual model loads when loading from `safetensors` - Updated logic for calculating **steps** when using base/hires/refiner workflows diff --git a/cli/lcm_convert.py b/cli/lcm-convert.py similarity index 62% rename from cli/lcm_convert.py rename to cli/lcm-convert.py index 73cefa6fd..c2d7c266b 100644 --- a/cli/lcm_convert.py +++ b/cli/lcm-convert.py @@ -1,3 +1,4 @@ +import os import argparse import torch from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPipelineForText2Image, LCMScheduler @@ -5,11 +6,13 @@ from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, AutoPi parser = argparse.ArgumentParser("lcm_convert") parser.add_argument("--name", help="Name of the new LCM model", type=str) parser.add_argument("--model", help="A model to convert", type=str) +parser.add_argument("--lora-scale", default=1.0, help="Strenght of the LCM", type=float) parser.add_argument("--huggingface", action="store_true", help="Use Hugging Face models instead of safetensors models") parser.add_argument("--upload", action="store_true", help="Upload the new LCM model to Hugging Face") -parser.add_argument("--no_save", action="store_true", help="Don't save the new LCM model to local disk") +parser.add_argument("--no-half", action="store_true", help="Convert the new LCM model to FP32") +parser.add_argument("--no-save", action="store_true", help="Don't save the new LCM model to local disk") parser.add_argument("--sdxl", action="store_true", help="Use SDXL models") -parser.add_argument("--ssd_1b", action="store_true", help="Use SSD-1B models") +parser.add_argument("--ssd-1b", action="store_true", help="Use SSD-1B models") args = parser.parse_args() @@ -28,15 +31,25 @@ elif args.ssd_1b: pipeline.load_lora_weights("latent-consistency/lcm-lora-ssd-1b") else: pipeline.load_lora_weights("latent-consistency/lcm-lora-sdv1-5") -pipeline.fuse_lora() +pipeline.fuse_lora(lora_scale=args.lora_scale) #components = pipeline.components #pipeline = LatentConsistencyModelPipeline(**components) -pipeline = pipeline.to(dtype=torch.float16) +if args.no_half: + pipeline = pipeline.to(dtype=torch.float32) +else: + pipeline = pipeline.to(dtype=torch.float16) print(pipeline) if not args.no_save: - pipeline.save_pretrained(args.name, variant="fp16") + os.makedirs(f"models--local--{args.name}/snapshots") + if args.no_half: + pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}") + else: + pipeline.save_pretrained(f"models--local--{args.name}/snapshots/{args.name}", variant="fp16") if args.upload: - pipeline.push_to_hub(args.name, variant="fp16") + if args.no_half: + pipeline.push_to_hub(args.name) + else: + pipeline.push_to_hub(args.name, variant="fp16") diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 6b1025d65..015a55aaa 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -41,7 +41,7 @@ config = { 'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, - 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, + 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, } samplers_data_diffusers = [ From 79af359b1b962a4bb313cbbb8ba3ef541805c20f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 10:04:12 -0500 Subject: [PATCH 41/43] minor updates --- .gitmodules | 54 ++++++++++--------- .../stable-diffusion-webui-rembg | 2 +- javascript/settings.js | 37 ++++++------- modules/sd_samplers_diffusers.py | 8 +-- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/.gitmodules b/.gitmodules index a6d3e9bf1..cc6c17569 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,34 +1,36 @@ [submodule "wiki"] path = wiki url = https://github.com/vladmandic/automatic.wiki -[submodule "extensions-builtin/sd-extension-system-info"] - path = extensions-builtin/sd-extension-system-info - url = https://github.com/vladmandic/sd-extension-system-info -[submodule "extensions-builtin/stable-diffusion-webui-images-browser"] - path = extensions-builtin/stable-diffusion-webui-images-browser - url = https://github.com/AlUlkesh/stable-diffusion-webui-images-browser - ignore = dirty -[submodule "modules/lora"] - path = modules/lora - url = https://github.com/kohya-ss/sd-scripts - ignore = dirty -[submodule "extensions-builtin/sd-webui-controlnet"] - path = extensions-builtin/sd-webui-controlnet - url = https://github.com/Mikubill/sd-webui-controlnet - ignore = dirty -[submodule "extensions-builtin/stable-diffusion-webui-rembg"] - path = extensions-builtin/stable-diffusion-webui-rembg - url = https://github.com/vladmandic/sd-extension-rembg - ignore = dirty -[submodule "extensions-builtin/sd-webui-agent-scheduler"] - path = extensions-builtin/sd-webui-agent-scheduler - url = https://github.com/ArtVentureX/sd-webui-agent-scheduler - ignore = dirty -[submodule "extensions-builtin/sd-extension-chainner"] - path = extensions-builtin/sd-extension-chainner - url = https://github.com/vladmandic/sd-extension-chainner ignore = dirty [submodule "modules/k-diffusion"] path = modules/k-diffusion url = https://github.com/crowsonkb/k-diffusion ignore = dirty +[submodule "modules/lora"] + path = modules/lora + url = https://github.com/kohya-ss/sd-scripts + ignore = dirty +[submodule "extensions-builtin/sd-extension-system-info"] + path = extensions-builtin/sd-extension-system-info + url = https://github.com/vladmandic/sd-extension-system-info + ignore = dirty +[submodule "extensions-builtin/sd-extension-chainner"] + path = extensions-builtin/sd-extension-chainner + url = https://github.com/vladmandic/sd-extension-chainner + ignore = dirty +[submodule "extensions-builtin/stable-diffusion-webui-rembg"] + path = extensions-builtin/stable-diffusion-webui-rembg + url = https://github.com/vladmandic/sd-extension-rembg + ignore = dirty +[submodule "extensions-builtin/stable-diffusion-webui-images-browser"] + path = extensions-builtin/stable-diffusion-webui-images-browser + url = https://github.com/AlUlkesh/stable-diffusion-webui-images-browser + ignore = dirty +[submodule "extensions-builtin/sd-webui-controlnet"] + path = extensions-builtin/sd-webui-controlnet + url = https://github.com/Mikubill/sd-webui-controlnet + ignore = dirty +[submodule "extensions-builtin/sd-webui-agent-scheduler"] + path = extensions-builtin/sd-webui-agent-scheduler + url = https://github.com/ArtVentureX/sd-webui-agent-scheduler + ignore = dirty diff --git a/extensions-builtin/stable-diffusion-webui-rembg b/extensions-builtin/stable-diffusion-webui-rembg index d5cd87bd4..b73dee3f3 160000 --- a/extensions-builtin/stable-diffusion-webui-rembg +++ b/extensions-builtin/stable-diffusion-webui-rembg @@ -1 +1 @@ -Subproject commit d5cd87bd434f1d82403ef740e0ab727afaf9dc96 +Subproject commit b73dee3f3fa99b1e7c7ee8dc6dad0176cb74e24c diff --git a/javascript/settings.js b/javascript/settings.js index dacc8bd87..93e4bebce 100644 --- a/javascript/settings.js +++ b/javascript/settings.js @@ -77,7 +77,7 @@ function markIfModified(setting_name, value) { tab_nav_indicator.classList.toggle('saved', saved.size > 0); if (changed_items.size > 0) tab_nav_indicator.title += `click to reset ${changed_items.size} unapplied changes in this tab\n`; if (saved.size > 0) tab_nav_indicator.title += `${saved.size} custom values\n${unsaved.size} default values}`; - elem.scrollIntoView({ behavior: 'smooth', block: 'center' }); // TODO why is scroll happening on every change if all pages are visible? + // elem.scrollIntoView({ behavior: 'smooth', block: 'center' }); // TODO why is scroll happening on every change if all pages are visible? } onAfterUiUpdate(async () => { @@ -105,9 +105,10 @@ onAfterUiUpdate(async () => { }, }); - const settings_search = gradioApp().querySelectorAll('#settings_search > label > textarea')[0]; - settings_search.oninput = (e) => { + const settingsSearch = gradioApp().querySelectorAll('#settings_search > label > textarea')[0]; + settingsSearch.oninput = (e) => { setTimeout(() => { + log('settingsSearch', e.target.value) showAllSettings(); gradioApp().querySelectorAll('#tab_settings .tabitem').forEach((section) => { section.querySelectorAll('.dirtyable').forEach((setting) => { @@ -131,23 +132,23 @@ onOptionsChanged(() => { function initSettings() { if (settingsInitialized) return; settingsInitialized = true; - const tab_nav_element = gradioApp().querySelector('#settings > .tab-nav'); - const tab_nav_buttons = gradioApp().querySelectorAll('#settings > .tab-nav > button'); - const tab_elements = gradioApp().querySelectorAll('#settings > div:not(.tab-nav)'); + const tabNavElements = gradioApp().querySelector('#settings > .tab-nav'); + const tabNavButtons = gradioApp().querySelectorAll('#settings > .tab-nav > button'); + const tabElements = gradioApp().querySelectorAll('#settings > div:not(.tab-nav)'); const observer = new MutationObserver((mutations) => { - const show_all_pages_dummy = gradioApp().getElementById('settings_show_all_pages'); - if (show_all_pages_dummy.style.display === 'none') { return; } - const mutation_on_style = (mut) => mut.type === 'attributes' && mut.attributeName === 'style'; - if (mutations.some(mutation_on_style)) showAllSettings(); + const showAllPages = gradioApp().getElementById('settings_show_all_pages'); + if (showAllPages.style.display === 'none') return; + const mutation = (mut) => mut.type === 'attributes' && mut.attributeName === 'style' + if (mutations.some(mutation)) showAllSettings(); }); - const tab_content_wrapper = document.createElement('div'); - tab_content_wrapper.className = 'tab-content'; - tab_nav_element.parentElement.insertBefore(tab_content_wrapper, tab_nav_element.nextSibling); - tab_elements.forEach((elem, index) => { - const tab_name = elem.id.replace('settings_', ''); - const indicator = gradioApp().getElementById(`modification_indicator_${tab_name}`); - tab_nav_element.insertBefore(indicator, tab_nav_buttons[index]); - tab_content_wrapper.appendChild(elem); + const tabContentWrapper = document.createElement('div'); + tabContentWrapper.className = 'tab-content'; + tabNavElements.parentElement.insertBefore(tabContentWrapper, tabNavElements.nextSibling); + tabElements.forEach((elem, index) => { + const tabName = elem.id.replace('settings_', ''); + const indicator = gradioApp().getElementById(`modification_indicator_${tabName}`); + tabNavElements.insertBefore(indicator, tabNavButtons[index]); + tabContentWrapper.appendChild(elem); observer.observe(elem, { attributes: true, attributeFilter: ['style'] }); }); log('initSettings'); diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 015a55aaa..9cd3ee806 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -73,10 +73,10 @@ class DiffusionSampler: return for key, value in config.get('All', {}).items(): # apply global defaults self.config[key] = value - shared.log.debug(f'Sampler: name={name} type=all config={self.config}') + # shared.log.debug(f'Sampler: name={name} type=all config={self.config}') for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults self.config[key] = value - shared.log.debug(f'Sampler: name={name} type=scheduler config={self.config}') + # shared.log.debug(f'Sampler: name={name} type=scheduler config={self.config}') if hasattr(model.scheduler, 'scheduler_config'): # find model defaults orig_config = model.scheduler.scheduler_config else: @@ -84,11 +84,11 @@ class DiffusionSampler: for key, value in orig_config.items(): # apply model defaults if key in self.config: self.config[key] = value - shared.log.debug(f'Sampler: name={name} type=model config={self.config}') + # shared.log.debug(f'Sampler: name={name} type=model config={self.config}') for key, value in kwargs.items(): # apply user args, if any if key in self.config: self.config[key] = value - shared.log.debug(f'Sampler: name={name} type=user config={self.config}') + # shared.log.debug(f'Sampler: name={name} type=user config={self.config}') # finally apply user preferences if shared.opts.schedulers_prediction_type != 'default': self.config['prediction_type'] = shared.opts.schedulers_prediction_type From f5987018c6c9a31fadecfcee44ff9704cbe2cbde Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 10:06:07 -0500 Subject: [PATCH 42/43] update changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9324bc7d..43167176f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2023-11-08 +## Update for 2023-11-10 - **Diffusers** - **LCM** support for any *SD 1.5* or *SD-XL* model! @@ -24,11 +24,14 @@ - Reworked parser when pasting previously generated images/prompts includes all `txt2img`, `img2img` and `override` params - Add refiner options to XYZ Grid + - Support custom upscalers in subfolders + - Support `--ckpt none` to skip loading a model - **Fixes** - Fix inpaint - Fix manual grid image save - Fix img2img init image save - More uniform models paths + - Safe scripts callback execution - Improve extension compatibility - Improve BF16 support From 056b04dcd0be9624b91a22033efe3cfe1bd0aa62 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Nov 2023 10:10:41 -0500 Subject: [PATCH 43/43] lint cleanup --- modules/ui_extra_networks_checkpoints.py | 2 +- modules/ui_extra_networks_hypernets.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 98aac49af..01902b376 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -14,7 +14,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): def refresh(self): shared.refresh_checkpoints() - def list_reference(self): + def list_reference(self): # pylint: disable=inconsistent-return-statements if shared.backend != shared.Backend.DIFFUSERS: return [] reference_models = shared.readfile(os.path.join('html', 'reference.json')) diff --git a/modules/ui_extra_networks_hypernets.py b/modules/ui_extra_networks_hypernets.py index 2fde91634..28189dad4 100644 --- a/modules/ui_extra_networks_hypernets.py +++ b/modules/ui_extra_networks_hypernets.py @@ -13,7 +13,6 @@ class ExtraNetworksPageHypernetworks(ui_extra_networks.ExtraNetworksPage): def list_items(self): for name, path in shared.hypernetworks.items(): try: - fn = os.path.splitext(path)[0] name = os.path.relpath(os.path.splitext(path)[0], shared.opts.hypernetwork_dir) yield { "type": 'Hypernetwork',