extension curation

This commit is contained in:
Vladimir Mandic
2023-10-13 15:30:51 -04:00
parent b55bb8c883
commit 74fb8fba3e
6 changed files with 104 additions and 95 deletions
+5 -3
View File
@@ -6,6 +6,8 @@ from modules.paths_internal import extensions_dir, extensions_builtin_dir
extensions = []
if not os.path.exists(extensions_dir):
os.makedirs(extensions_dir)
@@ -38,8 +40,8 @@ class Extension:
self.mtime = 0
self.ctime = 0
def read_info_from_repo(self):
if self.have_info_from_repo:
def read_info(self, force=False):
if self.have_info_from_repo and not force:
return
self.have_info_from_repo = True
repo = None
@@ -116,7 +118,7 @@ class Extension:
self.can_update = False
self.status = "latest"
def fetch_and_reset_hard(self, commit='origin'):
def git_fetch(self, commit='origin'):
repo = git.Repo(self.path)
# Fix: `error: Your local changes to the following files would be overwritten by merge`,
# because WSL2 Docker set 755 file permissions instead of 644, this results to the error.
+1 -1
View File
@@ -472,7 +472,7 @@ def atomically_save_image():
pnginfo_data = PngImagePlugin.PngInfo()
for k, v in params.pnginfo.items():
pnginfo_data.add_text(k, str(v))
image.save(fn, format=image_format, optimize=True, compress_level=9, pnginfo=pnginfo_data if shared.opts.image_metadata else None)
image.save(fn, format=image_format, compress_level=8, pnginfo=pnginfo_data if shared.opts.image_metadata else None)
elif image_format == 'JPEG':
if image.mode == 'RGBA':
shared.log.warning('Saving RGBA image as JPEG: Alpha channel will be lost')
+1 -1
View File
@@ -358,7 +358,7 @@ def parse_prompt_attention(text):
continue
res.append([part, 1.0])
except Exception as e:
log.error(f'Prompt parser: section={text[m.start():m.end()]} position={m.start()}:{m.end()} text={text} error={e}')
log.error(f'Prompt parser: section="{text[m.start():m.end()]}" position={m.start()}:{m.end()} text="{text}" error={e}')
for pos in round_brackets:
multiply_range(pos, round_bracket_multiplier)
for pos in square_brackets:
+7 -2
View File
@@ -49,7 +49,6 @@ def split_attention(layer: nn.Module, tile_size: int=256, min_tile_size: int=256
nhs = possible_tile_sizes(height, tile_size, min_tile_size, swap_size) # possible sub-grids that fit into the image
nws = possible_tile_sizes(width, tile_size, min_tile_size, swap_size)
make_ns = lambda: (nhs[random.randint(0, len(nhs) - 1)], nws[random.randint(0, len(nws) - 1)]) # pylint: disable=unnecessary-lambda-assignment
def reset_nhs():
nonlocal nws, make_ns, ar
ar = height / width # Aspect ratio
@@ -145,6 +144,9 @@ def context_hypertile_vae(p):
shared.log.warning('Hypertile UNet is not compatible with Sub-quadratic cross-attention optimization')
return nullcontext()
vae = getattr(p.sd_model, "vae", None) if shared.backend == shared.Backend.DIFFUSERS else getattr(p.sd_model, "first_stage_model", None)
if height % 8 != 0 or width % 8 != 0:
log.warning(f'Hypertile VAE disabled: width={width} height={height} are not divisible by 8')
return nullcontext()
if vae is None:
shared.log.warning('Hypertile VAE is enabled but no VAE model was found')
return nullcontext()
@@ -168,8 +170,11 @@ def context_hypertile_unet(p):
shared.log.warning('Hypertile UNet is not compatible with Sub-quadratic cross-attention optimization')
return nullcontext()
unet = getattr(p.sd_model, "unet", None) if shared.backend == shared.Backend.DIFFUSERS else getattr(p.sd_model.model, "diffusion_model", None)
if height % 8 != 0 or width % 8 != 0:
log.warning(f'Hypertile UNet disabled: width={width} height={height} are not divisible by 8')
return nullcontext()
if unet is None:
shared.log.warning('Hypertile Unet is enabled but no Unet model was found')
shared.log.warning('Hypertile UNet is enabled but no Unet model was found')
return nullcontext()
else:
shared.log.info(f'Applying hypertile: unet={shared.opts.hypertile_unet_tile}')
+1 -1
View File
@@ -516,7 +516,7 @@ options_templates.update(options_section(('system-paths', "System Paths"), {
options_templates.update(options_section(('saving-images', "Image Options"), {
"samples_save": OptionInfo(True, "Always save all generated images"),
"samples_format": OptionInfo('jpg', 'File format for generated images', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}),
"jpeg_quality": OptionInfo(90, "Quality for saved jpeg images", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}),
"jpeg_quality": OptionInfo(90, "Quality for saved images", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}),
"img_max_size_mp": OptionInfo(250, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}),
"webp_lossless": OptionInfo(False, "Use lossless compression for webp images"),
"save_selected_only": OptionInfo(True, "When using 'Save' button, only save a single selected image"),
+89 -87
View File
@@ -3,7 +3,7 @@ import os.path
import shutil
import errno
import html
from datetime import datetime
from datetime import datetime, timedelta
import git
import gradio as gr
from modules import extensions, shared, paths, errors
@@ -29,26 +29,22 @@ sort_ordering = {
}
def update_extension_list():
def get_installed(ext) -> extensions.Extension:
installed: extensions.Extension = [e for e in extensions.extensions if (e.remote or '').startswith(ext['url'].replace('.git', ''))]
return installed[0] if len(installed) > 0 else None
def list_extensions():
global extensions_list # pylint: disable=global-statement
try:
with open(os.path.join(paths.script_path, "html", "extensions.json"), "r", encoding="utf-8") as f:
extensions_list = json.loads(f.read())
# shared.log.debug(f'Extensions list loaded: {os.path.join(paths.script_path, "html", "extensions.json")}')
except Exception:
shared.log.debug(f'Extensions list failed to load: {os.path.join(paths.script_path, "html", "extensions.json")}')
extensions_list = shared.readfile(os.path.join(paths.script_path, "html", "extensions.json"))
found = []
for ext in extensions.extensions:
ext.read_info_from_repo()
ext.read_info()
for ext in extensions_list:
installed = [extension for extension in extensions.extensions
if extension.git_name == ext['name']
or extension.name == ext['name']
or (extension.remote or '').startswith(ext['url'].replace('.git', ''))]
if len(installed) > 0:
found.append(installed[0])
not_matched = [extension for extension in extensions.extensions if extension not in found]
for ext in not_matched:
installed = get_installed(ext)
if installed:
found.append(installed)
for ext in [e for e in extensions.extensions if e not in found]: # installed but not in index
entry = {
"name": ext.name or "",
"description": ext.description or "",
@@ -82,13 +78,13 @@ def apply_and_restart(disable_list, update_list, disable_all):
if ext.name not in update:
continue
try:
ext.fetch_and_reset_hard()
ext.git_fetch()
except Exception as e:
errors.display(e, f'extensions apply update: {ext.name}')
shared.opts.disabled_extensions = disabled
shared.opts.disable_all_extensions = disable_all
shared.opts.save(shared.config_filename)
shared.restart_server(restart=True)
# shared.restart_server(restart=True)
def check_updates(_id_task, disable_list, search_text, sort_column):
@@ -103,8 +99,8 @@ def check_updates(_id_task, disable_list, search_text, sort_column):
try:
ext.check_updates()
if ext.can_update:
ext.fetch_and_reset_hard()
ext.read_info_from_repo()
ext.git_fetch()
ext.read_info()
commit_date = ext.commit_date or 1577836800
shared.log.info(f'Extensions updated: {ext.name} {ext.commit_hash[:8]} {datetime.utcfromtimestamp(commit_date)}')
else:
@@ -116,7 +112,7 @@ def check_updates(_id_task, disable_list, search_text, sort_column):
except Exception as e:
errors.display(e, f'extensions check update: {ext.name}')
shared.state.nextjob()
return refresh_extensions_list_from_data(search_text, sort_column), "Extension update complete | Restart required"
return create_html(search_text, sort_column), "Extension update complete | Restart required"
def make_commit_link(commit_hash, remote, text=None):
@@ -155,8 +151,7 @@ def install_extension_from_url(dirname, url, branch_name, search_text, sort_colu
url = url.replace('.git', '')
try:
shutil.rmtree(tmpdir, True)
if not branch_name:
# if no branch is specified, use the default branch
if not branch_name: # if no branch is specified, use the default branch
with git.Repo.clone_from(url, tmpdir, filter=['blob:none']) as repo:
repo.remote().fetch()
for submodule in repo.submodules:
@@ -176,7 +171,7 @@ def install_extension_from_url(dirname, url, branch_name, search_text, sort_colu
from launch import run_extension_installer
run_extension_installer(target_dir)
extensions.list_extensions()
return [refresh_extensions_list_from_data(search_text, sort_column), html.escape(f"Extension installed: {target_dir} | Restart required")]
return [create_html(search_text, sort_column), html.escape(f"Extension installed: {target_dir} | Restart required")]
except Exception as e:
shared.log.error(f'Error installing extension: {url} {e}')
finally:
@@ -208,15 +203,15 @@ def uninstall_extension(extension_path, search_text, sort_column):
# extensions.extensions = [extension for extension in extensions.extensions if os.path.abspath(found.path) != os.path.abspath(extension_path)]
except Exception as e:
shared.log.warning(f'Extension uninstall failed: {found.path} {e}')
update_extension_list()
list_extensions()
global extensions_list # pylint: disable=global-statement
extensions_list = [ext for ext in extensions_list if ext['name'] != found.name]
shared.log.info(f'Extension uninstalled: {found.path}')
code = refresh_extensions_list_from_data(search_text, sort_column)
code = create_html(search_text, sort_column)
return code, f"Extension uninstalled: {found.path} | Restart required"
else:
shared.log.warning(f'Extension uninstall cannot find extension: {extension_path}')
code = refresh_extensions_list_from_data(search_text, sort_column)
code = create_html(search_text, sort_column)
return code, f"Extension uninstalled failed: {extension_path}"
@@ -229,8 +224,8 @@ def update_extension(extension_path, search_text, sort_column):
try:
ext.check_updates()
if ext.can_update:
ext.fetch_and_reset_hard()
ext.read_info_from_repo()
ext.git_fetch()
ext.read_info()
commit_date = ext.commit_date or 1577836800
shared.log.info(f'Extensions updated: {ext.name} {ext.commit_hash[:8]} {datetime.utcfromtimestamp(commit_date)}')
else:
@@ -244,7 +239,7 @@ def update_extension(extension_path, search_text, sort_column):
errors.display(e, f'extensions check update: {ext.name}')
shared.log.debug(f'Extensions update finish: {ext.name} {ext.commit_hash} {ext.commit_date}')
shared.state.nextjob()
return refresh_extensions_list_from_data(search_text, sort_column), f"Extension updated | {extension_path} | Restart required"
return create_html(search_text, sort_column), f"Extension updated | {extension_path} | Restart required"
def refresh_extensions_list(search_text, sort_column):
@@ -260,21 +255,22 @@ def refresh_extensions_list(search_text, sort_column):
shared.log.debug(f'Updated extensions list: {len(extensions_list)} {extensions_index}')
except Exception as e:
shared.log.warning(f'Updated extensions list failed: {extensions_index} {e}')
update_extension_list()
code = refresh_extensions_list_from_data(search_text, sort_column)
list_extensions()
code = create_html(search_text, sort_column)
return code, f'Extensions | {len(extensions.extensions)} registered | {len(extensions_list)} available'
def search_extensions(search_text, sort_column):
code = refresh_extensions_list_from_data(search_text, sort_column)
code = create_html(search_text, sort_column)
return code, f'Search | {search_text} | {sort_column}'
def refresh_extensions_list_from_data(search_text, sort_column):
def create_html(search_text, sort_column):
# shared.log.debug(f'Extensions manager: refresh list search="{search_text}" sort="{sort_column}"')
code = """
<table id="extensions">
<colgroup>
<col style="width: 1%; background: var(--table-border-color)">
<col style="width: 1%; background: var(--table-border-color)">
<col style="width: 20%; background: var(--table-border-color)">
<col style="width: 59%;">
@@ -284,6 +280,7 @@ def refresh_extensions_list_from_data(search_text, sort_column):
</colgroup>
<thead style="font-size: 110%; border-style: solid; border-bottom: 1px var(--button-primary-border-color) solid">
<tr>
<th>Status</th>
<th>Enabled</th>
<th>Extension</th>
<th>Description</th>
@@ -294,65 +291,46 @@ def refresh_extensions_list_from_data(search_text, sort_column):
</thead>
<tbody>"""
if len(extensions_list) == 0:
update_extension_list()
list_extensions()
for ext in extensions_list:
extension = [e for e in extensions.extensions
if (e.name == ext['name'])
or (e.git_name == ext['name'])
or (e.remote == ext['url'])]
if len(extension) > 0:
extension[0].read_info_from_repo()
ext['installed'] = len(extension) > 0
ext['commit_date'] = extension[0].commit_date if len(extension) > 0 else 1577836800
ext['is_builtin'] = extension[0].is_builtin if len(extension) > 0 else False
ext['version'] = extension[0].version if len(extension) > 0 else ''
ext['enabled'] = extension[0].enabled if len(extension) > 0 else ''
ext['remote'] = extension[0].remote if len(extension) > 0 else None
ext['path'] = extension[0].path if len(extension) > 0 else ''
installed = get_installed(ext)
ext['installed'] = installed is not None
ext['commit_date'] = installed.commit_date if installed is not None else 1577836800
ext['is_builtin'] = installed.is_builtin if installed is not None else False
ext['version'] = installed.version if installed is not None else ''
ext['enabled'] = installed.enabled if installed is not None else ''
ext['remote'] = installed.remote if installed is not None else None
ext['path'] = installed.path if installed is not None else ''
ext['sort_default'] = f"{'1' if ext['is_builtin'] else '0'}{'1' if ext['installed'] else '0'}{ext.get('updated', '2000-01-01T00:00')}"
sort_reverse, sort_function = sort_ordering[sort_column]
def dt(x: str):
val = ext.get(x, None)
if val is not None:
return datetime.fromisoformat(val[:-1]).strftime('%a %b%d %Y %H:%M')
else:
return "N/A"
return datetime.fromisoformat(val[:-1]).strftime('%a %b%d %Y %H:%M') if val is not None else "N/A"
stats = { 'processed': 0, 'enabled': 0, 'hidden': 0, 'installed': 0 }
for ext in sorted(extensions_list, key=sort_function, reverse=sort_reverse):
name = ext.get("name", "unknown")
added = dt('added')
created = dt('created')
pushed = dt('pushed')
updated = dt('updated')
url = ext.get('url', None)
size = ext.get('size', 0)
stars = ext.get('stars', 0)
issues = ext.get('issues', 0)
commits = ext.get('commits', 0)
description = ext.get("description", "")
installed = ext.get("installed", False)
enabled = ext.get("enabled", False)
path = ext.get("path", "")
remote = ext.get("remote", None)
commit_date = ext.get("commit_date", 1577836800) or 1577836800
update_available = (remote is not None) & (installed) & (datetime.utcfromtimestamp(commit_date + 60 * 60) < datetime.fromisoformat(ext.get('updated', '2000-01-01T00:00:00.000Z')[:-1]))
installed = get_installed(ext)
author = f"Author: {ext['url'].split('/')[3]}" if 'github' in ext['url'] else ''
updated = datetime.timestamp(datetime.fromisoformat(ext.get('updated', '2000-01-01T00:00:00.000Z')))
update_available = (installed is not None) and (ext['remote'] is not None) and (ext['commit_date'] + 60 * 60 < updated)
ext['sort_user'] = f"{'0' if ext['is_builtin'] else '1'}{'1' if ext['installed'] else '0'}{ext.get('name', '')}"
ext['sort_enabled'] = f"{'0' if ext['enabled'] else '1'}{'1' if ext['is_builtin'] else '0'}{'1' if ext['installed'] else '0'}{ext.get('updated', '2000-01-01T00:00')}"
ext['sort_update'] = f"{'1' if update_available else '0'}{'1' if ext['installed'] else '0'}{ext.get('updated', '2000-01-01T00:00')}"
timedelta = datetime.now() - datetime.fromisoformat(ext.get('created', '2000-01-01T00:00Z')[:-1])
ext['sort_trending'] = round(stars / max(timedelta.days, 5), 1)
delta = datetime.now() - datetime.fromisoformat(ext.get('created', '2000-01-01T00:00Z')[:-1])
ext['sort_trending'] = round(ext.get('stars', 0) / max(delta.days, 5), 1)
tags = ext.get("tags", [])
if not isinstance(tags, list):
tags = tags.split(' ')
tags_string = ' '.join(tags)
tags = tags + ["installed"] if installed else tags
tags = [t for t in tags if t.strip() != '']
if len([x for x in tags if x in hide_tags]) > 0:
continue
visible = 'table-row'
if search_text and search_text.strip():
if search_text.lower() not in html.escape(name).lower() and search_text.lower() not in html.escape(description).lower() and search_text.lower() not in html.escape(tags_string).lower():
if search_text:
s = search_text.strip().lower()
if s not in html.escape(ext.get("name", "unknown")).lower() and s not in html.escape(ext.get("description", "")).lower() and s not in html.escape(tags_string).lower() and s not in author.lower():
stats['hidden'] += 1
visible = 'none'
stats['processed'] += 1
@@ -362,33 +340,57 @@ def refresh_extensions_list_from_data(search_text, sort_column):
enabled_code = ''
if installed:
stats['installed'] += 1
if enabled:
if ext.get("enabled", False):
stats['enabled'] += 1
type_code = f"""<div class="type">{"SYSTEM" if ext['is_builtin'] else 'USER'}</div>"""
version_code = f"""<div class="version" style="background: {"--input-border-color-focus" if update_available else "inherit"}">{ext['version']}</div>"""
enabled_code = f"""<input class="gr-check-radio gr-checkbox" name="enable_{html.escape(name)}" type="checkbox" {'checked="checked"' if enabled else ''}>"""
masked_path = html.escape(path.replace('\\', '/'))
enabled_code = f"""<input class="gr-check-radio gr-checkbox" name="enable_{html.escape(ext.get("name", "unknown"))}" type="checkbox" {'checked="checked"' if ext.get("enabled", False) else ''}>"""
masked_path = html.escape(ext.get("path", "").replace('\\', '/'))
if not ext['is_builtin']:
install_code = f"""<button onclick="uninstall_extension(this, '{masked_path}')" class="lg secondary gradio-button custom-button extension-button">uninstall</button>"""
if update_available:
install_code += f"""<button onclick="update_extension(this, '{masked_path}')" class="lg secondary gradio-button custom-button extension-button">update</button>"""
else:
install_code = f"""<button onclick="install_extension(this, '{html.escape(url)}')" class="lg secondary gradio-button custom-button extension-button">install</button>"""
install_code = f"""<button onclick="install_extension(this, '{html.escape(ext.get('url', ''))}')" class="lg secondary gradio-button custom-button extension-button">install</button>"""
tags_text = ", ".join([f"<span class='extension-tag'>{x}</span>" for x in tags])
if ext.get('status', None) is None or type(ext['status']) == str: # old format
ext['status'] = 0
if ext['url'] is None or ext['url'] == '':
status = "<span style='cursor:pointer;color:#00C0FD' title='Local'>⬤</span>"
elif ext['status'] > 0:
if ext['status'] == 1:
status = "<span style='cursor:pointer;color:#00FD9C ' title='Verified'>⬤</span>"
elif ext['status'] == 2:
status = "<span style='cursor:pointer;color:#FFC300' title='Supported only with backend:Original'>⬤</span>"
elif ext['status'] == 3:
status = "<span style='cursor:pointer;color:#FFC300' title='Supported only with backend:Diffusers'>⬤</span>"
elif ext['status'] == 4:
status = f"<span style='cursor:pointer;color:#4E22FF' title=\"{ext.get('note', 'custom value')}\">⬤</span>"
elif ext['status'] == 5:
status = "<span style='cursor:pointer;color:#CE0000' title='Not supported'>⬤</span>"
else:
status = "<span style='cursor:pointer;color:#008EBC' title='Unknown status'>⬤</span>"
else:
if updated < datetime.timestamp(datetime.now() - timedelta(6*30)):
status = "<span style='cursor:pointer;color:#C000CF' title='unmaintained'>⬤</span>"
else:
status = "<span style='cursor:pointer;color:#7C7C7C'>⬤</span>"
code += f"""
<tr style="display: {visible}">
<td>{status}</td>
<td{' class="extension_status"' if ext['installed'] else ''}>{enabled_code}</td>
<td><a href="{html.escape(url)}" target="_blank" class="name">{html.escape(name)}</a><br>{tags_text}</td>
<td>{html.escape(description)}
<p class="info"><span class="date">Created {html.escape(created)} | Added {html.escape(added)} | Pushed {html.escape(pushed)} | Updated {html.escape(updated)}</span></p>
<p class="info"><span class="date">Stars {html.escape(str(stars))} | Size {html.escape(str(size))} | Commits {html.escape(str(commits))} | Issues {html.escape(str(issues))} | Trending {html.escape(str(ext['sort_trending']))}</span></p>
<td><a href="{html.escape(ext.get('url', ''))}" title={html.escape(ext.get('note', ''))} target="_blank" class="name">{html.escape(ext.get("name", "unknown"))}</a><br>{tags_text}</td>
<td>{html.escape(ext.get("description", ""))}
<p class="info"><span class="date">Created {html.escape(dt('created'))} | Added {html.escape(dt('added'))} | Pushed {html.escape(dt('pushed'))} | Updated {html.escape(dt('updated'))}</span></p>
<p class="info"><span class="date">{author} | Stars {html.escape(str(ext.get('stars', 0)))} | Size {html.escape(str(ext.get('size', 0)))} | Commits {html.escape(str(ext.get('commits', 0)))} | Issues {html.escape(str(ext.get('issues', 0)))} | Trending {html.escape(str(ext['sort_trending']))}</span></p>
</td>
<td>{type_code}</td>
<td>{version_code}</td>
<td>{install_code}</td>
</tr>"""
code += "</tbody></table>"
shared.log.debug(f'Extension list refresh: processed={stats["processed"]} installed={stats["installed"]} enabled={stats["enabled"]} disabled={stats["installed"] - stats["enabled"]} visible={stats["processed"] - stats["hidden"]} hidden={stats["hidden"]}')
shared.log.debug(f'Extension list: processed={stats["processed"]} installed={stats["installed"]} enabled={stats["enabled"]} disabled={stats["installed"] - stats["enabled"]} visible={stats["processed"] - stats["hidden"]} hidden={stats["hidden"]}')
return code
@@ -412,10 +414,10 @@ def create_ui():
sort_column = gr.Dropdown(value="default", label="Sort by", choices=list(sort_ordering.keys()), multiselect=False)
with gr.Column(scale=1):
refresh_extensions_button = gr.Button(value="Refresh extension list", variant="primary")
check = gr.Button(value="Update installed extensions", variant="primary")
apply = gr.Button(value="Apply changes & restart server", variant="primary")
update_extension_list()
extensions_table = gr.HTML(refresh_extensions_list_from_data(search_text.value, sort_column.value))
check = gr.Button(value="Update all installed", variant="primary")
apply = gr.Button(value="Apply changes", variant="primary")
list_extensions()
extensions_table = gr.HTML(create_html(search_text.value, sort_column.value))
check.click(
fn=wrap_gradio_gpu_call(check_updates, extra_outputs=[gr.update()]),
_js="extensions_check",