Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

redo ui model metadata

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-08-02 18:46:40 -04:00
parent e90ac68dcc
commit 265cb5e8ba
7 changed files with 160 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -19,6 +19,7 @@
- updated real-time hints, thanks @CalamitousFelicitousness
- updated *models -> current* tab
- updated *models -> list models* tab
- updated *models -> metadata* tab
- more css optimizations and styling
- **Offloading**
- changed **default** values for offloading based on detected gpu memory
javascript/sdnext.css
+4
View File
@@ -1974,6 +1974,10 @@ div:has(>#tab-gallery-folders) {
max-height: 50vh;
}
#civit_metadata {
overflow: auto;
}
.simple-table tr {
vertical-align: baseline;
}
modules/modelloader.py
+26 -34
View File
@@ -48,24 +48,6 @@ def hf_login(token=None):
return True
def download_civit_meta(model_path: str, model_id):
fn = os.path.splitext(model_path)[0] + '.json'
url = f'https://civitai.com/api/v1/models/{model_id}'
r = shared.req(url)
if r.status_code == 200:
try:
shared.writefile(r.json(), filename=fn, mode='w', silent=True)
msg = f'CivitAI download: id={model_id} url={url} file="{fn}"'
shared.log.info(msg)
return msg
except Exception as e:
msg = f'CivitAI download error: id={model_id} url={url} file="{fn}" {e}'
errors.display(e, 'CivitAI download error')
shared.log.error(msg)
return msg
return f'CivitAI download error: id={model_id} url={url} code={r.status_code}'
def save_video_frame(filepath: str):
from modules import video
try:
@@ -83,21 +65,38 @@ def save_video_frame(filepath: str):
return frame
def download_civit_meta(model_path: str, model_id):
fn = os.path.splitext(model_path)[0] + '.json'
url = f'https://civitai.com/api/v1/models/{model_id}'
r = shared.req(url)
if r.status_code == 200:
try:
data = r.json()
shared.writefile(data, filename=fn, mode='w', silent=True)
shared.log.info(f'CivitAI download: id={model_id} url={url} file="{fn}"')
return r.status_code, len(data), '' # code/size/note
except Exception as e:
errors.display(e, 'civitai meta')
shared.log.error(f'CivitAI meta: id={model_id} url={url} file="{fn}" {e}')
return r.status_code, '', str(e)
return r.status_code, '', ''
def download_civit_preview(model_path: str, preview_url: str):
global pbar # pylint: disable=global-statement
if model_path is None:
pbar = None
return ''
return 500, '', ''
ext = os.path.splitext(preview_url)[1]
preview_file = os.path.splitext(model_path)[0] + ext
is_video = preview_file.lower().endswith('.mp4')
is_json = preview_file.lower().endswith('.json')
if is_json:
shared.log.warning(f'CivitAI download: url="{preview_url}" skip json')
return 'CivitAI download error: JSON file'
return 500, '', 'exepected preview image got json'
if os.path.exists(preview_file):
return ''
res = f'CivitAI download: url={preview_url} file="{preview_file}"'
return 304, '', 'already exists'
# res = f'CivitAI download: url={preview_url} file="{preview_file}"'
r = shared.req(preview_url, stream=True)
total_size = int(r.headers.get('content-length', 0))
block_size = 16384 # 16KB blocks
@@ -116,21 +115,20 @@ def download_civit_preview(model_path: str, preview_url: str):
pbar.update(task, advance=block_size)
if written < 1024: # min threshold
os.remove(preview_file)
raise ValueError(f'removed invalid download: bytes={written}')
return 400, '', 'removed invalid download'
if is_video:
img = save_video_frame(preview_file)
else:
img = Image.open(preview_file)
except Exception as e:
# os.remove(preview_file)
res += f' error={e}'
shared.log.error(f'CivitAI download error: url={preview_url} file="{preview_file}" written={written} {e}')
return 500, '', str(e)
shared.state.end()
if img is None:
return res
shared.log.info(f'{res} size={total_size} image={img.size}')
return 500, '', 'image is none'
shared.log.info(f'CivitAI download: url={preview_url} file="{preview_file}" size={total_size} image={img.size}')
img.close()
return res
return 200, str(total_size), '' # code/size/note
download_pbar = None
@@ -201,12 +199,6 @@ def download_civit_model_thread(model_name: str, model_url: str, model_path: str
if written < 1024: # min threshold
os.remove(temp_file)
raise ValueError(f'removed invalid download: bytes={written}')
"""
if preview is not None:
preview_file = os.path.splitext(model_file)[0] + '.jpg'
preview.save(preview_file)
res += f' preview={preview_file}'
"""
except Exception as e:
shared.log.error(f'{res} {e}')
finally:
modules/models_civitai.py
+118 -55
View File
@@ -8,12 +8,12 @@ from modules.shared import log, opts, req, readfile, max_workers
data = []
selected_model = None
update_data = []
class CivitModel:
def __init__(self, name, fn, sha = None, meta = {}):
self.name = name
self.file = name
self.id = meta.get('id', 0)
self.fn = fn
self.sha = sha
@@ -25,28 +25,61 @@ class CivitModel:
self.latest_name = ''
self.url = None
self.status = 'Not found'
def array(self):
return [self.id, self.fn, self.name, self.versions, self.vername, self.latest, self.status]
def civit_update_metadata():
def create_update_metadata_table(rows: list[CivitModel]):
html = """
<table class="simple-table">
<thead">
<tr>
<th>ID</th>
<th>File</th>
<th>Name</th>
<th>Hash</th>
<th>Versions</th>
<th>Latest</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{tbody}
</tbody>
</table>
"""
tbody = ''
for row in rows:
try:
tbody += f"""
<tr>
<td>{row.id}</td>
<td>{row.file}</td>
<td>{row.name}</td>
<td>{row.sha}</td>
<td>{row.versions}</td>
<td>{row.latest}</td>
<td>{row.status}</td>
</tr>
"""
except Exception as e:
log.error(f'Model list: row={row} {e}')
return html.format(tbody=tbody)
log.debug('CivitAI update metadata: models')
from modules import ui_extra_networks, modelloader
res = []
pages = ui_extra_networks.get_pages('Model')
if len(pages) == 0:
return 'CivitAI update metadata: no models found'
page: ui_extra_networks.ExtraNetworksPage = pages[0]
table_data = []
update_data.clear()
results = []
all_hashes = [(item.get('hash', None) or 'XXXXXXXX').upper()[:8] for item in page.list_items()]
for item in page.list_items():
model = CivitModel(name=item['name'], fn=item['filename'], sha=item.get('hash', None), meta=item.get('metadata', {}))
if model.sha is None or len(model.sha) == 0:
res.append(f'CivitAI skip search: name="{model.name}" hash=None')
log.debug(f'CivitAI skip search: name="{model.name}" hash=None')
else:
r = req(f'https://civitai.com/api/v1/model-versions/by-hash/{model.sha}')
res.append(f'CivitAI search: name="{model.name}" hash={model.sha} status={r.status_code}')
log.debug(f'CivitAI search: name="{model.name}" hash={model.sha} status={r.status_code}')
if r.status_code == 200:
d = r.json()
model.id = d['modelId']
@@ -76,31 +109,9 @@ def civit_update_metadata():
else:
model.status = 'Available'
break
log.debug(res[-1])
update_data.append(model)
table_data.append(model.array())
yield gr.update(value=table_data), '<br>'.join([r for r in res if len(r) > 0])
return '<br>'.join([r for r in res if len(r) > 0])
def civit_update_select(evt: gr.SelectData, in_data):
global selected_model # pylint: disable=global-statement
try:
selected_model = next([m for m in update_data if m.fn == in_data[evt.index[0]][1]])
except Exception:
selected_model = None
if selected_model is None or selected_model.url is None or selected_model.status != 'Available':
return [gr.update(value='Model update not available'), gr.update(visible=False)]
else:
return [gr.update(), gr.update(visible=True)]
def civit_update_download():
if selected_model is None or selected_model.url is None or selected_model.status != 'Available':
return 'Model update not available'
if selected_model.latest_name is None or len(selected_model.latest_name) == 0:
model_name = f'{selected_model.name} {selected_model.latest}.safetensors'
else:
model_name = selected_model.latest_name
return civit_download_model(selected_model.url, model_name, model_path='', model_type='Model')
results.append(model)
yield create_update_metadata_table(results)
return create_update_metadata_table(results)
def civit_search_model(name, tag, model_type):
@@ -211,56 +222,106 @@ def civit_download_model(model_url: str, model_name: str, model_path: str, model
return res
def atomic_civit_search_metadata(item, res, rehash):
def atomic_civit_search_metadata(item, results):
from modules.modelloader import download_civit_preview, download_civit_meta
if item is None:
return
return results
meta = os.path.splitext(item['filename'])[0] + '.json'
has_meta = os.path.isfile(meta) and os.stat(meta).st_size > 0
if ('card-no-preview.png' in item['preview'] or not has_meta) and os.path.isfile(item['filename']):
sha = item.get('hash', None)
found = False
result = {
'id': '',
'name': item['name'],
'type': '',
'hash': '',
'code': '',
'size': '',
'note': '',
}
if sha is not None and len(sha) > 0:
r = req(f'https://civitai.com/api/v1/model-versions/by-hash/{sha}')
log.debug(f'CivitAI search: name="{item["name"]}" hash={sha} status={r.status_code}')
result['hash'] = sha
result['code'] = r.status_code
if r.status_code == 200:
d = r.json()
res.append(download_civit_meta(item['filename'], d['modelId']))
result['code'], result['size'], result['note'] = download_civit_meta(item['filename'], d['modelId'])
result['id'] = d['modelId']
result['type'] = 'metadata'
results.append(result)
if d.get('images') is not None:
for i in d['images']:
preview_url = i['url']
img_res = download_civit_preview(item['filename'], preview_url)
res.append(img_res)
if 'error' not in img_res:
result['code'], result['size'], result['note'] = download_civit_preview(item['filename'], i['url'])
if result['code'] == 200:
result['type'] = 'preview'
results.append(result)
found = True
break
if not found and rehash and os.stat(item['filename']).st_size < (1024 * 1024 * 1024):
if not found and os.stat(item['filename']).st_size < (1024 * 1024 * 1024):
from modules import hashes
sha = hashes.calculate_sha256(item['filename'], quiet=True)[:10]
r = req(f'https://civitai.com/api/v1/model-versions/by-hash/{sha}')
log.debug(f'CivitAI search: name="{item["name"]}" hash={sha} status={r.status_code}')
result['hash'] = sha
result['code'] = r.status_code
if r.status_code == 200:
d = r.json()
res.append(download_civit_meta(item['filename'], d['modelId']))
result['code'], result['size'], result['note'] = download_civit_meta(item['filename'], d['modelId'])
result['id'] = d['modelId']
result['type'] = 'metadata'
results.append(result)
if d.get('images') is not None:
for i in d['images']:
preview_url = i['url']
img_res = download_civit_preview(item['filename'], preview_url)
res.append(img_res)
if 'error' not in img_res:
result['code'], result['size'], result['note'] = download_civit_preview(item['filename'], i['url'])
if result['code'] == 200:
result['type'] = 'preview'
results.append(result)
found = True
break
if not found:
results.append(result)
def civit_search_metadata(title: str = None):
def create_search_metadata_table(rows):
html = """
<table class="simple-table">
<thead">
<tr><th>ID</th><th>Name</th><th>Type</th><th>Code</th><th>Hash</th><th>Size</th><th>Note</th></tr>
</thead>
<tbody>
{tbody}
</tbody>
</table>
"""
tbody = ''
for row in rows:
try:
tbody += f"""
<tr>
<td>{row['id']}</td>
<td>{row['name']}</td>
<td>{row['type']}</td>
<td>{row['code']}</td>
<td>{row['hash']}</td>
<td>{row['size']}</td>
<td>{row['note']}</td>
</tr>
"""
except Exception as e:
log.error(f'Model list: row={row} {e}')
return html.format(tbody=tbody)
def civit_search_metadata(rehash, title):
log.debug(f'CivitAI search metadata: type={title if type(title) == str else "all"}')
from modules.ui_extra_networks import get_pages
res = []
results = []
scanned, skipped = 0, 0
t0 = time.time()
candidates = []
re_skip = [r.strip() for r in opts.extra_networks_scan_skip.split(',') if len(r.strip()) > 0]
log.debug(f'CivitAI search metadata: skip={re_skip}')
log.debug(f'CivitAI search metadata: type={title if type(title) == str else "all"} skip={re_skip}')
for page in get_pages():
if type(title) == str:
if page.title != title:
@@ -275,16 +336,18 @@ def civit_search_metadata(rehash, title):
continue
scanned += 1
candidates.append(item)
# atomic_civit_search_metadata(item, res, rehash)
import concurrent
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_items = {}
for fn in candidates:
executor.submit(atomic_civit_search_metadata, fn, res, rehash)
atomic_civit_search_metadata(None, res, rehash)
future_items[executor.submit(atomic_civit_search_metadata, fn, results)] = fn
for future in concurrent.futures.as_completed(future_items):
future.result()
yield create_search_metadata_table(results)
t1 = time.time()
log.debug(f'CivitAI search metadata: scanned={scanned} skipped={skipped} time={t1-t0:.2f}')
txt = '<br>'.join([r for r in res if len(r) > 0])
return txt
return create_search_metadata_table(results)
def civitai_update_token(token):
modules/shared.py
+1 -1
View File
@@ -57,7 +57,7 @@ restricted_opts = {
"outdir_init_images"
}
resize_modes = ["None", "Fixed", "Crop", "Fill", "Outpaint", "Context aware"]
max_workers = 8
max_workers = 12
default_hfcache_dir = os.environ.get("SD_HFCACHEDIR", None) or os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')
sdnq_quant_modes = ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"]
state = shared_state.State()
modules/ui_extra_networks.py
+1 -1
View File
@@ -933,7 +933,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
def ui_scan_click(title):
from modules.models_civitai import civit_search_metadata
civit_search_metadata(True, title)
civit_search_metadata(title)
return ui_refresh_click(title)
def ui_save_click():
modules/ui_models.py
+9 -27
View File
@@ -68,7 +68,7 @@ def create_ui():
return [html, meta]
with gr.Row():
gr.HTML('<h2>&nbspAnalyze currently loaded model<br></h2>')
gr.HTML('<h2>Analyze currently loaded model<br></h2>')
with gr.Row():
model_analyze = gr.Button(value="Analyze", variant='primary')
with gr.Row():
@@ -127,7 +127,7 @@ def create_ui():
return html.format(tbody=tbody)
with gr.Row():
gr.HTML('<h2>&nbspList models <br></h2>')
gr.HTML('<h2>List all locally available models</h2><br>')
with gr.Row():
model_list_btn = gr.Button(value="List models", variant='primary')
model_checkhash_btn = gr.Button(value="Calculate missing hashes", variant='secondary')
@@ -138,35 +138,17 @@ def create_ui():
model_list_btn.click(fn=lambda: create_models_table(sd_models.checkpoints_list.values()), inputs=[], outputs=[model_table])
with gr.Tab(label="Metadata"):
from modules.models_civitai import civit_search_metadata, civit_update_metadata, civit_update_select, civit_update_download
from modules.models_civitai import civit_search_metadata, civit_update_metadata
with gr.Row():
gr.HTML('<h2>&nbspCivitAI fetch metadata<br></h2>')
gr.HTML('Fetches preview and metadata information for models with missing information<br>Models with existing previews and information are not updated<br>')
gr.HTML('<h2>Fetch model preview metadata</h2><br>')
with gr.Row():
civit_previews_btn = gr.Button(value="Start", variant='primary')
civit_previews_btn = gr.Button(value="Scan missing", variant='primary')
civit_update_btn = gr.Button(value="Update all", variant='primary')
with gr.Row():
civit_previews_rehash = gr.Checkbox(value=True, label="Check alternative hash")
civit_previews_btn.click(fn=civit_search_metadata, inputs=[civit_previews_rehash, civit_previews_rehash], outputs=[models_outcome])
civit_metadata = gr.HTML(value='', elem_id="civit_metadata")
civit_previews_btn.click(fn=civit_search_metadata, inputs=[], outputs=[civit_metadata])
civit_update_btn.click(fn=civit_update_metadata, inputs=[], outputs=[civit_metadata])
with gr.Row():
gr.HTML('<h2>&nbspScan CivitAI for information on latest available model versions<br></h2>')
with gr.Row():
civit_update_btn = gr.Button(value="Update", variant='primary')
with gr.Row():
gr.HTML('<h2>Update scan results</h2>')
with gr.Row():
civit_headers4 = ['ID', 'File', 'Name', 'Versions', 'Current', 'Latest', 'Update']
civit_types4 = ['number', 'str', 'str', 'number', 'str', 'str', 'str']
civit_widths4 = ['10%', '25%', '25%', '5%', '10%', '10%', '15%']
civit_results4 = gr.DataFrame(value=None, label=None, show_label=False, interactive=False, wrap=True, row_count=20, headers=civit_headers4, datatype=civit_types4, type='array', column_widths=civit_widths4)
with gr.Row():
gr.HTML('<h3>Select model from the list and download update if available</h3>')
with gr.Row():
civit_update_download_btn = gr.Button(value="Download", variant='primary', visible=False)
civit_update_btn.click(fn=civit_update_metadata, inputs=[], outputs=[civit_results4, models_outcome])
civit_results4.select(fn=civit_update_select, inputs=[civit_results4], outputs=[models_outcome, civit_update_download_btn])
civit_update_download_btn.click(fn=civit_update_download, inputs=[], outputs=[models_outcome])
with gr.Tab(label="Loader"):
from modules import ui_models_load