diff --git a/CHANGELOG.md b/CHANGELOG.md index 72e71faec..0b7dab10f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - updated real-time hints, thanks @CalamitousFelicitousness - updated *models -> current* tab - updated *models -> list models* tab + - updated *models -> metadata* tab - more css optimizations and styling - **Offloading** - changed **default** values for offloading based on detected gpu memory diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 1396cf034..ee55b8785 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -1974,6 +1974,10 @@ div:has(>#tab-gallery-folders) { max-height: 50vh; } +#civit_metadata { + overflow: auto; +} + .simple-table tr { vertical-align: baseline; } diff --git a/modules/modelloader.py b/modules/modelloader.py index 5ad1d540b..9813b5ee5 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -48,24 +48,6 @@ def hf_login(token=None): return True -def download_civit_meta(model_path: str, model_id): - fn = os.path.splitext(model_path)[0] + '.json' - url = f'https://civitai.com/api/v1/models/{model_id}' - r = shared.req(url) - if r.status_code == 200: - try: - shared.writefile(r.json(), filename=fn, mode='w', silent=True) - msg = f'CivitAI download: id={model_id} url={url} file="{fn}"' - shared.log.info(msg) - return msg - except Exception as e: - msg = f'CivitAI download error: id={model_id} url={url} file="{fn}" {e}' - errors.display(e, 'CivitAI download error') - shared.log.error(msg) - return msg - return f'CivitAI download error: id={model_id} url={url} code={r.status_code}' - - def save_video_frame(filepath: str): from modules import video try: @@ -83,21 +65,38 @@ def save_video_frame(filepath: str): return frame +def download_civit_meta(model_path: str, model_id): + fn = os.path.splitext(model_path)[0] + '.json' + url = f'https://civitai.com/api/v1/models/{model_id}' + r = shared.req(url) + if r.status_code == 200: + try: + data = r.json() + shared.writefile(data, filename=fn, mode='w', silent=True) + shared.log.info(f'CivitAI download: id={model_id} url={url} file="{fn}"') + return r.status_code, len(data), '' # code/size/note + except Exception as e: + errors.display(e, 'civitai meta') + shared.log.error(f'CivitAI meta: id={model_id} url={url} file="{fn}" {e}') + return r.status_code, '', str(e) + return r.status_code, '', '' + + def download_civit_preview(model_path: str, preview_url: str): global pbar # pylint: disable=global-statement if model_path is None: pbar = None - return '' + return 500, '', '' ext = os.path.splitext(preview_url)[1] preview_file = os.path.splitext(model_path)[0] + ext is_video = preview_file.lower().endswith('.mp4') is_json = preview_file.lower().endswith('.json') if is_json: shared.log.warning(f'CivitAI download: url="{preview_url}" skip json') - return 'CivitAI download error: JSON file' + return 500, '', 'exepected preview image got json' if os.path.exists(preview_file): - return '' - res = f'CivitAI download: url={preview_url} file="{preview_file}"' + return 304, '', 'already exists' + # res = f'CivitAI download: url={preview_url} file="{preview_file}"' r = shared.req(preview_url, stream=True) total_size = int(r.headers.get('content-length', 0)) block_size = 16384 # 16KB blocks @@ -116,21 +115,20 @@ def download_civit_preview(model_path: str, preview_url: str): pbar.update(task, advance=block_size) if written < 1024: # min threshold os.remove(preview_file) - raise ValueError(f'removed invalid download: bytes={written}') + return 400, '', 'removed invalid download' if is_video: img = save_video_frame(preview_file) else: img = Image.open(preview_file) except Exception as e: - # os.remove(preview_file) - res += f' error={e}' shared.log.error(f'CivitAI download error: url={preview_url} file="{preview_file}" written={written} {e}') + return 500, '', str(e) shared.state.end() if img is None: - return res - shared.log.info(f'{res} size={total_size} image={img.size}') + return 500, '', 'image is none' + shared.log.info(f'CivitAI download: url={preview_url} file="{preview_file}" size={total_size} image={img.size}') img.close() - return res + return 200, str(total_size), '' # code/size/note download_pbar = None @@ -201,12 +199,6 @@ def download_civit_model_thread(model_name: str, model_url: str, model_path: str if written < 1024: # min threshold os.remove(temp_file) raise ValueError(f'removed invalid download: bytes={written}') - """ - if preview is not None: - preview_file = os.path.splitext(model_file)[0] + '.jpg' - preview.save(preview_file) - res += f' preview={preview_file}' - """ except Exception as e: shared.log.error(f'{res} {e}') finally: diff --git a/modules/models_civitai.py b/modules/models_civitai.py index 9b2f5420e..15fc58011 100644 --- a/modules/models_civitai.py +++ b/modules/models_civitai.py @@ -8,12 +8,12 @@ from modules.shared import log, opts, req, readfile, max_workers data = [] selected_model = None -update_data = [] class CivitModel: def __init__(self, name, fn, sha = None, meta = {}): self.name = name + self.file = name self.id = meta.get('id', 0) self.fn = fn self.sha = sha @@ -25,28 +25,61 @@ class CivitModel: self.latest_name = '' self.url = None self.status = 'Not found' - def array(self): - return [self.id, self.fn, self.name, self.versions, self.vername, self.latest, self.status] def civit_update_metadata(): + def create_update_metadata_table(rows: list[CivitModel]): + html = """ + + + + + + + + + + + + + + {tbody} + +
IDFileNameHashVersionsLatestStatus
+ """ + tbody = '' + for row in rows: + try: + tbody += f""" + + {row.id} + {row.file} + {row.name} + {row.sha} + {row.versions} + {row.latest} + {row.status} + + """ + except Exception as e: + log.error(f'Model list: row={row} {e}') + return html.format(tbody=tbody) + log.debug('CivitAI update metadata: models') from modules import ui_extra_networks, modelloader - res = [] pages = ui_extra_networks.get_pages('Model') if len(pages) == 0: return 'CivitAI update metadata: no models found' page: ui_extra_networks.ExtraNetworksPage = pages[0] - table_data = [] - update_data.clear() + results = [] all_hashes = [(item.get('hash', None) or 'XXXXXXXX').upper()[:8] for item in page.list_items()] for item in page.list_items(): model = CivitModel(name=item['name'], fn=item['filename'], sha=item.get('hash', None), meta=item.get('metadata', {})) if model.sha is None or len(model.sha) == 0: - res.append(f'CivitAI skip search: name="{model.name}" hash=None') + log.debug(f'CivitAI skip search: name="{model.name}" hash=None') else: r = req(f'https://civitai.com/api/v1/model-versions/by-hash/{model.sha}') - res.append(f'CivitAI search: name="{model.name}" hash={model.sha} status={r.status_code}') + log.debug(f'CivitAI search: name="{model.name}" hash={model.sha} status={r.status_code}') if r.status_code == 200: d = r.json() model.id = d['modelId'] @@ -76,31 +109,9 @@ def civit_update_metadata(): else: model.status = 'Available' break - log.debug(res[-1]) - update_data.append(model) - table_data.append(model.array()) - yield gr.update(value=table_data), '
'.join([r for r in res if len(r) > 0]) - return '
'.join([r for r in res if len(r) > 0]) - -def civit_update_select(evt: gr.SelectData, in_data): - global selected_model # pylint: disable=global-statement - try: - selected_model = next([m for m in update_data if m.fn == in_data[evt.index[0]][1]]) - except Exception: - selected_model = None - if selected_model is None or selected_model.url is None or selected_model.status != 'Available': - return [gr.update(value='Model update not available'), gr.update(visible=False)] - else: - return [gr.update(), gr.update(visible=True)] - -def civit_update_download(): - if selected_model is None or selected_model.url is None or selected_model.status != 'Available': - return 'Model update not available' - if selected_model.latest_name is None or len(selected_model.latest_name) == 0: - model_name = f'{selected_model.name} {selected_model.latest}.safetensors' - else: - model_name = selected_model.latest_name - return civit_download_model(selected_model.url, model_name, model_path='', model_type='Model') + results.append(model) + yield create_update_metadata_table(results) + return create_update_metadata_table(results) def civit_search_model(name, tag, model_type): @@ -211,56 +222,106 @@ def civit_download_model(model_url: str, model_name: str, model_path: str, model return res -def atomic_civit_search_metadata(item, res, rehash): +def atomic_civit_search_metadata(item, results): from modules.modelloader import download_civit_preview, download_civit_meta if item is None: - return + return results meta = os.path.splitext(item['filename'])[0] + '.json' has_meta = os.path.isfile(meta) and os.stat(meta).st_size > 0 if ('card-no-preview.png' in item['preview'] or not has_meta) and os.path.isfile(item['filename']): sha = item.get('hash', None) found = False + result = { + 'id': '', + 'name': item['name'], + 'type': '', + 'hash': '', + 'code': '', + 'size': '', + 'note': '', + } if sha is not None and len(sha) > 0: r = req(f'https://civitai.com/api/v1/model-versions/by-hash/{sha}') log.debug(f'CivitAI search: name="{item["name"]}" hash={sha} status={r.status_code}') + result['hash'] = sha + result['code'] = r.status_code if r.status_code == 200: d = r.json() - res.append(download_civit_meta(item['filename'], d['modelId'])) + result['code'], result['size'], result['note'] = download_civit_meta(item['filename'], d['modelId']) + result['id'] = d['modelId'] + result['type'] = 'metadata' + results.append(result) if d.get('images') is not None: for i in d['images']: - preview_url = i['url'] - img_res = download_civit_preview(item['filename'], preview_url) - res.append(img_res) - if 'error' not in img_res: + result['code'], result['size'], result['note'] = download_civit_preview(item['filename'], i['url']) + if result['code'] == 200: + result['type'] = 'preview' + results.append(result) found = True break - if not found and rehash and os.stat(item['filename']).st_size < (1024 * 1024 * 1024): + if not found and os.stat(item['filename']).st_size < (1024 * 1024 * 1024): from modules import hashes sha = hashes.calculate_sha256(item['filename'], quiet=True)[:10] r = req(f'https://civitai.com/api/v1/model-versions/by-hash/{sha}') log.debug(f'CivitAI search: name="{item["name"]}" hash={sha} status={r.status_code}') + result['hash'] = sha + result['code'] = r.status_code if r.status_code == 200: d = r.json() - res.append(download_civit_meta(item['filename'], d['modelId'])) + result['code'], result['size'], result['note'] = download_civit_meta(item['filename'], d['modelId']) + result['id'] = d['modelId'] + result['type'] = 'metadata' + results.append(result) if d.get('images') is not None: for i in d['images']: - preview_url = i['url'] - img_res = download_civit_preview(item['filename'], preview_url) - res.append(img_res) - if 'error' not in img_res: + result['code'], result['size'], result['note'] = download_civit_preview(item['filename'], i['url']) + if result['code'] == 200: + result['type'] = 'preview' + results.append(result) found = True break + if not found: + results.append(result) + + +def civit_search_metadata(title: str = None): + def create_search_metadata_table(rows): + html = """ + + + + + + {tbody} + +
IDNameTypeCodeHashSizeNote
+ """ + tbody = '' + for row in rows: + try: + tbody += f""" + + {row['id']} + {row['name']} + {row['type']} + {row['code']} + {row['hash']} + {row['size']} + {row['note']} + + """ + except Exception as e: + log.error(f'Model list: row={row} {e}') + return html.format(tbody=tbody) -def civit_search_metadata(rehash, title): - log.debug(f'CivitAI search metadata: type={title if type(title) == str else "all"}') from modules.ui_extra_networks import get_pages - res = [] + results = [] scanned, skipped = 0, 0 t0 = time.time() candidates = [] re_skip = [r.strip() for r in opts.extra_networks_scan_skip.split(',') if len(r.strip()) > 0] - log.debug(f'CivitAI search metadata: skip={re_skip}') + log.debug(f'CivitAI search metadata: type={title if type(title) == str else "all"} skip={re_skip}') for page in get_pages(): if type(title) == str: if page.title != title: @@ -275,16 +336,18 @@ def civit_search_metadata(rehash, title): continue scanned += 1 candidates.append(item) - # atomic_civit_search_metadata(item, res, rehash) import concurrent with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_items = {} for fn in candidates: - executor.submit(atomic_civit_search_metadata, fn, res, rehash) - atomic_civit_search_metadata(None, res, rehash) + future_items[executor.submit(atomic_civit_search_metadata, fn, results)] = fn + for future in concurrent.futures.as_completed(future_items): + future.result() + yield create_search_metadata_table(results) + t1 = time.time() log.debug(f'CivitAI search metadata: scanned={scanned} skipped={skipped} time={t1-t0:.2f}') - txt = '
'.join([r for r in res if len(r) > 0]) - return txt + return create_search_metadata_table(results) def civitai_update_token(token): diff --git a/modules/shared.py b/modules/shared.py index f9edba6a4..6c971f4eb 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -57,7 +57,7 @@ restricted_opts = { "outdir_init_images" } resize_modes = ["None", "Fixed", "Crop", "Fill", "Outpaint", "Context aware"] -max_workers = 8 +max_workers = 12 default_hfcache_dir = os.environ.get("SD_HFCACHEDIR", None) or os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub') sdnq_quant_modes = ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"] state = shared_state.State() diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index b8c777f92..8640e285d 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -933,7 +933,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): def ui_scan_click(title): from modules.models_civitai import civit_search_metadata - civit_search_metadata(True, title) + civit_search_metadata(title) return ui_refresh_click(title) def ui_save_click(): diff --git a/modules/ui_models.py b/modules/ui_models.py index da5495c2f..f5290ba08 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -68,7 +68,7 @@ def create_ui(): return [html, meta] with gr.Row(): - gr.HTML('

 Analyze currently loaded model

') + gr.HTML('

Analyze currently loaded model

') with gr.Row(): model_analyze = gr.Button(value="Analyze", variant='primary') with gr.Row(): @@ -127,7 +127,7 @@ def create_ui(): return html.format(tbody=tbody) with gr.Row(): - gr.HTML('

 List models

') + gr.HTML('

List all locally available models


') with gr.Row(): model_list_btn = gr.Button(value="List models", variant='primary') model_checkhash_btn = gr.Button(value="Calculate missing hashes", variant='secondary') @@ -138,35 +138,17 @@ def create_ui(): model_list_btn.click(fn=lambda: create_models_table(sd_models.checkpoints_list.values()), inputs=[], outputs=[model_table]) with gr.Tab(label="Metadata"): - from modules.models_civitai import civit_search_metadata, civit_update_metadata, civit_update_select, civit_update_download + from modules.models_civitai import civit_search_metadata, civit_update_metadata with gr.Row(): - gr.HTML('

 CivitAI fetch metadata

') - gr.HTML('Fetches preview and metadata information for models with missing information
Models with existing previews and information are not updated
') + gr.HTML('

Fetch model preview metadata


') with gr.Row(): - civit_previews_btn = gr.Button(value="Start", variant='primary') + civit_previews_btn = gr.Button(value="Scan missing", variant='primary') + civit_update_btn = gr.Button(value="Update all", variant='primary') with gr.Row(): - civit_previews_rehash = gr.Checkbox(value=True, label="Check alternative hash") - civit_previews_btn.click(fn=civit_search_metadata, inputs=[civit_previews_rehash, civit_previews_rehash], outputs=[models_outcome]) + civit_metadata = gr.HTML(value='', elem_id="civit_metadata") + civit_previews_btn.click(fn=civit_search_metadata, inputs=[], outputs=[civit_metadata]) + civit_update_btn.click(fn=civit_update_metadata, inputs=[], outputs=[civit_metadata]) - with gr.Row(): - gr.HTML('

 Scan CivitAI for information on latest available model versions

') - with gr.Row(): - civit_update_btn = gr.Button(value="Update", variant='primary') - with gr.Row(): - gr.HTML('

Update scan results

') - with gr.Row(): - civit_headers4 = ['ID', 'File', 'Name', 'Versions', 'Current', 'Latest', 'Update'] - civit_types4 = ['number', 'str', 'str', 'number', 'str', 'str', 'str'] - civit_widths4 = ['10%', '25%', '25%', '5%', '10%', '10%', '15%'] - civit_results4 = gr.DataFrame(value=None, label=None, show_label=False, interactive=False, wrap=True, row_count=20, headers=civit_headers4, datatype=civit_types4, type='array', column_widths=civit_widths4) - with gr.Row(): - gr.HTML('

Select model from the list and download update if available

') - with gr.Row(): - civit_update_download_btn = gr.Button(value="Download", variant='primary', visible=False) - - civit_update_btn.click(fn=civit_update_metadata, inputs=[], outputs=[civit_results4, models_outcome]) - civit_results4.select(fn=civit_update_select, inputs=[civit_results4], outputs=[models_outcome, civit_update_download_btn]) - civit_update_download_btn.click(fn=civit_update_download, inputs=[], outputs=[models_outcome]) with gr.Tab(label="Loader"): from modules import ui_models_load