From 09cc58408947f833c42967d8c17bb76657d011a7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 Aug 2025 18:06:36 -0400 Subject: [PATCH] civitai downloader batch download all versions Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ extensions-builtin/sdnext-modernui | 2 +- javascript/civitai.js | 24 +++++++++++++++++++++--- javascript/sdnext.css | 8 ++++++++ modules/civitai/search_civitai.py | 14 +++++++++++--- modules/sd_models.py | 21 ++++++++++++--------- modules/ui_common.py | 12 ++++++++---- modules/ui_models.py | 20 ++++++++++++++------ 8 files changed, 77 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b82b0560b..85a6439c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ And (*as always*) many bugfixes and improvements to existing features! - configurable image fit in all image views - rewritten **CivitAI downloader** in *models -> civitai* + *hint*: you can enter model id in a search bar to pull information on specific model directly + *hint*: you can download individual versions or batch-download all-at-once! - redesigned **GPU monitor** - standard-ui: *system -> gpu monitor* - modern-ui: *aside -> console -> gpu monitor* diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 303612cd9..61d458572 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 303612cd94248463b1835ab2f10630fde0c7923e +Subproject commit 61d458572fbf6531c796180374b2df9124c43273 diff --git a/javascript/civitai.js b/javascript/civitai.js index 417c08584..c60909f4a 100644 --- a/javascript/civitai.js +++ b/javascript/civitai.js @@ -31,6 +31,7 @@ const modelDetailsHTML = ` Downloads{downloads} Author{creator} Description
{desc}
+ Download
All variants

@@ -114,9 +115,26 @@ async function modelCardClick(id) { function startCivitDownload(url, name, type) { log('startCivitDownload', { url, name, type }); - selectedURL = url; - selectedName = name; - selectedType = type; + selectedURL = [url]; + selectedName = [name]; + selectedType = [type]; + const civitDownloadBtn = gradioApp().getElementById('civitai_download_btn'); + if (civitDownloadBtn) civitDownloadBtn.click(); +} + +function startCivitAllDownload(evt) { + log('startCivitAllDownload', evt); + const versions = gradioApp().getElementById('model-versions-table').querySelectorAll('tr'); + selectedURL = []; + selectedName = []; + selectedType = []; + for (const version of versions) { + const parsed = version.querySelector('td:nth-child(1) div')?.getAttribute('onclick')?.match(/startCivitDownload\('([^']+)', '([^']+)', '([^']+)'\)/); + if (!parsed || parsed.length < 4) continue; + selectedURL.push(parsed[1]); + selectedName.push(parsed[2]); + selectedType.push(parsed[3]); + } const civitDownloadBtn = gradioApp().getElementById('civitai_download_btn'); if (civitDownloadBtn) civitDownloadBtn.click(); } diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 6e855e3c5..487bec576 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -2054,6 +2054,14 @@ div:has(>#tab-gallery-folders) { font-weight: bold; } +.div-link { + cursor: pointer; +} + +.div-link:hover { + background-color: var(--button-primary-background-fill); +} + .video-model-link { color: var(--button-primary-background-fill); font-weight: normal; diff --git a/modules/civitai/search_civitai.py b/modules/civitai/search_civitai.py index 12d74fce3..dda495028 100644 --- a/modules/civitai/search_civitai.py +++ b/modules/civitai/search_civitai.py @@ -139,7 +139,11 @@ def search_civitai( headers['Authorization'] = f'Bearer {token}' url = 'https://civitai.com/api/v1/models' - uri = f'{url}?{encoded}' + if query.isnumeric(): + uri = f'{url}/{query}' + else: + uri = f'{url}?{encoded}' + log.info(f'CivitAI request: uri="{uri}" dct={dct} token={token is not None}') result = requests.get(uri, headers=headers, timeout=60) @@ -149,7 +153,11 @@ def search_civitai( all_models: list[Model] = [] exact_models: list[Model] = [] - items = result.json().get('items', []) + dct = result.json() + if 'items' not in dct: + items = [dct] # single model + else: + items = dct.get('items', []) for item in items: all_models.append(Model(item)) @@ -189,7 +197,7 @@ def create_model_cards(all_models: list[Model]) -> str: previews = [] for version in model.versions: for image in version.images: - if image.url and len(image.url) > 0: + if image.url and len(image.url) > 0 and not image.url.lower().endswith('.mp4'): previews.append(image.url) if len(previews) == 0: previews = ['./sd_extra_networks/thumb?filename=html/card-no-preview.png'] diff --git a/modules/sd_models.py b/modules/sd_models.py index 50869eb72..cbafbea80 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -73,24 +73,26 @@ def copy_diffuser_options(new_pipe, orig_pipe): def set_vae_options(sd_model, vae=None, op:str='model', quiet:bool=False): + ops = {} if hasattr(sd_model, "vae"): if vae is not None: sd_model.vae = vae - shared.log.quiet(quiet, f'Setting {op}: component=VAE name="{sd_vae.loaded_vae_file}"') + ops['name'] = f"{sd_vae.loaded_vae_file}" if shared.opts.diffusers_vae_upcast != 'default': sd_model.vae.config.force_upcast = True if shared.opts.diffusers_vae_upcast == 'true' else False - shared.log.quiet(quiet, f'Setting {op}: component=VAE upcast={sd_model.vae.config.force_upcast}') + ops['upcast'] = sd_model.vae.config.force_upcast if shared.opts.no_half_vae and op not in {'decode', 'encode'}: devices.dtype_vae = torch.float32 sd_model.vae.to(devices.dtype_vae) - shared.log.quiet(quiet, f'Setting {op}: component=VAE no-half=True') - if hasattr(sd_model, "enable_vae_slicing"): + ops['no-half'] = True + if hasattr(sd_model, "enable_vae_slicing") and hasattr(sd_model, "disable_vae_slicing"): + ops['slicing'] = shared.opts.diffusers_vae_slicing if shared.opts.diffusers_vae_slicing: - shared.log.quiet(quiet, f'Setting {op}: component=VAE slicing=True') sd_model.enable_vae_slicing() else: sd_model.disable_vae_slicing() if hasattr(sd_model, "enable_vae_tiling") and hasattr(sd_model, "disable_vae_tiling"): + ops['tiling'] = shared.opts.diffusers_vae_tiling if shared.opts.diffusers_vae_tiling: if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'config') and hasattr(sd_model.vae.config, 'sample_size') and isinstance(sd_model.vae.config.sample_size, int): if getattr(sd_model.vae, "tile_sample_min_size_backup", None) is None: @@ -107,15 +109,16 @@ def set_vae_options(sd_model, vae=None, op:str='model', quiet:bool=False): sd_model.vae.tile_overlap_factor = float(shared.opts.diffusers_vae_tile_overlap) else: sd_model.vae.tile_overlap_factor = getattr(sd_model.vae, "tile_overlap_factor_backup", sd_model.vae.tile_overlap_factor) - shared.log.quiet(quiet, f'Setting {op}: component=VAE tiling=True tile={sd_model.vae.tile_sample_min_size} overlap={sd_model.vae.tile_overlap_factor}') - else: - shared.log.quiet(quiet, f'Setting {op}: component=VAE tiling=True') + ops['tile'] = sd_model.vae.tile_sample_min_size + ops['overlap'] = sd_model.vae.tile_overlap_factor sd_model.enable_vae_tiling() else: sd_model.disable_vae_tiling() if hasattr(sd_model, "vqvae"): - shared.log.quiet(quiet, f'Setting {op}: component=VQVAE upcast=True') + ops['upcast'] = True sd_model.vqvae.to(torch.float32) # vqvae is producing nans in fp16 + if not quiet and len(ops) > 0: + shared.log.quiet(quiet, f'Setting {op}: component=vae {ops}') def set_diffuser_options(sd_model, vae=None, op:str='model', offload:bool=True, quiet:bool=False): diff --git a/modules/ui_common.py b/modules/ui_common.py index 494c2bc8a..292adcdef 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -19,10 +19,14 @@ def gr_show(visible=True): def update_generation_info(generation_info, html_info, img_index): try: - generation_info = json.loads(generation_info) - if img_index < 0 or img_index >= len(generation_info["infotexts"]): - return html_info, generation_info - info = generation_info["infotexts"][img_index] + generation_json = json.loads(generation_info) + if len(generation_json["infotexts"]) == 0: + return html_info, 'no infotexts found' + if img_index == -1: + img_index = 0 + if img_index >= len(generation_json["infotexts"]): + return html_info, 'error fetching infotext' + info = generation_json["infotexts"][img_index] html_info_formatted = infotext_to_html(info) return html_info, html_info_formatted except Exception: diff --git a/modules/ui_models.py b/modules/ui_models.py index 5a7b58076..59753091b 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -470,6 +470,7 @@ def create_ui(): with gr.Tab(label="CivitAI", elem_id="models_civitai_tab"): from modules.civitai.search_civitai import search_civitai, create_model_cards, base_models + def civitai_search(civit_search_text, civit_search_tag, civit_nsfw, civit_type, civit_base, civit_token): results = search_civitai(query=civit_search_text, tag=civit_search_tag, nsfw=civit_nsfw, types=civit_type, base=civit_base, token=civit_token) html = create_model_cards(results) @@ -480,12 +481,13 @@ def create_ui(): opts.civitai_token = token opts.save() - def civitai_download(model_url, model_name, model_type, model_path, civit_token, model_output): + def civitai_download(model_urls, model_names, model_types, model_path, civit_token, model_output): from modules.civitai.download_civitai import download_civit_model - msg = f"

Initiating download

{model_name} | {model_type} | {model_url}

" - yield msg + model_output - download_civit_model(model_url, model_name, model_path, model_type, civit_token) - yield model_output + for model_url, model_name, model_type in zip(model_urls, model_names, model_types): + msg = f"

Initiating download

{model_name} | {model_type} | {model_url}

" + yield msg + model_output + download_civit_model(model_url, model_name, model_path, model_type, civit_token) + yield model_output with gr.Row(): gr.HTML('

Search & Download

') @@ -515,7 +517,13 @@ def create_ui(): civit_search_text.submit(fn=civitai_search, inputs=civit_inputs, outputs=[civitai_models_output]) civit_search_tag.submit(fn=civitai_search, inputs=civit_inputs, outputs=[civitai_models_output]) civit_token.change(fn=civitai_update_token, inputs=[civit_token], outputs=[]) - civit_download_btn.click(fn=civitai_download, _js="downloadCivitModel", inputs=[_dummy, _dummy, _dummy, civit_folder, civit_token, civitai_models_output], outputs=[civitai_models_output]) + civit_download_btn.click( + fn=civitai_download, + _js="downloadCivitModel", + inputs=[_dummy, _dummy, _dummy, civit_folder, civit_token, civitai_models_output], + outputs=[civitai_models_output], + show_progress=True, + ) with gr.Tab(label="Huggingface", elem_id="models_huggingface_tab"): from modules.models_hf import hf_search, hf_select, hf_download_model, hf_update_token