diff --git a/CHANGELOG.md b/CHANGELOG.md index b676ddf98..a792bff42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2023-12-25 +## Update for 2023-12-26 *Note*: based on `diffusers==0.25.0.dev0` @@ -52,6 +52,9 @@ - allow setting of resize method directly in image tab (previously via settings -> upscaler_for_img2img) - **General** + - new **onboarding** + if no models are found during startup, app will no longer ask to download default checkpoint + instead, it will show message in UI with options to change model path or download any of the reference checkpoints - support for **Torch 2.1.2** - **Process** create videos from batch or folder processing supports *GIF*, *PNG* and *MP4* with full interpolation, scene change detection, etc. @@ -98,6 +101,7 @@ - **chaiNNer** fix `NaN` issues due to autocast - **Upscale** increase limit from 4x to 8x given the quality of some upscalers - **Extra Networks** fix sort + - reduced default **CFG scale** from 6 to 4 to be more out-of-the-box compatibile with LCM/Turbo models - disable google fonts check on server startup - fix torchvision/basicsr compatibility - fix styles quick save diff --git a/SECURITY.md b/SECURITY.md index af9687f8c..9c1e11bb0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -32,5 +32,5 @@ Any code commit is validated before merge - Download extensions and themes indexes from automatically updated indexes - Download required packages and repositories from GitHub during installation/upgrade - Download installed/enabled extensions -- Download default model from official repository +- Download models from CivitAI and/or Huggingface when instructed by user - Submit benchmark info upon user interaction diff --git a/html/locale_en.json b/html/locale_en.json index 8bce40518..91789b742 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -43,17 +43,18 @@ "tabs": [ {"id":"","label":"Text","localized":"","hint":"Create image from text"}, {"id":"","label":"Image","localized":"","hint":"Create image from image"}, + {"id":"","label":"Control","localized":"","hint":"Create image with additional control"}, {"id":"","label":"Process","localized":"","hint":"Process existing image"}, - {"id":"","label":"Train","localized":"","hint":"Run training or model merging"}, + {"id":"","label":"Interrogate","localized":"","hint":"Run interrogate to get description of your image"}, + {"id":"","label":"Train","localized":"","hint":"Run training"}, {"id":"","label":"Models","localized":"","hint":"Convert or merge your models"}, - {"id":"","label":"Interrogator","localized":"","hint":"Run interrogate to get description of your image"}, - {"id":"","label":"System Info","localized":"","hint":"System information"}, {"id":"","label":"Agent Scheduler","localized":"","hint":"Enqueue your generate requests and run them in the background"}, {"id":"","label":"Image Browser","localized":"","hint":"Browse through your generated image database"}, {"id":"","label":"System","localized":"","hint":"System settings and information"}, + {"id":"","label":"System Info","localized":"","hint":"System information"}, {"id":"","label":"Settings","localized":"","hint":"Application settings"}, - {"id":"","label":"Extensions","localized":"","hint":"Application extensions"}, - {"id":"","label":"Script","localized":"","hint":"Addtional scripts to be used"} + {"id":"","label":"Script","localized":"","hint":"Addtional scripts to be used"}, + {"id":"","label":"Extensions","localized":"","hint":"Application extensions"} ], "action panel": [ {"id":"","label":"Generate","localized":"","hint":"Start processing"}, diff --git a/html/reference.json b/html/reference.json index 5c8d26378..dc33c0481 100644 --- a/html/reference.json +++ b/html/reference.json @@ -1,4 +1,24 @@ { + "DreamShaper SD 1.5 v8": { + "path": "dreamshaper_8.safetensors@https://civitai.com/api/download/models/128713", + "desc": "Showcase finetuned model based on Stable diffusion 1.5", + "preview": "dreamshaper_8.jpg" + }, + "DreamShaper SD XL Turbo": { + "path": "dreamshaperXL_turboDpmppSDE.safetensors@https://civitai.com/api/download/models/251662", + "desc": "Showcase finetuned model based on Stable diffusion XL", + "preview": "dreamshaperXL_turboDpmppSDE.jpg" + }, + "Juggernaut Reborn": { + "path": "juggernaut_reborn.safetensors@https://civitai.com/api/download/models/274039", + "desc": "Showcase finetuned model based on Stable diffusion 1.5", + "preview": "juggernaut_reborn.jpg" + }, + "Juggernaut XL v7 RunDiffusion": { + "path": "juggernautXL_v7Rundiffusion.safetensors@https://civitai.com/api/download/models/240840", + "desc": "Showcase finetuned model based on Stable diffusion XL", + "preview": "juggernautXL_v7Rundiffusion.jpg" + }, "RunwayML SD 1.5": { "path": "runwayml/stable-diffusion-v1-5", "desc": "Stable Diffusion 1.5 is the base model all other 1.5 checkpoint were trained from. It's a latent text-to-image diffusion model capable of generating photo-realistic images given any text input. The Stable-Diffusion-v1-5 checkpoint was initialized with the weights of the Stable-Diffusion-v1-2 checkpoint and subsequently fine-tuned on 595k steps at resolution 512x512.", diff --git a/installer.py b/installer.py index 1db105dca..2a15bb4c4 100644 --- a/installer.py +++ b/installer.py @@ -363,7 +363,7 @@ def check_torch(): log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}') torch_command = os.environ.get('TORCH_COMMAND', '') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') - install('onnxruntime', 'onnxruntime', ignore=True) + install('onnxruntime onnxruntimegpu', 'onnxruntime', ignore=True) if torch_command != '': pass elif allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): diff --git a/javascript/sdnext.css b/javascript/sdnext.css index b4d7fce93..86a6bcbdb 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -263,6 +263,9 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .processor-settings { padding: 0 !important; max-width: 300px; } .processor-group>div { flex-flow: wrap;gap: 1em; } +/* main info */ +.main-info { font-weight: var(--section-header-text-weight); color: var(--body-text-color-subdued); padding: 1em !important; margin-top: 2em !important; line-height: var(--line-lg) !important; } + /* loader */ .splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; } .motd { margin-top: 2em; color: var(--body-text-color-subdued); font-family: monospace; font-variant: all-petite-caps; } diff --git a/javascript/settings.js b/javascript/settings.js index 93e4bebce..a787161dc 100644 --- a/javascript/settings.js +++ b/javascript/settings.js @@ -108,7 +108,7 @@ onAfterUiUpdate(async () => { const settingsSearch = gradioApp().querySelectorAll('#settings_search > label > textarea')[0]; settingsSearch.oninput = (e) => { setTimeout(() => { - log('settingsSearch', e.target.value) + log('settingsSearch', e.target.value); showAllSettings(); gradioApp().querySelectorAll('#tab_settings .tabitem').forEach((section) => { section.querySelectorAll('.dirtyable').forEach((setting) => { @@ -129,6 +129,40 @@ onOptionsChanged(() => { }); }); +async function initModels() { + const warn = () => ` +

No models available

+ - Select a model from reference list to download or
+ - Set model path to a folder containing your models
+ Current model path: ${opts.ckpt_dir}
+ `; + const el = gradioApp().getElementById('main_info'); + const en = gradioApp().getElementById('txt2img_extra_networks'); + if (!el || !en) return; + const req = await fetch('/sdapi/v1/sd-models'); + const res = req.ok ? await req.json() : []; + log('initModels', res.length); + const ready = () => ` +

Ready

+ ${res.length} models available
+ `; + el.innerHTML = res.length > 0 ? ready() : warn(); + el.style.display = 'block'; + setTimeout(() => el.style.display = 'none', res.length === 0 ? 30000 : 1500); + if (res.length === 0) { + if (en.classList.contains('hide')) gradioApp().getElementById('txt2img_extra_networks_btn').click(); + const repeat = setInterval(() => { + const buttons = Array.from(gradioApp().querySelectorAll('#txt2img_model_subdirs > button')) || []; + const reference = buttons.find((b) => b.innerText === 'Reference'); + if (reference) { + clearInterval(repeat); + reference.click(); + log('enReferenceSelect'); + } + }, 100); + } +} + function initSettings() { if (settingsInitialized) return; settingsInitialized = true; @@ -138,7 +172,7 @@ function initSettings() { const observer = new MutationObserver((mutations) => { const showAllPages = gradioApp().getElementById('settings_show_all_pages'); if (showAllPages.style.display === 'none') return; - const mutation = (mut) => mut.type === 'attributes' && mut.attributeName === 'style' + const mutation = (mut) => mut.type === 'attributes' && mut.attributeName === 'style'; if (mutations.some(mutation)) showAllSettings(); }); const tabContentWrapper = document.createElement('div'); @@ -155,3 +189,4 @@ function initSettings() { } onUiLoaded(initSettings); +onUiLoaded(initModels); diff --git a/models/Reference/dreamshaperXL_turboDpmppSDE.jpg b/models/Reference/dreamshaperXL_turboDpmppSDE.jpg new file mode 100644 index 000000000..802d92593 Binary files /dev/null and b/models/Reference/dreamshaperXL_turboDpmppSDE.jpg differ diff --git a/models/Reference/dreamshaper_8.jpg b/models/Reference/dreamshaper_8.jpg new file mode 100644 index 000000000..369d540b3 Binary files /dev/null and b/models/Reference/dreamshaper_8.jpg differ diff --git a/models/Reference/juggernautXL_v7Rundiffusion.jpg b/models/Reference/juggernautXL_v7Rundiffusion.jpg new file mode 100644 index 000000000..cbce7cb32 Binary files /dev/null and b/models/Reference/juggernautXL_v7Rundiffusion.jpg differ diff --git a/models/Reference/juggernaut_reborn.jpg b/models/Reference/juggernaut_reborn.jpg new file mode 100644 index 000000000..f19b294e9 Binary files /dev/null and b/models/Reference/juggernaut_reborn.jpg differ diff --git a/modules/control/run.py b/modules/control/run.py index c67a78120..e4dc60dfe 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -9,12 +9,12 @@ from PIL import Image from modules.control import util from modules.control import unit from modules.control import processors -from modules.control import controlnets # lllyasviel ControlNet -from modules.control import controlnetsxs # VisLearn ControlNet-XS -from modules.control import controlnetslite # Kohya ControlLLLite -from modules.control import adapters # TencentARC T2I-Adapter -from modules.control import reference # ControlNet-Reference -from modules.control import ipadapter # IP-Adapter +from modules.control.units import controlnet # lllyasviel ControlNet +from modules.control.units import xs # VisLearn ControlNet-XS +from modules.control.units import lite # Kohya ControlLLLite +from modules.control.units import t2iadapter # TencentARC T2I-Adapter +from modules.control.units import reference # ControlNet-Reference +from modules.control.units import ipadapter # IP-Adapter from modules import devices, shared, errors, processing, images, sd_models, sd_samplers @@ -77,7 +77,7 @@ def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_genera inputs = [None] output_images: List[Image.Image] = [] # output images active_process: List[processors.Processor] = [] # all active preprocessors - active_model: List[Union[controlnets.ControlNet, controlnetsxs.ControlNetXS, adapters.Adapter]] = [] # all active models + active_model: List[Union[controlnet.ControlNet, xs.ControlNetXS, t2iadapter.Adapter]] = [] # all active models active_strength: List[float] = [] # strength factors for all active models active_start: List[float] = [] # start step for all active models active_end: List[float] = [] # end step for all active models @@ -177,7 +177,7 @@ def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_genera p.ops.append('control') has_models = False - selected_models: List[Union[controlnets.ControlNetModel, controlnetsxs.ControlNetXSModel, adapters.AdapterModel]] = None + selected_models: List[Union[controlnet.ControlNetModel, xs.ControlNetXSModel, t2iadapter.AdapterModel]] = None if unit_type == 'adapter' or unit_type == 'controlnet' or unit_type == 'xs' or unit_type == 'lite': if len(active_model) == 0: selected_models = None @@ -198,7 +198,7 @@ def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_genera p.extra_generation_params["Control mode"] = 'Adapter' p.extra_generation_params["Control conditioning"] = use_conditioning p.task_args['adapter_conditioning_scale'] = use_conditioning - instance = adapters.AdapterPipeline(selected_models, shared.sd_model) + instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: T2I-Adapter does not support separate init image') @@ -209,7 +209,7 @@ def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_genera p.task_args['control_guidance_start'] = active_start[0] if len(active_start) == 1 else list(active_start) p.task_args['control_guidance_end'] = active_end[0] if len(active_end) == 1 else list(active_end) p.task_args['guess_mode'] = p.guess_mode - instance = controlnets.ControlNetPipeline(selected_models, shared.sd_model) + instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model) pipe = instance.pipeline elif unit_type == 'xs' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet-XS' @@ -217,7 +217,7 @@ def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_genera p.controlnet_conditioning_scale = use_conditioning p.control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) p.control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) - instance = controlnetsxs.ControlNetXSPipeline(selected_models, shared.sd_model) + instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: ControlNet-XS does not support separate init image') @@ -225,7 +225,7 @@ def control_run(units: List[unit.Unit], inputs, inits, unit_type: str, is_genera p.extra_generation_params["Control mode"] = 'ControlLLLite' p.extra_generation_params["Control conditioning"] = use_conditioning p.controlnet_conditioning_scale = use_conditioning - instance = controlnetslite.ControlLLitePipeline(shared.sd_model) + instance = lite.ControlLLitePipeline(shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: ControlLLLite does not support separate init image') diff --git a/modules/control/test.py b/modules/control/test.py index ba87420ea..235cee0b4 100644 --- a/modules/control/test.py +++ b/modules/control/test.py @@ -49,25 +49,25 @@ def test_processors(image): def test_controlnets(prompt, negative, image): from modules import devices, sd_models - from modules.control import controlnets + from modules.control.units import controlnet if image is None: shared.log.error('Image not loaded') return None, None, None from PIL import ImageDraw, ImageFont images = [] - for model_id in controlnets.list_models(): + for model_id in controlnet.list_models(): if model_id is None: model_id = 'None' if shared.state.interrupted: continue output = image if model_id != 'None': - controlnet = controlnets.ControlNet(model_id=model_id, device=devices.device, dtype=devices.dtype) + controlnet = controlnet.ControlNet(model_id=model_id, device=devices.device, dtype=devices.dtype) if controlnet is None: shared.log.error(f'ControlNet load failed: id="{model_id}"') continue shared.log.info(f'Testing ControlNet: {model_id}') - pipe = controlnets.ControlNetPipeline(controlnet=controlnet.model, pipeline=shared.sd_model) + pipe = controlnet.ControlNetPipeline(controlnet=controlnet.model, pipeline=shared.sd_model) pipe.pipeline.to(device=devices.device, dtype=devices.dtype) sd_models.set_diffuser_options(pipe) try: @@ -101,25 +101,25 @@ def test_controlnets(prompt, negative, image): def test_adapters(prompt, negative, image): from modules import devices, sd_models - from modules.control import adapters + from modules.control.units import t2iadapter if image is None: shared.log.error('Image not loaded') return None, None, None from PIL import ImageDraw, ImageFont images = [] - for model_id in adapters.list_models(): + for model_id in t2iadapter.list_models(): if model_id is None: model_id = 'None' if shared.state.interrupted: continue output = image.copy() if model_id != 'None': - adapter = adapters.Adapter(model_id=model_id, device=devices.device, dtype=devices.dtype) + adapter = t2iadapter.Adapter(model_id=model_id, device=devices.device, dtype=devices.dtype) if adapter is None: shared.log.error(f'Adapter load failed: id="{model_id}"') continue shared.log.info(f'Testing Adapter: {model_id}') - pipe = adapters.AdapterPipeline(adapter=adapter.model, pipeline=shared.sd_model) + pipe = t2iadapter.AdapterPipeline(adapter=adapter.model, pipeline=shared.sd_model) pipe.pipeline.to(device=devices.device, dtype=devices.dtype) sd_models.set_diffuser_options(pipe) image = image.convert('L') if 'Canny' in model_id or 'Sketch' in model_id else image.convert('RGB') @@ -154,25 +154,25 @@ def test_adapters(prompt, negative, image): def test_xs(prompt, negative, image): from modules import devices, sd_models - from modules.control import controlnetsxs + from modules.control.units import xs if image is None: shared.log.error('Image not loaded') return None, None, None from PIL import ImageDraw, ImageFont images = [] - for model_id in controlnetsxs.list_models(): + for model_id in xs.list_models(): if model_id is None: model_id = 'None' if shared.state.interrupted: continue output = image if model_id != 'None': - xs = controlnetsxs.ControlNetXS(model_id=model_id, device=devices.device, dtype=devices.dtype) + xs = xs.ControlNetXS(model_id=model_id, device=devices.device, dtype=devices.dtype) if xs is None: shared.log.error(f'ControlNet-XS load failed: id="{model_id}"') continue shared.log.info(f'Testing ControlNet-XS: {model_id}') - pipe = controlnetsxs.ControlNetXSPipeline(controlnet=xs.model, pipeline=shared.sd_model) + pipe = xs.ControlNetXSPipeline(controlnet=xs.model, pipeline=shared.sd_model) pipe.pipeline.to(device=devices.device, dtype=devices.dtype) sd_models.set_diffuser_options(pipe) try: @@ -206,25 +206,25 @@ def test_xs(prompt, negative, image): def test_lite(prompt, negative, image): from modules import devices, sd_models - from modules.control import controlnetslite + from modules.control.units import lite if image is None: shared.log.error('Image not loaded') return None, None, None from PIL import ImageDraw, ImageFont images = [] - for model_id in controlnetslite.list_models(): + for model_id in lite.list_models(): if model_id is None: model_id = 'None' if shared.state.interrupted: continue output = image if model_id != 'None': - lite = controlnetslite.ControlLLLite(model_id=model_id, device=devices.device, dtype=devices.dtype) + lite = lite.ControlLLLite(model_id=model_id, device=devices.device, dtype=devices.dtype) if lite is None: shared.log.error(f'Control-LLite load failed: id="{model_id}"') continue shared.log.info(f'Testing ControlNet-XS: {model_id}') - pipe = controlnetslite.ControlLLitePipeline(pipeline=shared.sd_model) + pipe = lite.ControlLLitePipeline(pipeline=shared.sd_model) pipe.apply(controlnet=lite.model, image=image, conditioning=1.0) pipe.pipeline.to(device=devices.device, dtype=devices.dtype) sd_models.set_diffuser_options(pipe) diff --git a/modules/control/unit.py b/modules/control/unit.py index 3e52531f8..1df54b81b 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -2,11 +2,11 @@ from typing import Union from PIL import Image from modules.shared import log from modules.control import processors -from modules.control import controlnets -from modules.control import controlnetsxs -from modules.control import controlnetslite -from modules.control import adapters -from modules.control import reference # pylint: disable=unused-import +from modules.control.units import controlnet +from modules.control.units import xs +from modules.control.units import lite +from modules.control.units import t2iadapter +from modules.control.units import reference # pylint: disable=unused-import default_device = None @@ -45,8 +45,8 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c self.end = max(self.start, self.end) # processor always exists, adapter and controlnet are optional self.process: processors.Processor = processors.Processor() - self.adapter: adapters.Adapter = None - self.controlnet: Union[controlnets.ControlNet, controlnetsxs.ControlNetXS] = None + self.adapter: t2iadapter.Adapter = None + self.controlnet: Union[controlnet.ControlNet, xs.ControlNetXS] = None # map to input image self.input: Image = image_input self.override: Image = None @@ -106,13 +106,13 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c # actual init if self.type == 'adapter': - self.adapter = adapters.Adapter(device=default_device, dtype=default_dtype) + self.adapter = t2iadapter.Adapter(device=default_device, dtype=default_dtype) elif self.type == 'controlnet': - self.controlnet = controlnets.ControlNet(device=default_device, dtype=default_dtype) + self.controlnet = controlnet.ControlNet(device=default_device, dtype=default_dtype) elif self.type == 'xs': - self.controlnet = controlnetsxs.ControlNetXS(device=default_device, dtype=default_dtype) + self.controlnet = xs.ControlNetXS(device=default_device, dtype=default_dtype) elif self.type == 'lite': - self.controlnet = controlnetslite.ControlLLLite(device=default_device, dtype=default_dtype) + self.controlnet = lite.ControlLLLite(device=default_device, dtype=default_dtype) elif self.type == 'reference': pass else: diff --git a/modules/control/controlnets.py b/modules/control/units/controlnet.py similarity index 100% rename from modules/control/controlnets.py rename to modules/control/units/controlnet.py diff --git a/modules/control/ipadapter.py b/modules/control/units/ipadapter.py similarity index 100% rename from modules/control/ipadapter.py rename to modules/control/units/ipadapter.py diff --git a/modules/control/controlnetslite.py b/modules/control/units/lite.py similarity index 97% rename from modules/control/controlnetslite.py rename to modules/control/units/lite.py index 16a7505f2..9796f77f1 100644 --- a/modules/control/controlnetslite.py +++ b/modules/control/units/lite.py @@ -6,7 +6,7 @@ from PIL import Image from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline from modules.shared import log, opts from modules import errors -from modules.control.controlnetslite_model import ControlNetLLLite +from modules.control.units.lite_model import ControlNetLLLite what = 'ControlLLLite' @@ -130,6 +130,6 @@ class ControlLLitePipeline(): cn.apply(pipe=self.pipeline, cond=np.asarray(images[i % len(images)]), weight=weight[i % len(weight)]) def restore(self): - from modules.control.controlnetslite_model import clear_all_lllite + from modules.control.units.lite_model import clear_all_lllite clear_all_lllite() self.nets = [] diff --git a/modules/control/controlnetslite_model.py b/modules/control/units/lite_model.py similarity index 100% rename from modules/control/controlnetslite_model.py rename to modules/control/units/lite_model.py diff --git a/modules/control/reference.py b/modules/control/units/reference.py similarity index 100% rename from modules/control/reference.py rename to modules/control/units/reference.py diff --git a/modules/control/adapters.py b/modules/control/units/t2iadapter.py similarity index 100% rename from modules/control/adapters.py rename to modules/control/units/t2iadapter.py diff --git a/modules/control/controlnetsxs.py b/modules/control/units/xs.py similarity index 100% rename from modules/control/controlnetsxs.py rename to modules/control/units/xs.py diff --git a/modules/modelloader.py b/modules/modelloader.py index 05781f3a7..09fdf326b 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -294,8 +294,9 @@ def load_diffusers_models(model_path: str, command_path: str = None, clear=True) if os.path.exists(os.path.join(folder, 'hidden')): continue output.append(name) - except Exception as e: - shared.log.error(f"Error analyzing diffusers model: {folder} {e}") + except Exception: + # shared.log.error(f"Error analyzing diffusers model: {folder} {e}") + pass except Exception as e: shared.log.error(f"Error listing diffusers: {place} {e}") shared.log.debug(f'Scanning diffusers cache: {model_path} {command_path} items={len(output)} time={time.time()-t0:.2f}') @@ -339,6 +340,29 @@ def load_reference(name: str): return True +def load_civitai(model: str, url: str): + from modules import sd_models + name, _ext = os.path.splitext(model) + info = sd_models.get_closet_checkpoint_match(name) + if info is not None: + shared.log.debug(f'Reference model: {name}') + return name # already downloaded + else: + shared.log.debug(f'Reference model: {name} download start') + download_civit_model_thread(model_name=model, model_url=url, model_path='', model_type='safetensors', preview=None, token=None) + shared.log.debug(f'Reference model: {name} download complete') + sd_models.list_models() + info = sd_models.get_closet_checkpoint_match(name) + print('HERE1', info) + print('HERE2', name) + if info is not None: + shared.log.debug(f'Reference model: {name}') + return name # already downloaded + else: + shared.log.debug(f'Reference model: {name} not found') + return None + + cache_folders = {} cache_last = 0 cache_time = 1 diff --git a/modules/processing.py b/modules/processing.py index e2283b8ae..b40973bbb 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -565,13 +565,13 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No if index is None: index = position_in_batch + iteration * p.batch_size if all_prompts is None: - all_prompts = p.all_prompts + all_prompts = p.all_prompts or [p.prompt] if all_negative_prompts is None: - all_negative_prompts = p.all_negative_prompts + all_negative_prompts = p.all_negative_prompts or [p.negative_prompt] if all_seeds is None: - all_seeds = p.all_seeds + all_seeds = p.all_seeds or [p.seed] if all_subseeds is None: - all_subseeds = p.all_subseeds + all_subseeds = p.all_subseeds or [p.subseed] while len(all_prompts) <= index: all_prompts.append(all_prompts[-1]) while len(all_seeds) <= index: diff --git a/modules/sd_models.py b/modules/sd_models.py index 33130a2fc..85f8894d4 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -168,6 +168,7 @@ def list_models(): shared.log.info(f'Available models: path="{shared.opts.ckpt_dir}" items={len(checkpoints_list)} time={time.time()-t0:.2f}') checkpoints_list = dict(sorted(checkpoints_list.items(), key=lambda cp: cp[1].filename)) + """ if len(checkpoints_list) == 0: if not shared.cmd_opts.no_download: key = input('Download the default model? (y/N) ') @@ -185,7 +186,7 @@ def list_models(): checkpoint_info = CheckpointInfo(filename) if checkpoint_info.name is not None: checkpoint_info.register() - + """ def update_model_hashes(): txt = [] diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index dd24d42ae..88f511a5d 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -38,6 +38,8 @@ def single_sample_to_image(sample, approximation=None): warn_once('Unknown decode type, please reset preview method') approximation = 0 + if len(sample.shape) > 4: # likely unknown video latent (e.g. svd) + return Image.new(mode="RGB", size=(512, 512)) if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] if approximation == 0: # Simple diff --git a/modules/ui.py b/modules/ui.py index eddf1393d..77676790d 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -182,10 +182,10 @@ def create_advanced_inputs(tab): with gr.Accordion(open=False, label="Advanced", elem_id=f"{tab}_advanced", elem_classes=["small-accordion"]): with gr.Group(): with FormRow(): - cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id=f"{tab}_cfg_scale") + cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='CFG scale', value=4.0, elem_id=f"{tab}_cfg_scale") clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id=f"{tab}_clip_skip", interactive=True) with FormRow(): - image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id=f"{tab}_image_cfg_scale") + image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=4.0, elem_id=f"{tab}_image_cfg_scale") diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id=f"{tab}_image_cfg_rescale", visible=shared.backend == shared.Backend.DIFFUSERS) with gr.Group(): with FormRow(): @@ -648,6 +648,9 @@ def create_ui(startup_timer = None): ui_extra_networks.setup_ui(extra_networks_ui, txt2img_gallery) + with FormRow(): + gr.HTML(value="", elem_id="main_info", visible=False, elem_classes=["main-info"]) + timer.startup.record("ui-txt2img") import modules.img2img # pylint: disable=redefined-outer-name @@ -1136,9 +1139,9 @@ def create_ui(startup_timer = None): interfaces += [(img2img_interface, "Image", "img2img")] interfaces += [(control_interface, "Control", "control")] if control_interface is not None else [] interfaces += [(extras_interface, "Process", "process")] + interfaces += [(interrogate_interface, "Interrogate", "interrogate")] interfaces += [(train_interface, "Train", "train")] interfaces += [(models_interface, "Models", "models")] - interfaces += [(interrogate_interface, "Interrogate", "interrogate")] interfaces += script_callbacks.ui_tabs_callback() interfaces += [(settings_interface, "System", "system")] @@ -1224,10 +1227,13 @@ def create_ui(startup_timer = None): ) def reference_submit(model): - loaded = modelloader.load_reference(model) - if loaded: + if '@' not in model: # diffusers + loaded = modelloader.load_reference(model) return model if loaded else opts.sd_model_checkpoint - return loaded + else: # civitai + model, url = model.split('@') + loaded = modelloader.load_civitai(model, url) + return loaded if loaded is not None else opts.sd_model_checkpoint button_set_reference = gr.Button('Change reference', elem_id='change_reference', visible=False) button_set_reference.click( diff --git a/modules/ui_control.py b/modules/ui_control.py index 468aa3af7..9315426d4 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -1,13 +1,13 @@ import os import gradio as gr from modules.control import unit -from modules.control import controlnets # lllyasviel ControlNet -from modules.control import controlnetsxs # vislearn ControlNet-XS -from modules.control import controlnetslite # vislearn ControlNet-XS -from modules.control import adapters # TencentARC T2I-Adapter from modules.control import processors # patrickvonplaten controlnet_aux -from modules.control import reference # reference pipeline -from modules.control import ipadapter # reference pipeline +from modules.control.units import controlnet # lllyasviel ControlNet +from modules.control.units import xs # vislearn ControlNet-XS +from modules.control.units import lite # vislearn ControlNet-XS +from modules.control.units import t2iadapter # TencentARC T2I-Adapter +from modules.control.units import reference # reference pipeline +from modules.control.units import ipadapter # reference pipeline from modules import errors, shared, progress, sd_samplers, ui, ui_components, ui_symbols, ui_common, generation_parameters_copypaste, call_queue from modules.ui_components import FormRow, FormGroup @@ -24,18 +24,18 @@ debug('Trace: CONTROL') def initialize(): from modules import devices shared.log.debug(f'Control initialize: models={shared.opts.control_dir}') - controlnets.cache_dir = os.path.join(shared.opts.control_dir, 'controlnet') - controlnetsxs.cache_dir = os.path.join(shared.opts.control_dir, 'xs') - controlnetslite.cache_dir = os.path.join(shared.opts.control_dir, 'lite') - adapters.cache_dir = os.path.join(shared.opts.control_dir, 'adapter') + controlnet.cache_dir = os.path.join(shared.opts.control_dir, 'controlnet') + xs.cache_dir = os.path.join(shared.opts.control_dir, 'xs') + lite.cache_dir = os.path.join(shared.opts.control_dir, 'lite') + t2iadapter.cache_dir = os.path.join(shared.opts.control_dir, 'adapter') processors.cache_dir = os.path.join(shared.opts.control_dir, 'processor') unit.default_device = devices.device unit.default_dtype = devices.dtype os.makedirs(shared.opts.control_dir, exist_ok=True) - os.makedirs(controlnets.cache_dir, exist_ok=True) - os.makedirs(controlnetsxs.cache_dir, exist_ok=True) - os.makedirs(controlnetslite.cache_dir, exist_ok=True) - os.makedirs(adapters.cache_dir, exist_ok=True) + os.makedirs(controlnet.cache_dir, exist_ok=True) + os.makedirs(xs.cache_dir, exist_ok=True) + os.makedirs(lite.cache_dir, exist_ok=True) + os.makedirs(t2iadapter.cache_dir, exist_ok=True) os.makedirs(processors.cache_dir, exist_ok=True) @@ -324,8 +324,8 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): enabled_cb = gr.Checkbox(value= i==0, label="") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="ControlNet", choices=controlnets.list_models(), value='None') - ui_common.create_refresh_button(model_id, controlnets.list_models, lambda: {"choices": controlnets.list_models(refresh=True)}, 'refresh_control_models') + model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None') + ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, 'refresh_control_models') model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) @@ -369,8 +369,8 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): enabled_cb = gr.Checkbox(value= i==0, label="") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="ControlNet-XS", choices=controlnetsxs.list_models(), value='None') - ui_common.create_refresh_button(model_id, controlnetsxs.list_models, lambda: {"choices": controlnetsxs.list_models(refresh=True)}, 'refresh_control_models') + model_id = gr.Dropdown(label="ControlNet-XS", choices=xs.list_models(), value='None') + ui_common.create_refresh_button(model_id, xs.list_models, lambda: {"choices": xs.list_models(refresh=True)}, 'refresh_control_models') model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0) control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0) @@ -414,8 +414,8 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="Adapter", choices=adapters.list_models(), value='None') - ui_common.create_refresh_button(model_id, adapters.list_models, lambda: {"choices": adapters.list_models(refresh=True)}, 'refresh_adapter_models') + model_id = gr.Dropdown(label="Adapter", choices=t2iadapter.list_models(), value='None') + ui_common.create_refresh_button(model_id, t2iadapter.list_models, lambda: {"choices": t2iadapter.list_models(refresh=True)}, 'refresh_adapter_models') model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) @@ -454,8 +454,8 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') - model_id = gr.Dropdown(label="Model", choices=controlnetslite.list_models(), value='None') - ui_common.create_refresh_button(model_id, controlnetslite.list_models, lambda: {"choices": controlnetslite.list_models(refresh=True)}, 'refresh_lite_models') + model_id = gr.Dropdown(label="Model", choices=lite.list_models(), value='None') + ui_common.create_refresh_button(model_id, lite.list_models, lambda: {"choices": lite.list_models(refresh=True)}, 'refresh_lite_models') model_strength = gr.Slider(label="Strength", minimum=0.01, maximum=1.0, step=0.01, value=1.0-i/10) reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index 31ebd6d1f..7297099dc 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -16,19 +16,19 @@ from modules import scripts, processing, shared, devices image_encoder = None image_encoder_type = None loaded = None -ADAPTERS = [ - 'none', - 'ip-adapter_sd15', - 'ip-adapter_sd15_light', - 'ip-adapter-plus_sd15', - 'ip-adapter-plus-face_sd15', - 'ip-adapter-full-face_sd15', +ADAPTERS = { + 'None': 'none', + 'Base': 'ip-adapter_sd15', + 'Light': 'ip-adapter_sd15_light', + 'Plus': 'ip-adapter-plus_sd15', + 'Plus Face': 'ip-adapter-plus-face_sd15', + 'Full face': 'ip-adapter-full-face_sd15', + 'Base SXDL': 'ip-adapter_sdxl', # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072) - 'ip-adapter_sdxl', # 'sdxl_models/ip-adapter_sdxl_vit-h', # 'sdxl_models/ip-adapter-plus_sdxl_vit-h', # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h', -] +} class Script(scripts.Script): @@ -41,7 +41,7 @@ class Script(scripts.Script): def ui(self, _is_img2img): with gr.Accordion('IP Adapter', open=False, elem_id='ipadapter'): with gr.Row(): - adapter = gr.Dropdown(label='Adapter', choices=ADAPTERS, value='none') + adapter = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='none') scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5) with gr.Row(): image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512) @@ -50,6 +50,8 @@ class Script(scripts.Script): def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ from transformers import CLIPVisionModelWithProjection # overrides + adapter = ADAPTERS[adapter] + print('HERE', adapter) if hasattr(p, 'ip_adapter_name'): adapter = p.ip_adapter_name if hasattr(p, 'ip_adapter_scale'): @@ -96,6 +98,7 @@ class Script(scripts.Script): # main code subfolder = 'models' if 'sd15' in adapter else 'sdxl_models' + print('HERE2', subfolder) if adapter != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None: t0 = time.time() if loaded is not None: