diff --git a/CHANGELOG.md b/CHANGELOG.md index f215b84ec..2ecc7ef26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-10 +## Update for 2024-12-11 ### New models and integrations @@ -22,6 +22,9 @@ *recommended*: guidance scale 30 - [Depth](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev): ~23.8GB, replaces currently loaded model *recommended*: guidance scale 10 +- [Flux ControlNet LoRA](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora) + alternative to standard ControlNets, FLUX.1 also allows LoRA to help guide the generation process + both **Depth** and **Canny** LoRAs are available in standard control menus - [StabilityAI SD35 ControlNets]([sd3_medium](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets)) - In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI - [Style Aligned Image Generation](https://style-aligned-gen.github.io/) @@ -39,6 +42,10 @@ thanks @AI-Casanova - LoRA weights can be applied/unapplied as on each generate or they can store weights backups for later use this setting has large performance and resource implications, see [Offload](https://github.com/vladmandic/automatic/wiki/Offload) wiki for details + - LoRA name in prompt can now also be an absolute path to a LoRA file, even if LoRA is not indexed + example: `` + - LoRA name in prompt can now also be path to a LoRA file op `huggingface` + example: `` - **Model loader** improvements: - detect model components on model load fail - allow passing absolute path to model loader diff --git a/TODO.md b/TODO.md index 76c672260..63088d39f 100644 --- a/TODO.md +++ b/TODO.md @@ -17,6 +17,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - SANA: - LTX-Video: - TorchAO: +- ControlNetUnion/ControlNetPromax: ## Other diff --git a/installer.py b/installer.py index b020418e1..36254b66a 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '3335e2262d47e7d7e311a44dea7f454b5f01b643' # diffusers commit hash + sha = '914a585be8187ec0ad92fab4f072c992f8c297cd' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/control/run.py b/modules/control/run.py index 2fe13dd73..6ae7fb20c 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -332,7 +332,7 @@ def control_run(state: str = '', p.task_args['control_guidance_start'] = control_guidance_start p.task_args['control_guidance_end'] = control_guidance_end p.task_args['guess_mode'] = p.guess_mode - instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model) + instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p) pipe = instance.pipeline elif unit_type == 'xs' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet-XS' @@ -370,7 +370,6 @@ def control_run(state: str = '', debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}') return pipe - pipe = set_pipe() debug(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}') t1, t2, t3 = time.time(), 0, 0 diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 3f68a4896..3fa2d90eb 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -5,6 +5,7 @@ from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, FluxPi from modules.control.units import detect from modules.shared import log, opts, listdir from modules import errors, sd_models, devices, model_quant +from modules.processing import StableDiffusionProcessingControl what = 'ControlNet' @@ -75,6 +76,8 @@ predefined_f1 = { "InstantX Union": 'InstantX/FLUX.1-dev-Controlnet-Union', "InstantX Canny": 'InstantX/FLUX.1-dev-Controlnet-Canny', "JasperAI Depth": 'jasperai/Flux.1-dev-Controlnet-Depth', + "BlackForrestLabs Canny LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora/flux1-canny-dev-lora.safetensors', + "BlackForrestLabs Depth LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Depth-dev-lora/flux1-depth-dev-lora.safetensors', "JasperAI Surface Normals": 'jasperai/Flux.1-dev-Controlnet-Surface-Normals', "JasperAI Upscaler": 'jasperai/Flux.1-dev-Controlnet-Upscaler', "Shakker-Labs Union": 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro', @@ -162,12 +165,21 @@ class ControlNet(): self.model = None self.model_id = None - def get_class(self): + def get_class(self, model_id:str=''): import modules.shared if modules.shared.sd_model_type == 'sd': from diffusers import ControlNetModel as cls # pylint: disable=reimported config = 'lllyasviel/control_v11p_sd15_canny' elif modules.shared.sd_model_type == 'sdxl': + # TODO ControlNetUnion + """ + if 'union' in model_id.lower(): + from diffusers import ControlNetUnionModel as cls + config = 'xinsir/controlnet-union-sdxl-1.0' + else: + from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class + config = 'Eugeoter/noob-sdxl-controlnet-canny' + """ from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class config = 'Eugeoter/noob-sdxl-controlnet-canny' elif modules.shared.sd_model_type == 'f1': @@ -181,7 +193,7 @@ class ControlNet(): return None, None return cls, config - def load_safetensors(self, model_path): + def load_safetensors(self, model_id, model_path): name = os.path.splitext(model_path)[0] config_path = None if not os.path.exists(model_path): @@ -206,7 +218,7 @@ class ControlNet(): config_path = f'{name}.json' if config_path is not None: self.load_config['original_config_file '] = config_path - cls, config = self.get_class() + cls, config = self.get_class(model_id) if cls is None: log.error(f'Control {what} model load failed: unknown base model') else: @@ -228,18 +240,21 @@ class ControlNet(): if model_path is None: log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') return + if 'lora' in model_id.lower(): + self.model = model_path + return if model_id == self.model_id and not force: log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') return log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') + cls, _config = self.get_class(model_id) if model_path.endswith('.safetensors'): - self.load_safetensors(model_path) + self.load_safetensors(model_id, model_path) else: kwargs = {} if '/bin' in model_path: model_path = model_path.replace('/bin', '') self.load_config['use_safetensors'] = False - cls, _config = self.get_class() if cls is None: log.error(f'Control {what} model load failed: id="{model_id}" unknown base model') return @@ -271,7 +286,7 @@ class ControlNet(): self.model.to(self.device) t1 = time.time() self.model_id = model_id - log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}') return f'{what} loaded model: {model_id}' except Exception as e: log.error(f'Control {what} model load failed: id="{model_id}" error={e}') @@ -284,16 +299,30 @@ class ControlNetPipeline(): controlnet: Union[ControlNetModel, list[ControlNetModel]], pipeline: Union[StableDiffusionXLPipeline, StableDiffusionPipeline, FluxPipeline, StableDiffusion3Pipeline], dtype = None, + p: StableDiffusionProcessingControl = None, ): t0 = time.time() self.orig_pipeline = pipeline self.pipeline = None + + controlnets = controlnet if isinstance(controlnet, list) else [controlnet] + loras = [cn for cn in controlnets if isinstance(cn, str)] + controlnets = [cn for cn in controlnets if not isinstance(cn, str)] + if pipeline is None: log.error('Control model pipeline: model not loaded') return - elif detect.is_sdxl(pipeline): - from diffusers import StableDiffusionXLControlNetPipeline - self.pipeline = StableDiffusionXLControlNetPipeline( + elif detect.is_sdxl(pipeline) and len(controlnets) > 0: + from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetUnionPipeline + # TODO ControlNetUnion + """ + if controlnet.__class__.__name__ == 'ControlNetUnionModel': + cls = StableDiffusionXLControlNetUnionPipeline + else: + cls = StableDiffusionXLControlNetPipeline + """ + cls = StableDiffusionXLControlNetPipeline + self.pipeline = cls( vae=pipeline.vae, text_encoder=pipeline.text_encoder, text_encoder_2=pipeline.text_encoder_2, @@ -302,9 +331,9 @@ class ControlNetPipeline(): unet=pipeline.unet, scheduler=pipeline.scheduler, feature_extractor=getattr(pipeline, 'feature_extractor', None), - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) - elif detect.is_sd15(pipeline): + elif detect.is_sd15(pipeline) and len(controlnets) > 0: from diffusers import StableDiffusionControlNetPipeline self.pipeline = StableDiffusionControlNetPipeline( vae=pipeline.vae, @@ -315,10 +344,10 @@ class ControlNetPipeline(): feature_extractor=getattr(pipeline, 'feature_extractor', None), requires_safety_checker=False, safety_checker=None, - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) sd_models.move_model(self.pipeline, pipeline.device) - elif detect.is_f1(pipeline): + elif detect.is_f1(pipeline) and len(controlnets) > 0: from diffusers import FluxControlNetPipeline self.pipeline = FluxControlNetPipeline( vae=pipeline.vae.to(devices.device), @@ -328,9 +357,9 @@ class ControlNetPipeline(): tokenizer_2=pipeline.tokenizer_2, transformer=pipeline.transformer, scheduler=pipeline.scheduler, - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) - elif detect.is_sd3(pipeline): + elif detect.is_sd3(pipeline) and len(controlnets) > 0: from diffusers import StableDiffusion3ControlNetPipeline self.pipeline = StableDiffusion3ControlNetPipeline( vae=pipeline.vae, @@ -342,8 +371,18 @@ class ControlNetPipeline(): tokenizer_3=pipeline.tokenizer_3, transformer=pipeline.transformer, scheduler=pipeline.scheduler, - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) + elif len(loras) > 0: + self.pipeline = pipeline + for lora in loras: + log.debug(f'Control {what} pipeline: lora="{lora}"') + lora = lora.replace('/huggingface.co/', '') + self.pipeline.load_lora_weights(lora) + """ + if p is not None: + p.prompt += f'' + """ else: log.error(f'Control {what} pipeline: class={pipeline.__class__.__name__} unsupported model type') return @@ -353,6 +392,7 @@ class ControlNetPipeline(): return if dtype is not None: self.pipeline = self.pipeline.to(dtype) + if opts.diffusers_offload_mode == 'none': sd_models.move_model(self.pipeline, devices.device) from modules.sd_models import set_diffuser_offload @@ -362,5 +402,6 @@ class ControlNetPipeline(): log.debug(f'Control {what} pipeline: class={self.pipeline.__class__.__name__} time={t1-t0:.2f}') def restore(self): + self.pipeline.unload_lora_weights() self.pipeline = None return self.orig_pipeline diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 5d285af95..a38945072 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -207,11 +207,27 @@ def list_available_networks(): shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') +def network_download(name): + from huggingface_hub import hf_hub_download + if os.path.exists(name): + return network.NetworkOnDisk(name, name) + parts = name.split('/') + if len(parts) >= 5 and parts[1] == 'huggingface.co': + repo_id = f'{parts[2]}/{parts[3]}' + filename = '/'.join(parts[4:]) + fn = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=shared.opts.hfcache_dir) + return network.NetworkOnDisk(name, fn) + return None + + def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] + for i in range(len(names)): + if names[i].startswith('/'): + networks_on_disk[i] = network_download(names[i]) failed_to_load_networks = [] recompile_model = maybe_recompile_model(names, te_multipliers) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 3c59bbcf7..3b6f228ba 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -375,10 +375,10 @@ def process_decode(p: processing.StableDiffusionProcessing, output): elif hasattr(output, 'images'): results = output.images else: - shared.log.warning('Processing returned no results') + shared.log.warning('Processing: no results') results = [] else: - shared.log.warning('Processing returned no results') + shared.log.warning('Processing: no results') results = [] return results diff --git a/requirements.txt b/requirements.txt index 3b1b14c7d..d9eba6958 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ pi-heif # versioned safetensors==0.4.5 tensordict==0.1.2 -peft==0.13.1 +peft==0.14.0 httpx==0.24.1 compel==2.0.3 torchsde==0.2.6