Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-04 12:00:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

lora: absolute path, hf download, flux controlnet loras

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2024-12-11 18:10:51 -05:00
parent 8f21e96f73
commit bd540efb73
8 changed files with 87 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+8 -1
View File
@@ -1,6 +1,6 @@
# Change Log for SD.Next
## Update for 2024-12-10
## Update for 2024-12-11
### New models and integrations
@@ -22,6 +22,9 @@
*recommended*: guidance scale 30
- [Depth](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev): ~23.8GB, replaces currently loaded model
*recommended*: guidance scale 10
- [Flux ControlNet LoRA](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora)
alternative to standard ControlNets, FLUX.1 also allows LoRA to help guide the generation process
both **Depth** and **Canny** LoRAs are available in standard control menus
- [StabilityAI SD35 ControlNets]([sd3_medium](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets))
- In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI
- [Style Aligned Image Generation](https://style-aligned-gen.github.io/)
@@ -39,6 +42,10 @@
thanks @AI-Casanova
- LoRA weights can be applied/unapplied as on each generate or they can store weights backups for later use
this setting has large performance and resource implications, see [Offload](https://github.com/vladmandic/automatic/wiki/Offload) wiki for details
- LoRA name in prompt can now also be an absolute path to a LoRA file, even if LoRA is not indexed
example: `<lora:/test/folder/my-lora.safetensors:1.0>`
- LoRA name in prompt can now also be path to a LoRA file op `huggingface`
example: `<lora:/huggingface.co/vendor/repo/my-lora.safetensors:1.0>`
- **Model loader** improvements:
- detect model components on model load fail
- allow passing absolute path to model loader
TODO.md
+1
View File
@@ -17,6 +17,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
- SANA: <https://github.com/huggingface/diffusers/pull/9982>
- LTX-Video: <https://github.com/huggingface/diffusers/pull/10021> <https://huggingface.co/Lightricks/LTX-Video> <https://huggingface.co/spaces/Lightricks/LTX-Video-Playground/tree/main>
- TorchAO: <https://github.com/huggingface/diffusers/pull/10009>
- ControlNetUnion/ControlNetPromax: <https://github.com/huggingface/diffusers/pull/10131>
## Other
installer.py
+1 -1
View File
@@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None):
def check_diffusers():
if args.skip_all or args.skip_requirements:
return
sha = '3335e2262d47e7d7e311a44dea7f454b5f01b643' # diffusers commit hash
sha = '914a585be8187ec0ad92fab4f072c992f8c297cd' # diffusers commit hash
pkg = pkg_resources.working_set.by_key.get('diffusers', None)
minor = int(pkg.version.split('.')[1] if pkg is not None else 0)
cur = opts.get('diffusers_version', '') if minor > 0 else ''
modules/control/run.py
+1 -2
View File
@@ -332,7 +332,7 @@ def control_run(state: str = '',
p.task_args['control_guidance_start'] = control_guidance_start
p.task_args['control_guidance_end'] = control_guidance_end
p.task_args['guess_mode'] = p.guess_mode
instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model)
instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p)
pipe = instance.pipeline
elif unit_type == 'xs' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet-XS'
@@ -370,7 +370,6 @@ def control_run(state: str = '',
debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}')
return pipe
pipe = set_pipe()
debug(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}')
t1, t2, t3 = time.time(), 0, 0
modules/control/units/controlnet.py
+57 -16
View File
@@ -5,6 +5,7 @@ from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, FluxPi
from modules.control.units import detect
from modules.shared import log, opts, listdir
from modules import errors, sd_models, devices, model_quant
from modules.processing import StableDiffusionProcessingControl
what = 'ControlNet'
@@ -75,6 +76,8 @@ predefined_f1 = {
"InstantX Union": 'InstantX/FLUX.1-dev-Controlnet-Union',
"InstantX Canny": 'InstantX/FLUX.1-dev-Controlnet-Canny',
"JasperAI Depth": 'jasperai/Flux.1-dev-Controlnet-Depth',
"BlackForrestLabs Canny LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora/flux1-canny-dev-lora.safetensors',
"BlackForrestLabs Depth LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Depth-dev-lora/flux1-depth-dev-lora.safetensors',
"JasperAI Surface Normals": 'jasperai/Flux.1-dev-Controlnet-Surface-Normals',
"JasperAI Upscaler": 'jasperai/Flux.1-dev-Controlnet-Upscaler',
"Shakker-Labs Union": 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro',
@@ -162,12 +165,21 @@ class ControlNet():
self.model = None
self.model_id = None
def get_class(self):
def get_class(self, model_id:str=''):
import modules.shared
if modules.shared.sd_model_type == 'sd':
from diffusers import ControlNetModel as cls # pylint: disable=reimported
config = 'lllyasviel/control_v11p_sd15_canny'
elif modules.shared.sd_model_type == 'sdxl':
# TODO ControlNetUnion
"""
if 'union' in model_id.lower():
from diffusers import ControlNetUnionModel as cls
config = 'xinsir/controlnet-union-sdxl-1.0'
else:
from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class
config = 'Eugeoter/noob-sdxl-controlnet-canny'
"""
from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class
config = 'Eugeoter/noob-sdxl-controlnet-canny'
elif modules.shared.sd_model_type == 'f1':
@@ -181,7 +193,7 @@ class ControlNet():
return None, None
return cls, config
def load_safetensors(self, model_path):
def load_safetensors(self, model_id, model_path):
name = os.path.splitext(model_path)[0]
config_path = None
if not os.path.exists(model_path):
@@ -206,7 +218,7 @@ class ControlNet():
config_path = f'{name}.json'
if config_path is not None:
self.load_config['original_config_file '] = config_path
cls, config = self.get_class()
cls, config = self.get_class(model_id)
if cls is None:
log.error(f'Control {what} model load failed: unknown base model')
else:
@@ -228,18 +240,21 @@ class ControlNet():
if model_path is None:
log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id')
return
if 'lora' in model_id.lower():
self.model = model_path
return
if model_id == self.model_id and not force:
log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded')
return
log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"')
cls, _config = self.get_class(model_id)
if model_path.endswith('.safetensors'):
self.load_safetensors(model_path)
self.load_safetensors(model_id, model_path)
else:
kwargs = {}
if '/bin' in model_path:
model_path = model_path.replace('/bin', '')
self.load_config['use_safetensors'] = False
cls, _config = self.get_class()
if cls is None:
log.error(f'Control {what} model load failed: id="{model_id}" unknown base model')
return
@@ -271,7 +286,7 @@ class ControlNet():
self.model.to(self.device)
t1 = time.time()
self.model_id = model_id
log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}')
log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}')
return f'{what} loaded model: {model_id}'
except Exception as e:
log.error(f'Control {what} model load failed: id="{model_id}" error={e}')
@@ -284,16 +299,30 @@ class ControlNetPipeline():
controlnet: Union[ControlNetModel, list[ControlNetModel]],
pipeline: Union[StableDiffusionXLPipeline, StableDiffusionPipeline, FluxPipeline, StableDiffusion3Pipeline],
dtype = None,
p: StableDiffusionProcessingControl = None,
):
t0 = time.time()
self.orig_pipeline = pipeline
self.pipeline = None
controlnets = controlnet if isinstance(controlnet, list) else [controlnet]
loras = [cn for cn in controlnets if isinstance(cn, str)]
controlnets = [cn for cn in controlnets if not isinstance(cn, str)]
if pipeline is None:
log.error('Control model pipeline: model not loaded')
return
elif detect.is_sdxl(pipeline):
from diffusers import StableDiffusionXLControlNetPipeline
self.pipeline = StableDiffusionXLControlNetPipeline(
elif detect.is_sdxl(pipeline) and len(controlnets) > 0:
from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetUnionPipeline
# TODO ControlNetUnion
"""
if controlnet.__class__.__name__ == 'ControlNetUnionModel':
cls = StableDiffusionXLControlNetUnionPipeline
else:
cls = StableDiffusionXLControlNetPipeline
"""
cls = StableDiffusionXLControlNetPipeline
self.pipeline = cls(
vae=pipeline.vae,
text_encoder=pipeline.text_encoder,
text_encoder_2=pipeline.text_encoder_2,
@@ -302,9 +331,9 @@ class ControlNetPipeline():
unet=pipeline.unet,
scheduler=pipeline.scheduler,
feature_extractor=getattr(pipeline, 'feature_extractor', None),
controlnet=controlnet, # can be a list
controlnet=controlnets, # can be a list
)
elif detect.is_sd15(pipeline):
elif detect.is_sd15(pipeline) and len(controlnets) > 0:
from diffusers import StableDiffusionControlNetPipeline
self.pipeline = StableDiffusionControlNetPipeline(
vae=pipeline.vae,
@@ -315,10 +344,10 @@ class ControlNetPipeline():
feature_extractor=getattr(pipeline, 'feature_extractor', None),
requires_safety_checker=False,
safety_checker=None,
controlnet=controlnet, # can be a list
controlnet=controlnets, # can be a list
)
sd_models.move_model(self.pipeline, pipeline.device)
elif detect.is_f1(pipeline):
elif detect.is_f1(pipeline) and len(controlnets) > 0:
from diffusers import FluxControlNetPipeline
self.pipeline = FluxControlNetPipeline(
vae=pipeline.vae.to(devices.device),
@@ -328,9 +357,9 @@ class ControlNetPipeline():
tokenizer_2=pipeline.tokenizer_2,
transformer=pipeline.transformer,
scheduler=pipeline.scheduler,
controlnet=controlnet, # can be a list
controlnet=controlnets, # can be a list
)
elif detect.is_sd3(pipeline):
elif detect.is_sd3(pipeline) and len(controlnets) > 0:
from diffusers import StableDiffusion3ControlNetPipeline
self.pipeline = StableDiffusion3ControlNetPipeline(
vae=pipeline.vae,
@@ -342,8 +371,18 @@ class ControlNetPipeline():
tokenizer_3=pipeline.tokenizer_3,
transformer=pipeline.transformer,
scheduler=pipeline.scheduler,
controlnet=controlnet, # can be a list
controlnet=controlnets, # can be a list
)
elif len(loras) > 0:
self.pipeline = pipeline
for lora in loras:
log.debug(f'Control {what} pipeline: lora="{lora}"')
lora = lora.replace('/huggingface.co/', '')
self.pipeline.load_lora_weights(lora)
"""
if p is not None:
p.prompt += f'<lora:{lora}:1.0>'
"""
else:
log.error(f'Control {what} pipeline: class={pipeline.__class__.__name__} unsupported model type')
return
@@ -353,6 +392,7 @@ class ControlNetPipeline():
return
if dtype is not None:
self.pipeline = self.pipeline.to(dtype)
if opts.diffusers_offload_mode == 'none':
sd_models.move_model(self.pipeline, devices.device)
from modules.sd_models import set_diffuser_offload
@@ -362,5 +402,6 @@ class ControlNetPipeline():
log.debug(f'Control {what} pipeline: class={self.pipeline.__class__.__name__} time={t1-t0:.2f}')
def restore(self):
self.pipeline.unload_lora_weights()
self.pipeline = None
return self.orig_pipeline
modules/lora/networks.py
+16
View File
@@ -207,11 +207,27 @@ def list_available_networks():
shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}')
def network_download(name):
from huggingface_hub import hf_hub_download
if os.path.exists(name):
return network.NetworkOnDisk(name, name)
parts = name.split('/')
if len(parts) >= 5 and parts[1] == 'huggingface.co':
repo_id = f'{parts[2]}/{parts[3]}'
filename = '/'.join(parts[4:])
fn = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=shared.opts.hfcache_dir)
return network.NetworkOnDisk(name, fn)
return None
def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None):
networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names]
if any(x is None for x in networks_on_disk):
list_available_networks()
networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names]
for i in range(len(names)):
if names[i].startswith('/'):
networks_on_disk[i] = network_download(names[i])
failed_to_load_networks = []
recompile_model = maybe_recompile_model(names, te_multipliers)
modules/processing_diffusers.py
+2 -2
View File
@@ -375,10 +375,10 @@ def process_decode(p: processing.StableDiffusionProcessing, output):
elif hasattr(output, 'images'):
results = output.images
else:
shared.log.warning('Processing returned no results')
shared.log.warning('Processing: no results')
results = []
else:
shared.log.warning('Processing returned no results')
shared.log.warning('Processing: no results')
results = []
return results
requirements.txt
+1 -1
View File
@@ -34,7 +34,7 @@ pi-heif
# versioned
safetensors==0.4.5
tensordict==0.1.2
peft==0.13.1
peft==0.14.0
httpx==0.24.1
compel==2.0.3
torchsde==0.2.6