From ce2234178d760104ef6441192c24940eed67aa4a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 29 Oct 2024 11:02:39 -0400 Subject: [PATCH] add sd35 medium Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 +++++++- html/reference.json | 12 ++++++++++-- installer.py | 2 +- modules/model_sd3.py | 26 +++++++++++++++++++++++--- modules/modelloader.py | 2 +- wiki | 2 +- 6 files changed, 43 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c955f0ce3..cfa77c3bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-10-28 +## Update for 2024-10-29 improvements: - model selector: @@ -19,6 +19,7 @@ improvements: - handle missing model components during load - handle component preloading - native lora handler + - support for all sd35 variants: *medium/large/large-turbo* - gguf transformer loader (prototype) - samplers: - support for original k-diffusion samplers @@ -37,6 +38,11 @@ improvements: - OpenVINO: add accuracy option - ZLUDA: guess GPU arch - major model load refactor +- wiki: new articles + - [Gated Access Wiki](https://github.com/vladmandic/automatic/wiki/Gated) + - [Quantization Wiki](https://github.com/vladmandic/automatic/wiki/Quantization) + - [Offloading Wiki](https://github.com/vladmandic/automatic/wiki/Offload) + fixes: - fix send-to-control diff --git a/html/reference.json b/html/reference.json index 8d26433e7..4a549586f 100644 --- a/html/reference.json +++ b/html/reference.json @@ -119,11 +119,19 @@ "preview": "stabilityai--stable-diffusion-3.jpg", "extras": "sampler: Default, cfg_scale: 7.0" }, + "StabilityAI Stable Diffusion 3.5 Medium": { + "path": "stabilityai/stable-diffusion-3.5-medium", + "skip": true, + "variant": "fp16", + "desc": "Stable Diffusion 3.5 Medium is a Multimodal Diffusion Transformer with improvements (MMDiT-X) text-to-image model that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency.", + "preview": "stabilityai--stable-diffusion-3_5.jpg", + "extras": "sampler: Default, cfg_scale: 7.0" + }, "StabilityAI Stable Diffusion 3.5 Large": { "path": "stabilityai/stable-diffusion-3.5-large", "skip": true, "variant": "fp16", - "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", + "desc": "Stable Diffusion 3.5 Large is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency.", "preview": "stabilityai--stable-diffusion-3_5.jpg", "extras": "sampler: Default, cfg_scale: 7.0" }, @@ -131,7 +139,7 @@ "path": "stabilityai/stable-diffusion-3.5-large-turbo", "skip": true, "variant": "fp16", - "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", + "desc": "Stable Diffusion 3.5 Large Turbo is a Multimodal Diffusion Transformer (MMDiT) text-to-image model with Adversarial Diffusion Distillation (ADD) that features improved performance in image quality, typography, complex prompt understanding, and resource-efficiency, with a focus on fewer inference steps.", "preview": "stabilityai--stable-diffusion-3_5.jpg", "extras": "sampler: Default, cfg_scale: 7.0" }, diff --git a/installer.py b/installer.py index 207dde388..f65b0e58f 100644 --- a/installer.py +++ b/installer.py @@ -455,7 +455,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): # check diffusers version def check_diffusers(): - sha = '435f6b7e47c031f98b8374b1689e1abeb17bfdb6' + sha = '0d1d267b12e47b40b0e8f265339c76e0f45f8c49' pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/model_sd3.py b/modules/model_sd3.py index da99e6c4b..8e358b8b1 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -49,6 +49,24 @@ def load_overrides(kwargs, cache_dir): return kwargs +def create_bnb_config(kwargs): + if len(shared.opts.bnb_quantization) > 0: + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + from modules.model_quant import load_bnb + load_bnb('Load model: type=SD3') + bnb_config = diffusers.BitsAndBytesConfig( + load_in_8bit=shared.opts.bnb_quantization_type in ['fp8'], + load_in_4bit=shared.opts.bnb_quantization_type in ['nf4', 'fp4'], + bnb_4bit_quant_storage=shared.opts.bnb_quantization_storage, + bnb_4bit_quant_type=shared.opts.bnb_quantization_type, + bnb_4bit_compute_dtype=devices.dtype + ) + kwargs['quantization_config'] = bnb_config + shared.log.debug(f'Quantization: module=all type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + + return kwargs + + def load_quants(kwargs, repo_id, cache_dir): if len(shared.opts.bnb_quantization) > 0: from modules.model_quant import load_bnb @@ -127,7 +145,7 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): kwargs = load_quants(kwargs, repo_id, cache_dir) loader = diffusers.StableDiffusion3Pipeline.from_pretrained - if fn is not None and os.path.exists(fn): + if fn is not None and os.path.exists(fn) and os.path.isfile(fn): if fn.endswith('.safetensors'): loader = diffusers.StableDiffusion3Pipeline.from_single_file kwargs = load_missing(kwargs, fn, cache_dir) @@ -139,8 +157,10 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): else: kwargs['variant'] = 'fp16' - shared.log.debug(f'Load model: type=SD3 preloaded={list(kwargs)}') + shared.log.debug(f'Load model: type=SD3 kwargs={list(kwargs)}') + kwargs = create_bnb_config(kwargs) + print('HERE', repo_id, kwargs) pipe = loader( repo_id, torch_dtype=devices.dtype, @@ -148,5 +168,5 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): config=config, **kwargs, ) - devices.torch_gc() + devices.torch_gc(force=True) return pipe diff --git a/modules/modelloader.py b/modules/modelloader.py index 70ea7cabb..8eab91597 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -318,7 +318,7 @@ def load_diffusers_models(clear=True): def find_diffuser(name: str, full=False): repo = [r for r in diffuser_repos if name == r['name'] or name == r['friendly'] or name == r['path']] if len(repo) > 0: - return repo['name'] + return [repo[0]['name']] hf_api = hf.HfApi() models = list(hf_api.list_models(model_name=name, library=['diffusers'], full=True, limit=20, sort="downloads", direction=-1)) shared.log.debug(f'Searching diffusers models: {name} {len(models) > 0}') diff --git a/wiki b/wiki index b2d3110d4..6e278d8aa 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit b2d3110d42ef1417008295425437721b25369cb1 +Subproject commit 6e278d8aa2d559b9b46357086c341b9bd8cf33fd