diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd62512b..293fc5651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,18 @@ # Change Log for SD.Next -## Update for 2025-11-07 +## Update for 2025-11-08 - **Features** - - allow recursive inline wildcards using curly braces syntax - - simplify SDNQ pre-quantization saved config + - **wildcards**: allow recursive inline wildcards using curly braces syntax + - **sdnq**: simplify pre-quantization saved config + - **attention**: refactor settings and improve handling of attention mechanisms + - **lora**: separate fuse setting for native-vs-diffuser implementations + - **auth**: strong-enforce auth check on all api endpoints - **Fixes** - - hires strength save/load in metadata + - hires strength save/load in metadata, thanks @awsr + - fix imgi2img initial scale tab, thanks @awsr + - fix pony-v7 text-encoder + - detailer with face-restorers ## Update for 2025-11-06 diff --git a/cli/api-txt2img.js b/cli/api-txt2img.js index 8d0e9f5d1..7b0f6994a 100755 --- a/cli/api-txt2img.js +++ b/cli/api-txt2img.js @@ -30,10 +30,15 @@ async function main() { const headers = new Headers(); const body = JSON.stringify(sd_options); headers.set('Content-Type', 'application/json'); - if (sd_username && sd_password) headers.set({ Authorization: `Basic ${btoa('sd_username:sd_password')}` }); + if (sd_username && sd_password) { + // const credentials = btoa(`${sd_username}:${sd_password}`); + const credentials = Buffer.from(`${sd_username}:${sd_password}`).toString('base64'); + headers.set('Authorization', `Basic ${credentials}`); + } const res = await fetch(`${sd_url}/sdapi/v1/txt2img`, { method, headers, body }); if (res.status !== 200) { - console.log('Error', res.status); + const err = await res.text(); + console.log('Error', res.status, res.statusText, err); } else { const json = await res.json(); console.log('result:', json.info); diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 19a1b1b72..c724b86a7 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 19a1b1b722b768b821f67831a6d3ba83847bcb61 +Subproject commit c724b86a7a35208ee1132ab06c2ae5b7ed395824 diff --git a/extensions-builtin/sdnext-kanvas b/extensions-builtin/sdnext-kanvas deleted file mode 120000 index 994c8a3b0..000000000 --- a/extensions-builtin/sdnext-kanvas +++ /dev/null @@ -1 +0,0 @@ -/home/vlado/dev/kanvas \ No newline at end of file diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 452b7044b..f14464f3f 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 452b7044b85326436c3ac73e722cd1a1d482ecfb +Subproject commit f14464f3f7bb0de9ef854a40f7f56f8dd0378f42 diff --git a/javascript/login.js b/javascript/login.js index 3f2ab1f64..a9eeb0be9 100644 --- a/javascript/login.js +++ b/javascript/login.js @@ -4,21 +4,21 @@ const loginCSS = ` left: 0; width: 100%; height: 100%; - background: var(--background-fill-primary); - color: var(--body-text-color-subdued); + background: #222; + color: #ddd; font-family: monospace; z-index: 100; `; const loginHTML = ` -
+

Login

- + - +
- +
`; diff --git a/modules/api/api.py b/modules/api/api.py index a260c22ae..ca03f3433 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -4,7 +4,7 @@ from secrets import compare_digest from fastapi import FastAPI, APIRouter, Depends, Request from fastapi.security import HTTPBasic, HTTPBasicCredentials from fastapi.exceptions import HTTPException -from modules import errors, shared, postprocessing +from modules import errors, shared from modules.api import models, endpoints, script, helpers, server, generate, process, control, docs, gpu @@ -60,8 +60,8 @@ class Api: self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img) self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img) self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl) - self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage) - self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch) + self.add_api_route("/sdapi/v1/extra-single-image", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage) + self.add_api_route("/sdapi/v1/extra-batch-images", self.process.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch) self.add_api_route("/sdapi/v1/preprocess", self.process.post_preprocess, methods=["POST"]) self.add_api_route("/sdapi/v1/mask", self.process.post_mask, methods=["POST"]) self.add_api_route("/sdapi/v1/detect", self.process.post_detect, methods=["POST"]) @@ -117,17 +117,22 @@ class Api: from modules.civitai import api_civitai api_civitai.register_api() - - def add_api_route(self, path: str, endpoint, **kwargs): + def add_api_route(self, path: str, fn, **kwargs): + if self.credentials: + deps = list(kwargs.get('dependencies', [])) + deps.append(Depends(self.auth)) + kwargs['dependencies'] = deps if shared.opts.subpath is not None and len(shared.opts.subpath) > 0: - self.app.add_api_route(f'{shared.opts.subpath}{path}', endpoint, **kwargs) - self.app.add_api_route(path, endpoint, **kwargs) + self.app.add_api_route(f'{shared.opts.subpath}{path}', endpoint=fn, **kwargs) + self.app.add_api_route(path, endpoint=fn, **kwargs) def auth(self, credentials: HTTPBasicCredentials = Depends(HTTPBasic())): - # this is only needed for api-only since otherwise auth is handled in gradio/routes.py + if not self.credentials: + return True if credentials.username in self.credentials: if compare_digest(credentials.password, self.credentials[credentials.username]): return True + shared.log.error(f'API authentication: user="{credentials.username}" password="{credentials.password}"') raise HTTPException(status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": "Basic"}) def get_session_start(self, req: Request, agent: Optional[str] = None): @@ -136,27 +141,6 @@ class Api: shared.log.info(f'Browser session: user={user} client={req.client.host} agent={agent}') return {} - def set_upscalers(self, req: dict): - reqDict = vars(req) - reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None) - reqDict['extras_upscaler_2'] = reqDict.pop('upscaler_2', None) - return reqDict - - def extras_single_image_api(self, req: models.ReqProcessImage): - reqDict = self.set_upscalers(req) - reqDict['image'] = helpers.decode_base64_to_image(reqDict['image']) - with self.queue_lock: - result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict) - return models.ResProcessImage(image=helpers.encode_pil_to_base64(result[0][0]), html_info=result[1]) - - def extras_batch_images_api(self, req: models.ReqProcessBatch): - reqDict = self.set_upscalers(req) - image_list = reqDict.pop('imageList', []) - image_folder = [helpers.decode_base64_to_image(x.data) for x in image_list] - with self.queue_lock: - result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict) - return models.ResProcessBatch(images=list(map(helpers.encode_pil_to_base64, result[0])), html_info=result[1]) - def launch(self): config = { "listen": shared.cmd_opts.listen, diff --git a/modules/api/process.py b/modules/api/process.py index 3151907a5..c106a18e2 100644 --- a/modules/api/process.py +++ b/modules/api/process.py @@ -4,8 +4,8 @@ from pydantic import BaseModel, Field # pylint: disable=no-name-in-module from fastapi.responses import JSONResponse from fastapi.exceptions import HTTPException from modules.api.helpers import decode_base64_to_image, encode_pil_to_base64 -from modules import errors, shared -from modules.api import models +from modules import errors, shared, postprocessing +from modules.api import models, helpers processor = None # cached instance of processor @@ -175,3 +175,24 @@ class APIProcess(): raise HTTPException(status_code=400, detail="prompt enhancement: invalid type") res = models.ResPromptEnhance(prompt=prompt, seed=seed) return res + + def set_upscalers(self, req: dict): + reqDict = vars(req) + reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None) + reqDict['extras_upscaler_2'] = reqDict.pop('upscaler_2', None) + return reqDict + + def extras_single_image_api(self, req: models.ReqProcessImage): + reqDict = self.set_upscalers(req) + reqDict['image'] = helpers.decode_base64_to_image(reqDict['image']) + with self.queue_lock: + result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict) + return models.ResProcessImage(image=helpers.encode_pil_to_base64(result[0][0]), html_info=result[1]) + + def extras_batch_images_api(self, req: models.ReqProcessBatch): + reqDict = self.set_upscalers(req) + image_list = reqDict.pop('imageList', []) + image_folder = [helpers.decode_base64_to_image(x.data) for x in image_list] + with self.queue_lock: + result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict) + return models.ResProcessBatch(images=list(map(helpers.encode_pil_to_base64, result[0])), html_info=result[1]) diff --git a/modules/attention.py b/modules/attention.py new file mode 100644 index 000000000..1ad1b1d4b --- /dev/null +++ b/modules/attention.py @@ -0,0 +1,194 @@ +from typing import Optional +from functools import wraps +import torch +from modules import rocm +from modules.errors import log +from installer import install, installed + + +def set_dynamic_attention(): + try: + sdpa_pre_dyanmic_atten = torch.nn.functional.scaled_dot_product_attention + from modules.sd_hijack_dynamic_atten import dynamic_scaled_dot_product_attention + torch.nn.functional.scaled_dot_product_attention = dynamic_scaled_dot_product_attention + return sdpa_pre_dyanmic_atten + except Exception as err: + log.error(f'Torch attention: type="dynamic attention" {err}') + return None + +def set_triton_flash_attention(): + try: + from modules.flash_attn_triton_amd import interface_fa + sdpa_pre_triton_flash_atten = torch.nn.functional.scaled_dot_product_attention + @wraps(sdpa_pre_triton_flash_atten) + def sdpa_triton_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: + if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32: + if scale is None: + scale = query.shape[-1] ** (-0.5) + head_size_og = query.size(3) + if head_size_og % 8 != 0: + query = torch.nn.functional.pad(query, [0, 8 - head_size_og % 8]) + key = torch.nn.functional.pad(key, [0, 8 - head_size_og % 8]) + value = torch.nn.functional.pad(value, [0, 8 - head_size_og % 8]) + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + out_padded = torch.zeros_like(query) + interface_fa.fwd(query, key, value, out_padded, dropout_p, scale, is_causal) + return out_padded[..., :head_size_og].transpose(1, 2) + else: + if enable_gqa: + kwargs["enable_gqa"] = enable_gqa + return sdpa_pre_triton_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) + torch.nn.functional.scaled_dot_product_attention = sdpa_triton_flash_atten + log.debug('Torch attention: type="Triton Flash attention"') + except Exception as err: + log.error(f'Torch attention: type="Triton Flash attention" {err}') + +def set_ck_flash_attention(backend: str, device: torch.device): + try: + if backend == "rocm": + if not installed('flash-attn'): + log.info('Torch attention: type="CK Flash" building...') + agent = rocm.Agent(getattr(torch.cuda.get_device_properties(device), "gcnArchName", "gfx0000")) + install(rocm.get_flash_attention_command(agent), reinstall=True) + else: + install('flash-attn') + from flash_attn import flash_attn_func + sdpa_pre_flash_atten = torch.nn.functional.scaled_dot_product_attention + @wraps(sdpa_pre_flash_atten) + def sdpa_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: + if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32: + is_unsqueezed = False + if query.dim() == 3: + query = query.unsqueeze(0) + is_unsqueezed = True + if key.dim() == 3: + key = key.unsqueeze(0) + if value.dim() == 3: + value = value.unsqueeze(0) + if enable_gqa: + key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) + value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + attn_output = flash_attn_func(q=query, k=key, v=value, dropout_p=dropout_p, causal=is_causal, softmax_scale=scale).transpose(1, 2) + if is_unsqueezed: + attn_output = attn_output.squeeze(0) + return attn_output + else: + if enable_gqa: + kwargs["enable_gqa"] = enable_gqa + return sdpa_pre_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) + torch.nn.functional.scaled_dot_product_attention = sdpa_flash_atten + log.debug('Torch attention: type="CK Flash attention"') + except Exception as err: + log.error(f'Torch attention: type="CK Flash attention" {err}') + +def set_sage_attention(backend: str, device: torch.device): + try: + install('sageattention') + + use_cuda_backend = False + if (backend == "cuda") and (torch.cuda.get_device_capability(device) == (8, 6)): + use_cuda_backend = True # Detect GPU architecture - sm86 confirmed to need CUDA backend workaround as Sage Attention + Triton causes NaNs + try: + from sageattention import sageattn_qk_int8_pv_fp16_cuda + except: + use_cuda_backend = False + + if use_cuda_backend: + from sageattention import sageattn_qk_int8_pv_fp16_cuda + def sage_attn_impl(query, key, value, is_causal, scale): + return sageattn_qk_int8_pv_fp16_cuda( + q=query, k=key, v=value, + tensor_layout="HND", + is_causal=is_causal, + sm_scale=scale, + return_lse=False, + pv_accum_dtype="fp32", + ) + else: + from sageattention import sageattn + def sage_attn_impl(query, key, value, is_causal, scale): + return sageattn( + q=query, k=key, v=value, + attn_mask=None, + dropout_p=0.0, + is_causal=is_causal, + scale=scale, + ) + + sdpa_pre_sage_atten = torch.nn.functional.scaled_dot_product_attention + @wraps(sdpa_pre_sage_atten) + def sdpa_sage_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: + if (query.shape[-1] in {128, 96, 64}) and (attn_mask is None) and (query.dtype != torch.float32): + if enable_gqa: + key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) + value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) + + # Call pre-selected sage attention implementation + return sage_attn_impl(query, key, value, is_causal, scale) + else: + if enable_gqa: + kwargs["enable_gqa"] = enable_gqa + return sdpa_pre_sage_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) + torch.nn.functional.scaled_dot_product_attention = sdpa_sage_atten + log.debug(f'Torch attention: type="Sage attention" backend={"cuda" if use_cuda_backend else "auto"}') + except Exception as err: + log.error(f'Torch attention: type="Sage attention" {err}') + + +def set_diffusers_attention(pipe, quiet:bool=False): + from modules import shared + import diffusers.models.attention_processor as p + + def set_attn(pipe, attention, name:str=None): + if attention is None: + return + # other models uses their own attention processor + if getattr(pipe, "unet", None) is not None and hasattr(pipe.unet, "set_attn_processor"): + try: + pipe.unet.set_attn_processor(attention) + except Exception as e: + if 'Nunchaku' in pipe.unet.__class__.__name__: + pass + else: + shared.log.error(f'Torch attention: type="{name}" cls={attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}') + """ # each transformer typically has its own attention processor + if getattr(pipe, "transformer", None) is not None and hasattr(pipe.transformer, "set_attn_processor"): + try: + pipe.transformer.set_attn_processor(attention) + except Exception as e: + if 'Nunchaku' in pipe.transformer.__class__.__name__: + pass + else: + shared.log.error(f'Torch attention: type="{name}" cls={attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}') + """ + + shared.log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"') + if shared.opts.cross_attention_optimization == "Disabled": + pass # do nothing + elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers + # set_attn(pipe, p.AttnProcessor2_0(), name="Scaled-Dot-Product") + pass + elif shared.opts.cross_attention_optimization == "xFormers": + if hasattr(pipe, 'enable_xformers_memory_efficient_attention'): + pipe.enable_xformers_memory_efficient_attention() + else: + shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") + elif shared.opts.cross_attention_optimization == "Batch matrix-matrix": + set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix") + elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM": + from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM + set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM") + + if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"): + if shared.opts.attention_slicing: + pipe.enable_attention_slicing() + else: + pipe.disable_attention_slicing() + shared.log.debug(f"Torch attention: slicing={shared.opts.attention_slicing}") + + pipe.current_attn_name = shared.opts.cross_attention_optimization diff --git a/modules/devices.py b/modules/devices.py index 8945c0a58..f8ef7d3d1 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -1,14 +1,10 @@ -from typing import Optional - import os import sys import time import contextlib -from functools import wraps import torch -from modules import rocm +from modules import rocm, attention from modules.errors import log, display, install as install_traceback -from installer import install, installed debug = os.environ.get('SD_DEVICE_DEBUG', None) is not None @@ -462,148 +458,30 @@ def set_sdpa_params(): log.warning(f'Torch attention: type="sdpa" {err}') try: - torch.backends.cuda.enable_flash_sdp('Flash attention' in opts.sdp_options) - torch.backends.cuda.enable_mem_efficient_sdp('Memory attention' in opts.sdp_options) - torch.backends.cuda.enable_math_sdp('Math attention' in opts.sdp_options) + torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options) + torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options) + torch.backends.cuda.enable_math_sdp('Math' in opts.sdp_options) if hasattr(torch.backends.cuda, "allow_fp16_bf16_reduction_math_sdp"): # only valid for torch >= 2.5 torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True) - log.debug(f'Torch attention: type="sdpa" opts={opts.sdp_options}') + log.debug(f'Torch attention: type="sdpa" kernels={opts.sdp_options} overrides={opts.sdp_overrides}') except Exception as err: log.warning(f'Torch attention: type="sdpa" {err}') # Stack hijcaks in reverse order. This gives priority to the last added hijack. # If the last hijack is not compatible, it will use the one before it and so on. - if 'Dynamic attention' in opts.sdp_options: - try: - global sdpa_pre_dyanmic_atten # pylint: disable=global-statement - sdpa_pre_dyanmic_atten = torch.nn.functional.scaled_dot_product_attention - from modules.sd_hijack_dynamic_atten import dynamic_scaled_dot_product_attention - torch.nn.functional.scaled_dot_product_attention = dynamic_scaled_dot_product_attention - except Exception as err: - log.error(f'Torch attention: type="dynamic attention" {err}') + if 'Dynamic attention' in opts.sdp_overrides: + global sdpa_pre_dyanmic_atten # pylint: disable=global-statement + sdpa_pre_dyanmic_atten = attention.set_dynamic_attention() - if 'Triton Flash attention' in opts.sdp_options: - try: - if backend in {"zluda", "rocm"}: - from modules.flash_attn_triton_amd import interface_fa - sdpa_pre_triton_flash_atten = torch.nn.functional.scaled_dot_product_attention - @wraps(sdpa_pre_triton_flash_atten) - def sdpa_triton_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: - if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32: - if scale is None: - scale = query.shape[-1] ** (-0.5) - head_size_og = query.size(3) - if head_size_og % 8 != 0: - query = torch.nn.functional.pad(query, [0, 8 - head_size_og % 8]) - key = torch.nn.functional.pad(key, [0, 8 - head_size_og % 8]) - value = torch.nn.functional.pad(value, [0, 8 - head_size_og % 8]) - query = query.transpose(1, 2) - key = key.transpose(1, 2) - value = value.transpose(1, 2) - out_padded = torch.zeros_like(query) - interface_fa.fwd(query, key, value, out_padded, dropout_p, scale, is_causal) - return out_padded[..., :head_size_og].transpose(1, 2) - else: - if enable_gqa: - kwargs["enable_gqa"] = enable_gqa - return sdpa_pre_triton_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) - torch.nn.functional.scaled_dot_product_attention = sdpa_triton_flash_atten - log.debug('Torch attention: type="triton flash attention"') - except Exception as err: - log.error(f'Torch attention: type="triton flash attention" {err}') + if 'Triton Flash attention' in opts.sdp_overrides: + attention.set_triton_flash_attention() - if 'CK Flash attention' in opts.sdp_options: - try: - if backend == "rocm": - if not installed('flash-attn'): - log.info('Building CK Flash attention...') - agent = rocm.Agent(getattr(torch.cuda.get_device_properties(device), "gcnArchName", "gfx0000")) - install(rocm.get_flash_attention_command(agent), reinstall=True) - else: - install('flash-attn') - from flash_attn import flash_attn_func - sdpa_pre_flash_atten = torch.nn.functional.scaled_dot_product_attention - @wraps(sdpa_pre_flash_atten) - def sdpa_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: - if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32: - is_unsqueezed = False - if query.dim() == 3: - query = query.unsqueeze(0) - is_unsqueezed = True - if key.dim() == 3: - key = key.unsqueeze(0) - if value.dim() == 3: - value = value.unsqueeze(0) - if enable_gqa: - key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) - value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) - query = query.transpose(1, 2) - key = key.transpose(1, 2) - value = value.transpose(1, 2) - attn_output = flash_attn_func(q=query, k=key, v=value, dropout_p=dropout_p, causal=is_causal, softmax_scale=scale).transpose(1, 2) - if is_unsqueezed: - attn_output = attn_output.squeeze(0) - return attn_output - else: - if enable_gqa: - kwargs["enable_gqa"] = enable_gqa - return sdpa_pre_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) - torch.nn.functional.scaled_dot_product_attention = sdpa_flash_atten - log.debug('Torch attention: type="ck flash attention"') - except Exception as err: - log.error(f'Torch attention: type="ck flash attention" {err}') - - if 'Sage attention' in opts.sdp_options: - try: - install('sageattention') - from sageattention import sageattn, sageattn_qk_int8_pv_fp16_cuda - - use_cuda_backend = False - if (backend == "cuda") and (torch.cuda.get_device_capability(device) == (8, 6)): - use_cuda_backend = True # Detect GPU architecture - sm86 confirmed to need CUDA backend workaround as Sage Attention + Triton causes NaNs - - if use_cuda_backend: - log.debug('Torch attention: type=SageAttention backend=cuda') - def sage_attn_impl(query, key, value, is_causal, scale): - return sageattn_qk_int8_pv_fp16_cuda( - q=query, k=key, v=value, - tensor_layout="HND", - is_causal=is_causal, - sm_scale=scale, - return_lse=False, - pv_accum_dtype="fp32", - ) - else: - log.debug('Torch attention: type=SageAttention backend=auto') - def sage_attn_impl(query, key, value, is_causal, scale): - return sageattn( - q=query, k=key, v=value, - attn_mask=None, - dropout_p=0.0, - is_causal=is_causal, - scale=scale, - ) - - sdpa_pre_sage_atten = torch.nn.functional.scaled_dot_product_attention - @wraps(sdpa_pre_sage_atten) - def sdpa_sage_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: - if (query.shape[-1] in {128, 96, 64}) and (attn_mask is None) and (query.dtype != torch.float32): - if enable_gqa: - key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) - value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) - - # Call pre-selected sage attention implementation - return sage_attn_impl(query, key, value, is_causal, scale) - else: - if enable_gqa: - kwargs["enable_gqa"] = enable_gqa - return sdpa_pre_sage_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) - torch.nn.functional.scaled_dot_product_attention = sdpa_sage_atten - log.debug('Torch attention: type="sage attention"') - except Exception as err: - log.error(f'Torch attention: type="sage attention" {err}') + if 'CK Flash attention' in opts.sdp_overrides: + attention.set_ck_flash_attention(backend, device) + if 'Sage attention' in opts.sdp_overrides: + attention.set_sage_attention(backend, device) from importlib.metadata import version try: diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 99ab20d15..e6e6a37b2 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -235,7 +235,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): infotext(p) prompt(p) if has_changed and len(include) == 0: # print only once - shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}') + shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_native else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}') def deactivate(self, p): if len(lora_diffusers.diffuser_loaded) > 0: diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index f18922040..9ded1a590 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -20,7 +20,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is not None or bias_backup is not None: - if (shared.opts.lora_fuse_diffusers and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_diffusers and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly + if (shared.opts.lora_fuse_native and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_native and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly weights_backup = None bias_backup = None self.network_weights_backup = weights_backup @@ -33,15 +33,15 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n if bnb is None: bnb = model_quant.load_bnb('Network load: type=LoRA', silent=True) if bnb is not None: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: self.network_weights_backup = True else: self.network_weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) self.quant_state, self.quant_type, self.blocksize = weight.quant_state, weight.quant_type, weight.blocksize else: - self.network_weights_backup = weight.clone().to(devices.cpu) if not shared.opts.lora_fuse_diffusers else True + self.network_weights_backup = weight.clone().to(devices.cpu) if not shared.opts.lora_fuse_native else True else: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: self.network_weights_backup = True else: self.network_weights_backup = weight.clone().to(devices.cpu) @@ -61,7 +61,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n if bias_backup is None: if getattr(self, 'bias', None) is not None: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: self.network_bias_backup = True else: bias_backup = self.bias.clone() diff --git a/modules/lora/lora_diffusers.py b/modules/lora/lora_diffusers.py index a97272370..eb1515ca0 100644 --- a/modules/lora/lora_diffusers.py +++ b/modules/lora/lora_diffusers.py @@ -54,7 +54,7 @@ def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale t0 = time.time() name = name.replace(".", "_") sd_model: diffusers.DiffusionPipeline = getattr(shared.sd_model, "pipe", shared.sd_model) - shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') + shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}') if not hasattr(sd_model, 'load_lora_weights'): shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') return None diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 85c66208d..de3e9bfe0 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -128,7 +128,7 @@ def load_safetensors(name, network_on_disk: network.NetworkOnDisk) -> Union[netw if l.debug: shared.log.debug(f'Network load: type=LoRA name="{name}" unmatched={keys_failed_to_match}') else: - shared.log.debug(f'Network load: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} dtypes={dtypes} fuse={shared.opts.lora_fuse_diffusers}') + shared.log.debug(f'Network load: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} dtypes={dtypes} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}') if len(matched_networks) == 0: return None lora_cache[name] = net @@ -303,7 +303,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non errors.display(e, 'LoRA') if len(l.loaded_networks) > 0 and l.debug: - shared.log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)}') + shared.log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}') if recompile_model: shared.log.info("Network load: type=LoRA recompiling model") diff --git a/modules/lora/networks.py b/modules/lora/networks.py index f03063e2d..4294615c9 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -49,7 +49,7 @@ def network_activate(include=[], exclude=[]): continue backup_size += network_backup_weights(module, network_layer_name, wanted_names) batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device) else: network_apply_weights(module, batch_updown, batch_ex_bias, device=device) @@ -68,14 +68,14 @@ def network_activate(include=[], exclude=[]): pbar.remove_task(task) # hide progress bar for no action l.timer.activate += time.time() - t0 if l.debug and len(l.loaded_networks) > 0: - shared.log.debug(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={round(backup_size/1024/1024/1024, 2)} fuse={shared.opts.lora_fuse_diffusers} device={device} time={l.timer.summary}') + shared.log.debug(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={round(backup_size/1024/1024/1024, 2)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers} device={device} time={l.timer.summary}') modules.clear() if len(applied_layers) > 0 or shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") def network_deactivate(include=[], exclude=[]): - if not shared.opts.lora_fuse_diffusers or shared.opts.lora_force_diffusers: + if not shared.opts.lora_fuse_native or shared.opts.lora_force_diffusers: return if len(l.previously_loaded_networks) == 0: return @@ -112,7 +112,7 @@ def network_deactivate(include=[], exclude=[]): pbar.update(task, advance=1) continue batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, use_previous=True) - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device, deactivate=True) else: network_apply_weights(module, batch_updown, batch_ex_bias, device=device, deactivate=True) @@ -125,7 +125,7 @@ def network_deactivate(include=[], exclude=[]): l.timer.deactivate = time.time() - t0 if l.debug and len(l.previously_loaded_networks) > 0: - shared.log.debug(f'Network deactivate: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} fuse={shared.opts.lora_fuse_diffusers} time={l.timer.summary}') + shared.log.debug(f'Network deactivate: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers} time={l.timer.summary}') modules.clear() if len(applied_layers) > 0 or shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index e6ceb1794..250944e22 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -257,6 +257,7 @@ class YoloRestorer(Detailer): models_used = [] np_images = [] annotated = Image.fromarray(np_image) + image = None for i, model_val in enumerate(models): if ':' in model_val: @@ -271,9 +272,10 @@ class YoloRestorer(Detailer): shared.log.warning(f'Detailer: model="{name}" not loaded') continue - if name.endswith('.fp16'): + if name.endswith('.fp16'): # run gfpgan or codeformer directly and skip detailer processing from modules.postprocess import restorer np_image = restorer.restore(np_image, name, model, p.detailer_strength) + image = Image.fromarray(np_image) continue image = Image.fromarray(np_image) @@ -403,7 +405,8 @@ class YoloRestorer(Detailer): p.image_mask = blend([np.array(m) for m in mask_all]) p.image_mask = Image.fromarray(p.image_mask) - np_images.append(np.array(image)) + if image is not None: + np_images.append(np.array(image)) if shared.opts.detailer_save and annotated is not None: np_images.append(annotated) # save debug image with boxes return np_images diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index c2d852f25..6bca30a3f 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -5,7 +5,7 @@ import numpy as np import torch import torchvision.transforms.functional as TF from PIL import Image -from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks +from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks, attention from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed @@ -497,7 +497,7 @@ def update_pipeline(sd_model, p: processing.StableDiffusionProcessing): orig_pipeline = sd_model # processed ONNX pipeline should not be replaced with original pipeline. if getattr(sd_model, "current_attn_name", None) != shared.opts.cross_attention_optimization: shared.log.info(f"Setting attention optimization: {shared.opts.cross_attention_optimization}") - sd_models.set_diffusers_attention(sd_model) + attention.set_diffusers_attention(sd_model) return sd_model diff --git a/modules/sd_models.py b/modules/sd_models.py index b40cf4aea..2c3e9f0f0 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -10,7 +10,7 @@ import diffusers.loaders.single_file_utils import torch import huggingface_hub as hf from installer import log -from modules import timer, paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_compile, sd_detect, model_quant, sd_hijack_te, sd_hijack_accelerate, sd_hijack_safetensors +from modules import timer, paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_compile, sd_detect, model_quant, sd_hijack_te, sd_hijack_accelerate, sd_hijack_safetensors, attention from modules.memstats import memory_stats from modules.modeldata import model_data from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closest_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import @@ -130,7 +130,7 @@ def set_diffuser_options(sd_model, vae=None, op:str='model', offload:bool=True, clear_caches() set_vae_options(sd_model, vae, op, quiet) - set_diffusers_attention(sd_model, quiet) + attention.set_diffusers_attention(sd_model, quiet) if shared.opts.diffusers_fuse_projections and hasattr(sd_model, 'fuse_qkv_projections'): try: @@ -1157,60 +1157,6 @@ def set_diffuser_pipe(pipe, new_pipe_type): return pipe -def set_diffusers_attention(pipe, quiet:bool=False): - import diffusers.models.attention_processor as p - - def set_attn(pipe, attention, name:str=None, quiet:bool=False): - if attention is None: - return - # other models uses their own attention processor - if pipe.__class__.__name__.startswith("StableDiffusion") and getattr(pipe, "unet", None) is not None and hasattr(pipe.unet, "set_attn_processor"): - try: - pipe.unet.set_attn_processor(attention) - except Exception as e: - if 'Nunchaku' in pipe.unet.__class__.__name__: - pass - else: - shared.log.error(f"Attention: {name if name is not None else attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}") - elif not quiet: - shared.log.warning(f"Attention: {name if name is not None else attention.__class__.__name__} is not compatible with {pipe.__class__.__name__}") - - # if hasattr(pipe, 'pipe'): - # set_diffusers_attention(pipe.pipe) - - if 'Control' in pipe.__class__.__name__ or 'Adapter' in pipe.__class__.__name__ or not (pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet")): - if shared.opts.cross_attention_optimization not in {"Scaled-Dot-Product", "Disabled"}: - shared.log.warning(f"Attention: {shared.opts.cross_attention_optimization} is not compatible with {pipe.__class__.__name__}") - else: - pipe.current_attn_name = shared.opts.cross_attention_optimization - return - - shared.log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"') - if shared.opts.cross_attention_optimization == "Disabled": - pass # do nothing - elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers - set_attn(pipe, p.AttnProcessor2_0(), name="Scaled-Dot-Product", quiet=True) - elif shared.opts.cross_attention_optimization == "xFormers": - if hasattr(pipe, 'enable_xformers_memory_efficient_attention'): - pipe.enable_xformers_memory_efficient_attention() - else: - shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") - elif shared.opts.cross_attention_optimization == "Batch matrix-matrix": - set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix") - elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM": - from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM - set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM") - - if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"): - if shared.opts.attention_slicing: - pipe.enable_attention_slicing() - else: - pipe.disable_attention_slicing() - shared.log.debug(f"Attention: slicing={shared.opts.attention_slicing}") - - pipe.current_attn_name = shared.opts.cross_attention_optimization - - def add_noise_pred_to_diffusers_callback(pipe): if not hasattr(pipe, "_callback_tensor_inputs"): return pipe diff --git a/modules/shared.py b/modules/shared.py index 24aad5271..8af5135e2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -136,7 +136,7 @@ def list_samplers(): return modules.sd_samplers.all_samplers -startup_offload_mode, startup_offload_min_gpu, startup_offload_max_gpu, startup_cross_attention, startup_sdp_options, startup_sdp_choices, startup_offload_always, startup_offload_never = get_default_modes(cmd_opts=cmd_opts, mem_stat=mem_stat) +startup_offload_mode, startup_offload_min_gpu, startup_offload_max_gpu, startup_cross_attention, startup_sdp_options, startup_sdp_choices, startup_sdp_override_options, startup_sdp_override_choices, startup_offload_always, startup_offload_never = get_default_modes(cmd_opts=cmd_opts, mem_stat=mem_stat) options_templates.update(options_section(('sd', "Model Loading"), { "sd_backend": OptionInfo('diffusers', "Execution backend", gr.Radio, {"choices": ['diffusers', 'original'], "visible": False }), @@ -296,13 +296,13 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), - "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}), - "attention_": OptionInfo("

Cross Attention

", "", gr.HTML), + "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}), + "sdp_options": OptionInfo(startup_sdp_options, "SDP kernels", gr.CheckboxGroup, {"choices": startup_sdp_choices}), + "sdp_overrides": OptionInfo(startup_sdp_override_options, "SDP overrides", gr.CheckboxGroup, {"choices": startup_sdp_override_choices}), "attention_slicing": OptionInfo('Default', "Attention slicing", gr.Radio, {"choices": ['Default', 'Enabled', 'Disabled']}), - "sdp_options": OptionInfo(startup_sdp_options, "SDP options", gr.CheckboxGroup, {"choices": startup_sdp_choices}), "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }), - "dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}), - "dynamic_attention_trigger_rate": OptionInfo(1, "Dynamic Attention trigger rate in GB", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4)*2, "step": 0.01}), + "dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}), + "dynamic_attention_trigger_rate": OptionInfo(1, "Dynamic Attention trigger rate", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4)*2, "step": 0.01}), })) options_templates.update(options_section(('backends', "Backend Settings"), { @@ -711,14 +711,15 @@ options_templates.update(options_section(('extra_networks', "Networks"), { "extra_networks_lora_sep": OptionInfo("

LoRA

", "", gr.HTML), "extra_networks_default_multiplier": OptionInfo(1.0, "Default strength", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}), - "lora_fuse_diffusers": OptionInfo(True, "LoRA fuse directly to model"), "lora_force_reload": OptionInfo(False, "LoRA force reload always"), "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA load using Diffusers method"), - "lora_maybe_diffusers": OptionInfo(False, "LoRA load using Diffusers method for selected models", gr.Checkbox, {"visible": False}), + "lora_fuse_native": OptionInfo(True, "LoRA native fuse with model"), + "lora_fuse_diffusers": OptionInfo(False, "LoRA diffusers fuse with model"), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(1, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}), "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"), "lora_quant": OptionInfo("NF4","LoRA precision when quantized", gr.Radio, {"choices": ["NF4", "FP4"]}), + "lora_maybe_diffusers": OptionInfo(False, "LoRA load using Diffusers method for selected models", gr.Checkbox, {"visible": False}), "extra_networks_styles_sep": OptionInfo("

Styles

", "", gr.HTML), "extra_networks_styles": OptionInfo(True, "Show reference styles"), diff --git a/modules/shared_defaults.py b/modules/shared_defaults.py index d3993d2e9..ff5baea89 100644 --- a/modules/shared_defaults.py +++ b/modules/shared_defaults.py @@ -40,16 +40,17 @@ def get_default_modes(cmd_opts, mem_stat): default_cross_attention = "Scaled-Dot-Product" + default_sdp_override_options = [] if devices.backend == "zluda": default_sdp_options = ['Math attention', 'Dynamic attention'] elif devices.backend in {"rocm", "directml", "cpu", "mps"}: - default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention'] + default_sdp_options = ['Flash', 'Memory', 'Math'] + default_sdp_override_options = ['Dynamic attention'] else: - default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention'] + default_sdp_options = ['Flash', 'Memory', 'Math'] - default_sdp_choices = ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention', 'CK Flash attention', 'Sage attention'] - if devices.backend in {"rocm", "zluda"}: - default_sdp_choices.insert(4, 'Triton Flash attention') # insert after Dynamic attention + default_sdp_choices = ['Flash', 'Memory', 'Math'] + default_sdp_override_choices = ['Dynamic attention', 'CK Flash attention', 'Triton Flash attention', 'Sage attention'] return ( default_offload_mode, @@ -58,6 +59,8 @@ def get_default_modes(cmd_opts, mem_stat): default_cross_attention, default_sdp_options, default_sdp_choices, + default_sdp_override_options, + default_sdp_override_choices, default_diffusers_offload_always, - default_diffusers_offload_never + default_diffusers_offload_never, ) diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index a22e66901..a89046df0 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -281,7 +281,7 @@ def create_ui(): (enable_hr, "Second pass"), (enable_hr, "Refine"), (denoising_strength, "Denoising strength"), - (denoising_strength, "Hires strength"), + (hr_denoising_strength, "Hires strength"), (hr_sampler_index, "Hires sampler"), (hr_resize_mode, "Hires mode"), (hr_resize_context, "Hires context"), diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 7b95ce282..ffa0fbcfb 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -339,7 +339,7 @@ def create_resize_inputs(tab, images, accordion=True, latent=False, non_zero=Tru with gr.Row(visible=True) as _resize_group: with gr.Column(elem_id=f"{tab}_column_size"): - selected_scale_tab = gr.State(value=0) # pylint: disable=abstract-class-instantiated + selected_scale_tab = gr.State(value=0 if tab != 'img2img' else 1) # pylint: disable=abstract-class-instantiated with gr.Tabs(elem_id=f"{tab}_scale_tabs", selected=0 if non_zero else 1): with gr.Tab(label="Fixed", id=0, elem_id=f"{tab}_scale_tab_fixed") as tab_scale_to: with gr.Row(elem_id=f"{tab}_resize_row_fixed"): diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index f3e108666..8497491c9 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -35,7 +35,7 @@ def create_ui(): guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('txt2img') vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('txt2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('txt2img') - enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') + enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.yolo.ui('txt2img') override_settings = ui_common.create_override_inputs('txt2img') state = gr.Textbox(value='', visible=False) @@ -61,7 +61,7 @@ def create_ui(): clip_skip, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, height, width, - enable_hr, denoising_strength, + enable_hr, hr_denoising_strength, hr_scale, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio, @@ -132,8 +132,7 @@ def create_ui(): # second pass (enable_hr, "Second pass"), (enable_hr, "Refine"), - (denoising_strength, "Denoising strength"), - (denoising_strength, "Hires strength"), + (hr_denoising_strength, "Hires strength"), (hr_sampler_index, "Hires sampler"), (hr_resize_mode, "Hires mode"), (hr_resize_context, "Hires context"), diff --git a/pipelines/model_auraflow.py b/pipelines/model_auraflow.py index 957875dfa..ee9e1b633 100644 --- a/pipelines/model_auraflow.py +++ b/pipelines/model_auraflow.py @@ -14,7 +14,7 @@ def load_auraflow(checkpoint_info, diffusers_load_config=None): shared.log.debug(f'Load model: type=AuraFlow repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') transformer = generic.load_transformer(repo_id, cls_name=diffusers.AuraFlowTransformer2DModel, load_config=diffusers_load_config) - text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.UMT5EncoderModel, load_config=diffusers_load_config) # auraflow uses EleutherAI/pile-t5-xl + text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.UMT5EncoderModel, load_config=diffusers_load_config, allow_shared=False) # auraflow uses EleutherAI/pile-t5-xl pipe = diffusers.AuraFlowPipeline.from_pretrained( repo_id,