Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4381 from vladmandic/dev

Browse Source 
refresh branch
This commit is contained in:
Vladimir Mandic
2025-11-08 11:59:06 -05:00
committed by GitHub
parent a4aef23f39 31f09cf266
commit e7e41d30cf
24 changed files with 316 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+10 -4
View File
@@ -1,12 +1,18 @@
# Change Log for SD.Next
## Update for 2025-11-07
## Update for 2025-11-08
- **Features**
- allow recursive inline wildcards using curly braces syntax
- simplify SDNQ pre-quantization saved config
- **wildcards**: allow recursive inline wildcards using curly braces syntax
- **sdnq**: simplify pre-quantization saved config
- **attention**: refactor settings and improve handling of attention mechanisms
- **lora**: separate fuse setting for native-vs-diffuser implementations
- **auth**: strong-enforce auth check on all api endpoints
- **Fixes**
- hires strength save/load in metadata
- hires strength save/load in metadata, thanks @awsr
- fix imgi2img initial scale tab, thanks @awsr
- fix pony-v7 text-encoder
- detailer with face-restorers
## Update for 2025-11-06
cli/api-txt2img.js
+7 -2
View File
@@ -30,10 +30,15 @@ async function main() {
const headers = new Headers();
const body = JSON.stringify(sd_options);
headers.set('Content-Type', 'application/json');
if (sd_username && sd_password) headers.set({ Authorization: `Basic ${btoa('sd_username:sd_password')}` });
if (sd_username && sd_password) {
// const credentials = btoa(`${sd_username}:${sd_password}`);
const credentials = Buffer.from(`${sd_username}:${sd_password}`).toString('base64');
headers.set('Authorization', `Basic ${credentials}`);
}
const res = await fetch(`${sd_url}/sdapi/v1/txt2img`, { method, headers, body });
if (res.status !== 200) {
console.log('Error', res.status);
const err = await res.text();
console.log('Error', res.status, res.statusText, err);
} else {
const json = await res.json();
console.log('result:', json.info);
extensions-builtin/sd-extension-system-info
+1 -1
Submodule extensions-builtin/sd-extension-system-info updated: 19a1b1b722...c724b86a7a
extensions-builtin/sdnext-kanvas
-1
View File
@@ -1 +0,0 @@
/home/vlado/dev/kanvas
extensions-builtin/sdnext-modernui
+1 -1
Submodule extensions-builtin/sdnext-modernui updated: 452b7044b8...f14464f3f7
javascript/login.js
+6 -6
View File
@@ -4,21 +4,21 @@ const loginCSS = `
left: 0;
width: 100%;
height: 100%;
background: var(--background-fill-primary);
color: var(--body-text-color-subdued);
background: #222;
color: #ddd;
font-family: monospace;
z-index: 100;
`;
const loginHTML = `
<div id="loginDiv" style="margin: 15% auto; max-width: 200px; padding: 2em; background: var(--background-fill-secondary);">
<div id="loginDiv" style="margin: 15% auto; max-width: 200px; padding: 2em; background: #444; border-radius: 4px; filter: drop-shadow(2px 4px 6px black);">
<h2>Login</h2>
<label for="username" style="margin-top: 0.5em">Username</label>
<input type="text" id="loginUsername" name="username" style="width: 92%; padding: 0.5em; margin-top: 0.5em">
<input type="text" id="loginUsername" name="username" style="width: 92%; padding: 0.5em; margin-top: 0.5em; border-radius: 4px;">
<label for="password" style="margin-top: 0.5em">Password</label>
<input type="text" id="loginPassword" name="password" style="width: 92%; padding: 0.5em; margin-top: 0.5em">
<input type="text" id="loginPassword" name="password" style="width: 92%; padding: 0.5em; margin-top: 0.5em; border-radius: 4px;">
<div id="loginStatus" style="margin-top: 0.5em"></div>
<button type="submit" style="width: 100%; padding: 0.5em; margin-top: 0.5em; background: var(--button-primary-background-fill); color: var(--button-primary-text-color); border: var(--button-primary-border-color);">Login</button>
<button type="submit" style="width: 100%; padding: 0.5em; margin-top: 0.5em; background: #366; color: #ddd; border: none; border-radius: 4px; filter: drop-shadow(2px 4px 6px black);">Login</button>
</div>
`;
modules/api/api.py
+13 -29
View File
@@ -4,7 +4,7 @@ from secrets import compare_digest
from fastapi import FastAPI, APIRouter, Depends, Request
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from fastapi.exceptions import HTTPException
from modules import errors, shared, postprocessing
from modules import errors, shared
from modules.api import models, endpoints, script, helpers, server, generate, process, control, docs, gpu
@@ -60,8 +60,8 @@ class Api:
self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img)
self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img)
self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl)
self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage)
self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch)
self.add_api_route("/sdapi/v1/extra-single-image", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage)
self.add_api_route("/sdapi/v1/extra-batch-images", self.process.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch)
self.add_api_route("/sdapi/v1/preprocess", self.process.post_preprocess, methods=["POST"])
self.add_api_route("/sdapi/v1/mask", self.process.post_mask, methods=["POST"])
self.add_api_route("/sdapi/v1/detect", self.process.post_detect, methods=["POST"])
@@ -117,17 +117,22 @@ class Api:
from modules.civitai import api_civitai
api_civitai.register_api()
def add_api_route(self, path: str, endpoint, **kwargs):
def add_api_route(self, path: str, fn, **kwargs):
if self.credentials:
deps = list(kwargs.get('dependencies', []))
deps.append(Depends(self.auth))
kwargs['dependencies'] = deps
if shared.opts.subpath is not None and len(shared.opts.subpath) > 0:
self.app.add_api_route(f'{shared.opts.subpath}{path}', endpoint, **kwargs)
self.app.add_api_route(path, endpoint, **kwargs)
self.app.add_api_route(f'{shared.opts.subpath}{path}', endpoint=fn, **kwargs)
self.app.add_api_route(path, endpoint=fn, **kwargs)
def auth(self, credentials: HTTPBasicCredentials = Depends(HTTPBasic())):
# this is only needed for api-only since otherwise auth is handled in gradio/routes.py
if not self.credentials:
return True
if credentials.username in self.credentials:
if compare_digest(credentials.password, self.credentials[credentials.username]):
return True
shared.log.error(f'API authentication: user="{credentials.username}" password="{credentials.password}"')
raise HTTPException(status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": "Basic"})
def get_session_start(self, req: Request, agent: Optional[str] = None):
@@ -136,27 +141,6 @@ class Api:
shared.log.info(f'Browser session: user={user} client={req.client.host} agent={agent}')
return {}
def set_upscalers(self, req: dict):
reqDict = vars(req)
reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None)
reqDict['extras_upscaler_2'] = reqDict.pop('upscaler_2', None)
return reqDict
def extras_single_image_api(self, req: models.ReqProcessImage):
reqDict = self.set_upscalers(req)
reqDict['image'] = helpers.decode_base64_to_image(reqDict['image'])
with self.queue_lock:
result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict)
return models.ResProcessImage(image=helpers.encode_pil_to_base64(result[0][0]), html_info=result[1])
def extras_batch_images_api(self, req: models.ReqProcessBatch):
reqDict = self.set_upscalers(req)
image_list = reqDict.pop('imageList', [])
image_folder = [helpers.decode_base64_to_image(x.data) for x in image_list]
with self.queue_lock:
result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict)
return models.ResProcessBatch(images=list(map(helpers.encode_pil_to_base64, result[0])), html_info=result[1])
def launch(self):
config = {
"listen": shared.cmd_opts.listen,
modules/api/process.py
+23 -2
View File
@@ -4,8 +4,8 @@ from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
from fastapi.responses import JSONResponse
from fastapi.exceptions import HTTPException
from modules.api.helpers import decode_base64_to_image, encode_pil_to_base64
from modules import errors, shared
from modules.api import models
from modules import errors, shared, postprocessing
from modules.api import models, helpers
processor = None # cached instance of processor
@@ -175,3 +175,24 @@ class APIProcess():
raise HTTPException(status_code=400, detail="prompt enhancement: invalid type")
res = models.ResPromptEnhance(prompt=prompt, seed=seed)
return res
def set_upscalers(self, req: dict):
reqDict = vars(req)
reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None)
reqDict['extras_upscaler_2'] = reqDict.pop('upscaler_2', None)
return reqDict
def extras_single_image_api(self, req: models.ReqProcessImage):
reqDict = self.set_upscalers(req)
reqDict['image'] = helpers.decode_base64_to_image(reqDict['image'])
with self.queue_lock:
result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict)
return models.ResProcessImage(image=helpers.encode_pil_to_base64(result[0][0]), html_info=result[1])
def extras_batch_images_api(self, req: models.ReqProcessBatch):
reqDict = self.set_upscalers(req)
image_list = reqDict.pop('imageList', [])
image_folder = [helpers.decode_base64_to_image(x.data) for x in image_list]
with self.queue_lock:
result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict)
return models.ResProcessBatch(images=list(map(helpers.encode_pil_to_base64, result[0])), html_info=result[1])
modules/attention.py
+194
View File
@@ -0,0 +1,194 @@
from typing import Optional
from functools import wraps
import torch
from modules import rocm
from modules.errors import log
from installer import install, installed
def set_dynamic_attention():
try:
sdpa_pre_dyanmic_atten = torch.nn.functional.scaled_dot_product_attention
from modules.sd_hijack_dynamic_atten import dynamic_scaled_dot_product_attention
torch.nn.functional.scaled_dot_product_attention = dynamic_scaled_dot_product_attention
return sdpa_pre_dyanmic_atten
except Exception as err:
log.error(f'Torch attention: type="dynamic attention" {err}')
return None
def set_triton_flash_attention():
try:
from modules.flash_attn_triton_amd import interface_fa
sdpa_pre_triton_flash_atten = torch.nn.functional.scaled_dot_product_attention
@wraps(sdpa_pre_triton_flash_atten)
def sdpa_triton_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32:
if scale is None:
scale = query.shape[-1] ** (-0.5)
head_size_og = query.size(3)
if head_size_og % 8 != 0:
query = torch.nn.functional.pad(query, [0, 8 - head_size_og % 8])
key = torch.nn.functional.pad(key, [0, 8 - head_size_og % 8])
value = torch.nn.functional.pad(value, [0, 8 - head_size_og % 8])
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)
out_padded = torch.zeros_like(query)
interface_fa.fwd(query, key, value, out_padded, dropout_p, scale, is_causal)
return out_padded[..., :head_size_og].transpose(1, 2)
else:
if enable_gqa:
kwargs["enable_gqa"] = enable_gqa
return sdpa_pre_triton_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_triton_flash_atten
log.debug('Torch attention: type="Triton Flash attention"')
except Exception as err:
log.error(f'Torch attention: type="Triton Flash attention" {err}')
def set_ck_flash_attention(backend: str, device: torch.device):
try:
if backend == "rocm":
if not installed('flash-attn'):
log.info('Torch attention: type="CK Flash" building...')
agent = rocm.Agent(getattr(torch.cuda.get_device_properties(device), "gcnArchName", "gfx0000"))
install(rocm.get_flash_attention_command(agent), reinstall=True)
else:
install('flash-attn')
from flash_attn import flash_attn_func
sdpa_pre_flash_atten = torch.nn.functional.scaled_dot_product_attention
@wraps(sdpa_pre_flash_atten)
def sdpa_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32:
is_unsqueezed = False
if query.dim() == 3:
query = query.unsqueeze(0)
is_unsqueezed = True
if key.dim() == 3:
key = key.unsqueeze(0)
if value.dim() == 3:
value = value.unsqueeze(0)
if enable_gqa:
key = key.repeat_interleave(query.size(-3)//key.size(-3), -3)
value = value.repeat_interleave(query.size(-3)//value.size(-3), -3)
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)
attn_output = flash_attn_func(q=query, k=key, v=value, dropout_p=dropout_p, causal=is_causal, softmax_scale=scale).transpose(1, 2)
if is_unsqueezed:
attn_output = attn_output.squeeze(0)
return attn_output
else:
if enable_gqa:
kwargs["enable_gqa"] = enable_gqa
return sdpa_pre_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_flash_atten
log.debug('Torch attention: type="CK Flash attention"')
except Exception as err:
log.error(f'Torch attention: type="CK Flash attention" {err}')
def set_sage_attention(backend: str, device: torch.device):
try:
install('sageattention')
use_cuda_backend = False
if (backend == "cuda") and (torch.cuda.get_device_capability(device) == (8, 6)):
use_cuda_backend = True # Detect GPU architecture - sm86 confirmed to need CUDA backend workaround as Sage Attention + Triton causes NaNs
try:
from sageattention import sageattn_qk_int8_pv_fp16_cuda
except:
use_cuda_backend = False
if use_cuda_backend:
from sageattention import sageattn_qk_int8_pv_fp16_cuda
def sage_attn_impl(query, key, value, is_causal, scale):
return sageattn_qk_int8_pv_fp16_cuda(
q=query, k=key, v=value,
tensor_layout="HND",
is_causal=is_causal,
sm_scale=scale,
return_lse=False,
pv_accum_dtype="fp32",
)
else:
from sageattention import sageattn
def sage_attn_impl(query, key, value, is_causal, scale):
return sageattn(
q=query, k=key, v=value,
attn_mask=None,
dropout_p=0.0,
is_causal=is_causal,
scale=scale,
)
sdpa_pre_sage_atten = torch.nn.functional.scaled_dot_product_attention
@wraps(sdpa_pre_sage_atten)
def sdpa_sage_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
if (query.shape[-1] in {128, 96, 64}) and (attn_mask is None) and (query.dtype != torch.float32):
if enable_gqa:
key = key.repeat_interleave(query.size(-3)//key.size(-3), -3)
value = value.repeat_interleave(query.size(-3)//value.size(-3), -3)
# Call pre-selected sage attention implementation
return sage_attn_impl(query, key, value, is_causal, scale)
else:
if enable_gqa:
kwargs["enable_gqa"] = enable_gqa
return sdpa_pre_sage_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_sage_atten
log.debug(f'Torch attention: type="Sage attention" backend={"cuda" if use_cuda_backend else "auto"}')
except Exception as err:
log.error(f'Torch attention: type="Sage attention" {err}')
def set_diffusers_attention(pipe, quiet:bool=False):
from modules import shared
import diffusers.models.attention_processor as p
def set_attn(pipe, attention, name:str=None):
if attention is None:
return
# other models uses their own attention processor
if getattr(pipe, "unet", None) is not None and hasattr(pipe.unet, "set_attn_processor"):
try:
pipe.unet.set_attn_processor(attention)
except Exception as e:
if 'Nunchaku' in pipe.unet.__class__.__name__:
pass
else:
shared.log.error(f'Torch attention: type="{name}" cls={attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}')
""" # each transformer typically has its own attention processor
if getattr(pipe, "transformer", None) is not None and hasattr(pipe.transformer, "set_attn_processor"):
try:
pipe.transformer.set_attn_processor(attention)
except Exception as e:
if 'Nunchaku' in pipe.transformer.__class__.__name__:
pass
else:
shared.log.error(f'Torch attention: type="{name}" cls={attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}')
"""
shared.log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"')
if shared.opts.cross_attention_optimization == "Disabled":
pass # do nothing
elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers
# set_attn(pipe, p.AttnProcessor2_0(), name="Scaled-Dot-Product")
pass
elif shared.opts.cross_attention_optimization == "xFormers":
if hasattr(pipe, 'enable_xformers_memory_efficient_attention'):
pipe.enable_xformers_memory_efficient_attention()
else:
shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}")
elif shared.opts.cross_attention_optimization == "Batch matrix-matrix":
set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix")
elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM":
from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM
set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM")
if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"):
if shared.opts.attention_slicing:
pipe.enable_attention_slicing()
else:
pipe.disable_attention_slicing()
shared.log.debug(f"Torch attention: slicing={shared.opts.attention_slicing}")
pipe.current_attn_name = shared.opts.cross_attention_optimization
modules/devices.py
+14 -136
View File
@@ -1,14 +1,10 @@
from typing import Optional
import os
import sys
import time
import contextlib
from functools import wraps
import torch
from modules import rocm
from modules import rocm, attention
from modules.errors import log, display, install as install_traceback
from installer import install, installed
debug = os.environ.get('SD_DEVICE_DEBUG', None) is not None
@@ -462,148 +458,30 @@ def set_sdpa_params():
log.warning(f'Torch attention: type="sdpa" {err}')
try:
torch.backends.cuda.enable_flash_sdp('Flash attention' in opts.sdp_options)
torch.backends.cuda.enable_mem_efficient_sdp('Memory attention' in opts.sdp_options)
torch.backends.cuda.enable_math_sdp('Math attention' in opts.sdp_options)
torch.backends.cuda.enable_flash_sdp('Flash' in opts.sdp_options)
torch.backends.cuda.enable_mem_efficient_sdp('Memory' in opts.sdp_options)
torch.backends.cuda.enable_math_sdp('Math' in opts.sdp_options)
if hasattr(torch.backends.cuda, "allow_fp16_bf16_reduction_math_sdp"): # only valid for torch >= 2.5
torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True)
log.debug(f'Torch attention: type="sdpa" opts={opts.sdp_options}')
log.debug(f'Torch attention: type="sdpa" kernels={opts.sdp_options} overrides={opts.sdp_overrides}')
except Exception as err:
log.warning(f'Torch attention: type="sdpa" {err}')
# Stack hijcaks in reverse order. This gives priority to the last added hijack.
# If the last hijack is not compatible, it will use the one before it and so on.
if 'Dynamic attention' in opts.sdp_options:
try:
global sdpa_pre_dyanmic_atten # pylint: disable=global-statement
sdpa_pre_dyanmic_atten = torch.nn.functional.scaled_dot_product_attention
from modules.sd_hijack_dynamic_atten import dynamic_scaled_dot_product_attention
torch.nn.functional.scaled_dot_product_attention = dynamic_scaled_dot_product_attention
except Exception as err:
log.error(f'Torch attention: type="dynamic attention" {err}')
if 'Dynamic attention' in opts.sdp_overrides:
global sdpa_pre_dyanmic_atten # pylint: disable=global-statement
sdpa_pre_dyanmic_atten = attention.set_dynamic_attention()
if 'Triton Flash attention' in opts.sdp_options:
try:
if backend in {"zluda", "rocm"}:
from modules.flash_attn_triton_amd import interface_fa
sdpa_pre_triton_flash_atten = torch.nn.functional.scaled_dot_product_attention
@wraps(sdpa_pre_triton_flash_atten)
def sdpa_triton_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32:
if scale is None:
scale = query.shape[-1] ** (-0.5)
head_size_og = query.size(3)
if head_size_og % 8 != 0:
query = torch.nn.functional.pad(query, [0, 8 - head_size_og % 8])
key = torch.nn.functional.pad(key, [0, 8 - head_size_og % 8])
value = torch.nn.functional.pad(value, [0, 8 - head_size_og % 8])
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)
out_padded = torch.zeros_like(query)
interface_fa.fwd(query, key, value, out_padded, dropout_p, scale, is_causal)
return out_padded[..., :head_size_og].transpose(1, 2)
else:
if enable_gqa:
kwargs["enable_gqa"] = enable_gqa
return sdpa_pre_triton_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_triton_flash_atten
log.debug('Torch attention: type="triton flash attention"')
except Exception as err:
log.error(f'Torch attention: type="triton flash attention" {err}')
if 'Triton Flash attention' in opts.sdp_overrides:
attention.set_triton_flash_attention()
if 'CK Flash attention' in opts.sdp_options:
try:
if backend == "rocm":
if not installed('flash-attn'):
log.info('Building CK Flash attention...')
agent = rocm.Agent(getattr(torch.cuda.get_device_properties(device), "gcnArchName", "gfx0000"))
install(rocm.get_flash_attention_command(agent), reinstall=True)
else:
install('flash-attn')
from flash_attn import flash_attn_func
sdpa_pre_flash_atten = torch.nn.functional.scaled_dot_product_attention
@wraps(sdpa_pre_flash_atten)
def sdpa_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
if query.shape[-1] <= 128 and attn_mask is None and query.dtype != torch.float32:
is_unsqueezed = False
if query.dim() == 3:
query = query.unsqueeze(0)
is_unsqueezed = True
if key.dim() == 3:
key = key.unsqueeze(0)
if value.dim() == 3:
value = value.unsqueeze(0)
if enable_gqa:
key = key.repeat_interleave(query.size(-3)//key.size(-3), -3)
value = value.repeat_interleave(query.size(-3)//value.size(-3), -3)
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)
attn_output = flash_attn_func(q=query, k=key, v=value, dropout_p=dropout_p, causal=is_causal, softmax_scale=scale).transpose(1, 2)
if is_unsqueezed:
attn_output = attn_output.squeeze(0)
return attn_output
else:
if enable_gqa:
kwargs["enable_gqa"] = enable_gqa
return sdpa_pre_flash_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_flash_atten
log.debug('Torch attention: type="ck flash attention"')
except Exception as err:
log.error(f'Torch attention: type="ck flash attention" {err}')
if 'Sage attention' in opts.sdp_options:
try:
install('sageattention')
from sageattention import sageattn, sageattn_qk_int8_pv_fp16_cuda
use_cuda_backend = False
if (backend == "cuda") and (torch.cuda.get_device_capability(device) == (8, 6)):
use_cuda_backend = True # Detect GPU architecture - sm86 confirmed to need CUDA backend workaround as Sage Attention + Triton causes NaNs
if use_cuda_backend:
log.debug('Torch attention: type=SageAttention backend=cuda')
def sage_attn_impl(query, key, value, is_causal, scale):
return sageattn_qk_int8_pv_fp16_cuda(
q=query, k=key, v=value,
tensor_layout="HND",
is_causal=is_causal,
sm_scale=scale,
return_lse=False,
pv_accum_dtype="fp32",
)
else:
log.debug('Torch attention: type=SageAttention backend=auto')
def sage_attn_impl(query, key, value, is_causal, scale):
return sageattn(
q=query, k=key, v=value,
attn_mask=None,
dropout_p=0.0,
is_causal=is_causal,
scale=scale,
)
sdpa_pre_sage_atten = torch.nn.functional.scaled_dot_product_attention
@wraps(sdpa_pre_sage_atten)
def sdpa_sage_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: Optional[torch.FloatTensor] = None, dropout_p: float = 0.0, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
if (query.shape[-1] in {128, 96, 64}) and (attn_mask is None) and (query.dtype != torch.float32):
if enable_gqa:
key = key.repeat_interleave(query.size(-3)//key.size(-3), -3)
value = value.repeat_interleave(query.size(-3)//value.size(-3), -3)
# Call pre-selected sage attention implementation
return sage_attn_impl(query, key, value, is_causal, scale)
else:
if enable_gqa:
kwargs["enable_gqa"] = enable_gqa
return sdpa_pre_sage_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_sage_atten
log.debug('Torch attention: type="sage attention"')
except Exception as err:
log.error(f'Torch attention: type="sage attention" {err}')
if 'CK Flash attention' in opts.sdp_overrides:
attention.set_ck_flash_attention(backend, device)
if 'Sage attention' in opts.sdp_overrides:
attention.set_sage_attention(backend, device)
from importlib.metadata import version
try:
modules/lora/extra_networks_lora.py
+1 -1
View File
@@ -235,7 +235,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
infotext(p)
prompt(p)
if has_changed and len(include) == 0: # print only once
shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}')
shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_native else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}')
def deactivate(self, p):
if len(lora_diffusers.diffuser_loaded) > 0:
modules/lora/lora_apply.py
+5 -5
View File
@@ -20,7 +20,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n
weights_backup = getattr(self, "network_weights_backup", None)
bias_backup = getattr(self, "network_bias_backup", None)
if weights_backup is not None or bias_backup is not None:
if (shared.opts.lora_fuse_diffusers and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_diffusers and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly
if (shared.opts.lora_fuse_native and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_native and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly
weights_backup = None
bias_backup = None
self.network_weights_backup = weights_backup
@@ -33,15 +33,15 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n
if bnb is None:
bnb = model_quant.load_bnb('Network load: type=LoRA', silent=True)
if bnb is not None:
if shared.opts.lora_fuse_diffusers:
if shared.opts.lora_fuse_native:
self.network_weights_backup = True
else:
self.network_weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,)
self.quant_state, self.quant_type, self.blocksize = weight.quant_state, weight.quant_type, weight.blocksize
else:
self.network_weights_backup = weight.clone().to(devices.cpu) if not shared.opts.lora_fuse_diffusers else True
self.network_weights_backup = weight.clone().to(devices.cpu) if not shared.opts.lora_fuse_native else True
else:
if shared.opts.lora_fuse_diffusers:
if shared.opts.lora_fuse_native:
self.network_weights_backup = True
else:
self.network_weights_backup = weight.clone().to(devices.cpu)
@@ -61,7 +61,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n
if bias_backup is None:
if getattr(self, 'bias', None) is not None:
if shared.opts.lora_fuse_diffusers:
if shared.opts.lora_fuse_native:
self.network_bias_backup = True
else:
bias_backup = self.bias.clone()
modules/lora/lora_diffusers.py
+1 -1
View File
@@ -54,7 +54,7 @@ def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale
t0 = time.time()
name = name.replace(".", "_")
sd_model: diffusers.DiffusionPipeline = getattr(shared.sd_model, "pipe", shared.sd_model)
shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}')
shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}')
if not hasattr(sd_model, 'load_lora_weights'):
shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora')
return None
modules/lora/lora_load.py
+2 -2
View File
@@ -128,7 +128,7 @@ def load_safetensors(name, network_on_disk: network.NetworkOnDisk) -> Union[netw
if l.debug:
shared.log.debug(f'Network load: type=LoRA name="{name}" unmatched={keys_failed_to_match}')
else:
shared.log.debug(f'Network load: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} dtypes={dtypes} fuse={shared.opts.lora_fuse_diffusers}')
shared.log.debug(f'Network load: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} dtypes={dtypes} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}')
if len(matched_networks) == 0:
return None
lora_cache[name] = net
@@ -303,7 +303,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non
errors.display(e, 'LoRA')
if len(l.loaded_networks) > 0 and l.debug:
shared.log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)}')
shared.log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}')
if recompile_model:
shared.log.info("Network load: type=LoRA recompiling model")
modules/lora/networks.py
+5 -5
View File
@@ -49,7 +49,7 @@ def network_activate(include=[], exclude=[]):
continue
backup_size += network_backup_weights(module, network_layer_name, wanted_names)
batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name)
if shared.opts.lora_fuse_diffusers:
if shared.opts.lora_fuse_native:
network_apply_direct(module, batch_updown, batch_ex_bias, device=device)
else:
network_apply_weights(module, batch_updown, batch_ex_bias, device=device)
@@ -68,14 +68,14 @@ def network_activate(include=[], exclude=[]):
pbar.remove_task(task) # hide progress bar for no action
l.timer.activate += time.time() - t0
if l.debug and len(l.loaded_networks) > 0:
shared.log.debug(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={round(backup_size/1024/1024/1024, 2)} fuse={shared.opts.lora_fuse_diffusers} device={device} time={l.timer.summary}')
shared.log.debug(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={round(backup_size/1024/1024/1024, 2)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers} device={device} time={l.timer.summary}')
modules.clear()
if len(applied_layers) > 0 or shared.opts.diffusers_offload_mode == "sequential":
sd_models.set_diffuser_offload(sd_model, op="model")
def network_deactivate(include=[], exclude=[]):
if not shared.opts.lora_fuse_diffusers or shared.opts.lora_force_diffusers:
if not shared.opts.lora_fuse_native or shared.opts.lora_force_diffusers:
return
if len(l.previously_loaded_networks) == 0:
return
@@ -112,7 +112,7 @@ def network_deactivate(include=[], exclude=[]):
pbar.update(task, advance=1)
continue
batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, use_previous=True)
if shared.opts.lora_fuse_diffusers:
if shared.opts.lora_fuse_native:
network_apply_direct(module, batch_updown, batch_ex_bias, device=device, deactivate=True)
else:
network_apply_weights(module, batch_updown, batch_ex_bias, device=device, deactivate=True)
@@ -125,7 +125,7 @@ def network_deactivate(include=[], exclude=[]):
l.timer.deactivate = time.time() - t0
if l.debug and len(l.previously_loaded_networks) > 0:
shared.log.debug(f'Network deactivate: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} fuse={shared.opts.lora_fuse_diffusers} time={l.timer.summary}')
shared.log.debug(f'Network deactivate: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers} time={l.timer.summary}')
modules.clear()
if len(applied_layers) > 0 or shared.opts.diffusers_offload_mode == "sequential":
sd_models.set_diffuser_offload(sd_model, op="model")
modules/postprocess/yolo.py
+5 -2
View File
@@ -257,6 +257,7 @@ class YoloRestorer(Detailer):
models_used = []
np_images = []
annotated = Image.fromarray(np_image)
image = None
for i, model_val in enumerate(models):
if ':' in model_val:
@@ -271,9 +272,10 @@ class YoloRestorer(Detailer):
shared.log.warning(f'Detailer: model="{name}" not loaded')
continue
if name.endswith('.fp16'):
if name.endswith('.fp16'): # run gfpgan or codeformer directly and skip detailer processing
from modules.postprocess import restorer
np_image = restorer.restore(np_image, name, model, p.detailer_strength)
image = Image.fromarray(np_image)
continue
image = Image.fromarray(np_image)
@@ -403,7 +405,8 @@ class YoloRestorer(Detailer):
p.image_mask = blend([np.array(m) for m in mask_all])
p.image_mask = Image.fromarray(p.image_mask)
np_images.append(np.array(image))
if image is not None:
np_images.append(np.array(image))
if shared.opts.detailer_save and annotated is not None:
np_images.append(annotated) # save debug image with boxes
return np_images
modules/processing_diffusers.py
+2 -2
View File
@@ -5,7 +5,7 @@ import numpy as np
import torch
import torchvision.transforms.functional as TF
from PIL import Image
from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks
from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks, attention
from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name
from modules.processing_args import set_pipeline_args
from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
@@ -497,7 +497,7 @@ def update_pipeline(sd_model, p: processing.StableDiffusionProcessing):
orig_pipeline = sd_model # processed ONNX pipeline should not be replaced with original pipeline.
if getattr(sd_model, "current_attn_name", None) != shared.opts.cross_attention_optimization:
shared.log.info(f"Setting attention optimization: {shared.opts.cross_attention_optimization}")
sd_models.set_diffusers_attention(sd_model)
attention.set_diffusers_attention(sd_model)
return sd_model
modules/sd_models.py
+2 -56
View File
@@ -10,7 +10,7 @@ import diffusers.loaders.single_file_utils
import torch
import huggingface_hub as hf
from installer import log
from modules import timer, paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_compile, sd_detect, model_quant, sd_hijack_te, sd_hijack_accelerate, sd_hijack_safetensors
from modules import timer, paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_compile, sd_detect, model_quant, sd_hijack_te, sd_hijack_accelerate, sd_hijack_safetensors, attention
from modules.memstats import memory_stats
from modules.modeldata import model_data
from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closest_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import
@@ -130,7 +130,7 @@ def set_diffuser_options(sd_model, vae=None, op:str='model', offload:bool=True,
clear_caches()
set_vae_options(sd_model, vae, op, quiet)
set_diffusers_attention(sd_model, quiet)
attention.set_diffusers_attention(sd_model, quiet)
if shared.opts.diffusers_fuse_projections and hasattr(sd_model, 'fuse_qkv_projections'):
try:
@@ -1157,60 +1157,6 @@ def set_diffuser_pipe(pipe, new_pipe_type):
return pipe
def set_diffusers_attention(pipe, quiet:bool=False):
import diffusers.models.attention_processor as p
def set_attn(pipe, attention, name:str=None, quiet:bool=False):
if attention is None:
return
# other models uses their own attention processor
if pipe.__class__.__name__.startswith("StableDiffusion") and getattr(pipe, "unet", None) is not None and hasattr(pipe.unet, "set_attn_processor"):
try:
pipe.unet.set_attn_processor(attention)
except Exception as e:
if 'Nunchaku' in pipe.unet.__class__.__name__:
pass
else:
shared.log.error(f"Attention: {name if name is not None else attention.__class__.__name__} pipe={pipe.__class__.__name__} {e}")
elif not quiet:
shared.log.warning(f"Attention: {name if name is not None else attention.__class__.__name__} is not compatible with {pipe.__class__.__name__}")
# if hasattr(pipe, 'pipe'):
# set_diffusers_attention(pipe.pipe)
if 'Control' in pipe.__class__.__name__ or 'Adapter' in pipe.__class__.__name__ or not (pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet")):
if shared.opts.cross_attention_optimization not in {"Scaled-Dot-Product", "Disabled"}:
shared.log.warning(f"Attention: {shared.opts.cross_attention_optimization} is not compatible with {pipe.__class__.__name__}")
else:
pipe.current_attn_name = shared.opts.cross_attention_optimization
return
shared.log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"')
if shared.opts.cross_attention_optimization == "Disabled":
pass # do nothing
elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers
set_attn(pipe, p.AttnProcessor2_0(), name="Scaled-Dot-Product", quiet=True)
elif shared.opts.cross_attention_optimization == "xFormers":
if hasattr(pipe, 'enable_xformers_memory_efficient_attention'):
pipe.enable_xformers_memory_efficient_attention()
else:
shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}")
elif shared.opts.cross_attention_optimization == "Batch matrix-matrix":
set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix")
elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM":
from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM
set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM")
if shared.opts.attention_slicing != "Default" and hasattr(pipe, "enable_attention_slicing") and hasattr(pipe, "disable_attention_slicing"):
if shared.opts.attention_slicing:
pipe.enable_attention_slicing()
else:
pipe.disable_attention_slicing()
shared.log.debug(f"Attention: slicing={shared.opts.attention_slicing}")
pipe.current_attn_name = shared.opts.cross_attention_optimization
def add_noise_pred_to_diffusers_callback(pipe):
if not hasattr(pipe, "_callback_tensor_inputs"):
return pipe
modules/shared.py
+9 -8
View File
@@ -136,7 +136,7 @@ def list_samplers():
return modules.sd_samplers.all_samplers
startup_offload_mode, startup_offload_min_gpu, startup_offload_max_gpu, startup_cross_attention, startup_sdp_options, startup_sdp_choices, startup_offload_always, startup_offload_never = get_default_modes(cmd_opts=cmd_opts, mem_stat=mem_stat)
startup_offload_mode, startup_offload_min_gpu, startup_offload_max_gpu, startup_cross_attention, startup_sdp_options, startup_sdp_choices, startup_sdp_override_options, startup_sdp_override_choices, startup_offload_always, startup_offload_never = get_default_modes(cmd_opts=cmd_opts, mem_stat=mem_stat)
options_templates.update(options_section(('sd', "Model Loading"), {
"sd_backend": OptionInfo('diffusers', "Execution backend", gr.Radio, {"choices": ['diffusers', 'original'], "visible": False }),
@@ -296,13 +296,13 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
"diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}),
"cross_attention_sep": OptionInfo("<h2>Cross Attention</h2>", "", gr.HTML),
"cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}),
"attention_": OptionInfo("<h2>Cross Attention</h2>", "", gr.HTML),
"cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention method", gr.Radio, lambda: {"choices": shared_items.list_crossattention()}),
"sdp_options": OptionInfo(startup_sdp_options, "SDP kernels", gr.CheckboxGroup, {"choices": startup_sdp_choices}),
"sdp_overrides": OptionInfo(startup_sdp_override_options, "SDP overrides", gr.CheckboxGroup, {"choices": startup_sdp_override_choices}),
"attention_slicing": OptionInfo('Default', "Attention slicing", gr.Radio, {"choices": ['Default', 'Enabled', 'Disabled']}),
"sdp_options": OptionInfo(startup_sdp_options, "SDP options", gr.CheckboxGroup, {"choices": startup_sdp_choices}),
"xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }),
"dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}),
"dynamic_attention_trigger_rate": OptionInfo(1, "Dynamic Attention trigger rate in GB", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4)*2, "step": 0.01}),
"dynamic_attention_slice_rate": OptionInfo(0.5, "Dynamic Attention slicing rate", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4), "step": 0.01}),
"dynamic_attention_trigger_rate": OptionInfo(1, "Dynamic Attention trigger rate", gr.Slider, {"minimum": 0.01, "maximum": max(gpu_memory,4)*2, "step": 0.01}),
}))
options_templates.update(options_section(('backends', "Backend Settings"), {
@@ -711,14 +711,15 @@ options_templates.update(options_section(('extra_networks', "Networks"), {
"extra_networks_lora_sep": OptionInfo("<h2>LoRA</h2>", "", gr.HTML),
"extra_networks_default_multiplier": OptionInfo(1.0, "Default strength", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}),
"lora_fuse_diffusers": OptionInfo(True, "LoRA fuse directly to model"),
"lora_force_reload": OptionInfo(False, "LoRA force reload always"),
"lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA load using Diffusers method"),
"lora_maybe_diffusers": OptionInfo(False, "LoRA load using Diffusers method for selected models", gr.Checkbox, {"visible": False}),
"lora_fuse_native": OptionInfo(True, "LoRA native fuse with model"),
"lora_fuse_diffusers": OptionInfo(False, "LoRA diffusers fuse with model"),
"lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
"lora_in_memory_limit": OptionInfo(1, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}),
"lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"),
"lora_quant": OptionInfo("NF4","LoRA precision when quantized", gr.Radio, {"choices": ["NF4", "FP4"]}),
"lora_maybe_diffusers": OptionInfo(False, "LoRA load using Diffusers method for selected models", gr.Checkbox, {"visible": False}),
"extra_networks_styles_sep": OptionInfo("<h2>Styles</h2>", "", gr.HTML),
"extra_networks_styles": OptionInfo(True, "Show reference styles"),
modules/shared_defaults.py
+9 -6
View File
@@ -40,16 +40,17 @@ def get_default_modes(cmd_opts, mem_stat):
default_cross_attention = "Scaled-Dot-Product"
default_sdp_override_options = []
if devices.backend == "zluda":
default_sdp_options = ['Math attention', 'Dynamic attention']
elif devices.backend in {"rocm", "directml", "cpu", "mps"}:
default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention']
default_sdp_options = ['Flash', 'Memory', 'Math']
default_sdp_override_options = ['Dynamic attention']
else:
default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention']
default_sdp_options = ['Flash', 'Memory', 'Math']
default_sdp_choices = ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention', 'CK Flash attention', 'Sage attention']
if devices.backend in {"rocm", "zluda"}:
default_sdp_choices.insert(4, 'Triton Flash attention') # insert after Dynamic attention
default_sdp_choices = ['Flash', 'Memory', 'Math']
default_sdp_override_choices = ['Dynamic attention', 'CK Flash attention', 'Triton Flash attention', 'Sage attention']
return (
default_offload_mode,
@@ -58,6 +59,8 @@ def get_default_modes(cmd_opts, mem_stat):
default_cross_attention,
default_sdp_options,
default_sdp_choices,
default_sdp_override_options,
default_sdp_override_choices,
default_diffusers_offload_always,
default_diffusers_offload_never
default_diffusers_offload_never,
)
modules/ui_img2img.py
+1 -1
View File
@@ -281,7 +281,7 @@ def create_ui():
(enable_hr, "Second pass"),
(enable_hr, "Refine"),
(denoising_strength, "Denoising strength"),
(denoising_strength, "Hires strength"),
(hr_denoising_strength, "Hires strength"),
(hr_sampler_index, "Hires sampler"),
(hr_resize_mode, "Hires mode"),
(hr_resize_context, "Hires context"),
modules/ui_sections.py
+1 -1
View File
@@ -339,7 +339,7 @@ def create_resize_inputs(tab, images, accordion=True, latent=False, non_zero=Tru
with gr.Row(visible=True) as _resize_group:
with gr.Column(elem_id=f"{tab}_column_size"):
selected_scale_tab = gr.State(value=0) # pylint: disable=abstract-class-instantiated
selected_scale_tab = gr.State(value=0 if tab != 'img2img' else 1) # pylint: disable=abstract-class-instantiated
with gr.Tabs(elem_id=f"{tab}_scale_tabs", selected=0 if non_zero else 1):
with gr.Tab(label="Fixed", id=0, elem_id=f"{tab}_scale_tab_fixed") as tab_scale_to:
with gr.Row(elem_id=f"{tab}_resize_row_fixed"):
modules/ui_txt2img.py
+3 -4
View File
@@ -35,7 +35,7 @@ def create_ui():
guidance_name, guidance_scale, guidance_rescale, guidance_start, guidance_stop, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_guidance.create_guidance_inputs('txt2img')
vae_type, tiling, hidiffusion, clip_skip = ui_sections.create_advanced_inputs('txt2img')
hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('txt2img')
enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img')
enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img')
detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, detailer_resolution = shared.yolo.ui('txt2img')
override_settings = ui_common.create_override_inputs('txt2img')
state = gr.Textbox(value='', visible=False)
@@ -61,7 +61,7 @@ def create_ui():
clip_skip,
seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
height, width,
enable_hr, denoising_strength,
enable_hr, hr_denoising_strength,
hr_scale, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y,
refiner_steps, refiner_start, refiner_prompt, refiner_negative,
hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundary, hdr_color_picker, hdr_tint_ratio,
@@ -132,8 +132,7 @@ def create_ui():
# second pass
(enable_hr, "Second pass"),
(enable_hr, "Refine"),
(denoising_strength, "Denoising strength"),
(denoising_strength, "Hires strength"),
(hr_denoising_strength, "Hires strength"),
(hr_sampler_index, "Hires sampler"),
(hr_resize_mode, "Hires mode"),
(hr_resize_context, "Hires context"),
pipelines/model_auraflow.py
+1 -1
View File
@@ -14,7 +14,7 @@ def load_auraflow(checkpoint_info, diffusers_load_config=None):
shared.log.debug(f'Load model: type=AuraFlow repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
transformer = generic.load_transformer(repo_id, cls_name=diffusers.AuraFlowTransformer2DModel, load_config=diffusers_load_config)
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.UMT5EncoderModel, load_config=diffusers_load_config) # auraflow uses EleutherAI/pile-t5-xl
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.UMT5EncoderModel, load_config=diffusers_load_config, allow_shared=False) # auraflow uses EleutherAI/pile-t5-xl
pipe = diffusers.AuraFlowPipeline.from_pretrained(
repo_id,