Merge branch 'dev' into dev

This commit is contained in:
Vladimir Mandic
2025-10-23 10:13:10 -04:00
committed by GitHub
44 changed files with 286 additions and 106 deletions
+1
View File
@@ -41,6 +41,7 @@ tunableop_results*.csv
/*.txt
/*.mp3
/*.lnk
/*.swp
!webui.bat
!webui.sh
!package.json
+10 -1
View File
@@ -1,6 +1,6 @@
# Change Log for SD.Next
## Update for 2025-10-19
## Update for 2025-10-23
- **Features**
- **offline mode**: enable in *settings -> hugginface*
@@ -13,9 +13,18 @@
- improved SDNQ SVD and low-bit matmul performance
- **Other**
- **scheduler** add base and max shift parameters for flow-matching samplers
- enhance `--optional` flag to pre-install optional packages
- add `[lora]` to recognized filename patterns
- **Fixes**
- startup error with `--profile` enabled if using `--skip`
- restore orig init image for each batch sequence
- fix modernui hints layout
- fix `wan-2.2-a14b` stage selection
- fix `wan-2.2-5b` vae decode
- disabling live preview should not disable progress updates
- video tab create `params.txt` with metadata
- fix full-screen image-viewer toolbar actions with control tab
- improve filename sanitization
## Update for 2025-10-18
+20 -12
View File
@@ -386,6 +386,9 @@ def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True):
t_start = time.time()
originalArg = arg
arg = arg.replace('>=', '==')
if opts.get('offline_mode', False):
log.warning('Offline mode enabled')
return
package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force-reinstall", "").replace(" ", " ").strip()
uv = uv and args.uv and not package.startswith('git+')
pipCmd = "uv pip" if uv else "pip"
@@ -608,7 +611,7 @@ def check_diffusers():
if args.skip_git:
install('diffusers')
return
sha = '23ebbb4bc81a17ebea17cb7cb94f301199e49a7f' # diffusers commit hash
sha = 'b3e56e71fb7c73601851bb83e7583f113f563d26' # diffusers commit hash
# if args.use_rocm or args.use_zluda or args.use_directml:
# sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
pkg = pkg_resources.working_set.by_key.get('diffusers', None)
@@ -1270,18 +1273,23 @@ def install_insightface():
def install_optional():
t_start = time.time()
log.info('Installing optional requirements...')
install('--no-build-isolation git+https://github.com/Disty0/BasicSR@23c1fb6f5c559ef5ce7ad657f2fa56e41b121754', 'basicsr')
install('--no-build-isolation git+https://github.com/Disty0/GFPGAN@ae0f7e44fafe0ef4716f3c10067f8f379b74c21c', 'gfpgan')
install('clean-fid', quiet=True)
install('pillow-jxl-plugin==1.3.4', ignore=True, quiet=True)
install('optimum-quanto==0.2.7', ignore=True, quiet=True)
install('torchao==0.10.0', ignore=True, quiet=True)
install('bitsandbytes==0.47.0', ignore=True, quiet=True)
install('nvidia-ml-py', ignore=True, quiet=True)
install('ultralytics==8.3.40', ignore=True, quiet=True)
install('Cython', ignore=True, quiet=True)
install('--no-build-isolation git+https://github.com/Disty0/BasicSR@23c1fb6f5c559ef5ce7ad657f2fa56e41b121754', 'basicsr', ignore=True, quiet=True)
install('--no-build-isolation git+https://github.com/Disty0/GFPGAN@ae0f7e44fafe0ef4716f3c10067f8f379b74c21c', 'gfpgan', ignore=True, quiet=True)
install('av', ignore=True, quiet=True)
install('gguf', ignore=True)
install('beautifulsoup4', ignore=True, quiet=True)
install('bitsandbytes==0.47.0', ignore=True, quiet=True)
install('clean-fid', ignore=True, quiet=True)
install('clip_interrogator==0.6.0', ignore=True, quiet=True)
install('Cython', ignore=True, quiet=True)
install('gguf', ignore=True, quiet=True)
install('git+https://github.com/tencent-ailab/IP-Adapter.git', 'ip_adapter', ignore=True, quiet=True)
install('hf_transfer', ignore=True, quiet=True)
install('hf_xet', ignore=True, quiet=True)
install('nvidia-ml-py', ignore=True, quiet=True)
install('optimum-quanto==0.2.7', ignore=True, quiet=True)
install('pillow-jxl-plugin==1.3.4', ignore=True, quiet=True)
install('torchao==0.10.0', ignore=True, quiet=True)
install('ultralytics==8.3.40', ignore=True, quiet=True)
try:
import gguf
scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts')
+11 -3
View File
@@ -52,12 +52,15 @@ function modalImageSwitch(offset) {
}
function modalSaveImage(event) {
if (gradioApp().getElementById('tab_txt2img').style.display !== 'none') gradioApp().getElementById('save_txt2img').click();
else if (gradioApp().getElementById('tab_img2img').style.display !== 'none') gradioApp().getElementById('save_img2img').click();
else if (gradioApp().getElementById('tab_process').style.display !== 'none') gradioApp().getElementById('save_extras').click();
const tabName = getENActiveTab();
const saveBtn = gradioApp().getElementById(`save_${tabName}`);
log('modalSaveImage', tabName, saveBtn);
if (saveBtn) saveBtn.click();
modalImageSwitch(0);
}
function modalKeyHandler(event) {
log('modalKeyHandler', event.key);
switch (event.key) {
case 's':
modalSaveImage();
@@ -158,6 +161,7 @@ function modalZoomToggle(event) {
const modalImage = gradioApp().getElementById('modalImage');
modalZoomSet(modalImage, !modalImage.classList.contains('modalImageFullscreen'));
event.stopPropagation();
modalImageSwitch(0);
}
function modalTileToggle(event) {
@@ -172,12 +176,15 @@ function modalTileToggle(event) {
modal.style.setProperty('background-image', `url(${modalImage.src})`);
}
event.stopPropagation();
modalImageSwitch(0);
}
function modalResetInstance(event) {
const modalImage = document.getElementById('modalImage');
previewInstance.dispose();
previewInstance = panzoom(modalImage, { zoomSpeed: 0.05, minZoom: 0.1, maxZoom: 5.0, filterKey: (/* e, dx, dy, dz */) => true });
event.stopPropagation();
modalImageSwitch(0);
}
function modalToggleParams(event) {
@@ -188,6 +195,7 @@ function modalToggleParams(event) {
modalExif.style.display = 'none';
}
event.stopPropagation();
modalImageSwitch(0);
}
function galleryClickEventHandler(event) {
+1 -1
View File
@@ -132,7 +132,7 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres
};
const start = (id_task, id_live_preview) => { // eslint-disable-line no-shadow
if (!opts.live_previews_enable || opts.live_preview_refresh_period === 0 || opts.show_progress_every_n_steps === 0) return;
if (opts.live_preview_refresh_period === 0) return;
const request_id = document.hidden ? -1 : id_live_preview;
const onProgressHandler = (res) => {
+1 -1
View File
@@ -108,7 +108,7 @@ def search_civitai(
global models # pylint: disable=global-statement
import requests
from urllib.parse import urlencode
install('bs4') # Ensure BeautifulSoup is installed
install('beautifulsoup4')
if len(query) == 0:
log.error('CivitAI: empty query')
@@ -27,7 +27,7 @@ class DepthAnythingDetector:
PrepareForNet()])
@classmethod
def from_pretrained(cls, pretrained_model_or_path: str, cache_dir: str) -> str:
def from_pretrained(cls, pretrained_model_or_path: str, cache_dir: str, local_files_only=False) -> str:
from modules.control.proc.depth_anything.dpt import DPT_DINOv2
import huggingface_hub as hf
model = (
@@ -40,7 +40,7 @@ class DepthAnythingDetector:
.to(devices.device)
.eval()
)
model_path = hf.hf_hub_download(repo_id=pretrained_model_or_path, filename="pytorch_model.bin", cache_dir=cache_dir)
model_path = hf.hf_hub_download(repo_id=pretrained_model_or_path, filename="pytorch_model.bin", cache_dir=cache_dir, local_files_only=local_files_only)
model_dict = torch.load(model_path)
model.load_state_dict(model_dict)
return cls(model)
+1 -1
View File
@@ -51,7 +51,7 @@ def check_dependencies():
status = [installed(p, reload=False, quiet=True) for p in packages]
debug(f'DWPose required={packages} status={status}')
if not all(status):
log.info(f'Installing DWPose dependencies: {packages}')
log.info(f'Installing dependencies: for=dwpose packages={packages}')
cmd = 'install --upgrade --no-deps --force-reinstall '
pkgs = ' '.join(packages)
pip(cmd + pkgs, ignore=False, quiet=True, uv=False)
+2 -2
View File
@@ -60,12 +60,12 @@ class HEDdetector:
self.model = model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None, local_files_only=False):
filename = filename or "ControlNetHED.pth"
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
model = ControlNetHED_Apache2()
model.load_state_dict(torch.load(model_path, map_location='cpu'))
model.float().eval()
+3 -3
View File
@@ -20,13 +20,13 @@ class LeresDetector:
self.pix2pixmodel = pix2pixmodel
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, pix2pix_filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, pix2pix_filename=None, cache_dir=None, local_files_only=False):
filename = filename or "res101.pth"
pix2pix_filename = pix2pix_filename or "latest_net_G.pth"
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
checkpoint = torch.load(model_path, map_location=torch.device('cpu'))
model = RelDepthModel(backbone='resnext101')
model.load_state_dict(strip_prefix_if_present(checkpoint['depth_model'], "module."), strict=True)
@@ -34,7 +34,7 @@ class LeresDetector:
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, pix2pix_filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, pix2pix_filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, pix2pix_filename, cache_dir=cache_dir, local_files_only=local_files_only)
opt = TestOptions().parse()
if not torch.cuda.is_available():
opt.gpu_ids = [] # cpu mode
+3 -3
View File
@@ -95,7 +95,7 @@ class LineartDetector:
self.model_coarse = coarse_model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, coarse_filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, coarse_filename=None, cache_dir=None, local_files_only=False):
filename = filename or "sk_model.pth"
coarse_filename = coarse_filename or "sk_model2.pth"
@@ -103,8 +103,8 @@ class LineartDetector:
model_path = os.path.join(pretrained_model_or_path, filename)
coarse_model_path = os.path.join(pretrained_model_or_path, coarse_filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
coarse_model_path = hf_hub_download(pretrained_model_or_path, coarse_filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
coarse_model_path = hf_hub_download(pretrained_model_or_path, coarse_filename, cache_dir=cache_dir, local_files_only=local_files_only)
model = Generator(3, 1, 3)
model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))
+2 -2
View File
@@ -117,12 +117,12 @@ class LineartAnimeDetector:
self.model = model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None, local_files_only=False):
filename = filename or "netG.pth"
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False)
net = UnetGenerator(3, 1, 8, 64, norm_layer=norm_layer, use_dropout=False)
ckpt = torch.load(model_path)
+2 -2
View File
@@ -17,7 +17,7 @@ class MidasDetector:
self.model = model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, model_type="dpt_hybrid", filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, model_type="dpt_hybrid", filename=None, cache_dir=None, local_files_only=False):
if pretrained_model_or_path == "lllyasviel/ControlNet":
filename = filename or "annotator/ckpts/dpt_hybrid-midas-501f0c75.pt"
else:
@@ -25,7 +25,7 @@ class MidasDetector:
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
model = MiDaSInference(model_type=model_type, model_path=model_path)
return cls(model)
+2 -2
View File
@@ -16,7 +16,7 @@ class MLSDdetector:
self.model = model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None, local_files_only=False):
if pretrained_model_or_path == "lllyasviel/ControlNet":
filename = filename or "annotator/ckpts/mlsd_large_512_fp32.pth"
else:
@@ -24,7 +24,7 @@ class MLSDdetector:
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
model = MobileV2_MLSD_Large()
model.load_state_dict(torch.load(model_path), strict=True)
model.eval()
+2 -2
View File
@@ -33,12 +33,12 @@ class NormalBaeDetector:
self.norm = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None, local_files_only=False):
filename = filename or "scannet.pt"
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
args = types.SimpleNamespace()
args.mode = 'client'
args.architecture = 'BN'
+4 -4
View File
@@ -76,7 +76,7 @@ class OpenposeDetector:
self.face_estimation = face_estimation
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, hand_filename=None, face_filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, hand_filename=None, face_filename=None, cache_dir=None, local_files_only=False):
if pretrained_model_or_path == "lllyasviel/ControlNet":
filename = filename or "annotator/ckpts/body_pose_model.pth"
@@ -96,9 +96,9 @@ class OpenposeDetector:
hand_model_path = os.path.join(pretrained_model_or_path, hand_filename)
face_model_path = os.path.join(face_pretrained_model_or_path, face_filename)
else:
body_model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
hand_model_path = hf_hub_download(pretrained_model_or_path, hand_filename, cache_dir=cache_dir)
face_model_path = hf_hub_download(face_pretrained_model_or_path, face_filename, cache_dir=cache_dir)
body_model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
hand_model_path = hf_hub_download(pretrained_model_or_path, hand_filename, cache_dir=cache_dir, local_files_only=local_files_only)
face_model_path = hf_hub_download(face_pretrained_model_or_path, face_filename, cache_dir=cache_dir, local_files_only=local_files_only)
body_estimation = Body(body_model_path)
hand_estimation = Hand(hand_model_path)
+2 -2
View File
@@ -16,12 +16,12 @@ class PidiNetDetector:
self.model = model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, filename=None, cache_dir=None, local_files_only=False):
filename = filename or "table5_pidinet.pth"
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
model = pidinet()
model.load_state_dict({k.replace('module.', ''): v for k, v in torch.load(model_path)['state_dict'].items()})
model.eval()
@@ -23,12 +23,12 @@ class SamDetector:
self.model = mask_generator
@classmethod
def from_pretrained(cls, model_path, filename, model_type, cache_dir=None):
def from_pretrained(cls, model_path, filename, model_type, cache_dir=None, local_files_only=False):
"""
Possible model_type : vit_h, vit_l, vit_b, vit_t
download weights from https://github.com/facebookresearch/segment-anything
"""
model_path = hf_hub_download(model_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(model_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
sam = sam_model_registry[model_type](checkpoint=model_path)
sam.to(devices.device)
mask_generator = SamAutomaticMaskGenerator(sam)
+2 -2
View File
@@ -20,12 +20,12 @@ class ZoeDetector:
self.model = model
@classmethod
def from_pretrained(cls, pretrained_model_or_path, model_type="zoedepth", filename=None, cache_dir=None):
def from_pretrained(cls, pretrained_model_or_path, model_type="zoedepth", filename=None, cache_dir=None, local_files_only=False):
filename = filename or "ZoeD_M12_N.pt"
if os.path.isdir(pretrained_model_or_path):
model_path = os.path.join(pretrained_model_or_path, filename)
else:
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir)
model_path = hf_hub_download(pretrained_model_or_path, filename, cache_dir=cache_dir, local_files_only=local_files_only)
if model_type == "zoedepth":
model_cls = ZoeDepth
elif model_type == "zoedepth_nk":
+8 -2
View File
@@ -181,9 +181,15 @@ class Processor():
self.model = None
self.processor_id = processor_id
devices.torch_gc(force=True, reason='processor')
# self.override = None
# devices.torch_gc()
self.load_config = { 'cache_dir': cache_dir }
from modules.shared import opts
if opts.offline_mode:
self.load_config["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
def config(self, processor_id = None):
if processor_id is not None:
+6
View File
@@ -205,6 +205,12 @@ class ControlNet():
self.load_config = { 'cache_dir': cache_dir }
if load_config is not None:
self.load_config.update(load_config)
if opts.offline_mode:
self.load_config["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
if model_id is not None:
self.load()
+8 -1
View File
@@ -108,8 +108,15 @@ class ControlLLLite():
self.model = ControlNetLLLite(model_path)
else:
import huggingface_hub as hf
offline_config = {}
if opts.offline_mode:
offline_config["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
folder, filename = os.path.split(model_path)
model_path = hf.hf_hub_download(repo_id=folder, filename=f'{filename}.safetensors', cache_dir=cache_dir)
model_path = hf.hf_hub_download(repo_id=folder, filename=f'{filename}.safetensors', cache_dir=cache_dir, **offline_config)
self.model = ControlNetLLLite(model_path)
if self.device is not None:
self.model.to(self.device)
+10 -1
View File
@@ -2,7 +2,7 @@ import os
import time
from typing import Union
import threading
from diffusers import pipelines, StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline # pylint: disable=unused-import
from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline # pylint: disable=unused-import
from installer import log
from modules import errors, sd_models
from modules.control.units import detect
@@ -104,6 +104,13 @@ class Adapter():
return
model_path, model_args = all_models[model_id]
self.load_config.update(model_args)
from modules.shared import opts
if opts.offline_mode:
self.load_config["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
if model_path is None:
log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id')
return
@@ -168,6 +175,7 @@ class AdapterPipeline():
adapter=adapter,
)
sd_models.move_model(self.pipeline, pipeline.device)
sd_models.apply_balanced_offload(self.pipeline, force=True)
elif detect.is_sd15(pipeline):
self.pipeline = StableDiffusionAdapterPipeline(
vae=pipeline.vae,
@@ -181,6 +189,7 @@ class AdapterPipeline():
adapter=adapter,
)
sd_models.move_model(self.pipeline, pipeline.device)
sd_models.apply_balanced_offload(self.pipeline, force=True)
else:
log.error(f'Control {what} pipeline: class={pipeline.__class__.__name__} unsupported model type')
return
+8
View File
@@ -100,6 +100,12 @@ class ControlNetXS():
# log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded')
return
self.load_config['time_embedding_mix'] = time_embedding_mix
if opts.offline_mode:
self.load_config["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}')
if model_path.endswith('.safetensors'):
self.model = ControlNetXSModel.from_single_file(model_path, **self.load_config)
@@ -140,6 +146,7 @@ class ControlNetXSPipeline():
controlnet=controlnet, # can be a list
)
sd_models.move_model(self.pipeline, pipeline.device)
sd_models.apply_balanced_offload(self.pipeline, force=True)
elif detect.is_sd15(pipeline):
self.pipeline = StableDiffusionControlNetXSPipeline(
vae=pipeline.vae,
@@ -153,6 +160,7 @@ class ControlNetXSPipeline():
controlnet=controlnet, # can be a list
)
sd_models.move_model(self.pipeline, pipeline.device)
sd_models.apply_balanced_offload(self.pipeline, force=True)
else:
log.error(f'Control {what} pipeline: class={pipeline.__class__.__name__} unsupported model type')
return
+17 -8
View File
@@ -1,3 +1,4 @@
import os
import time
from modules import shared, devices, errors, sd_models, sd_checkpoint, model_quant
@@ -128,23 +129,31 @@ def load_model(variant:str=None, pipeline:str=None, text_encoder:str=None, text_
sd_models.hf_auth_check(model["text_encoder"]["repo"])
sd_models.hf_auth_check(model["text_encoder_2"]["repo"])
offline_config = {}
if shared.opts.offline_mode:
offline_config["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
shared.log.debug(f'FramePack load: module=llm {model["text_encoder"]}')
load_args, quant_args = model_quant.get_dit_args({}, module='TE', device_map=True)
text_encoder = LlamaModel.from_pretrained(model["text_encoder"]["repo"], subfolder=model["text_encoder"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **load_args, **quant_args)
tokenizer = LlamaTokenizerFast.from_pretrained(model["tokenizer"]["repo"], subfolder=model["tokenizer"]["subfolder"], cache_dir=shared.opts.hfcache_dir)
text_encoder = LlamaModel.from_pretrained(model["text_encoder"]["repo"], subfolder=model["text_encoder"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **load_args, **quant_args, **offline_config)
tokenizer = LlamaTokenizerFast.from_pretrained(model["tokenizer"]["repo"], subfolder=model["tokenizer"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **offline_config)
text_encoder.requires_grad_(False)
text_encoder.eval()
sd_models.move_model(text_encoder, devices.cpu)
shared.log.debug(f'FramePack load: module=te {model["text_encoder_2"]}')
text_encoder_2 = CLIPTextModel.from_pretrained(model["text_encoder_2"]["repo"], subfolder=model["text_encoder_2"]["subfolder"], torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir)
tokenizer_2 = CLIPTokenizer.from_pretrained(model["pipeline"]["repo"], subfolder='tokenizer_2', cache_dir=shared.opts.hfcache_dir)
text_encoder_2 = CLIPTextModel.from_pretrained(model["text_encoder_2"]["repo"], subfolder=model["text_encoder_2"]["subfolder"], torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, **offline_config)
tokenizer_2 = CLIPTokenizer.from_pretrained(model["pipeline"]["repo"], subfolder='tokenizer_2', cache_dir=shared.opts.hfcache_dir, **offline_config)
text_encoder_2.requires_grad_(False)
text_encoder_2.eval()
sd_models.move_model(text_encoder_2, devices.cpu)
shared.log.debug(f'FramePack load: module=vae {model["vae"]}')
vae = AutoencoderKLHunyuanVideo.from_pretrained(model["vae"]["repo"], subfolder=model["vae"]["subfolder"], torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir)
vae = AutoencoderKLHunyuanVideo.from_pretrained(model["vae"]["repo"], subfolder=model["vae"]["subfolder"], torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, **offline_config)
vae.requires_grad_(False)
vae.eval()
vae.enable_slicing()
@@ -152,8 +161,8 @@ def load_model(variant:str=None, pipeline:str=None, text_encoder:str=None, text_
sd_models.move_model(vae, devices.cpu)
shared.log.debug(f'FramePack load: module=encoder {model["feature_extractor"]} model={model["image_encoder"]}')
feature_extractor = SiglipImageProcessor.from_pretrained(model["feature_extractor"]["repo"], subfolder=model["feature_extractor"]["subfolder"], cache_dir=shared.opts.hfcache_dir)
image_encoder = SiglipVisionModel.from_pretrained(model["image_encoder"]["repo"], subfolder=model["image_encoder"]["subfolder"], torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir)
feature_extractor = SiglipImageProcessor.from_pretrained(model["feature_extractor"]["repo"], subfolder=model["feature_extractor"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **offline_config)
image_encoder = SiglipVisionModel.from_pretrained(model["image_encoder"]["repo"], subfolder=model["image_encoder"]["subfolder"], torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, **offline_config)
image_encoder.requires_grad_(False)
image_encoder.eval()
sd_models.move_model(image_encoder, devices.cpu)
@@ -161,7 +170,7 @@ def load_model(variant:str=None, pipeline:str=None, text_encoder:str=None, text_
shared.log.debug(f'FramePack load: module=transformer {model["transformer"]}')
dit_repo = model["transformer"]["repo"]
load_args, quant_args = model_quant.get_dit_args({}, module='Model', device_map=True)
transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained(dit_repo, subfolder=model["transformer"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **load_args, **quant_args)
transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained(dit_repo, subfolder=model["transformer"]["subfolder"], cache_dir=shared.opts.hfcache_dir, **load_args, **quant_args, **offline_config)
transformer.high_quality_fp32_output_for_inference = False
transformer.requires_grad_(False)
transformer.eval()
+16 -2
View File
@@ -308,12 +308,26 @@ def worker(
if is_last_section:
break
total_generated_frames, _video_filename = save_video(history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate, pbar=pbar, stream=stream, metadata=metadata)
total_generated_frames, _video_filename = save_video(
None,
history_pixels,
mp4_fps,
mp4_codec,
mp4_opt,
mp4_ext,
mp4_sf,
mp4_video,
mp4_frames,
mp4_interpolate,
pbar=pbar,
stream=stream,
metadata=metadata,
)
except AssertionError:
shared.log.info('FramePack: interrupted')
if shared.opts.keep_incomplete:
save_video(history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate=0, stream=stream, metadata=metadata)
save_video(None, history_pixels, mp4_fps, mp4_codec, mp4_opt, mp4_ext, mp4_sf, mp4_video, mp4_frames, mp4_interpolate=0, stream=stream, metadata=metadata)
except Exception as e:
shared.log.error(f'FramePack: {e}')
errors.display(e, 'FramePack')
+1 -1
View File
@@ -3,7 +3,6 @@ import io
import os
from PIL import Image
import gradio as gr
from modules.paths import params_path
from modules import shared, gr_tempdir, script_callbacks, images
from modules.infotext import parse, mapping, quote, unquote # pylint: disable=unused-import
@@ -204,6 +203,7 @@ def create_override_settings_dict(text_pairs):
def connect_paste(button, local_paste_fields, input_comp, override_settings_component, tabname):
def paste_func(prompt):
from modules.paths import params_path
if prompt is None or len(prompt.strip()) == 0:
if os.path.exists(params_path):
with open(params_path, "r", encoding="utf8") as file:
+25 -8
View File
@@ -1,6 +1,7 @@
import re
import os
import time
import unicodedata
import uuid
import string
import hashlib
@@ -23,8 +24,8 @@ NOTHING = object()
class FilenameGenerator:
replacements = {
'width': lambda self: self.image.width,
'height': lambda self: self.image.height,
'width': lambda self: self.width,
'height': lambda self: self.height,
'batch_number': lambda self: self.batch_number,
'iter_number': lambda self: self.iter_number,
'num': lambda self: NOTHING if self.p.n_iter == 1 and self.p.batch_size == 1 else self.p.iteration * self.p.batch_size + self.p.batch_index + 1,
@@ -32,8 +33,8 @@ class FilenameGenerator:
'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'),
'datetime': lambda self, *args: self.datetime(*args), # accepts formats: [datetime], [datetime<Format>], [datetime<Format><Time Zone>]
'hasprompt': lambda self, *args: self.hasprompt(*args), # accepts formats:[hasprompt<prompt1|default><prompt2>..]
'hash': lambda self: self.image_hash(),
'image_hash': lambda self: self.image_hash(),
'hash': lambda self: self.image_hash() if self.image is not None else '',
'image_hash': lambda self: self.image_hash() if self.image is not None else '',
'timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
'epoch': lambda self: int(time.time()),
'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
@@ -44,6 +45,8 @@ class FilenameGenerator:
'model_type': lambda self: shared.sd_model_type if shared.sd_loaded else '',
'model_hash': lambda self: shared.sd_model.sd_checkpoint_info.shorthash if shared.sd_loaded and getattr(shared.sd_model, 'sd_checkpoint_info', None) is not None else '',
'lora': lambda self: self.p and getattr(self.p, 'extra_generation_params', {}).get('LoRA networks', ''),
'prompt': lambda self: self.prompt_full(),
'prompt_no_styles': lambda self: self.prompt_no_style(),
'prompt_words': lambda self: self.prompt_words(),
@@ -61,11 +64,11 @@ class FilenameGenerator:
}
default_time_format = '%Y%m%d%H%M%S'
def __init__(self, p, seed, prompt, image, grid=False):
def __init__(self, p, seed, prompt, image=None, grid=False, width=None, height=None):
if p is None:
debug('Filename generator init skip')
else:
debug(f'Filename generator init: {seed} {prompt}')
debug(f'Filename generator init: seed={seed} prompt="{prompt}"')
self.p = p
if seed is not None and int(seed) > 0:
self.seed = seed
@@ -82,6 +85,8 @@ class FilenameGenerator:
if isinstance(self.prompt, list):
self.prompt = ' '.join(self.prompt)
self.image = image
self.width = width if width is not None else (image.width if image is not None else (p.width if p is not None else 0))
self.height = height if height is not None else (image.height if image is not None else (p.height if p is not None else 0))
if not grid:
self.batch_number = NOTHING if self.p is None or getattr(self.p, 'batch_size', 1) == 1 else (self.p.batch_index + 1 if hasattr(self.p, 'batch_index') else NOTHING)
self.iter_number = NOTHING if self.p is None or getattr(self.p, 'n_iter', 1) == 1 else (self.p.iteration + 1 if hasattr(self.p, 'iteration') else NOTHING)
@@ -162,14 +167,25 @@ class FilenameGenerator:
return sanitized
def sanitize(self, filename):
invalid_chars = '\'"|?*\n\t\r' # <https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file>
# starting reference: <https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file>
invalid_chars = (
"#<>/\\\"'`" # ASCII quote and backtick
"’‚‛\u2018\u2019\u201B" # smart single quotes and variants
"\u02BB" # modifier letter turned comma (ʻ)
"\u201C\u201D\u201F" # smart double quotes and variants
"|?*^%$\u00A0\u2013\u2014\n\t\r" # pipes, wildcards, percent, currency, NBSP, dashes, control chars
)
invalid_folder = ':'
invalid_files = ['CON', 'PRN', 'AUX', 'NUL', 'NULL', 'COM0', 'COM1', 'LPT0', 'LPT1']
invalid_prefix = ', '
invalid_suffix = '.,_ '
fn, ext = os.path.splitext(filename)
fn, ext = os.path.splitext(unicodedata.normalize('NFKC', filename))
fn = fn.strip()
ext = ext.strip()
parts = Path(fn).parts
newparts = []
# for ch in filename:
# print(repr(ch), hex(ord(ch)), unicodedata.name(ch, 'UNKNOWN'), ch in invalid_chars)
for i, part in enumerate(parts):
part = part.translate({ ord(x): '_' for x in invalid_chars })
if i > 0 or (len(part) >= 2 and part[1] != invalid_folder): # skip drive, otherwise remove
@@ -179,6 +195,7 @@ class FilenameGenerator:
[part := part.replace(word, '_') for word in invalid_files] # pylint: disable=expression-not-assigned
newparts.append(part)
fn = str(Path(*newparts))
fn = fn.replace(' ', ' ').strip()
max_length = max(256 - len(ext), os.statvfs(__file__).f_namemax - 32 if hasattr(os, 'statvfs') else 256 - len(ext))
while len(os.path.abspath(fn)) > max_length:
fn = fn[:-1]
+8 -4
View File
@@ -190,14 +190,15 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st
if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}':
jobid = shared.state.begin('Load encoder')
try:
offline_config = { 'local_files_only': True } if shared.opts.offline_mode else {}
if shared.sd_model_type == 'sd3':
image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir)
image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, **offline_config)
else:
if clip_subfolder is None:
image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True)
image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True, **offline_config)
shared.log.debug(f'IP adapter load: encoder="{clip_repo}" cls={pipe.image_encoder.__class__.__name__}')
else:
image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True)
image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True, **offline_config)
shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}')
sd_models.clear_caches()
image_encoder = model_quant.do_post_load_quant(image_encoder, allow=True)
@@ -220,8 +221,9 @@ def load_feature_extractor(pipe):
if pipe.feature_extractor is None:
try:
jobid = shared.state.begin('Load extractor')
offline_config = { 'local_files_only': True } if shared.opts.offline_mode else {}
if shared.sd_model_type == 'sd3':
feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir)
feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, **offline_config)
else:
feature_extractor = transformers.CLIPImageProcessor()
if hasattr(pipe, 'register_modules'):
@@ -343,6 +345,8 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt
kwargs['weight_name'] = names if len(names) > 1 else names[0]
if len(revisions) > 0:
kwargs['revision'] = revisions[0]
if shared.opts.offline_mode:
kwargs["local_files_only"] = True
pipe.load_ip_adapter(repos, **kwargs)
adapters_loaded = names
if hasattr(p, 'ip_adapter_layers'):
+3
View File
@@ -89,6 +89,8 @@ def run_ltx(task_id,
shared.state.job_count = 1
p = processing.StableDiffusionProcessingVideo(
video_engine=engine,
video_model=model,
prompt=prompt,
negative_prompt=negative,
styles=styles,
@@ -247,6 +249,7 @@ def run_ltx(task_id,
timer.process.add('offload', t11 - t10)
num_frames, video_file = save_video(
p=p,
pixels=frames,
mp4_fps=mp4_fps,
mp4_codec=mp4_codec,
+4 -2
View File
@@ -428,9 +428,11 @@ class StableDiffusionProcessing:
class StableDiffusionProcessingVideo(StableDiffusionProcessing):
def __init__(self, **kwargs):
self.prompt_template: str = None
self.frames: int = 1
self.frames: int = kwargs.pop('frames', 1)
self.vae_tile_frames: int = kwargs.pop('vae_tile_frames', 0)
self.video_engine: str = kwargs.pop('video_engine', None)
self.video_model: str = kwargs.pop('video_model', None)
self.scheduler_shift: float = 0.0
self.vae_tile_frames: int = 0
debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access
super().__init__(**kwargs)
+2 -2
View File
@@ -92,9 +92,9 @@ def api_progress(req: ProgressRequest):
id_live_preview = -1
textinfo = "Queued..." if queued else "Waiting..."
debug_log(f'Preview: job={shared.state.job} active={active} progress={step}/{steps}/{progress} image={shared.state.current_image_sampling_step} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} job={shared.state.preview_job} elapsed={elapsed:.3f}')
debug_log(f'Preview: job={shared.state.job} active={active} progress={step}/{steps}/{progress} image={shared.state.current_image_sampling_step} request={id_live_preview} last={shared.state.id_live_preview} job={shared.state.preview_job} elapsed={elapsed:.3f}')
if shared.opts.live_previews_enable and active and (req.id_live_preview != -1):
if active and (req.id_live_preview != -1):
have_image = shared.state.set_current_image()
if have_image and shared.state.current_image is not None:
buffered = io.BytesIO()
+2 -2
View File
@@ -409,7 +409,7 @@ def report_model_stats(module_name, module):
shared.log.error(f'Module stats: name={module_name} {e}')
def apply_balanced_offload(sd_model=None, exclude=[]):
def apply_balanced_offload(sd_model=None, exclude=[], force=False):
global offload_hook_instance # pylint: disable=global-statement
if shared.opts.diffusers_offload_mode != "balanced":
return sd_model
@@ -424,7 +424,7 @@ def apply_balanced_offload(sd_model=None, exclude=[]):
return sd_model
cached = True
checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else sd_model.__class__.__name__
if (offload_hook_instance is None) or (offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory) or (offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory) or (checkpoint_name != offload_hook_instance.checkpoint_name):
if force or (offload_hook_instance is None) or (offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory) or (offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory) or (checkpoint_name != offload_hook_instance.checkpoint_name):
cached = False
offload_hook_instance = OffloadHook(checkpoint_name)
+1 -1
View File
@@ -124,7 +124,7 @@ def images_tensor_to_samples(image, approximation=None, model=None):
def store_latent(decoded):
shared.state.current_latent = decoded
if shared.opts.live_previews_enable and shared.opts.show_progress_every_n_steps > 0 and shared.state.sampling_step % shared.opts.show_progress_every_n_steps == 0:
if shared.opts.show_progress_every_n_steps > 0 and shared.state.sampling_step % shared.opts.show_progress_every_n_steps == 0:
if not shared.parallel_processing_allowed:
image = sample_to_image(decoded)
shared.state.assign_current_image(image)
+2 -2
View File
@@ -259,7 +259,7 @@ options_templates.update(options_section(('vae_encoder', "Variational Auto Encod
"sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list),
"diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}),
"no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half-vae)"),
"diffusers_vae_slicing": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE slicing", gr.Checkbox),
"diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox),
"diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram, "VAE tiling", gr.Checkbox),
"diffusers_vae_tile_size": OptionInfo(0, "VAE tile size", gr.Slider, {"minimum": 0, "maximum": 4096, "step": 8 }),
"diffusers_vae_tile_overlap": OptionInfo(0.25, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.95, "step": 0.05 }),
@@ -591,7 +591,7 @@ options_templates.update(options_section(('ui', "User Interface"), {
}))
options_templates.update(options_section(('live-preview', "Live Previews"), {
"show_progress_every_n_steps": OptionInfo(1, "Live preview display period", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}),
"show_progress_every_n_steps": OptionInfo(1, "Live preview display period", gr.Slider, {"minimum": 0, "maximum": 20, "step": 1}),
"show_progress_type": OptionInfo("TAESD", "Live preview method", gr.Radio, {"choices": ["Simple", "Approximate", "TAESD", "Full VAE"]}),
"live_preview_refresh_period": OptionInfo(500, "Progress update period", gr.Slider, {"minimum": 0, "maximum": 5000, "step": 25}),
"taesd_variant": OptionInfo(shared_items.sd_taesd_items()[0], "TAESD variant", gr.Dropdown, {"choices": shared_items.sd_taesd_items()}),
+1 -1
View File
@@ -260,7 +260,7 @@ class State:
if self.job == 'VAE' or self.job == 'Upscale': # avoid generating preview while vae is running
return False
from modules.shared import opts, cmd_opts
if cmd_opts.lowvram or self.api or (not opts.live_previews_enable) or (opts.show_progress_every_n_steps <= 0):
if cmd_opts.lowvram or self.api or (opts.show_progress_every_n_steps <= 0):
return False
if (not self.disable_preview) and (abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps):
return self.do_set_current_image()
+12 -2
View File
@@ -38,7 +38,7 @@ def create_ui():
mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf = video_ui.create_ui_outputs()
with gr.Tab('Models', id='video-core-tab') as video_core_tab:
from modules.video_models import video_ui
video_ui.create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed, reuse_seed)
engine, model, steps, sampler_index = video_ui.create_ui(prompt, negative, styles, overrides, init_image, init_strength, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, width, height, frames, seed, reuse_seed)
with gr.Tab('FramePack', id='framepack-tab') as framepack_tab:
from modules.framepack import framepack_ui
framepack_ui.create_ui(prompt, negative, styles, overrides, init_image, last_image, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf)
@@ -48,8 +48,18 @@ def create_ui():
paste_fields = [
(prompt, "Prompt"), # cannot add more fields as they are not defined yet
(negative, "Negative prompt"),
(width, "Width"),
(height, "Height"),
(frames, "Frames"),
(seed, "Seed"),
(styles, "Styles"),
(steps, "Steps"),
(sampler_index, "Sampler"),
(engine, "Engine"),
(model, "Model"),
]
generation_parameters_copypaste.add_paste_fields("video", None, paste_fields, overrides)
generation_parameters_copypaste.add_paste_fields("video", None, paste_fields)
bindings = generation_parameters_copypaste.ParamBinding(paste_button=paste, tabname="video", source_text_component=prompt, source_image_component=None)
generation_parameters_copypaste.register_paste_params_button(bindings)
+17 -3
View File
@@ -1,3 +1,4 @@
import os
import copy
import time
from modules import shared, errors, sd_models, sd_checkpoint, model_quant, devices, sd_hijack_te, sd_hijack_vae
@@ -11,6 +12,8 @@ def load_model(selected: models_def.Model):
if selected is None or selected.te_cls is None or selected.dit_cls is None:
return ''
global loaded_model # pylint: disable=global-statement
if not shared.sd_loaded:
loaded_model = None
if loaded_model == selected.name:
return ''
sd_models.unload_model_weights()
@@ -20,7 +23,15 @@ def load_model(selected: models_def.Model):
video_cache.apply_teacache_patch(selected.dit_cls)
# overrides
kwargs = video_overrides.load_override(selected)
offline_args = {}
if shared.opts.offline_mode:
offline_args["local_files_only"] = True
os.environ['HF_HUB_OFFLINE'] = '1'
else:
os.environ.pop('HF_HUB_OFFLINE', None)
os.unsetenv('HF_HUB_OFFLINE')
kwargs = video_overrides.load_override(selected, **offline_args)
# text encoder
try:
@@ -51,7 +62,8 @@ def load_model(selected: models_def.Model):
revision=selected.te_revision or selected.repo_revision,
cache_dir=shared.opts.hfcache_dir,
**load_args,
**quant_args
**quant_args,
**offline_args,
)
except Exception as e:
shared.log.error(f'video load: module=te cls={selected.te_cls.__name__} {e}')
@@ -70,7 +82,8 @@ def load_model(selected: models_def.Model):
revision=selected.dit_revision or selected.repo_revision,
cache_dir=shared.opts.hfcache_dir,
**load_args,
**quant_args
**quant_args,
**offline_args,
)
else:
shared.log.debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" module="{dit_folder}" folder="{dit_folder}" cls={selected.dit_cls.__name__} skip')
@@ -95,6 +108,7 @@ def load_model(selected: models_def.Model):
cache_dir=shared.opts.hfcache_dir,
torch_dtype=devices.dtype,
**kwargs,
**offline_args,
)
except Exception as e:
shared.log.error(f'video load: module=pipe repo="{selected.repo}" cls={selected.repo_cls.__name__} {e}')
+9 -6
View File
@@ -8,25 +8,25 @@ from modules.video_models.models_def import Model
debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None else lambda *args, **kwargs: None
def load_override(selected: Model):
def load_override(selected: Model, **load_args):
kwargs = {}
# Allegro
if 'Allegro T2V' in selected.name:
kwargs['vae'] = diffusers.AutoencoderKLAllegro.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir)
kwargs['vae'] = diffusers.AutoencoderKLAllegro.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args)
# LTX
if 'LTXVideo 0.9.5 I2V' in selected.name:
kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir)
kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args)
# WAN
if 'WAN 2.1 14B' in selected.name:
kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir)
if 'A14B' in selected.name or '14B VACE' in selected.name:
kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir, **load_args)
if ('A14B' in selected.name) or ('14B VACE' in selected.name):
if shared.opts.model_wan_stage == 'combined':
kwargs['boundary_ratio'] = shared.opts.model_wan_boundary
elif shared.opts.model_wan_stage == 'high noise':
kwargs['transformer_2'] = None
kwargs['boundary_ratio'] = 0.0
elif shared.opts.model_wan_stage == 'low noise':
kwargs['boundary_ratio'] = 1.0
kwargs['boundary_ratio'] = 1000.0
kwargs['transformer'] = None
debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}')
return kwargs
@@ -61,3 +61,6 @@ def set_overrides(p: processing.StableDiffusionProcessingVideo, selected: Model)
if 'WanVACEPipeline' in cls:
if (getattr(p, 'init_images', None) is not None) and (len(p.init_images) > 0):
p.task_args['reference_images'] = p.init_images
# WAN 2.2-5B
if 'WAN 2.2 5B' in selected.name:
shared.sd_model.vae.disable_tiling()
+4 -1
View File
@@ -28,6 +28,8 @@ def generate(*args, **kwargs):
p = processing.StableDiffusionProcessingVideo(
sd_model=shared.sd_model,
video_engine=engine,
video_model=model,
prompt=prompt,
negative_prompt=negative,
styles=styles,
@@ -103,7 +105,7 @@ def generate(*args, **kwargs):
if hasattr(shared.sd_model.scheduler.config, 'use_dynamic_shifting'):
shared.sd_model.scheduler.config.use_dynamic_shifting = dynamic_shift
shared.sd_model.scheduler.register_to_config(use_dynamic_shifting = dynamic_shift)
if hasattr(shared.sd_model.scheduler.config, 'flow_shift'):
if hasattr(shared.sd_model.scheduler.config, 'flow_shift') and sampler_shift >= 0:
shared.sd_model.scheduler.config.flow_shift = sampler_shift
shared.sd_model.scheduler.register_to_config(flow_shift = sampler_shift)
shared.sd_model.default_scheduler = copy.deepcopy(shared.sd_model.scheduler)
@@ -138,6 +140,7 @@ def generate(*args, **kwargs):
# video_file = images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=video_duration, loop=video_loop, pad=video_pad, interpolate=video_interpolate) # legacy video save from list of images
pixels = video_save.images_to_tensor(processed.images)
_num_frames, video_file = video_save.save_video(
p=p,
pixels=pixels,
mp4_fps=mp4_fps,
mp4_codec=mp4_codec,
+45 -7
View File
@@ -1,18 +1,52 @@
import os
import time
import datetime
import cv2
import numpy as np
import torch
import einops
from modules import shared, errors ,timer, rife
from modules import shared, errors ,timer, rife, processing
from modules.video_models.video_utils import check_av
def get_video_filename(frames:int, codec:str):
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
output_filename = os.path.join(shared.opts.outdir_video, f'{timestamp}-{codec}-f{frames}')
return output_filename
def get_video_filename(p:processing.StableDiffusionProcessingVideo):
from modules.images_namegen import FilenameGenerator
namegen = FilenameGenerator(p, seed=p.seed if p is not None else 0, prompt=p.prompt if p is not None else '')
filename = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]")
if shared.opts.save_to_dirs:
dirname = namegen.apply(shared.opts.directories_filename_pattern or "[prompt_words]")
dirname = os.path.join(shared.opts.outdir_video, dirname, filename)
else:
dirname = shared.opts.outdir_video
if not os.path.exists(dirname):
os.makedirs(dirname, exist_ok=True)
filename = os.path.join(dirname, filename)
filename = namegen.sequence(filename)
filename = namegen.sanitize(filename)
return filename
def save_params(p, filename: str = None):
from modules.paths import params_path
if p is None:
dct = {}
else:
# sampler_index, sampler_shift, dynamic_shift, guidance_scale, guidance_true, init_image, init_strength, last_image, vae_type, vae_tile_frames, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, vlm_enhance, vlm_model, vlm_system_prompt, override_settings = args
dct = {
"Prompt": p.prompt,
"Negative prompt": p.negative_prompt,
"Steps": p.steps,
"Sampler": p.sampler_name,
"Seed": p.seed,
"Engine": p.video_engine,
"Model": p.video_model,
"Frames": p.frames,
"Size": f"{p.width}x{p.height}",
"Styles": ','.join(p.styles) if isinstance(p.styles, list) else p.styles,
}
params = ', '.join([f'{k}: {v}' for k, v in dct.items() if v is not None and v != ''])
fn = filename if filename is not None else params_path
with open(fn, "w", encoding="utf8") as file:
file.write(params)
def images_to_tensor(images):
@@ -71,6 +105,7 @@ def atomic_save_video(filename, tensor:torch.Tensor, fps:float=24, codec:str='li
def save_video(
p:processing.StableDiffusionProcessingVideo,
pixels:torch.Tensor,
mp4_fps:int=24,
mp4_codec:str='libx264',
@@ -111,7 +146,10 @@ def save_video(
x = einops.rearrange(x, '(m n) c t h w -> t (m h) (n w) c', n=n)
x = x.contiguous()
output_filename = get_video_filename(t, mp4_codec)
output_filename = get_video_filename(p)
if shared.opts.save_txt:
save_params(p, f'{output_filename}.txt')
save_params(p)
if mp4_sf:
fn = f'{output_filename}.safetensors'
+2 -1
View File
@@ -127,7 +127,7 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la
generate = gr.Button('Generate', elem_id="video_generate_btn", variant='primary', visible=False)
with gr.Row():
engine = gr.Dropdown(label='Engine', choices=list(models_def.models), value='None', elem_id="video_engine")
model = gr.Dropdown(label='Model', choices=[''], value=None, elem_id="video_model")
model = gr.Dropdown(label='Model', choices=[''], value='None', elem_id="video_model")
btn_load = ToolButton(ui_symbols.loading, elem_id="video_model_load")
with gr.Row():
url = gr.HTML(label='Model URL', elem_id='video_model_url', value='<br><br>')
@@ -197,3 +197,4 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la
show_progress=False,
)
generate.click(**video_dict)
return [engine, model, steps, sampler_index]
+1 -1
View File
@@ -74,7 +74,7 @@ def load_wan(checkpoint_info, diffusers_load_config={}):
elif shared.opts.model_wan_stage == 'low noise' or shared.opts.model_wan_stage == 'second':
transformer = None
transformer_2 = load_transformer(repo_id, diffusers_load_config, 'transformer_2')
boundary_ratio = 1.0
boundary_ratio = 1000.0
elif shared.opts.model_wan_stage == 'combined' or shared.opts.model_wan_stage == 'both':
transformer = load_transformer(repo_id, diffusers_load_config, 'transformer')
transformer_2 = load_transformer(repo_id, diffusers_load_config, 'transformer_2')