diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 7abade093..5b00eec65 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -7,6 +7,12 @@ on: jobs: lint: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + flags: + - --debug --test --uv + - --debug --test steps: - name: checkout-code uses: actions/checkout@main @@ -27,5 +33,5 @@ jobs: msg: apply code formatting and linting auto-fixes - name: test-startup run: | - export COMMANDLINE_ARGS="--debug --test" + export COMMANDLINE_ARGS="${{ matrix.flags }}" python launch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bffa9703..2ded1b665 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,21 @@ # Change Log for SD.Next -## Update for 2024-06-28 +TODO: +- +- Requires `diffusers==0.30.0` +- Alpha Lumina +## Update for 2024-07-01 + +- support for **HunyuanDiT 1.2** +- add support for [uv](https://pypi.org/project/uv/), extremely fast installer, thanks @Yoinky3000! + to use, simply add `--uv` to your command line params - enable `florence` VLM for all platforms, thanks @lshqqytiger! - fix executing extensions with zero params - fix nncf for lora, thanks @Disty0! - fix diffusers version detection for SD3 - fix current step for higher order samplers +- fix control input type video - add SD3 with FP16 T5 to list of detected models - multiple ModernUI fixes diff --git a/README.md b/README.md index 6b7733b98..cdbd0fa7e 100644 --- a/README.md +++ b/README.md @@ -64,31 +64,31 @@ For screenshots and informations on other available themes, see [Themes Wiki](ht Additional models will be added as they become available and there is public interest in them -- [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* -- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models) -- [StabilityAI Stable Diffusion 3 Medium](https://stability.ai/news/stable-diffusion-3-medium) +- [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* +- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models) +- [StabilityAI Stable Diffusion 3 Medium](https://stability.ai/news/stable-diffusion-3-medium) - [StabilityAI Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) Base, XT 1.0, XT 1.1 -- [LCM: Latent Consistency Models](https://github.com/openai/consistency_models) -- [Playground](https://huggingface.co/playgroundai/playground-v2-256px-base) *v1, v2 256, v2 512, v2 1024 and latest v2.5* +- [LCM: Latent Consistency Models](https://github.com/openai/consistency_models) +- [Playground](https://huggingface.co/playgroundai/playground-v2-256px-base) *v1, v2 256, v2 512, v2 1024 and latest v2.5* - [Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite* - [aMUSEd 256](https://huggingface.co/amused/amused-256) 256 and 512 -- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) -- [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B) -- [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL* -- [Kandinsky](https://github.com/ai-forever/Kandinsky-2) *2.1 and 2.2 and latest 3.0* -- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large* -- [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma) -- [Warp Wuerstchen](https://huggingface.co/blog/wuertschen) +- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) +- [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B) +- [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL* +- [Kandinsky](https://github.com/ai-forever/Kandinsky-2) *2.1 and 2.2 and latest 3.0* +- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large* +- [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma) +- [Warp Wuerstchen](https://huggingface.co/blog/wuertschen) - [Tenecent HunyuanDiT](https://github.com/Tencent/HunyuanDiT) - [Tsinghua UniDiffusion](https://github.com/thu-ml/unidiffuser) - [DeepFloyd IF](https://github.com/deep-floyd/IF) *Medium and Large* - [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) - [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* -- [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/) +- [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/) - [KOALA 700M](https://github.com/youngwanLEE/sdxl-koala) -- [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) +- [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) - [SDXS](https://github.com/IDKiro/sdxs) -- [Hyper-SD](https://huggingface.co/ByteDance/Hyper-SD) +- [Hyper-SD](https://huggingface.co/ByteDance/Hyper-SD) Also supported are modifiers such as: @@ -226,6 +226,7 @@ List of available parameters, run `webui --help` for the full & up-to-date list: --version Print version information --ignore Ignore any errors and attempt to continue --safe Run in safe mode with no user extensions + --uv Use uv as installer, default: False Logging options: --log LOG Set log file, default: None diff --git a/TODO.md b/TODO.md index 0647494dc..fd704b838 100644 --- a/TODO.md +++ b/TODO.md @@ -11,7 +11,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - diffusers public callbacks - include reference styles - lora: sc lora, dora, etc -- sd3 controlnet: ## Experimental diff --git a/html/reference.json b/html/reference.json index a3f62e7e1..c8eaee3f0 100644 --- a/html/reference.json +++ b/html/reference.json @@ -182,13 +182,20 @@ "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" }, - "Tencent HunyuanDiT 1.1": { - "path": "Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers", + "Tencent HunyuanDiT 1.2": { + "path": "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers", "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.", "preview": "Tencent-Hunyuan-HunyuanDiT.jpg", "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" }, - + + "AlphaVLLM Lumina Next SFT": { + "path": "Alpha-VLLM/Lumina-Next-SFT-diffusers", + "desc": "The Lumina-Next-SFT is a Next-DiT model containing 2B parameters and utilizes Gemma-2B as the text encoder, enhanced through high-quality supervised fine-tuning (SFT).", + "preview": "Alpha-VLLM-Lumina-Next-SFT-diffusers.jpg", + "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" + }, + "Kandinsky 2.1": { "path": "kandinsky-community/kandinsky-2-1", "desc": "Kandinsky 2.1 is a text-conditional diffusion model based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", diff --git a/installer.py b/installer.py index a00a12624..707af8a7a 100644 --- a/installer.py +++ b/installer.py @@ -52,6 +52,7 @@ args = Dot({ 'reinstall': False, 'version': False, 'ignore': False, + 'uv': False, }) git_commit = "unknown" submodules_commit = { @@ -235,22 +236,25 @@ def uninstall(package, quiet = False): @lru_cache() -def pip(arg: str, ignore: bool = False, quiet: bool = False): +def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True): + uv = uv and args.uv + pipCmd = "uv pip" if uv else "pip" arg = arg.replace('>=', '==') if not quiet and '-r ' not in arg: - log.info(f'Install: package="{arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}"') + log.info(f'Install: package="{arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}" mode={"uv" if uv else "pip"}') env_args = os.environ.get("PIP_EXTRA_ARGS", "") - log.debug(f'Running: pip="{pip_log}{arg} {env_args}"') - result = subprocess.run(f'"{sys.executable}" -m pip {pip_log}{arg} {env_args}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + all_args = f'{pip_log}{arg} {env_args}'.strip() + log.debug(f'Running: {pipCmd}="{all_args}"') + result = subprocess.run(f'"{sys.executable}" -m {pipCmd} {all_args}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) txt = result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stderr) > 0: txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore") txt = txt.strip() - debug(f'Install pip: {txt}') + debug(f'Install {pipCmd}: {txt}') if result.returncode != 0 and not ignore: global errors # pylint: disable=global-statement errors += 1 - log.error(f'Error running pip: {arg}') + log.error(f'Error running {pipCmd}: {arg}') log.debug(f'Pip output: {txt}') return txt @@ -264,7 +268,7 @@ def install(package, friendly: str = None, ignore: bool = False, reinstall: bool quick_allowed = False if args.reinstall or reinstall or not installed(package, friendly, quiet=quiet): deps = '' if not no_deps else '--no-deps ' - res = pip(f"install --upgrade {deps}{package}", ignore=ignore) + res = pip(f"install{' --upgrade' if not args.uv else ''} {deps}{package}", ignore=ignore, uv=package != "uv") try: import imp # pylint: disable=deprecated-module imp.reload(pkg_resources) @@ -1223,6 +1227,7 @@ def add_args(parser): group.add_argument('--version', default = False, action='store_true', help = "Print version information") group.add_argument('--ignore', default = os.environ.get("SD_IGNORE",False), action='store_true', help = "Ignore any errors and attempt to continue") group.add_argument('--safe', default = os.environ.get("SD_SAFE",False), action='store_true', help = "Run in safe mode with no user extensions") + group.add_argument('--uv', default = os.environ.get("SD_UV",False), action='store_true', help = "Use uv instead of pip to install the packages") group = parser.add_argument_group('Logging options') group.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s") diff --git a/launch.py b/launch.py index f1d8b7ec5..1db8a81f6 100755 --- a/launch.py +++ b/launch.py @@ -204,6 +204,8 @@ def main(): installer.log.info(f'Platform: {installer.print_dict(installer.get_platform())}') if not args.skip_env: installer.set_environment() + if args.uv: + installer.install("uv", "uv") installer.check_torch() installer.check_onnx() installer.check_diffusers() diff --git a/models/Reference/Alpha-VLLM-Lumina-Next-SFT-diffusers.jpg b/models/Reference/Alpha-VLLM-Lumina-Next-SFT-diffusers.jpg new file mode 100644 index 000000000..e252bff5b Binary files /dev/null and b/models/Reference/Alpha-VLLM-Lumina-Next-SFT-diffusers.jpg differ diff --git a/modules/control/run.py b/modules/control/run.py index cb4c121ca..ba736a4a8 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -282,67 +282,72 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini else: pass - debug(f'Control: run type={unit_type} models={has_models}') - if has_models: - p.ops.append('control') - p.extra_generation_params["Control mode"] = unit_type # overriden later with pretty-print - p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning] - p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start] - p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end] - p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) - p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]]) - p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']]) - p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']]) - if unit_type == 't2i adapter' and has_models: - p.extra_generation_params["Control mode"] = 'T2I-Adapter' - p.task_args['adapter_conditioning_scale'] = control_conditioning - instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: T2I-Adapter does not support separate init image') - elif unit_type == 'controlnet' and has_models: - p.extra_generation_params["Control mode"] = 'ControlNet' - p.task_args['controlnet_conditioning_scale'] = control_conditioning - p.task_args['control_guidance_start'] = control_guidance_start - p.task_args['control_guidance_end'] = control_guidance_end - p.task_args['guess_mode'] = p.guess_mode - instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model) - pipe = instance.pipeline - elif unit_type == 'xs' and has_models: - p.extra_generation_params["Control mode"] = 'ControlNet-XS' - p.controlnet_conditioning_scale = control_conditioning - p.control_guidance_start = control_guidance_start - p.control_guidance_end = control_guidance_end - instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: ControlNet-XS does not support separate init image') - elif unit_type == 'lite' and has_models: - p.extra_generation_params["Control mode"] = 'ControlLLLite' - p.controlnet_conditioning_scale = control_conditioning - instance = lite.ControlLLitePipeline(shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: ControlLLLite does not support separate init image') - elif unit_type == 'reference' and has_models: - p.extra_generation_params["Control mode"] = 'Reference' - p.extra_generation_params["Control attention"] = p.attention - p.task_args['reference_attn'] = 'Attention' in p.attention - p.task_args['reference_adain'] = 'Adain' in p.attention - p.task_args['attention_auto_machine_weight'] = p.query_weight - p.task_args['gn_auto_machine_weight'] = p.adain_weight - p.task_args['style_fidelity'] = p.fidelity - instance = reference.ReferencePipeline(shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: ControlNet-XS does not support separate init image') - else: # run in txt2img/img2img mode - if len(active_strength) > 0: - p.strength = active_strength[0] - pipe = shared.sd_model - instance = None + def set_pipe(): + global pipe, instance # pylint: disable=global-statement + pipe = None + if has_models: + p.ops.append('control') + p.extra_generation_params["Control mode"] = unit_type # overriden later with pretty-print + p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning] + p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start] + p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end] + p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) + p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]]) + p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']]) + p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']]) + if unit_type == 't2i adapter' and has_models: + p.extra_generation_params["Control mode"] = 'T2I-Adapter' + p.task_args['adapter_conditioning_scale'] = control_conditioning + instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: T2I-Adapter does not support separate init image') + elif unit_type == 'controlnet' and has_models: + p.extra_generation_params["Control mode"] = 'ControlNet' + p.task_args['controlnet_conditioning_scale'] = control_conditioning + p.task_args['control_guidance_start'] = control_guidance_start + p.task_args['control_guidance_end'] = control_guidance_end + p.task_args['guess_mode'] = p.guess_mode + instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model) + pipe = instance.pipeline + elif unit_type == 'xs' and has_models: + p.extra_generation_params["Control mode"] = 'ControlNet-XS' + p.controlnet_conditioning_scale = control_conditioning + p.control_guidance_start = control_guidance_start + p.control_guidance_end = control_guidance_end + instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlNet-XS does not support separate init image') + elif unit_type == 'lite' and has_models: + p.extra_generation_params["Control mode"] = 'ControlLLLite' + p.controlnet_conditioning_scale = control_conditioning + instance = lite.ControlLLitePipeline(shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlLLLite does not support separate init image') + elif unit_type == 'reference' and has_models: + p.extra_generation_params["Control mode"] = 'Reference' + p.extra_generation_params["Control attention"] = p.attention + p.task_args['reference_attn'] = 'Attention' in p.attention + p.task_args['reference_adain'] = 'Adain' in p.attention + p.task_args['attention_auto_machine_weight'] = p.query_weight + p.task_args['gn_auto_machine_weight'] = p.adain_weight + p.task_args['style_fidelity'] = p.fidelity + instance = reference.ReferencePipeline(shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlNet-XS does not support separate init image') + else: # run in txt2img/img2img mode + if len(active_strength) > 0: + p.strength = active_strength[0] + pipe = shared.sd_model + instance = None + debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}') + return pipe + pipe = set_pipe() debug(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}') t1, t2, t3 = time.time(), 0, 0 status = True @@ -383,6 +388,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini codec = util.decode_fourcc(video.get(cv2.CAP_PROP_FOURCC)) status, frame = video.read() if status: + shared.state.frame_count = 1 + frames // (video_skip_frames + 1) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) shared.log.debug(f'Control: input video: path={inputs} frames={frames} fps={fps} size={w}x{h} codec={codec}') except Exception as e: @@ -390,6 +396,9 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini return [], '', '', 'Error: video open failed' while status: + if pipe is None: # pipe may have been reset externally + pipe = set_pipe() + debug(f'Control pipeline reinit: class={pipe.__class__.__name__}') processed_image = None if frame is not None: inputs = [Image.fromarray(frame)] # cv2 to pil @@ -426,9 +435,10 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini else: debug(f'Control Init image: {i % len(inits) + 1} of {len(inits)}') init_image = inits[i % len(inits)] - index += 1 if video is not None and index % (video_skip_frames + 1) != 0: + index += 1 continue + index += 1 # resize before if resize_mode_before != 0 and resize_name_before != 'None': @@ -593,10 +603,11 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini output = None script_run = False if pipe is not None: # run new pipeline - pipe.restore_pipeline = restore_pipeline + if not hasattr(pipe, 'restore_pipeline') and video is None: + pipe.restore_pipeline = restore_pipeline debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}') - debug(f'Control exec pipeline: p={vars(p)}') - debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}') + # debug(f'Control exec pipeline: p={vars(p)}') + # debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}') if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode sd_models.move_model(pipe.vae, devices.device) diff --git a/modules/control/units/xs_pipe.py b/modules/control/units/xs_pipe.py index 7e717b542..30cd8cef0 100644 --- a/modules/control/units/xs_pipe.py +++ b/modules/control/units/xs_pipe.py @@ -26,8 +26,7 @@ from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, StableDiffus from diffusers.models import AutoencoderKL, UNet2DConditionModel from diffusers.models.attention_processor import ( AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, + FusedAttnProcessor2_0, XFormersAttnProcessor, ) from diffusers.models.lora import adjust_lora_scale_text_encoder @@ -652,8 +651,7 @@ class StableDiffusionXLControlNetXSPipeline( ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, ), ) # if xformers or torch_2_0 is used attention block does not need diff --git a/modules/onnx_impl/__init__.py b/modules/onnx_impl/__init__.py index 7e23e72e6..8ca39defa 100644 --- a/modules/onnx_impl/__init__.py +++ b/modules/onnx_impl/__init__.py @@ -3,7 +3,6 @@ import numpy as np import torch import diffusers import onnxruntime as ort -import optimum.onnxruntime initialized = False @@ -201,6 +200,8 @@ def initialize_onnx(): return try: # may fail on onnx import import onnx # pylint: disable=unused-import + import optimum.onnxruntime + optimum.onnxruntime.modeling_diffusion._ORTDiffusionModelPart.to = ORTDiffusionModelPart_to # pylint: disable=protected-access from .execution_providers import ExecutionProvider, TORCH_DEVICE_TO_EP, available_execution_providers if devices.backend == "rocm": TORCH_DEVICE_TO_EP["cuda"] = ExecutionProvider.ROCm @@ -234,8 +235,6 @@ def initialize_onnx(): diffusers.ORTStableDiffusionXLPipeline = diffusers.OnnxStableDiffusionXLPipeline # Huggingface model compatibility diffusers.ORTStableDiffusionXLImg2ImgPipeline = diffusers.OnnxStableDiffusionXLImg2ImgPipeline - optimum.onnxruntime.modeling_diffusion._ORTDiffusionModelPart.to = ORTDiffusionModelPart_to # pylint: disable=protected-access - log.debug(f'ONNX: version={ort.__version__} provider={opts.onnx_execution_provider}, available={available_execution_providers}') except Exception as e: log.error(f'ONNX failed to initialize: {e}') diff --git a/modules/onnx_impl/pipelines/__init__.py b/modules/onnx_impl/pipelines/__init__.py index bb0d9b010..a257ab695 100644 --- a/modules/onnx_impl/pipelines/__init__.py +++ b/modules/onnx_impl/pipelines/__init__.py @@ -4,11 +4,8 @@ import shutil import tempfile from abc import ABCMeta from typing import Type, Tuple, List, Any, Dict -from packaging import version import torch import diffusers -import onnxruntime as ort -import optimum.onnxruntime from installer import log, install from modules import shared from modules.paths import sd_configs_path, models_path @@ -23,7 +20,6 @@ from modules.onnx_impl.execution_providers import ExecutionProvider, EP_TO_NAME, SUBMODELS_SD = ("text_encoder", "unet", "vae_encoder", "vae_decoder",) SUBMODELS_SDXL = ("text_encoder", "text_encoder_2", "unet", "vae_encoder", "vae_decoder",) SUBMODELS_SDXL_REFINER = ("text_encoder_2", "unet", "vae_encoder", "vae_decoder",) - SUBMODELS_LARGE = ("text_encoder_2", "unet",) @@ -37,6 +33,7 @@ class PipelineBase(TorchCompatibleModule, diffusers.DiffusionPipeline, metaclass self.model_type = self.__class__.__name__ def to(self, *args, **kwargs): + import optimum.onnxruntime if self.__class__ == OnnxRawPipeline: # cannot move pipeline which is not preprocessed. return self diff --git a/modules/pag/pipe_sdxl.py b/modules/pag/pipe_sdxl.py index 429384ea3..82ae06c07 100644 --- a/modules/pag/pipe_sdxl.py +++ b/modules/pag/pipe_sdxl.py @@ -5,7 +5,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import torch import torch.nn.functional as F -from packaging import version from transformers import ( CLIPImageProcessor, @@ -26,8 +25,6 @@ from diffusers.models import AutoencoderKL, ImageProjection, UNet2DConditionMode from diffusers.models.attention_processor import ( AttnProcessor2_0, FusedAttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, XFormersAttnProcessor, ) from diffusers.models.lora import adjust_lora_scale_text_encoder @@ -943,8 +940,6 @@ class StableDiffusionXLPAGPipeline( ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, FusedAttnProcessor2_0, ), ) diff --git a/modules/processing.py b/modules/processing.py index d453e0b15..34a34a0e3 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -388,6 +388,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: devices.torch_gc() if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: + print('HERE RESTORE') shared.sd_model.restore_pipeline() t1 = time.time() diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index b4be9940b..ef1eb9d69 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -104,7 +104,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): clip_skip=p.clip_skip, desc='Base', ) - shared.state.sampling_steps = base_args.get('prior_num_inference_steps', None) or base_args.get('num_inference_steps', None) or p.steps + shared.state.sampling_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None) if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1: p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta output = None @@ -215,7 +215,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): desc='Hires', ) shared.state.job = 'HiRes' - shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or hires_args.get('num_inference_steps', None) or p.steps + shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or p.steps or hires_args.get('num_inference_steps', None) try: sd_models_compile.check_deepcache(enable=True) output = shared.sd_model(**hires_args) # pylint: disable=not-callable @@ -280,7 +280,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): clip_skip=p.clip_skip, desc='Refiner', ) - shared.state.sampling_steps = refiner_args.get('prior_num_inference_steps', None) or refiner_args.get('num_inference_steps', None) or p.steps + shared.state.sampling_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None) try: if 'requires_aesthetics_score' in shared.sd_refiner.config: # sdxl-model needs false and sdxl-refiner needs true shared.sd_refiner.register_to_config(requires_aesthetics_score = getattr(shared.sd_refiner, 'tokenizer', None) is None) diff --git a/modules/progress.py b/modules/progress.py index f5573374b..aeef195b4 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -62,7 +62,7 @@ def progressapi(req: ProgressRequest): paused = shared.state.paused if not active: return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, textinfo="Queued..." if queued else "Waiting...") - shared.state.job_count = max(shared.state.job_count, shared.state.job_no) + shared.state.job_count = max(shared.state.frame_count, shared.state.job_count, shared.state.job_no) batch_x = max(shared.state.job_no, 0) batch_y = max(shared.state.job_count, 1) step_x = max(shared.state.sampling_step, 0) diff --git a/modules/shared_state.py b/modules/shared_state.py index 79ee20f19..82f8d21dc 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -12,6 +12,7 @@ class State: job = "" job_no = 0 job_count = 0 + frame_count = 0 total_jobs = 0 job_timestamp = '0' sampling_step = 0 @@ -71,6 +72,7 @@ class State: self.interrupted = False self.job = title self.job_count = -1 + self.frame_count = -1 self.job_no = 0 self.job_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") self.paused = False @@ -93,6 +95,7 @@ class State: self.job = "" self.job_count = 0 self.job_no = 0 + self.frame_count = 0 self.paused = False self.interrupted = False self.skipped = False diff --git a/modules/xadapter/pipeline_sd_xl_adapter.py b/modules/xadapter/pipeline_sd_xl_adapter.py index 65e04ecab..757681972 100644 --- a/modules/xadapter/pipeline_sd_xl_adapter.py +++ b/modules/xadapter/pipeline_sd_xl_adapter.py @@ -24,8 +24,7 @@ from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInver from diffusers.models import AutoencoderKL from diffusers.models.attention_processor import ( AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, + FusedAttnProcessor2_0, XFormersAttnProcessor, ) from diffusers.schedulers import KarrasDiffusionSchedulers @@ -558,8 +557,7 @@ class StableDiffusionXLAdapterPipeline(DiffusionPipeline, FromSingleFileMixin, L ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, ), ) # if xformers or torch_2_0 is used attention block does not need diff --git a/modules/xadapter/pipeline_sd_xl_adapter_controlnet.py b/modules/xadapter/pipeline_sd_xl_adapter_controlnet.py index ddb334568..f0f27fa69 100644 --- a/modules/xadapter/pipeline_sd_xl_adapter_controlnet.py +++ b/modules/xadapter/pipeline_sd_xl_adapter_controlnet.py @@ -30,8 +30,7 @@ from diffusers.models import AutoencoderKL, ControlNetModel from diffusers.models.attention_processor import ( AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, + FusedAttnProcessor2_0, XFormersAttnProcessor, ) from diffusers.schedulers import KarrasDiffusionSchedulers @@ -572,8 +571,7 @@ class StableDiffusionXLAdapterControlnetPipeline(DiffusionPipeline, FromSingleFi ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, ), ) # if xformers or torch_2_0 is used attention block does not need diff --git a/modules/xadapter/pipeline_sd_xl_adapter_controlnet_img2img.py b/modules/xadapter/pipeline_sd_xl_adapter_controlnet_img2img.py index d1cf59033..c2dadfbfc 100644 --- a/modules/xadapter/pipeline_sd_xl_adapter_controlnet_img2img.py +++ b/modules/xadapter/pipeline_sd_xl_adapter_controlnet_img2img.py @@ -31,8 +31,7 @@ from diffusers.models import AutoencoderKL, ControlNetModel from diffusers.models.attention_processor import ( AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, + FusedAttnProcessor2_0, XFormersAttnProcessor, ) from diffusers.schedulers import KarrasDiffusionSchedulers @@ -571,8 +570,7 @@ class StableDiffusionXLAdapterControlnetI2IPipeline(DiffusionPipeline, FromSingl ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, ), ) # if xformers or torch_2_0 is used attention block does not need diff --git a/scripts/demofusion.py b/scripts/demofusion.py index 95e58a74e..f7cdfe543 100644 --- a/scripts/demofusion.py +++ b/scripts/demofusion.py @@ -8,7 +8,7 @@ from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokeniz from diffusers.image_processor import VaeImageProcessor from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin from diffusers.models import AutoencoderKL, UNet2DConditionModel -from diffusers.models.attention_processor import AttnProcessor2_0, LoRAAttnProcessor2_0, LoRAXFormersAttnProcessor, XFormersAttnProcessor +from diffusers.models.attention_processor import AttnProcessor2_0, FusedAttnProcessor2_0, XFormersAttnProcessor from diffusers.models.lora import adjust_lora_scale_text_encoder from diffusers.schedulers import KarrasDiffusionSchedulers from diffusers.utils import is_accelerate_available, is_accelerate_version @@ -484,8 +484,7 @@ class DemoFusionSDXLPipeline(DiffusionPipeline, FromSingleFileMixin, LoraLoaderM ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, ), ) # if xformers or torch_2_0 is used attention block does not need diff --git a/scripts/differential_diffusion.py b/scripts/differential_diffusion.py index b48e0f6e6..36aa17eca 100644 --- a/scripts/differential_diffusion.py +++ b/scripts/differential_diffusion.py @@ -22,8 +22,7 @@ from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInver from diffusers.models import AutoencoderKL, UNet2DConditionModel from diffusers.models.attention_processor import ( AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, + FusedAttnProcessor2_0, XFormersAttnProcessor, ) from diffusers.configuration_utils import FrozenDict @@ -631,8 +630,7 @@ class StableDiffusionXLDiffImg2ImgPipeline(DiffusionPipeline, FromSingleFileMixi ( AttnProcessor2_0, XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, ), ) # if xformers or torch_2_0 is used attention block does not need diff --git a/wiki b/wiki index 8c44b3055..a9fd0bcb7 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 8c44b305543f8b612b3b0fbc935ac40997360588 +Subproject commit a9fd0bcb711184e5df032c99158945e3cbcee72e