diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0864556..857177ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,6 @@ # Change Log for SD.Next -## Update for 2023-10-14 - -- Final strech of the DEV branch before merge to master: requires pending `diffusers==0.22.0` +## Update for 2023-10-16 This is a major release, with many changes and new functionality... @@ -151,9 +149,10 @@ or even free speedups and quality improvements (regardless of which workflows yo - updated installer to support detect `ROCm` *5.4/5.5/5.6/5.7* - support for `torch-rocm-5.7` - **xFormers**: - - default updated to *0.0.22* - - note that latest xformers are still not compatible with standard torch 2.1.0 with cuda 12.1 - either downgrade torch to 2.0.1 with cuda 11.8 or build xformers manually + - default updated to *0.0.23* + - note that latest xformers are still not compatible with cuda 12.1 + recommended to use torch 2.1.0 with cuda 11.8 + - added cmd param `--use-xformers` to force usage of exformers - **GC**: - custom garbage collect threshold to reduce vram memory usage, thanks @Disty0 see *settings -> compute -> gc* @@ -166,16 +165,17 @@ or even free speedups and quality improvements (regardless of which workflows yo - [Free-U](https://github.com/ChenyangSi/FreeU): new! available for *diffusers* and *original* backends improve generations quality at no cost (other than finding params that work for you) - thanks @ljleb + *note: temporarily disabled for diffusers pending release of diffusers==0.22* + thanks @ljleb - [Token Merging](https://github.com/dbolya/tomesd): not new, but updated available for *diffusers* and *original* backends speed-up your generations by merging redundant tokens speed up will depend on how aggressive you want to be with token merging - **Batch mode** new option *settings -> inference -> batch mode* - when using img2img process batch, process multiple images in batch in parallel + when using img2img process batch, optionally process multiple images in batch in parallel thanks @Symbiomatrix -- **NSFW** +- **NSFW Detection/Censor** - install extension: [NudeNet](https://github.com/vladmandic/sd-extension-nudenet) body part detection, image metadata, advanced censoring, etc... works for *text*, *image* and *process* workflows diff --git a/installer.py b/installer.py index 79b282f52..90ae210b3 100644 --- a/installer.py +++ b/installer.py @@ -359,9 +359,12 @@ def check_torch(): xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if torch_command != '': pass - elif allow_cuda and (shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): - log.info('nVidia CUDA toolkit detected') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu121') + elif allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): + log.info('nVidia CUDA toolkit detected: nvidia-smi present') + if not args.use_xformers: + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu121') + else: + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu118') xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre xformers<0.0.24' if opts.get('cross_attention_optimization', '') == 'xFormers' else 'none') elif allow_rocm and (shutil.which('rocminfo') is not None or os.path.exists('/opt/rocm/bin/rocminfo') or os.path.exists('/dev/kfd')): log.info('AMD ROCm toolkit detected') @@ -488,8 +491,10 @@ def check_torch(): import torch import xformers if torch.__version__ != '2.0.1+cu118' and xformers.__version__ in ['0.0.22', '0.0.21', '0.0.20']: - log.warning(f'Likely incompatible torch with: xformers=={xformers.__version__} installed: torch=={torch.__version__} required: torch==2.0.1+cu118 - build xformers manually or downgrade torch') - elif not args.experimental: + log.warning(f'Likely incompatible torch with: xformers=={xformers.__version__} installed: torch=={torch.__version__} required: torch==2.1.0+cu118 - build xformers manually or downgrade torch') + if 'cu118' not in torch.__version__: + log.warning(f'Likely incompatible Cuda with: xformers=={xformers.__version__} installed: torch=={torch.__version__} required: torch==2.1.0+cu118 - build xformers manually or downgrade torch') + elif not args.experimental and not args.use_xformers: x = pkg_resources.working_set.by_key.get('xformers', None) if x is not None: log.warning(f'Not used, uninstalling: {x}') @@ -646,11 +651,14 @@ def install_extensions(): if not args.skip_extensions: run_extension_installer(os.path.join(folder, ext)) pkg_resources._initialize_master_working_set() # pylint: disable=protected-access - updated = [f'{p.project_name}=={p._version}' for p in pkg_resources.working_set] # pylint: disable=protected-access,not-an-iterable - diff = [x for x in updated if x not in pkgs] - pkgs = updated - if len(diff) > 0: - log.info(f'Extension installed packages: {ext} {diff}') + try: + updated = [f'{p.project_name}=={p._version}' for p in pkg_resources.working_set] # pylint: disable=protected-access,not-an-iterable + diff = [x for x in updated if x not in pkgs] + pkgs = updated + if len(diff) > 0: + log.info(f'Extension installed packages: {ext} {diff}') + except Exception as e: + log.error(f'Extension installed unknown package: {e}') log.info(f'Extensions enabled: {extensions_enabled}') if len(extensions_duplicates) > 0: log.warning(f'Extensions duplicates: {extensions_duplicates}') @@ -913,6 +921,7 @@ def add_args(parser): group.add_argument("--use-ipex", default = os.environ.get("SD_USEIPEX",False), action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") group.add_argument("--use-cuda", default = os.environ.get("SD_USECUDA",False), action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") group.add_argument("--use-rocm", default = os.environ.get("SD_USEROCM",False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s") + group.add_argument("--use-xformers", default = os.environ.get("SD_USEXFORMERS",False), action='store_true', help="Force use xFormers cross-optimization, default: %(default)s") group.add_argument('--skip-requirements', default = os.environ.get("SD_SKIPREQUIREMENTS",False), action='store_true', help = "Skips checking and installing requirements, default: %(default)s") group.add_argument('--skip-extensions', default = os.environ.get("SD_SKIPEXTENSION",False), action='store_true', help = "Skips running individual extension installers, default: %(default)s") group.add_argument('--skip-git', default = os.environ.get("SD_SKIPGIT",False), action='store_true', help = "Skips running all GIT operations, default: %(default)s") diff --git a/modules/interrogate.py b/modules/interrogate.py index d95209792..e2c6f9577 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -86,7 +86,7 @@ class InterrogateModels: def load_blip_model(self): self.create_fake_fairscale() - import models.blip + import models.blip # pylint: disable=no-name-in-module files = modelloader.load_models( model_path=os.path.join(paths.models_path, "BLIP"), @@ -95,7 +95,7 @@ class InterrogateModels: download_name='model_base_caption_capfilt_large.pth', ) - blip_model = models.blip.blip_decoder(pretrained=files[0], image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) + blip_model = models.blip.blip_decoder(pretrained=files[0], image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) # pylint: disable=c-extension-no-member blip_model.eval() return blip_model diff --git a/modules/ldsr/sd_hijack_ddpm_v1.py b/modules/ldsr/sd_hijack_ddpm_v1.py index fe86425fd..2cf506990 100644 --- a/modules/ldsr/sd_hijack_ddpm_v1.py +++ b/modules/ldsr/sd_hijack_ddpm_v1.py @@ -1412,7 +1412,6 @@ class DiffusionWrapperV1(pl.LightningModule): class Layout2ImgDiffusionV1(LatentDiffusionV1): - # TODO: move all layout-specific hacks to this class def __init__(self, cond_stage_key, *args, **kwargs): assert cond_stage_key == 'coordinates_bbox', 'Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"' super().__init__(*args, cond_stage_key=cond_stage_key, **kwargs) diff --git a/modules/modelloader.py b/modules/modelloader.py index 2a4a88ccf..f5a4e1f35 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -228,7 +228,8 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config shared.log.error(f"Diffusers no pipeline folder: {hub_id}") return None try: - model_info_dict = hf.model_info(hub_id).cardData if pipeline_dir is not None else None # pylint: disable=no-member # TODO Diffusers is this real error? + # TODO diffusers is this real error? + model_info_dict = hf.model_info(hub_id).cardData if pipeline_dir is not None else None # pylint: disable=no-member except Exception: model_info_dict = None if model_info_dict is not None and "prior" in model_info_dict: # some checkpoints need to be downloaded as "hidden" as they just serve as pre- or post-pipelines of other pipelines diff --git a/modules/processing.py b/modules/processing.py index e1cd956eb..6705db2a1 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -1223,7 +1223,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): image = images.resize_image(3, image, self.width, self.height) if shared.backend == shared.Backend.DIFFUSERS: unprocessed.append(image) - self.init_images = [image] # TODO assign early for diffusers + self.init_images = [image] # assign early for diffusers if image_mask is not None: if self.inpainting_fill != 1: image = modules.masking.fill(image, latent_mask) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 40dfac131..1bcb959e9 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -201,7 +201,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro return task_args def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): - + if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} @@ -323,8 +323,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op - # p.extra_generation_params['Sampler options'] = '' # TODO sampler_options + sd_samplers.create_sampler(sampler.name, shared.sd_model) + # TODO extra_generation_params add sampler options + # p.extra_generation_params['Sampler options'] = '' p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__ @@ -406,7 +407,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - sd_samplers.create_sampler(sampler.name, shared.sd_model) # TODO(Patrick): For wrapped pipelines this is currently a no-op + sd_samplers.create_sampler(sampler.name, shared.sd_model) hires_args = set_pipeline_args( model=shared.sd_model, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, @@ -442,7 +443,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro sampler = sd_samplers.all_samplers_map.get(p.latent_sampler, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") - sd_samplers.create_sampler(sampler.name, shared.sd_refiner) # TODO(Patrick): For wrapped pipelines this is currently a no-op + sd_samplers.create_sampler(sampler.name, shared.sd_refiner) if shared.state.interrupted or shared.state.skipped: return results diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index 6be3c5aac..09e763408 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -122,7 +122,7 @@ def get_learned_conditioning_prompt_schedules(prompts, steps): before, after, _, when = args yield before or () if step <= when else after def alternate(self, args): - yield next(args[(step - 1)%len(args)]) + yield next(args[(step - 1)%len(args)]) # pylint: disable=stop-iteration-return def start(self, args): def flatten(x): if type(x) == str: diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index b4b9cd6a2..0bd73d911 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -73,7 +73,7 @@ def split_attention(layer: nn.Module, tile_size: int=256, min_tile_size: int=256 out = forward(x, *args[1:], **kwargs) return out if x.ndim == 4: # VAE - # TODO: VAE breaks for diffusers when using non-standard sizes + # TODO hyperlink vae breaks for diffusers when using non-standard sizes if nh * nw > 1: x = rearrange(x, "b c (nh h) (nw w) -> (b nh nw) c h w", nh=nh, nw=nw) out = forward(x, *args[1:], **kwargs) diff --git a/modules/sd_models.py b/modules/sd_models.py index 78c527514..8d6776394 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -639,7 +639,7 @@ def detect_pipeline(f: str, op: str = 'model'): shared.log.info(f'Diffusers: {op}="{guess}" class={pipeline.__name__} file="{f}" size={size}MB') except Exception as e: shared.log.error(f'Error loading diffusers pipeline: model={f} {e}') - + if pipeline is None: shared.log.warning(f'Autodetect: pipeline not recognized: {guess}: {op}={f} size={size}') pipeline = diffusers.StableDiffusionPipeline @@ -774,7 +774,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No "requires_safety_checker": False, "load_safety_checker": False, "load_connected_pipeline": True, - # "use_safetensors": True, # TODO(PVP) - we can't enable this for all checkpoints just yet + # todo use_safetensors cant enable for all checkpoints just yet } if shared.opts.diffusers_model_load_variant == 'default': if devices.dtype == torch.float16: diff --git a/modules/sd_samplers_cfg_denoiser.py b/modules/sd_samplers_cfg_denoiser.py index 56c16aafe..c80a80e50 100644 --- a/modules/sd_samplers_cfg_denoiser.py +++ b/modules/sd_samplers_cfg_denoiser.py @@ -1,4 +1,4 @@ -# TODO: implementation missing +# TODO cfg_denoiser implementation missing import torch from modules import prompt_parser, devices, sd_samplers_common @@ -94,7 +94,8 @@ class CFGDenoiser(torch.nn.Module): if state.interrupted or state.skipped: raise sd_samplers_common.InterruptedException - # if sd_samplers_common.apply_refiner(self): # TODO implementation missing + # TODO cfg_scale implementation missing + # if sd_samplers_common.apply_refiner(self): # cond = self.sampler.sampler_extra_args['cond'] # uncond = self.sampler.sampler_extra_args['uncond'] diff --git a/modules/shared.py b/modules/shared.py index 3b79efb20..ad4db47e9 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -643,8 +643,10 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad']}), - "pad_cond_uncond": OptionInfo(True, "Pad prompt and negative prompt to be same length", gr.Checkbox, {"visible": False}), # TODO implementation missing - "batch_cond_uncond": OptionInfo(True, "Do conditional and unconditional denoising in one batch", gr.Checkbox, {"visible": False}), # TODO implementation missing + # TODO pad_cond_uncond implementation missing + "pad_cond_uncond": OptionInfo(True, "Pad prompt and negative prompt to be same length", gr.Checkbox, {"visible": False}), + # TODO batch_cond-uncond implementation missing + "batch_cond_uncond": OptionInfo(True, "Do conditional and unconditional denoising in one batch", gr.Checkbox, {"visible": False}), })) options_templates.update(options_section(('postprocessing', "Postprocessing"), { @@ -900,9 +902,11 @@ if cmd_opts.backend is None: else: backend = Backend.DIFFUSERS if cmd_opts.use_openvino or cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL opts.data['sd_backend'] = 'diffusers' if backend == Backend.DIFFUSERS else 'original' +if cmd_opts.use_xformers: + opts.data['cross_attention_optimization'] = 'xFormers' opts.data['uni_pc_lower_order_final'] = opts.schedulers_use_loworder # compatibility opts.data['uni_pc_order'] = opts.schedulers_solver_order # compatibility -log.info(f'Engine: backend={backend} compute={devices.backend} mode={devices.inference_context.__name__} device={devices.get_optimal_device_name()}') +log.info(f'Engine: backend={backend} compute={devices.backend} mode={devices.inference_context.__name__} device={devices.get_optimal_device_name()} cross-optimization="{opts.cross_attention_optimization}"') log.info(f'Device: {print_dict(devices.get_gpu_info())}') prompt_styles = modules.styles.StyleDatabase(opts) diff --git a/modules/ui.py b/modules/ui.py index bf74f6963..5a7573be8 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -327,7 +327,7 @@ def create_sampler_and_steps_selection(choices, tabname): sampler_options = gr.CheckboxGroup(label='Sampler options', choices=choices, value=values, type='value') with FormRow(elem_classes=['flex-break']): opts.data['schedulers_sigma'] = opts.data.get('schedulers_sigma', 'default') - sampler_algo = gr.Radio(label='Sigma algorithm', choices=['default', 'karras', 'exponential', 'polyexponential', 'vp'], value=opts.data['schedulers_sigma'], type='value') + sampler_algo = gr.Radio(label='Sigma algorithm', choices=['default', 'karras', 'exponential', 'polyexponential'], value=opts.data['schedulers_sigma'], type='value') sampler_options.change(fn=set_sampler_original_options, inputs=[sampler_options, sampler_algo], outputs=[]) sampler_algo.change(fn=set_sampler_original_options, inputs=[sampler_options, sampler_algo], outputs=[]) else: @@ -850,7 +850,8 @@ def create_ui(startup_timer = None): (diffusers_guidance_rescale, "CFG rescale"), (tiling, "Tiling"), (mask_blur, "Mask blur"), - (scale_by, "UNKNOWN"), # TODO scale_by + # TODO scale_by add to paste fields + (scale_by, "UNKNOWN"), # from txt2img (hr_force, "Hires force"), (hr_scale, "Hires upscale"),