diff --git a/CHANGELOG.md b/CHANGELOG.md index acbe327ac..e98307f2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log for SD.Next +## Update for 07/11/2023: + +- fixes in extra-networks, diffusers samplers + ## Update for 07/10/2023 Service release with some fixes and enhancements: diff --git a/TODO.md b/TODO.md index bb0ff288c..86dd63e14 100644 --- a/TODO.md +++ b/TODO.md @@ -64,3 +64,4 @@ Tech that can be integrated as part of the core workflow... - [sd-xl lora](https://civitai.com/models/104913/fcstyledxl) - sd-xl img2img with configurable steps - change backend pipeline on-the-fly +- rename repo diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 75ba093d4..e4767fac5 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 75ba093d46b37f2d01fa4207680a4fa6417b01c3 +Subproject commit e4767fac5850376ce3bc7bd76bc60d08172b10ca diff --git a/installer.py b/installer.py index 720cd067a..fcbfb5e29 100644 --- a/installer.py +++ b/installer.py @@ -801,4 +801,7 @@ def read_options(): global opts # pylint: disable=global-statement if os.path.isfile(args.config): with open(args.config, "r", encoding="utf8") as file: - opts = json.load(file) + try: + opts = json.load(file) + except Exception as e: + log.error(f'Error reading options file: {file} {e}') diff --git a/modules/processing.py b/modules/processing.py index b0ed5969b..378b569bf 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -18,7 +18,7 @@ from installer import git_commit import modules.sd_hijack from modules import devices, prompt_parser, masking, sd_samplers, lowvram, generation_parameters_copypaste, script_callbacks, extra_networks, sd_vae_approx, scripts, sd_samplers_common # pylint: disable=unused-import from modules.sd_hijack import model_hijack -from modules.shared import opts, cmd_opts, state, log, backend, Backend +from modules.shared import opts, cmd_opts, state, log, Backend import modules.shared as shared import modules.paths as paths import modules.face_restoration @@ -149,7 +149,6 @@ class StableDiffusionProcessing: self.is_hr_pass = False opts.data['clip_skip'] = clip_skip - @property def sd_model(self): return shared.sd_model @@ -223,7 +222,7 @@ class StableDiffusionProcessing: source_image = devices.cond_cast_float(source_image) # HACK: Using introspection as the Depth2Image model doesn't appear to uniquely # identify itself with a field common to all models. The conditioning_key is also hybrid. - if backend == Backend.DIFFUSERS: + if shared.backend == Backend.DIFFUSERS: log.warning('Diffusers not implemented: img2img_image_conditioning') if isinstance(self.sd_model, LatentDepth2ImageDiffusion): return self.depth2img_image_conditioning(source_image) @@ -562,7 +561,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: seed = get_fixed_seed(p.seed) subseed = get_fixed_seed(p.subseed) - if backend == Backend.ORIGINAL: + if shared.backend == Backend.ORIGINAL: modules.sd_hijack.model_hijack.apply_circular(p.tiling) modules.sd_hijack.model_hijack.clear_comments() comments = {} @@ -613,11 +612,11 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: cache[0] = (required_prompts, steps) return cache[1] - ema_scope_context = p.sd_model.ema_scope if backend == Backend.ORIGINAL else nullcontext + ema_scope_context = p.sd_model.ema_scope if shared.backend == Backend.ORIGINAL else nullcontext with torch.no_grad(), ema_scope_context(): with devices.autocast(): p.init(p.all_prompts, p.all_seeds, p.all_subseeds) - if shared.opts.live_previews_enable and opts.show_progress_type == "Approximate NN" and backend == Backend.ORIGINAL: + if shared.opts.live_previews_enable and opts.show_progress_type == "Approximate NN" and shared.backend == Backend.ORIGINAL: sd_vae_approx.model() if state.job_count == -1: state.job_count = p.n_iter @@ -655,7 +654,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if p.n_iter > 1: shared.state.job = f"Batch {n+1} out of {p.n_iter}" - if backend == Backend.ORIGINAL: + if shared.backend == Backend.ORIGINAL: uc = get_conds_with_caching(prompt_parser.get_learned_conditioning, negative_prompts, p.steps * step_multiplier, cached_uc) c = get_conds_with_caching(prompt_parser.get_multicond_learned_conditioning, prompts, p.steps * step_multiplier, cached_c) if len(model_hijack.comments) > 0: @@ -682,7 +681,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: x_samples_ddim = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0) del samples_ddim - elif backend == Backend.DIFFUSERS: + elif shared.backend == Backend.DIFFUSERS: generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] if (not hasattr(shared.sd_model.scheduler, 'name')) or (shared.sd_model.scheduler.name != p.sampler_name): @@ -749,7 +748,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: unload_diffusers_lora() else: - raise ValueError(f"Unknown backend {backend}") + raise ValueError(f"Unknown backend {shared.backend}") if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: lowvram.send_everything_to_cpu() @@ -759,7 +758,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for i, x_sample in enumerate(x_samples_ddim): p.batch_index = i - if backend == Backend.ORIGINAL: + if shared.backend == Backend.ORIGINAL: x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) x_sample = x_sample.astype(np.uint8) else: @@ -874,7 +873,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.applied_old_hires_behavior_to = None def init(self, all_prompts, all_seeds, all_subseeds): - if backend == Backend.DIFFUSERS: + if shared.backend == Backend.DIFFUSERS: sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) self.width = self.width or 512 @@ -947,7 +946,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.restore_faces = orig2 images.save_image(image, self.outpath_samples, "", seeds[index], prompts[index], opts.samples_format, info=info, suffix="-before-highres-fix") - if backend == Backend.DIFFUSERS: + if shared.backend == Backend.DIFFUSERS: sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) @@ -1041,9 +1040,9 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): def init(self, all_prompts, all_seeds, all_subseeds): image_mask = self.image_mask - if backend == Backend.DIFFUSERS and image_mask is None: + if shared.backend == Backend.DIFFUSERS and image_mask is None: sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - elif backend == Backend.DIFFUSERS and image_mask is not None: + elif shared.backend == Backend.DIFFUSERS and image_mask is not None: sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING) self.sd_model.dtype = self.sd_model.unet.dtype @@ -1117,7 +1116,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): image = 2. * image - 1. image = image.to(shared.device) - if backend == Backend.ORIGINAL: + if shared.backend == Backend.ORIGINAL: self.init_latent = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(image)) else: # TODO Diffusers don't pre-encode the latents for diffusers to allow the UI to stay general for different model types @@ -1142,7 +1141,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.image_conditioning = self.img2img_image_conditioning(image, self.init_latent, image_mask) def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): - if backend == Backend.DIFFUSERS: + if shared.backend == Backend.DIFFUSERS: if self.init_mask is None: # pylint: disable=no-member sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) else: diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index be7e90cc5..f03a3d8f2 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -181,7 +181,7 @@ class StableDiffusionModelHijack: shared.log.warning("Model compile skipped: IPEX Method is for Intel GPU's with OneAPI") elif opts.cuda_compile and opts.cuda_compile_mode != 'none' and shared.backend == shared.Backend.ORIGINAL: try: - import torch._dynamo # pylint: disable=unused-import + import torch._dynamo # pylint: disable=unused-import,redefined-outer-name log_level = logging.WARNING if opts.cuda_compile_verbose else logging.CRITICAL # pylint: disable=protected-access torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access torch._dynamo.config.verbose = opts.cuda_compile_verbose # pylint: disable=protected-access @@ -210,6 +210,8 @@ class StableDiffusionModelHijack: self.layers = flatten(m) def undo_hijack(self, m): + if not hasattr(m, 'cond_stage_model'): + return # not ldm model if type(m.cond_stage_model) == xlmr.BertSeriesModelWithTransformation: m.cond_stage_model = m.cond_stage_model.wrapped diff --git a/modules/sd_models.py b/modules/sd_models.py index 075d3b714..9543db97a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -196,6 +196,9 @@ def get_closet_checkpoint_match(search_string): if checkpoint_info is not None: return checkpoint_info found = sorted([info for info in checkpoints_list.values() if search_string in info.title], key=lambda x: len(x.title)) + if found: + return found[0] + found = sorted([info for info in checkpoints_list.values() if search_string.split(' ')[0] in info.title], key=lambda x: len(x.title)) if found: return found[0] return None @@ -217,12 +220,12 @@ def model_hash(filename): def select_checkpoint(op='model'): - if op == 'model': - model_checkpoint = shared.opts.sd_model_checkpoint - elif op == 'dict': + if op == 'dict': model_checkpoint = shared.opts.sd_model_dict elif op == 'refiner': model_checkpoint = shared.opts.data.get('sd_model_refiner', None) + else: + model_checkpoint = shared.opts.sd_model_checkpoint if model_checkpoint is None or model_checkpoint == 'None': return None checkpoint_info = get_closet_checkpoint_match(model_checkpoint) @@ -401,7 +404,8 @@ def load_model_weights(model: torch.nn.Module, checkpoint_info: CheckpointInfo, model.first_stage_model = vae if depth_model: model.depth_model = depth_model - devices.dtype_unet = model.model.diffusion_model.dtype + # devices.dtype_unet = model.model.diffusion_model.dtype + model.model.diffusion_model.to(devices.dtype_unet) model.first_stage_model.to(devices.dtype_vae) # clean up cache if limit is reached while len(checkpoints_loaded) > shared.opts.sd_checkpoint_cache: @@ -497,7 +501,6 @@ class ModelData: return self.sd_model def set_sd_model(self, v): - shared.log.debug(f"Class model: {v}") self.sd_model = v def get_sd_refiner(self): @@ -565,6 +568,15 @@ class PriorPipeline: return result +def change_backend(): + shared.log.info(f'Pipeline changed: {shared.backend}') + unload_model_weights() + checkpoints_loaded.clear() + list_models() + from modules.sd_samplers import list_samplers + list_samplers(shared.backend) + + def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument import torch # pylint: disable=reimported,redefined-outer-name if timer is None: @@ -768,6 +780,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No sd_model.sd_checkpoint_info = checkpoint_info # pylint: disable=attribute-defined-outside-init sd_model.sd_model_checkpoint = checkpoint_info.filename # pylint: disable=attribute-defined-outside-init sd_model.sd_model_hash = checkpoint_info.hash # pylint: disable=attribute-defined-outside-init + sd_model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {elapsed} {remaining}', ncols=80, colour='#327fba') if op == 'refiner' and shared.opts.diffusers_move_refiner: shared.log.debug('Moving refiner model to CPU') sd_model.to("cpu") diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 140286064..133720685 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -1,16 +1,32 @@ from modules import sd_samplers_compvis, sd_samplers_kdiffusion, sd_samplers_diffusers, shared from modules.sd_samplers_common import samples_to_image_grid, sample_to_image # pylint: disable=unused-import -from modules.shared import backend, Backend -if backend == Backend.ORIGINAL: - all_samplers = [*sd_samplers_kdiffusion.samplers_data_k_diffusion, *sd_samplers_compvis.samplers_data_compvis] -else: - all_samplers = [*sd_samplers_diffusers.samplers_data_diffusers] -all_samplers_map = {x.name: x for x in all_samplers} + +all_samplers = [] +all_samplers = [] +all_samplers_map = {} samplers = all_samplers samplers_for_img2img = all_samplers samplers_map = {} +def list_samplers(backend_name = shared.backend): + global all_samplers # pylint: disable=global-statement + global all_samplers_map # pylint: disable=global-statement + global samplers # pylint: disable=global-statement + global samplers_for_img2img # pylint: disable=global-statement + global samplers_map # pylint: disable=global-statement + if backend_name == shared.Backend.ORIGINAL: + all_samplers = [*sd_samplers_kdiffusion.samplers_data_k_diffusion, *sd_samplers_compvis.samplers_data_compvis] + else: + all_samplers = [*sd_samplers_diffusers.samplers_data_diffusers] + all_samplers_map = {x.name: x for x in all_samplers} + samplers = all_samplers + samplers_for_img2img = all_samplers + samplers_map = {} + shared.log.debug(f'Enumerated samplers: {len(all_samplers)}') + +list_samplers() + def find_sampler_config(name): if name is not None and name != 'None': @@ -25,11 +41,11 @@ def create_sampler(name, model): if config is None: shared.log.error(f'Attempting to use unknown sampler: {name}') config = all_samplers[0] - if backend == Backend.ORIGINAL: + if shared.backend == shared.Backend.ORIGINAL: sampler = config.constructor(model) sampler.config = config return sampler - elif backend == Backend.DIFFUSERS: + elif shared.backend == shared.Backend.DIFFUSERS: sampler = config.constructor(model) model.scheduler = sampler.sampler return sampler.sampler diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 4d1e43f2c..a8ae55a9d 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -49,7 +49,10 @@ samplers_data_diffusers = [ class DiffusionSampler: def __init__(self, name, constructor, model, **kwargs): + self.config = {} self.config = config['All'].copy() + if not hasattr(model, 'scheduler'): + return for key, value in config.get(name, {}).items(): # diffusers defaults self.config[key] = value for key, value in model.scheduler.config.items(): # model defaults diff --git a/modules/shared.py b/modules/shared.py index 420c925cf..be62eea5d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -192,6 +192,7 @@ class State: state = State() state.server_start = time.time() + backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL log.info(f'Pipeline: {backend}') @@ -308,6 +309,7 @@ else: # cuda cross_attention_optimization_default ="Scaled-Dot-Product" options_templates.update(options_section(('sd', "Stable Diffusion"), { + "sd_backend": OptionInfo("original", "Stable Diffusion backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }), "sd_checkpoint_autoload": OptionInfo(True, "Stable Diffusion checkpoint autoload on server start"), "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_model_refiner": OptionInfo('None', "Stable Diffusion refiner", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), @@ -321,7 +323,6 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), { "prompt_mean_norm": OptionInfo(True, "Prompt attention mean normalization"), "comma_padding_backtrack": OptionInfo(20, "Prompt padding for long prompts", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }), "sd_disable_ckpt": OptionInfo(False, "Disallow usage of checkpoints in ckpt format"), - "sd_backend": OptionInfo("original", "Stable Diffusion backend (experimental)", gr.Radio, lambda: {"choices": ["original", "diffusers"] }), })) options_templates.update(options_section(('optimizations', "Optimizations"), { @@ -490,6 +491,10 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), "show_samplers": OptionInfo(["Euler a", "UniPC", "DEIS", "DDIM", "DPM 1S", "DPM 2M", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "force_latent_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), + 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), + 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), + 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final"), })) if backend == Backend.ORIGINAL: @@ -505,10 +510,6 @@ if backend == Backend.ORIGINAL: 's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}), 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma"), - 'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}), - 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}), - 'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), - 'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final"), })) elif backend == Backend.DIFFUSERS: options_templates.update(options_section(('sampler-params', "Sampler Settings"), { @@ -524,10 +525,6 @@ elif backend == Backend.DIFFUSERS: 's_noise': OptionInfo(1.0, "sigma noise", gr.Number, { "visible": False}), 'eta_noise_seed_delta': OptionInfo(0, "Noise seed delta (eta)", gr.Number, {"precision": 0}, { "visible": False}), 'always_discard_next_to_last_sigma': OptionInfo(False, "Always discard next-to-last sigma", gr.Checkbox, { "visible": False}), - #'uni_pc_variant': OptionInfo("bh1", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"]}, { "visible": False}), - #'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"]}, { "visible": False}), - #'uni_pc_order': OptionInfo(3, "UniPC order (must be < sampling steps)", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}, { "visible": False}), - #'uni_pc_lower_order_final': OptionInfo(True, "UniPC lower order final", { "visible": False}), # diffuser specific "schedulers_prediction_type": OptionInfo("default", "Samplers override model prediction type", gr.Radio, lambda: {"choices": ['default', 'epsilon', 'sample', 'v-prediction']}), "schedulers_beta_schedule": OptionInfo("default", "Samplers override beta schedule", gr.Radio, lambda: {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), @@ -754,6 +751,7 @@ opts = Options() config_filename = cmd_opts.config opts.load(config_filename) cmd_opts = cmd_args.compatibility_args(opts, cmd_opts) +opts.data['sd_backend'] = 'original' if backend == Backend.ORIGINAL else 'diffusers' prompt_styles = modules.styles.StyleDatabase(opts.styles_dir) cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure @@ -918,10 +916,7 @@ def get_version(): return version -class Shared(sys.modules[__name__].__class__): - # this class is here to provide sd_model field as a property, so that it can be created and loaded on demand rather than at program startup. - sd_model_val = None - +class Shared(sys.modules[__name__].__class__): # this class is here to provide sd_model field as a property, so that it can be created and loaded on demand rather than at program startup. @property def sd_model(self): import modules.sd_models # pylint: disable=W0621 @@ -942,6 +937,10 @@ class Shared(sys.modules[__name__].__class__): import modules.sd_models # pylint: disable=W0621 modules.sd_models.model_data.set_sd_refiner(value) + @property + def backend(self): + return Backend.ORIGINAL if opts.data['sd_backend'] == 'original' else Backend.DIFFUSERS + sd_model = None sd_refiner = None diff --git a/webui.py b/webui.py index 1b8da00c7..84b56ccc3 100644 --- a/webui.py +++ b/webui.py @@ -180,6 +180,7 @@ def load_model(): shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='model')), call=False) shared.opts.onchange("sd_model_refiner", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='refiner')), call=False) shared.opts.onchange("sd_model_dict", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='dict')), call=False) + shared.opts.onchange("sd_backend", wrap_queued_call(lambda: modules.sd_models.change_backend()), call=False) startup_timer.record("checkpoint")