diff --git a/CHANGELOG.md b/CHANGELOG.md index 5163f37f5..8fd7796ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,24 @@ # Change Log for SD.Next -## Update for 2023-11-26 +## Update for 2023-11-28 Note: Release pending `diffusers==0.24` - **Diffusers** - - **IP adapter** - - Lightweight implementation of T2I adapters which can guide generation towards specific image style - - Supports most T2I models, not limited to SD - **HDR latent control**, based on [article](https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space#long-prompts-at-high-guidance-scales-becoming-possible) - In *Advanced* params - Allows control of *latent clamping*, *color centering* and *range maximimization* - Supported by *XYZ grid* + - **IP adapter** + - Lightweight implementation of T2I adapters which can guide generation towards specific image style + - Supports most T2I models, not limited to SD 1.5 + - Models are auto-downloaded on first use + - For IP adapter support in Original backend, use standard *ControlNet* extension + - **AnimateDiff** + - Lightweight implementation of AnimateDiff basic models *(1.4, 1.5, 1.5.2)* + - Supports SD 1.5 only + - Models are auto-downloaded on first use + - For AnimateDiff support in Original backend, use standard *AnimateDiff* extension - **Kandinsky 3** support - download using built-in model downloader or simply select from networks -> reference - this model is absolutely massive at 27.5GB at fp16, so be patient diff --git a/html/locale_en.json b/html/locale_en.json index a475df59e..27d4beaa7 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -71,7 +71,7 @@ "extra networks": [ {"id":"","label":"UI position","localized":"","hint":"Location of extra networks"}, {"id":"","label":"cover","localized":"","hint":"cover full area"}, - {"id":"","label":"inline","localized":"","hint":"inline with all additional elelemtns (scrollable)"}, + {"id":"","label":"inline","localized":"","hint":"inline with all additional elements (scrollable)"}, {"id":"","label":"sidebar","localized":"","hint":"sidebar on the right side of the screen"}, {"id":"","label":"UI height (%)","localized":"","hint":""}, {"id":"","label":"UI sidebar width (%)","localized":"","hint":""}, diff --git a/javascript/sdnext.css b/javascript/sdnext.css index ed212bf44..d10878a62 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -95,7 +95,7 @@ div#extras_scale_to_tab div.form{ flex-direction: row; } #img2img_sketch, #img2maskimg, #inpaint_sketch { overflow: overlay !important; resize: auto; background: var(--panel-background-fill); z-index: 5; } .image-buttons button{ min-width: auto; } .infotext { overflow-wrap: break-word; line-height: 1.5em; } -.infotext > p { padding-left: 1em; text-indent: -1em; white-space: pre; } +.infotext > p { padding-left: 1em; text-indent: -1em; white-space: pre; text-wrap: pretty; } .tooltip { display: block; position: fixed; top: 1em; right: 1em; padding: 0.5em; background: var(--input-background-fill); color: var(--body-text-color); border: 1pt solid var(--button-primary-border-color); width: 22em; min-height: 1.3em; font-size: 0.8em; transition: opacity 0.2s ease-in; pointer-events: none; opacity: 0; z-index: 999; } .tooltip-show { opacity: 0.9; } @@ -318,7 +318,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt /* maintain single column for from image operations on larger mobile devices */ #img2img_interface, #img2img_results, #img2img_footer p {text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;} /* fix inpaint image display being too large for mobile displays */ - #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; alignment-baseline:after-edge !important;overflow: auto !important;resize: none !important;} + #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; alignment-baseline:after-edge !important; overflow: auto !important; resize: none !important; } #img2maskimg canvas { width: auto !important; max-height: 100% !important; height: auto !important; } /* fix from text/image UI elements to prevent them from moving around within the UI */ diff --git a/modules/images.py b/modules/images.py index 9c8def630..fb15c3e3b 100644 --- a/modules/images.py +++ b/modules/images.py @@ -425,6 +425,20 @@ class FilenameGenerator: debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}') return fn + def sequence(self, x, dirname, basename): + if shared.opts.save_images_add_number or '[seq]' in x: + if '[seq]' not in x: + x = os.path.join(os.path.dirname(x), f"[seq]-{os.path.basename(x)}") + basecount = get_next_sequence_number(dirname, basename) + for i in range(9999): + seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}" + filename = x.replace('[seq]', seq) + if not os.path.exists(filename): + debug(f'Prompt sequence: input="{x}" seq={seq} output="{filename}"') + x = filename + break + return x + def apply(self, x): res = '' for m in re_pattern.finditer(x): @@ -591,18 +605,7 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share dirname = os.path.dirname(params.filename) if dirname is not None and len(dirname) > 0: os.makedirs(dirname, exist_ok=True) - # sequence - if shared.opts.save_images_add_number or '[seq]' in params.filename: - if '[seq]' not in params.filename: - params.filename = os.path.join(os.path.dirname(params.filename), f"[seq]-{os.path.basename(params.filename)}") - basecount = get_next_sequence_number(dirname, basename) - for i in range(9999): - seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}" - filename = params.filename.replace('[seq]', seq) - if not os.path.exists(filename): - debug(f'Prompt sequence: input="{params.filename}" seq={seq} output="{filename}"') - params.filename = filename - break + params.filename = namegen.sequence(params.filename, dirname, basename) # callbacks script_callbacks.before_image_saved_callback(params) exifinfo = params.pnginfo.get('UserComment', '') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index fef1c5c31..a89fb7b98 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -385,8 +385,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro def update_sampler(sd_model, second_pass=False): sampler_selection = p.latent_sampler if second_pass else p.sampler_name - is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers - if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default' and is_karras_compatible: + # is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers + if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default': sampler = sd_samplers.all_samplers_map.get(sampler_selection, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") diff --git a/modules/shared.py b/modules/shared.py index 2ca3ce5c7..791faaa7b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -159,7 +159,7 @@ def list_samplers(): def temp_disable_extensions(): disable_safe = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-agent-scheduler', 'clip-interrogator-ext', 'stable-diffusion-webui-rembg', 'sd-extension-chainner', 'stable-diffusion-webui-images-browser'] - disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris'] + disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff'] disable_original = [] disabled = [] if cmd_opts.safe: @@ -958,6 +958,8 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s model_type = 'sd' elif "LatentConsistencyModel" in self.sd_model.__class__.__name__: model_type = 'sd' # lcm is compatible with sd + elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__: + model_type = 'sd' # ad is compatible with sd elif "Kandinsky" in self.sd_model.__class__.__name__: model_type = 'kandinsky' else: diff --git a/modules/styles.py b/modules/styles.py index 427e76ea9..be5259f2c 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -60,7 +60,7 @@ def apply_styles_to_extra(p, style: Style): v = type(orig)(v) setattr(p, k, v) fields.append(f'{k}={v}') - log.info(f'Applying style: name={style.name} extra={fields}') + log.info(f'Applying style: name="{style.name}" extra={fields}') class StyleDatabase: diff --git a/modules/ui_common.py b/modules/ui_common.py index 69dcb2551..d63c1ce7a 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -35,7 +35,7 @@ def plaintext_to_html(text): def infotext_to_html(text): res = parse_generation_parameters(text) - prompt = res.get('Prompt', '').replace('\n', '
\n') + prompt = res.get('Prompt', '') negative = res.get('Negative prompt', '') res.pop('Prompt', None) res.pop('Negative prompt', None) diff --git a/scripts/animatediff.py b/scripts/animatediff.py new file mode 100644 index 000000000..e25aa2d86 --- /dev/null +++ b/scripts/animatediff.py @@ -0,0 +1,168 @@ +""" +Lightweight AnimateDiff implementation in Diffusers +Docs: +TODO: +- Use latents and update VAE decode +- SDXL +- MP4 with RIFE +- IP Adapter +- Custom models +- Custom LORAs +- Enable second pass +- TemporalDiff: https://huggingface.co/CiaraRowles/TemporalDiff/tree/main +- AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main +""" + +import os +import gradio as gr +import diffusers +from modules import scripts, processing, shared, devices, sd_models + + +# config +ADAPTERS = { + 'None': None, + 'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4', + 'Motion 1.5': 'guoyww/animatediff-motion-adapter-v1-5', + 'Motion 1.5.2' :'guoyww/animatediff-motion-adapter-v1-5-2', + 'TemporalDiff': 'vladmandic/temporaldiff', + 'AnimateFace': 'vladmandic/animateface', +} +LORAS = { + 'None': None, + 'Zoom-in': 'guoyww/animatediff-motion-lora-zoom-in', + 'Zoom-out': 'guoyww/animatediff-motion-lora-zoom-out', + 'Pan-left': 'guoyww/animatediff-motion-lora-pan-left', + 'Pan-right': 'guoyww/animatediff-motion-lora-pan-right', + 'Tilt-up': 'guoyww/animatediff-motion-lora-tilt-up', + 'Tilt-down': 'guoyww/animatediff-motion-lora-tilt-down', + 'Roll-left': 'guoyww/animatediff-motion-lora-rolling-anticlockwise', + 'Roll-right': 'guoyww/animatediff-motion-lora-rolling-clockwise', +} + +# state +motion_adapter = None +loaded_adapter = None +orig_pipe = None + + +def set_adapter(name: str = None): + if shared.sd_model is None: + return + if shared.backend != shared.Backend.DIFFUSERS: + shared.log.warning('AnimateDiff: not in diffusers mode') + return + if shared.sd_model_type != 'sd': + shared.log.warning(f'AnimateDiff: unsupported model type: {shared.sd_model.__class__.__name__}') + return + global motion_adapter, loaded_adapter, orig_pipe # pylint: disable=global-statement + adapter_name = name if name is not None and isinstance(name, str) else loaded_adapter + if adapter_name is None or adapter_name == 'None' or shared.sd_model is None: + motion_adapter = None + loaded_adapter = None + if orig_pipe is not None: + shared.log.debug(f'AnimateDiff restore pipeline: adapter="{loaded_adapter}"') + shared.sd_model = orig_pipe + orig_pipe = None + return + if motion_adapter is not None and loaded_adapter == adapter_name: + shared.log.info(f'AnimateDiff cache: adapter="{adapter_name}"') + return + try: + shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"') + motion_adapter = diffusers.MotionAdapter.from_pretrained(adapter_name, cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, low_cpu_mem_usage=False, device_map=None) + motion_adapter.to(shared.device) + sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter') + loaded_adapter = adapter_name + + new_pipe = diffusers.AnimateDiffPipeline( + vae=shared.sd_model.vae, + text_encoder=shared.sd_model.text_encoder, + tokenizer=shared.sd_model.tokenizer, + unet=shared.sd_model.unet, + scheduler=shared.sd_model.scheduler, + motion_adapter=motion_adapter, + ) + orig_pipe = shared.sd_model + new_pipe.sd_checkpoint_info = shared.sd_model.sd_checkpoint_info + new_pipe.sd_model_hash = shared.sd_model.sd_model_hash + new_pipe.sd_model_checkpoint = shared.sd_model.sd_checkpoint_info.filename + new_pipe.is_sdxl = False + new_pipe.is_sd2 = False + new_pipe.is_sd1 = True + shared.sd_model = new_pipe + shared.sd_model.to(shared.device) + sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') + shared.log.debug(f'AnimateDiff create pipeline: adapter="{loaded_adapter}"') + except Exception as e: + motion_adapter = None + loaded_adapter = None + shared.log.error(f'AnimateDiff load error: adapter="{adapter_name}" {e}') + + +class Script(scripts.Script): + def title(self): + return 'AnimateDiff' + + def show(self, _is_img2img): + return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'): + with gr.Row(): + adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None') + frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16) + with gr.Row(): + lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None') + strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0) + with gr.Row(): + override = gr.Checkbox(label='Override sampler', value=False) + with gr.Row(): + create_gif = gr.Checkbox(label='Create GIF', value=False) + loop = gr.Checkbox(label='Loop', value=True) + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2) + return [adapter_index, frames, lora_index, strength, override, create_gif, duration, loop] + + def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, override, create_gif, duration, loop): # pylint: disable=arguments-differ, unused-argument + adapter = ADAPTERS[adapter_index] + lora = LORAS[lora_index] + set_adapter(adapter) + if motion_adapter is None: + return + shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} sampler={override} gif={create_gif}') + p.extra_generation_params['AnimateDiff'] = loaded_adapter + if override: + shared.sd_model.scheduler = diffusers.DDIMScheduler.from_pretrained('SG161222/Realistic_Vision_V5.1_noVAE', subfolder="scheduler", clip_sample=False, timestep_spacing="linspace", steps_offset=1) + if lora is not None and lora != 'None': + shared.sd_model.load_lora_weights(lora, adapter_name=lora) + shared.sd_model.set_adapters([lora], adapter_weights=[strength]) + p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}' + p.do_not_save_grid = True + p.task_args = { + 'num_frames': frames, + 'output_type': 'np', # TODO: AnimateDiff use latents and update vae_decode + 'num_inference_steps': p.steps, + } + + def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, override, create_gif, duration, loop): # pylint: disable=arguments-differ, unused-argument + if not create_gif or len(processed.images) < 2: + return + from modules.images import FilenameGenerator + image = processed.images[0] + namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image) + fn = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]") + fn = namegen.sanitize(os.path.join(shared.opts.outdir_save, fn)) + fn = namegen.sequence(fn, shared.opts.outdir_save, '') + images = processed.images[1:] + if loop: + images += processed.images[::-1] + image.save( + f'{fn}.gif', + save_all = True, + append_images = images, + optimize = False, + duration = 1000.0 * duration / frames, + loop = 0 if loop else 1, + ) + shared.log.info(f'AnimateDiff saved: file="{fn}" frames={len(images) + 1} duration={duration} loop={loop}') diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index 33e5f623a..ba81bee5a 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -1,12 +1,16 @@ """ -lightweight ip-adapter applied to existing pipeline -- downloads image_encoder or first usage (2.5GB) -- introduced via: https://github.com/huggingface/diffusers/pull/5713 -- ip adapters: https://huggingface.co/h94/IP-Adapter +Lightweight IP-Adapter applied to existing pipeline in Diffusers +- Downloads image_encoder or first usage (2.5GB) +- Introduced via: https://github.com/huggingface/diffusers/pull/5713 +- IP adapters: https://huggingface.co/h94/IP-Adapter +TODO: +- Additional IP addapters +- SD/SDXL autodetect +- Support for AnimateDiff """ import gradio as gr -from modules import scripts, processing +from modules import scripts, processing, shared, devices image_encoder = None @@ -34,7 +38,8 @@ class Script(scripts.Script): return 'IP Adapter' def show(self, is_img2img): - return scripts.AlwaysVisible + + return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False # return signature is array of gradio components def ui(self, _is_img2img): @@ -46,10 +51,9 @@ class Script(scripts.Script): image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512) return [adapter, scale, image] - def before_process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ + def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ import torch from transformers import CLIPVisionModelWithProjection - from modules import shared, devices # init code global loaded # pylint: disable=global-statement @@ -104,6 +108,3 @@ class Script(scripts.Script): shared.sd_model.set_ip_adapter_scale(scale) p.task_args = { 'ip_adapter_image': p.batch_size * [image] } p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}' - - def after_process(self, _p: processing.StableDiffusionProcessing): # pylint: disable=arguments-differ - pass diff --git a/wiki b/wiki index 0a871354e..931082304 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 0a871354eeac24412a44c00f8da3b7fa767ee857 +Subproject commit 931082304da70e7683d993e28a1765c8cb6844c1