initial lightweight animatediff

CHANGELOG.md

@@ -1,17 +1,24 @@
 # Change Log for SD.Next
 
-## Update for 2023-11-26
+## Update for 2023-11-28
 
 Note: Release pending `diffusers==0.24`
 
 - **Diffusers**
-  - **IP adapter**
-    - Lightweight implementation of T2I adapters which can guide generation towards specific image style  
-    - Supports most T2I models, not limited to SD  
   - **HDR latent control**, based on [article](https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space#long-prompts-at-high-guidance-scales-becoming-possible)  
     - In *Advanced* params
     - Allows control of *latent clamping*, *color centering* and *range maximimization*  
     - Supported by *XYZ grid*  
+  - **IP adapter**
+    - Lightweight implementation of T2I adapters which can guide generation towards specific image style  
+    - Supports most T2I models, not limited to SD 1.5  
+    - Models are auto-downloaded on first use
+    - For IP adapter support in Original backend, use standard *ControlNet* extension  
+  - **AnimateDiff**
+    - Lightweight implementation of AnimateDiff basic models *(1.4, 1.5, 1.5.2)*
+    - Supports SD 1.5 only  
+    - Models are auto-downloaded on first use
+    - For AnimateDiff support in Original backend, use standard *AnimateDiff* extension  
   - **Kandinsky 3** support  
     - download using built-in model downloader or simply select from networks -> reference  
     - this model is absolutely massive at 27.5GB at fp16, so be patient  

html/locale_en.json

@@ -71,7 +71,7 @@
 "extra networks": [
   {"id":"","label":"UI position","localized":"","hint":"Location of extra networks"},
   {"id":"","label":"cover","localized":"","hint":"cover full area"},
-  {"id":"","label":"inline","localized":"","hint":"inline with all additional elelemtns (scrollable)"},
+  {"id":"","label":"inline","localized":"","hint":"inline with all additional elements (scrollable)"},
   {"id":"","label":"sidebar","localized":"","hint":"sidebar on the right side of the screen"},
   {"id":"","label":"UI height (%)","localized":"","hint":""},
   {"id":"","label":"UI sidebar width (%)","localized":"","hint":""},

javascript/sdnext.css

@@ -95,7 +95,7 @@ div#extras_scale_to_tab div.form{ flex-direction: row; }
 #img2img_sketch, #img2maskimg, #inpaint_sketch { overflow: overlay !important; resize: auto; background: var(--panel-background-fill); z-index: 5; }
 .image-buttons button{ min-width: auto; }
 .infotext { overflow-wrap: break-word; line-height: 1.5em; }
-.infotext > p { padding-left: 1em; text-indent: -1em; white-space: pre; }
+.infotext > p { padding-left: 1em; text-indent: -1em; white-space: pre; text-wrap: pretty; }
 .tooltip { display: block; position: fixed; top: 1em; right: 1em; padding: 0.5em; background: var(--input-background-fill); color: var(--body-text-color); border: 1pt solid var(--button-primary-border-color);
   width: 22em; min-height: 1.3em; font-size: 0.8em; transition: opacity 0.2s ease-in; pointer-events: none; opacity: 0; z-index: 999; }
 .tooltip-show { opacity: 0.9; }
@@ -318,7 +318,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt
     /* maintain single column for from image operations on larger mobile devices */
     #img2img_interface, #img2img_results, #img2img_footer p {text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;}
     /* fix inpaint image display being too large for mobile displays */
-    #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; alignment-baseline:after-edge !important;overflow: auto !important;resize: none !important;}
+    #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; alignment-baseline:after-edge !important; overflow: auto !important; resize: none !important; }
     #img2maskimg canvas { width: auto !important; max-height: 100% !important; height: auto !important; }
 
     /* fix from text/image UI elements to prevent them from moving around within the UI */

modules/images.py

@@ -425,6 +425,20 @@ class FilenameGenerator:
         debug(f'Filename sanitize: input="{filename}" parts={parts} output="{fn}" ext={ext} max={max_length} len={len(fn)}')
         return fn
 
+    def sequence(self, x, dirname, basename):
+        if shared.opts.save_images_add_number or '[seq]' in x:
+            if '[seq]' not in x:
+                x = os.path.join(os.path.dirname(x), f"[seq]-{os.path.basename(x)}")
+            basecount = get_next_sequence_number(dirname, basename)
+            for i in range(9999):
+                seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}"
+                filename = x.replace('[seq]', seq)
+                if not os.path.exists(filename):
+                    debug(f'Prompt sequence: input="{x}" seq={seq} output="{filename}"')
+                    x = filename
+                    break
+        return x
+
     def apply(self, x):
         res = ''
         for m in re_pattern.finditer(x):
@@ -591,18 +605,7 @@ def save_image(image, path, basename='', seed=None, prompt=None, extension=share
     dirname = os.path.dirname(params.filename)
     if dirname is not None and len(dirname) > 0:
         os.makedirs(dirname, exist_ok=True)
-    # sequence
-    if shared.opts.save_images_add_number or '[seq]' in params.filename:
-        if '[seq]' not in params.filename:
-            params.filename = os.path.join(os.path.dirname(params.filename), f"[seq]-{os.path.basename(params.filename)}")
-        basecount = get_next_sequence_number(dirname, basename)
-        for i in range(9999):
-            seq = f"{basecount + i:05}" if basename == '' else f"{basename}-{basecount + i:04}"
-            filename = params.filename.replace('[seq]', seq)
-            if not os.path.exists(filename):
-                debug(f'Prompt sequence: input="{params.filename}" seq={seq} output="{filename}"')
-                params.filename = filename
-                break
+    params.filename = namegen.sequence(params.filename, dirname, basename)
     # callbacks
     script_callbacks.before_image_saved_callback(params)
     exifinfo = params.pnginfo.get('UserComment', '')

modules/processing_diffusers.py

@@ -385,8 +385,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
 
     def update_sampler(sd_model, second_pass=False):
         sampler_selection = p.latent_sampler if second_pass else p.sampler_name
-        is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers
-        if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default' and is_karras_compatible:
+        # is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers
+        if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default':
             sampler = sd_samplers.all_samplers_map.get(sampler_selection, None)
             if sampler is None:
                 sampler = sd_samplers.all_samplers_map.get("UniPC")

modules/shared.py

@@ -159,7 +159,7 @@ def list_samplers():
 
 def temp_disable_extensions():
     disable_safe = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-agent-scheduler', 'clip-interrogator-ext', 'stable-diffusion-webui-rembg', 'sd-extension-chainner', 'stable-diffusion-webui-images-browser']
-    disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris']
+    disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff']
     disable_original = []
     disabled = []
     if cmd_opts.safe:
@@ -958,6 +958,8 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s
                 model_type = 'sd'
             elif "LatentConsistencyModel" in self.sd_model.__class__.__name__:
                 model_type = 'sd' # lcm is compatible with sd
+            elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__:
+                model_type = 'sd' # ad is compatible with sd
             elif "Kandinsky" in self.sd_model.__class__.__name__:
                 model_type = 'kandinsky'
             else:

modules/styles.py

@@ -60,7 +60,7 @@ def apply_styles_to_extra(p, style: Style):
                 v = type(orig)(v)
             setattr(p, k, v)
             fields.append(f'{k}={v}')
-    log.info(f'Applying style: name={style.name} extra={fields}')
+    log.info(f'Applying style: name="{style.name}" extra={fields}')
 
 
 class StyleDatabase:

modules/ui_common.py

@@ -35,7 +35,7 @@ def plaintext_to_html(text):
 
 def infotext_to_html(text):
     res = parse_generation_parameters(text)
-    prompt = res.get('Prompt', '').replace('\n', '<br>\n')
+    prompt = res.get('Prompt', '')
     negative = res.get('Negative prompt', '')
     res.pop('Prompt', None)
     res.pop('Negative prompt', None)

scripts/animatediff.py

@@ -0,0 +1,168 @@
+"""
+Lightweight AnimateDiff implementation in Diffusers
+Docs: <https://huggingface.co/docs/diffusers/api/pipelines/animatediff>
+TODO:
+- Use latents and update VAE decode
+- SDXL
+- MP4 with RIFE
+- IP Adapter
+- Custom models
+- Custom LORAs
+- Enable second pass
+- TemporalDiff: https://huggingface.co/CiaraRowles/TemporalDiff/tree/main
+- AnimateFace: https://huggingface.co/nlper2022/animatediff_face_512/tree/main
+"""
+
+import os
+import gradio as gr
+import diffusers
+from modules import scripts, processing, shared, devices, sd_models
+
+
+# config
+ADAPTERS = {
+    'None': None,
+    'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4',
+    'Motion 1.5': 'guoyww/animatediff-motion-adapter-v1-5',
+    'Motion 1.5.2' :'guoyww/animatediff-motion-adapter-v1-5-2',
+    'TemporalDiff': 'vladmandic/temporaldiff',
+    'AnimateFace': 'vladmandic/animateface',
+}
+LORAS = {
+    'None': None,
+    'Zoom-in': 'guoyww/animatediff-motion-lora-zoom-in',
+    'Zoom-out': 'guoyww/animatediff-motion-lora-zoom-out',
+    'Pan-left': 'guoyww/animatediff-motion-lora-pan-left',
+    'Pan-right': 'guoyww/animatediff-motion-lora-pan-right',
+    'Tilt-up': 'guoyww/animatediff-motion-lora-tilt-up',
+    'Tilt-down': 'guoyww/animatediff-motion-lora-tilt-down',
+    'Roll-left': 'guoyww/animatediff-motion-lora-rolling-anticlockwise',
+    'Roll-right': 'guoyww/animatediff-motion-lora-rolling-clockwise',
+}
+
+# state
+motion_adapter = None
+loaded_adapter = None
+orig_pipe = None
+
+
+def set_adapter(name: str = None):
+    if shared.sd_model is None:
+        return
+    if shared.backend != shared.Backend.DIFFUSERS:
+        shared.log.warning('AnimateDiff: not in diffusers mode')
+        return
+    if shared.sd_model_type != 'sd':
+        shared.log.warning(f'AnimateDiff: unsupported model type: {shared.sd_model.__class__.__name__}')
+        return
+    global motion_adapter, loaded_adapter, orig_pipe # pylint: disable=global-statement
+    adapter_name = name if name is not None and isinstance(name, str) else loaded_adapter
+    if adapter_name is None or adapter_name == 'None' or shared.sd_model is None:
+        motion_adapter = None
+        loaded_adapter = None
+        if orig_pipe is not None:
+            shared.log.debug(f'AnimateDiff restore pipeline: adapter="{loaded_adapter}"')
+            shared.sd_model = orig_pipe
+            orig_pipe = None
+        return
+    if motion_adapter is not None and loaded_adapter == adapter_name:
+        shared.log.info(f'AnimateDiff cache: adapter="{adapter_name}"')
+        return
+    try:
+        shared.log.info(f'AnimateDiff load: adapter="{adapter_name}"')
+        motion_adapter = diffusers.MotionAdapter.from_pretrained(adapter_name, cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, low_cpu_mem_usage=False, device_map=None)
+        motion_adapter.to(shared.device)
+        sd_models.set_diffuser_options(motion_adapter, vae=None, op='adapter')
+        loaded_adapter = adapter_name
+
+        new_pipe = diffusers.AnimateDiffPipeline(
+            vae=shared.sd_model.vae,
+            text_encoder=shared.sd_model.text_encoder,
+            tokenizer=shared.sd_model.tokenizer,
+            unet=shared.sd_model.unet,
+            scheduler=shared.sd_model.scheduler,
+            motion_adapter=motion_adapter,
+        )
+        orig_pipe = shared.sd_model
+        new_pipe.sd_checkpoint_info = shared.sd_model.sd_checkpoint_info
+        new_pipe.sd_model_hash = shared.sd_model.sd_model_hash
+        new_pipe.sd_model_checkpoint = shared.sd_model.sd_checkpoint_info.filename
+        new_pipe.is_sdxl = False
+        new_pipe.is_sd2 = False
+        new_pipe.is_sd1 = True
+        shared.sd_model = new_pipe
+        shared.sd_model.to(shared.device)
+        sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model')
+        shared.log.debug(f'AnimateDiff create pipeline: adapter="{loaded_adapter}"')
+    except Exception as e:
+        motion_adapter = None
+        loaded_adapter = None
+        shared.log.error(f'AnimateDiff load error: adapter="{adapter_name}" {e}')
+
+
+class Script(scripts.Script):
+    def title(self):
+        return 'AnimateDiff'
+
+    def show(self, _is_img2img):
+        return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False
+
+    # return signature is array of gradio components
+    def ui(self, _is_img2img):
+        with gr.Accordion('AnimateDiff', open=False, elem_id='animatediff'):
+            with gr.Row():
+                adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None')
+                frames = gr.Slider(label='Frames', minimum=1, maximum=32, step=1, value=16)
+            with gr.Row():
+                lora_index = gr.Dropdown(label='Lora', choices=list(LORAS), value='None')
+                strength = gr.Slider(label='Strength', minimum=0.0, maximum=2.0, step=0.05, value=1.0)
+            with gr.Row():
+                override = gr.Checkbox(label='Override sampler', value=False)
+            with gr.Row():
+                create_gif = gr.Checkbox(label='Create GIF', value=False)
+                loop = gr.Checkbox(label='Loop', value=True)
+                duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2)
+        return [adapter_index, frames, lora_index, strength, override, create_gif, duration, loop]
+
+    def process(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lora_index, strength, override, create_gif, duration, loop): # pylint: disable=arguments-differ, unused-argument
+        adapter = ADAPTERS[adapter_index]
+        lora = LORAS[lora_index]
+        set_adapter(adapter)
+        if motion_adapter is None:
+            return
+        shared.log.debug(f'AnimateDiff: adapter="{adapter}" lora="{lora}" strength={strength} sampler={override} gif={create_gif}')
+        p.extra_generation_params['AnimateDiff'] = loaded_adapter
+        if override:
+            shared.sd_model.scheduler = diffusers.DDIMScheduler.from_pretrained('SG161222/Realistic_Vision_V5.1_noVAE', subfolder="scheduler", clip_sample=False, timestep_spacing="linspace", steps_offset=1)
+        if lora is not None and lora != 'None':
+            shared.sd_model.load_lora_weights(lora, adapter_name=lora)
+            shared.sd_model.set_adapters([lora], adapter_weights=[strength])
+            p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}'
+        p.do_not_save_grid = True
+        p.task_args = {
+            'num_frames': frames,
+            'output_type': 'np', # TODO: AnimateDiff use latents and update vae_decode
+            'num_inference_steps': p.steps,
+        }
+
+    def postprocess(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, adapter_index, frames, lora_index, strength, override, create_gif, duration, loop): # pylint: disable=arguments-differ, unused-argument
+        if not create_gif or len(processed.images) < 2:
+            return
+        from modules.images import FilenameGenerator
+        image = processed.images[0]
+        namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image)
+        fn = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]")
+        fn = namegen.sanitize(os.path.join(shared.opts.outdir_save, fn))
+        fn = namegen.sequence(fn, shared.opts.outdir_save, '')
+        images = processed.images[1:]
+        if loop:
+            images += processed.images[::-1]
+        image.save(
+            f'{fn}.gif',
+            save_all = True,
+            append_images = images,
+            optimize = False,
+            duration = 1000.0 * duration / frames,
+            loop = 0 if loop else 1,
+        )
+        shared.log.info(f'AnimateDiff saved: file="{fn}" frames={len(images) + 1} duration={duration} loop={loop}')

scripts/ipadapter.py

@@ -1,12 +1,16 @@
 """
-lightweight ip-adapter applied to existing pipeline
-- downloads image_encoder or first usage (2.5GB)
-- introduced via: https://github.com/huggingface/diffusers/pull/5713
-- ip adapters: https://huggingface.co/h94/IP-Adapter
+Lightweight IP-Adapter applied to existing pipeline in Diffusers
+- Downloads image_encoder or first usage (2.5GB)
+- Introduced via: https://github.com/huggingface/diffusers/pull/5713
+- IP adapters: https://huggingface.co/h94/IP-Adapter
+TODO:
+- Additional IP addapters
+- SD/SDXL autodetect
+- Support for AnimateDiff
 """
 
 import gradio as gr
-from modules import scripts, processing
+from modules import scripts, processing, shared, devices
 
 
 image_encoder = None
@@ -34,7 +38,8 @@ class Script(scripts.Script):
         return 'IP Adapter'
 
     def show(self, is_img2img):
-        return scripts.AlwaysVisible
+
+        return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False
 
     # return signature is array of gradio components
     def ui(self, _is_img2img):
@@ -46,10 +51,9 @@ class Script(scripts.Script):
                 image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)
         return [adapter, scale, image]
 
-    def before_process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ
+    def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ
         import torch
         from transformers import CLIPVisionModelWithProjection
-        from modules import shared, devices
 
         # init code
         global loaded # pylint: disable=global-statement
@@ -104,6 +108,3 @@ class Script(scripts.Script):
         shared.sd_model.set_ip_adapter_scale(scale)
         p.task_args = { 'ip_adapter_image': p.batch_size * [image] }
         p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}'
-
-    def after_process(self, _p: processing.StableDiffusionProcessing): # pylint: disable=arguments-differ
-        pass