diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b31ac6c..7d5332f97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ Major refactor of [FLUX.1](https://blackforestlabs.ai/announcing-black-forest-la And few video related goodies: - [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b) 2b and 5b variants with support for *text-to-video* and *video-to-video*! -- **AnimateDiff** prompt travel! - create video which travels between different prompts +- **AnimateDiff** prompt travel and long context windows! + create video which travels between different prompts and at long video lengths! Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;) Plus tons of minor items and fixes - see [changelog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) for details! @@ -61,12 +61,16 @@ Plus tons of minor items and fixes - see [changelog](https://github.com/vladmand - simply select in *scripts -> cogvideox* - as with any video modules, includes additional frame interpolation using RIFE - if init video is used, it will be automatically resized and interpolated to desired number of frames -- **AnimateDiff prompt travel**: - create video which travels between different prompts at different steps! - example prompt: - > 0: dog - > 5: cat - > 10: bird +- **AnimateDiff**: + - **prompt travel** + create video which travels between different prompts at different steps! + example prompt: + > 0: dog + > 5: cat + > 10: bird + - support for **v3** model (finally) + - support for **LCM** model + - support for rolling context window, automatically enabled if frames > 16 - **HDR** high-dynamic-range image create for SD and SDXL create hdr images from in multiple exposures by latent-space modifications during generation use via *scripts -> hdr* diff --git a/scripts/animatediff.py b/scripts/animatediff.py index ac596cbfa..3e8eff3c5 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -8,7 +8,7 @@ from modules import scripts, processing, shared, devices, sd_models # config ADAPTERS = { 'None': None, - 'Motion 1.5 v3' :'vladmandic/animatediff-v3', + 'Motion 1.5 v3' :'diffusers/animatediff-motion-adapter-v1-5-3', 'Motion 1.5 v2' :'guoyww/animatediff-motion-adapter-v1-5-2', 'Motion 1.5 v1': 'guoyww/animatediff-motion-adapter-v1-5', 'Motion 1.4': 'guoyww/animatediff-motion-adapter-v1-4', @@ -31,6 +31,7 @@ LORAS = { 'Tilt-down': 'guoyww/animatediff-motion-lora-tilt-down', 'Roll-left': 'guoyww/animatediff-motion-lora-rolling-anticlockwise', 'Roll-right': 'guoyww/animatediff-motion-lora-rolling-clockwise', + 'LCM': 'wangfuyun/AnimateLCM/AnimateLCM_sd15_t2v_lora.safetensors' } # state @@ -126,34 +127,26 @@ def set_adapter(adapter_name: str = 'None'): shared.log.error(f'AnimateDiff load error: adapter="{adapter_name}" {e}') -def set_scheduler(p, override_scheduler: bool = False): - if override_scheduler: - shared.log.debug('AnimateDiff: override scheduler') +def set_scheduler(p, model, override: bool = False): + if override: p.sampler_name = 'Default' - shared.sd_model.scheduler = diffusers.DDIMScheduler( - beta_start=0.00085, - beta_end=0.012, - beta_schedule="linear", - clip_sample=False, - num_train_timesteps=1000, - rescale_betas_zero_snr=False, - set_alpha_to_one=True, - steps_offset=0, - timestep_spacing="linspace", - trained_betas=None, - ) + if 'LCM' in model: + shared.sd_model.scheduler = diffusers.LCMScheduler.from_config(shared.sd_model.scheduler.config) + else: + shared.sd_model.scheduler = diffusers.DDIMScheduler.from_config(shared.sd_model.scheduler.config) + shared.log.debug(f'AnimateDiff: scheduler={shared.sd_model.scheduler.__class__.__name__}') def set_prompt(p): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) prompts = p.prompt.split('\n') - if all(':' in x.lower() for x in prompts): + try: prompt = {} for line in prompts: k, v = line.split(':') prompt[int(k.strip())] = v.strip() - else: + except Exception: prompt = p.prompt shared.log.debug(f'AnimateDiff prompt: {prompt}') p.task_args['prompt'] = prompt @@ -163,7 +156,12 @@ def set_prompt(p): def set_lora(p, lora, strength): if lora is not None and lora != 'None': shared.log.debug(f'AnimateDiff: lora="{lora}" strength={strength}') - shared.sd_model.load_lora_weights(lora, adapter_name=lora) + if lora.endswith('.safetensors'): + fn = os.path.basename(lora) + lora = lora.replace(f'/{fn}', '') + shared.sd_model.load_lora_weights(lora, weight_name=fn, adapter_name=lora) + else: + shared.sd_model.load_lora_weights(lora, adapter_name=lora) shared.sd_model.set_adapters([lora], adapter_weights=[strength]) p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}' @@ -184,16 +182,9 @@ def set_free_init(method, iters, order, spatial, temporal): def set_free_noise(frames): context_length = 16 context_stride = 4 - shared.log.debug(f'AnimateDiff free noise: frames={frames} context={context_length} stride={context_stride}') - shared.sd_model.enable_free_noise(context_length=context_length, context_stride=context_stride) - # shared.sd_model.unet.enable_attn_chunking(context_length) # Temporal chunking across batch_size x num_frames - # shared.sd_model.unet.enable_motion_module_chunking((512 // 8 // 4) ** 2) # Spatial chunking across batch_size x latent height x latent width - # shared.sd_model.unet.enable_resnet_chunking(context_length) - # shared.sd_model.unet.enable_forward_chunking(context_length) - # pipe.enable_free_noise(context_length=context_length, context_stride=context_stride) - # shared.sd_model.enable_free_noise_chunked_inference() - # pipe.unet.enable_forward_chunking(context_length) - + if frames >= context_length: + shared.log.debug(f'AnimateDiff free noise: frames={frames} context={context_length} stride={context_stride}') + shared.sd_model.enable_free_noise(context_length=context_length, context_stride=context_stride) class Script(scripts.Script): @@ -218,7 +209,7 @@ class Script(scripts.Script): gr.HTML("  AnimateDiff
") with gr.Row(): adapter_index = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='None') - frames = gr.Slider(label='Frames', minimum=1, maximum=64, step=1, value=16) + frames = gr.Slider(label='Frames', minimum=1, maximum=256, step=1, value=16) with gr.Row(): override_scheduler = gr.Checkbox(label='Override sampler', value=True) with gr.Row(): @@ -252,7 +243,7 @@ class Script(scripts.Script): set_adapter(adapter) if motion_adapter is None: return - set_scheduler(p, override_scheduler) + set_scheduler(p, adapter, override_scheduler) set_lora(p, lora, strength) set_free_init(fi_method, fi_iters, fi_order, fi_spatial, fi_temporal) set_free_noise(frames) diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index 4c960dac4..6cbb71467 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -70,6 +70,7 @@ class Script(scripts.Script): shared.sd_model = None shared.sd_model = diffusers.CogVideoXPipeline.from_pretrained(model, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir) shared.sd_model.sd_checkpoint_info = sd_models.CheckpointInfo(model) + shared.sd_model.sd_model_hash = '' shared.sd_model.sd_model_checkpoint = model except Exception as e: shared.log.error(f'Loading CogVideoX: {e}')