mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 09:14:35 +02:00
add amused model support and update reference list
This commit is contained in:
Vladimir Mandic
+19
-16
@@ -2,8 +2,6 @@
|
||||
|
||||
## Update for 2023-12-27
|
||||
|
||||
*Note*: based on `diffusers==0.25.0.dev0`
|
||||
|
||||
- **Control**
|
||||
- native implementation of all image control methods:
|
||||
**ControlNet**, **ControlNet XS**, **Control LLLite**, **T2I Adapters** and **IP Adapters**
|
||||
@@ -13,6 +11,25 @@
|
||||
- for details and list of supported models and workflows, see Wiki documentation:
|
||||
<https://github.com/vladmandic/automatic/wiki/Control>
|
||||
- **Diffusers**
|
||||
- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) model support
|
||||
- small and fast version of **SDXL**, only 3.1GB in size!
|
||||
- select from *networks -> reference*
|
||||
- [aMUSEd 256](https://huggingface.co/amused/amused-256) and [aMUSEd 512](https://huggingface.co/amused/amused-512) model support
|
||||
- lightweigt models that excel at fast image generation
|
||||
- *note*: must select: settings -> diffusers -> generator device: unset
|
||||
- select from *networks -> reference*
|
||||
- [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support
|
||||
- comparable to SD15 and SD-XL, trained from scratch for highly aesthetic images
|
||||
- simply select from *networks -> reference* and use as usual
|
||||
- [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/)
|
||||
- img2img model that can replace subjects in images using prompt keywords
|
||||
- download and load by selecting from *networks -> reference -> blip diffusion*
|
||||
- in image tab, select `blip diffusion` script
|
||||
- [DemoFusion](https://github.com/PRIS-CV/DemoFusion) run your SDXL generations at any resolution!
|
||||
- in **Text** tab select *script* -> *demofusion*
|
||||
- *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions
|
||||
in the future, expect more optimizations, especially related to offloading/slicing/tiling,
|
||||
but at the moment this is pretty much experimental-only
|
||||
- **AnimateDiff**
|
||||
- can now be used with *second pass* - enhance, upscale and hires your videos!
|
||||
- **IP Adapter**
|
||||
@@ -30,20 +47,6 @@
|
||||
*if you know of any other t2v models you'd like to see supported, let me know!*
|
||||
- models are auto-downloaded on first use
|
||||
- *note*: current base model will be unloaded to free up resources
|
||||
- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) support
|
||||
- small and fast version of **SDXL**, only 3.1GB in size!
|
||||
- select from *networks -> reference*
|
||||
- [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support
|
||||
- simply select from *networks -> reference* and use as usual
|
||||
- [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/)
|
||||
- img2img model that can replace subjects in images using prompt keywords
|
||||
- download and load by selecting from *networks -> reference -> blip diffusion*
|
||||
- in image tab, select `blip diffusion` script
|
||||
- [DemoFusion](https://github.com/PRIS-CV/DemoFusion) run your SDXL generations at any resolution!
|
||||
- in **Text** tab select *script* -> *demofusion*
|
||||
- *note*: GPU VRAM limits do not automatically go away so be careful when using it with large resolutions
|
||||
in the future, expect more optimizations, especially related to offloading/slicing/tiling,
|
||||
but at the moment this is pretty much experimental-only
|
||||
- **Prompt scheduling** now implemented for Diffusers backend, thanks @AI-Casanova
|
||||
- **Custom pipelines** contribute by adding your own custom pipelines!
|
||||
- for details, see fully documented example:
|
||||
|
||||
@@ -144,6 +144,16 @@
|
||||
"desc": "DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model, that can generate pictures with new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. It is modular and composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024.",
|
||||
"preview": "DeepFloyd--IF-I-M-v1.0.jpg"
|
||||
},
|
||||
"aMUSEd 256": {
|
||||
"path": "amused/amused-256",
|
||||
"desc": "Amused is a lightweight text to image model based off of the muse architecture. Amused is particularly useful in applications that require a lightweight and fast model such as generating many images quickly at once.",
|
||||
"preview": "amused--amused-256.jpg"
|
||||
},
|
||||
"aMUSEd 512": {
|
||||
"path": "amused/amused-512",
|
||||
"desc": "Amused is a lightweight text to image model based off of the muse architecture. Amused is particularly useful in applications that require a lightweight and fast model such as generating many images quickly at once.",
|
||||
"preview": "amused--amused-512.jpg"
|
||||
},
|
||||
"Tsinghua UniDiffuser": {
|
||||
"path": "thu-ml/unidiffuser-v1",
|
||||
"desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.",
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
@@ -85,7 +85,7 @@ class OverrideItem(NamedTuple):
|
||||
message: Optional[str]
|
||||
|
||||
opts_override_table = {
|
||||
"diffusers_generator_device": OverrideItem("cpu", None, "DirectML does not support torch Generator API"),
|
||||
"diffusers_generator_device": OverrideItem("CPU", None, "DirectML does not support torch Generator API"),
|
||||
"diffusers_model_cpu_offload": OverrideItem(False, None, "Diffusers model CPU offloading does not support DirectML devices"),
|
||||
"diffusers_seq_cpu_offload": OverrideItem(False, lambda opts: opts.diffusers_pipeline != "Stable Diffusion XL", "Diffusers sequential CPU offloading is available only on StableDiffusionXLPipeline with DirectML devices"),
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ class UpscalerSD(Upscaler):
|
||||
if model is None:
|
||||
return img
|
||||
seeds = [torch.randint(0, 2 ** 32, (1,)).item() for _ in range(1)]
|
||||
generator_device = devices.cpu if shared.opts.diffusers_generator_device == "cpu" else devices.device
|
||||
generator_device = devices.cpu if shared.opts.diffusers_generator_device == "CPU" else devices.device
|
||||
generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds]
|
||||
args = {
|
||||
'prompt': '',
|
||||
|
||||
@@ -213,8 +213,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
signature = inspect.signature(type(model).__call__)
|
||||
possible = signature.parameters.keys()
|
||||
debug(f'Diffusers pipeline possible: {possible}')
|
||||
generator_device = devices.cpu if shared.opts.diffusers_generator_device == "cpu" else shared.device
|
||||
generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds]
|
||||
if shared.opts.diffusers_generator_device == "Unset":
|
||||
generator_device = None
|
||||
generator = None
|
||||
else:
|
||||
generator_device = devices.cpu if shared.opts.diffusers_generator_device == "CPU" else shared.device
|
||||
generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds]
|
||||
prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2)
|
||||
parser = 'Fixed attention'
|
||||
if shared.opts.prompt_attention != 'Fixed attention' and 'StableDiffusion' in model.__class__.__name__:
|
||||
@@ -247,7 +251,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
args['noise_sampler_seed'] = seeds[0]
|
||||
if 'guidance_scale' in possible:
|
||||
args['guidance_scale'] = p.cfg_scale
|
||||
if 'generator' in possible:
|
||||
if 'generator' in possible and generator is not None:
|
||||
args['generator'] = generator
|
||||
if 'output_type' in possible:
|
||||
args['output_type'] = 'np'
|
||||
@@ -363,6 +367,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
|
||||
def update_sampler(sd_model, second_pass=False):
|
||||
sampler_selection = p.latent_sampler if second_pass else p.sampler_name
|
||||
# is_karras_compatible = sd_model.__class__.__init__.__annotations__.get("scheduler", None) == diffusers.schedulers.scheduling_utils.KarrasDiffusionSchedulers
|
||||
if sd_model.__class__.__name__ in ['AmusedPipeline']:
|
||||
return # models with their own schedulers
|
||||
if hasattr(sd_model, 'scheduler') and sampler_selection != 'Default':
|
||||
sampler = sd_samplers.all_samplers_map.get(sampler_selection, None)
|
||||
if sampler is None:
|
||||
|
||||
@@ -756,6 +756,8 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'):
|
||||
devices.dtype_vae = torch.float32
|
||||
sd_model.vae.to(devices.dtype_vae)
|
||||
shared.log.debug(f'Setting {op} VAE: name={sd_vae.loaded_vae_file} upcast={sd_model.vae.config.get("force_upcast", None)}')
|
||||
if hasattr(sd_model, "vqvae"):
|
||||
sd_model.vqvae.to(torch.float32) # vqvae is producing nans in fp16
|
||||
if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'):
|
||||
sd_model.enable_xformers_memory_efficient_attention()
|
||||
|
||||
|
||||
+1
-1
@@ -373,7 +373,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
|
||||
"diffusers_move_unet": OptionInfo(True, "Move base model to CPU when using VAE"),
|
||||
"diffusers_move_refiner": OptionInfo(True, "Move refiner model to CPU when not in use"),
|
||||
"diffusers_extract_ema": OptionInfo(True, "Use model EMA weights when possible"),
|
||||
"diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, {"choices": ["default", "cpu"]}),
|
||||
"diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}),
|
||||
"diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload (--medvram)"),
|
||||
"diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload (--lowvram)"),
|
||||
"diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}),
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ requests==2.31.0
|
||||
tqdm==4.66.1
|
||||
accelerate==0.25.0
|
||||
opencv-contrib-python-headless==4.8.1.78
|
||||
diffusers==0.24.0
|
||||
diffusers==0.25.0
|
||||
einops==0.4.1
|
||||
gradio==3.43.2
|
||||
huggingface_hub==0.20.1
|
||||
|
||||
Reference in New Issue
Block a user