diff --git a/CHANGELOG.md b/CHANGELOG.md index 6028c2982..7e65fb85b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-02-11 +## Update for 2025-02-12 - **User Interface** - **Hints** @@ -14,9 +14,13 @@ - **UI**: - force browser cache-invalidate on page load - **Models** + - [AlphaVLLM Lumina 2](https://github.com/Alpha-VLLM/Lumina-Image-2.0) + new foundation model for image generation based o Gemma-2-2B text encoder and a flow-based diffusion transformer + simply select from *networks -> models -> reference* - [Ostris Flex.1-Alpha](https://huggingface.co/ostris/Flex.1-alpha) - originally based on Flux.1-Schnell, but retrained and with different architecture - result is model smaller than Flux.1-Dev, but with similar capabilities + originally based on *Flux.1-Schnell*, but retrained and with different architecture + result is model smaller than *Flux.1-Dev*, but with similar capabilities + simply select from *networks -> models -> reference* - **Docs** - New [Outpaint](https://github.com/vladmandic/sdnext/wiki/Outpaint) step-by-step guide - Updated [Docker](https://github.com/vladmandic/sdnext/wiki/Docker) guide diff --git a/README.md b/README.md index a50d04241..ef356ed19 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ ## SD.Next Features All individual features are not listed here, instead check [ChangeLog](CHANGELOG.md) for full list of changes +- Fully localized: + ▹ **English | Chinese | Russian | Spanish | German | French | Italian | Portuguese | Japanese | Korean** - Multiple UIs! ▹ **Standard | Modern** - Multiple [diffusion models](https://vladmandic.github.io/sdnext-docs/Model-Support/)! diff --git a/html/reference.json b/html/reference.json index 83849c3c0..4cc2edb28 100644 --- a/html/reference.json +++ b/html/reference.json @@ -309,6 +309,13 @@ "skip": true, "extras": "sampler: Default" }, + "AlphaVLLM Lumina 2": { + "path": "Alpha-VLLM/Lumina-Image-2.0", + "desc": "A Unified and Efficient Image Generative Model. Lumina-Image-2.0 is a 2 billion parameter flow-based diffusion transformer capable of generating images from text descriptions.", + "preview": "Alpha-VLLM--Lumina-Image-2.0.jpg", + "skip": true, + "extras": "sampler: Default" + }, "Kwai Kolors": { "path": "Kwai-Kolors/Kolors-diffusers", diff --git a/installer.py b/installer.py index aae9a6988..d9367fcec 100644 --- a/installer.py +++ b/installer.py @@ -531,7 +531,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all or args.skip_git: return - sha = 'f63d32233f402bd603da8f3aa385aecb9c3d8809' # diffusers commit hash + sha = '067eab1b3aaf4d09f85edf21d8b147e0980c662a' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/models/Reference/Alpha-VLLM--Lumina-Image-2.0.jpg b/models/Reference/Alpha-VLLM--Lumina-Image-2.0.jpg new file mode 100755 index 000000000..5c71625bf Binary files /dev/null and b/models/Reference/Alpha-VLLM--Lumina-Image-2.0.jpg differ diff --git a/modules/model_lumina.py b/modules/model_lumina.py index ca5f2e7b8..9dff4dccd 100644 --- a/modules/model_lumina.py +++ b/modules/model_lumina.py @@ -1,3 +1,4 @@ +import transformers import diffusers @@ -22,3 +23,23 @@ def load_lumina(_checkpoint_info, diffusers_load_config={}): ) devices.torch_gc() return pipe + + +def load_lumina2(checkpoint_info, diffusers_load_config={}): + from modules import shared, devices, sd_models, model_quant + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: + model_quant.load_bnb(f'Load model: type=Lumina quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=Lumina quant={quant_args}') + kwargs = {} + repo_id = sd_models.path_to_repo(checkpoint_info.name) + if ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): + kwargs['transformer'] = diffusers.Lumina2Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, **quant_args) + if ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + kwargs['text_encoder'] = transformers.AutoModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, **quant_args) + sd_model = diffusers.Lumina2Text2ImgPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config, **quant_args, **kwargs) + return sd_model diff --git a/modules/model_quant.py b/modules/model_quant.py index d7f9016a5..64005ebc4 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -30,8 +30,6 @@ def get_quant(name): return 'none' - - def create_bnb_config(kwargs = None, allow_bnb: bool = True): from modules import shared, devices if len(shared.opts.bnb_quantization) > 0 and allow_bnb: diff --git a/modules/modeldata.py b/modules/modeldata.py index 63130d041..0105771e5 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -31,6 +31,8 @@ def get_model_type(pipe): model_type = 'f1' elif "Mochi" in name: model_type = 'mochi' + elif "Lumina2" in name: + model_type = 'lumina2' elif "Lumina" in name: model_type = 'lumina' elif "OmniGen" in name: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index de30063e8..f011fd77a 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -69,6 +69,8 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'Sana' if 'lumina-next' in f.lower(): guess = 'Lumina-Next' + if 'lumina-image-2' in f.lower(): + guess = 'Lumina2' if 'kolors' in f.lower(): guess = 'Kolors' if 'auraflow' in f.lower(): @@ -101,6 +103,8 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'FLUX' if 'StableDiffusion3' in pipeline.__name__: guess = 'Stable Diffusion 3' + if 'Lumina2' in pipeline.__name__: + guess = 'Lumina 2' # switch for specific variant if guess == 'Stable Diffusion' and 'inpaint' in f.lower(): guess = 'Stable Diffusion Inpaint' diff --git a/modules/sd_models.py b/modules/sd_models.py index 8851c5cbd..a9e916d74 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -290,6 +290,9 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' elif model_type in ['FLUX']: from modules.model_flux import load_flux sd_model = load_flux(checkpoint_info, diffusers_load_config) + elif model_type in ['Lumina 2']: + from modules.model_lumina import load_lumina2 + sd_model = load_lumina2(checkpoint_info, diffusers_load_config) elif model_type in ['Stable Diffusion 3']: from modules.model_sd3 import load_sd3 shared.log.debug(f'Load {op}: model="Stable Diffusion 3"') @@ -314,7 +317,7 @@ def load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_c files = shared.walk_files(checkpoint_info.path, ['.safetensors', '.bin', '.ckpt']) if 'variant' not in diffusers_load_config and any('diffusion_pytorch_model.fp16' in f for f in files): # deal with diffusers lack of variant fallback when loading diffusers_load_config['variant'] = 'fp16' - if model_type is not None and pipeline is not None and 'ONNX' in model_type: # forced pipeline + if (model_type is not None) and (pipeline is not None) and ('ONNX' in model_type): # forced pipeline try: sd_model = pipeline.from_pretrained(checkpoint_info.path) except Exception as e: diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 62bc4f55c..0b50c5c6c 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -9,7 +9,7 @@ from modules import shared, devices, processing, images, sd_vae_approx, sd_vae_t SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } -flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana'] +flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2'] warned = False queue_lock = threading.Lock() diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 63740051a..667683eb0 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -306,6 +306,5 @@ class DiffusionSampler: if name == 'DC Solver': if not hasattr(self.sampler, 'dc_ratios'): pass - # self.sampler.dc_ratios = self.sampler.cascade_polynomial_regression(test_CFG=6.0, test_NFE=10, cpr_path='tmp/sd2.1.npy') # shared.log.debug_log(f'Sampler: class="{self.sampler.__class__.__name__}" config={self.sampler.config}') self.sampler.name = name diff --git a/scripts/mixture_of_diffusers.py b/scripts/mixture_of_diffusers.py index e3d6d20c6..21139e6ce 100644 --- a/scripts/mixture_of_diffusers.py +++ b/scripts/mixture_of_diffusers.py @@ -1,6 +1,8 @@ import gradio as gr -from modules import scripts, processing, shared +from modules import scripts, processing, shared, sd_models +max_xtiles = 4 +max_ytiles = 4 class Script(scripts.Script): def __init__(self): @@ -13,9 +15,31 @@ class Script(scripts.Script): def show(self, is_img2img): return shared.native + def update_ui(self, x, y): + updates = [] + for i in range(max_xtiles): + for j in range(max_ytiles): + updates.append(gr.update(visible=(i < x) and (j < y))) + return updates + def ui(self, _is_img2img): # ui elements with gr.Row(): gr.HTML('  Mixture-of-Diffusers
') + with gr.Row(): + x_tiles = gr.Slider(minimum=1, maximum=max_xtiles, default=1, label="X-axis tiles") + y_tiles = gr.Slider(minimum=1, maximum=max_ytiles, default=1, label="Y-axis tiles") + with gr.Row(): + tile_width = gr.Number(minimum=1, maximum=2048, value=1024, label="Tile width") + tile_height = gr.Number(minimum=1, maximum=2048, value=1024, label="Tile height") + with gr.Row(): + overlap_width = gr.Number(minimum=1, maximum=512, value=128, label="Overlap width") + overlap_height = gr.Number(minimum=1, maximum=512, value=128, label="Overlap height") + with gr.Row(): + prompts = [] + for i in range(max_xtiles*max_ytiles): + prompts.append(gr.Textbox('', label=f"Tile prompt: x={i%max_xtiles} y={i//max_ytiles}", placeholder='Prompt for tile', visible=False)) + x_tiles.change(fn=self.update_ui, inputs=[x_tiles, y_tiles], outputs=prompts) + y_tiles.change(fn=self.update_ui, inputs=[x_tiles, y_tiles], outputs=prompts) return [] def run(self, p: processing.StableDiffusionProcessing): # pylint: disable=arguments-differ, unused-argument @@ -24,10 +48,13 @@ class Script(scripts.Script): shared.log.warning(f'MoD: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') return None self.orig_pipe = shared.sd_model + from modules.mod import StableDiffusionXLTilingPipeline + shared.sd_model = sd_models.switch_pipe(StableDiffusionXLTilingPipeline, shared.sd_model) + sd_models.set_diffuser_options(shared.sd_model) + sd_models.apply_balanced_offload(shared.sd_model) shared.log.info(f'MoD: ') - def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed): # pylint: disable=arguments-differ, unused-argument if self.orig_pipe is None: return processed