diff --git a/TODO.md b/TODO.md index fafbea8c3..ad43e6d06 100644 --- a/TODO.md +++ b/TODO.md @@ -10,10 +10,8 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates -- SD35 IPAdapter: - Flux IPAdapter: - Flux NF4: -- LTX-Video: - GGUF: ## Other @@ -21,3 +19,18 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - IPAdapter negative: - Control API enhance scripts compatibility - PixelSmith: + +## Code TODO + +- python 3.12.4 or higher cause a mess with pydantic +- enable ROCm for windows when available +- enable full VAE mode for resize-latent +- remove duplicate mask params +- fix flux loader for civitai nf4 models +- implement model in-memory caching +- hypertile vae breaks for diffusers when using non-standard sizes +- forcing reloading entire model as loading transformers only leads to massive memory usage +- lora-direct with bnb +- make lora for quantized flux +- control script process +- monkey-patch for modernui missing tabs.select event diff --git a/extensions-builtin/Lora/lora_extract.py b/extensions-builtin/Lora/lora_extract.py index c2e0a275b..1d92f3c6e 100644 --- a/extensions-builtin/Lora/lora_extract.py +++ b/extensions-builtin/Lora/lora_extract.py @@ -182,19 +182,6 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: Handle quant for Flux - # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: - # for name, module in shared.sd_model.transformer.named_modules(): - # if "norm" in name and "linear" not in name: - # continue - # weights_backup = getattr(module, "network_weights_backup", None) - # if weights_backup is None: - # continue - # module.svdhandler = SVDHandler() - # module.svdhandler.network_name = "lora_transformer_" + name.replace(".", "_") - # module.svdhandler.decompose(module.weight, weights_backup) - # module.svdhandler.findrank(rank, rank_ratio) - lora_state_dict = {} for sub in ['text_encoder', 'text_encoder_2', 'unet', 'transformer']: submodel = getattr(shared.sd_model, sub, None) diff --git a/installer.py b/installer.py index 40ba7cea0..d186cc5ac 100644 --- a/installer.py +++ b/installer.py @@ -552,7 +552,7 @@ def install_rocm_zluda(): log.info(msg) torch_command = '' if sys.platform == "win32": - # TODO after ROCm for Windows is released + # TODO enable ROCm for windows when available if args.device_id is not None: if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index c2e0a275b..c7deb2530 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -182,7 +182,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: Handle quant for Flux + # TODO: make lora for quantized flux # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: # for name, module in shared.sd_model.transformer.named_modules(): # if "norm" in name and "linear" not in name: diff --git a/modules/model_flux.py b/modules/model_flux.py index ac1370ef7..759fc9b1e 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -149,7 +149,7 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) quant_args = model_quant.create_ao_config(quant_args) if not quant_args: - return + return kwargs model_quant.load_bnb(f'Load model: type=FLUX quant={quant_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) @@ -208,7 +208,7 @@ def load_transformer(file_path): # triggered by opts.sd_unet change _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer - elif 'nf4' in quant: # TODO right now this is not working for civitai published nf4 models + elif 'nf4' in quant: # TODO fix flux loader for civitai nf4 models from modules.model_flux_nf4 import load_flux_nf4 _transformer, _text_encoder_2 = load_flux_nf4(file_path) if _transformer is not None: diff --git a/modules/model_sana.py b/modules/model_sana.py index 414f9b74d..b9f56c7c6 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -13,7 +13,7 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_ao_config(quant_args) load_args = kwargs.copy() if not quant_args: - return + return kwargs model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args} args={load_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 2842661bd..e72ecd75c 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -55,7 +55,7 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) quant_args = model_quant.create_ao_config(quant_args) if not quant_args: - return + return kwargs model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) diff --git a/modules/processing_class.py b/modules/processing_class.py index 7a7d9cd36..45ca70815 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -170,7 +170,7 @@ class StableDiffusionProcessing: self.image_cfg_scale = image_cfg_scale self.scale_by = scale_by self.mask = mask - self.image_mask = mask # TODO duplciate mask params + self.image_mask = mask # TODO remove duplicate mask params self.latent_mask = latent_mask self.mask_blur = mask_blur self.inpainting_fill = inpainting_fill diff --git a/modules/sd_models.py b/modules/sd_models.py index 5939bccbd..f7edc6ddf 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1495,7 +1495,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', unload_model_weights(op=op) sd_model = None timer = Timer() - # TODO implement caching after diffusers implement state_dict loading + # TODO implement model in-memory caching state_dict = get_checkpoint_state_dict(checkpoint_info, timer) if not shared.native else None checkpoint_config = sd_models_config.find_checkpoint_config(state_dict, checkpoint_info) timer.record("config") diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index 16540019c..20a7d7de2 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -515,7 +515,7 @@ def torchao_quantization(sd_model): if fn is None: shared.log.error(f"Quantization: type=TorchAO type={shared.opts.torchao_quantization_type} not supported") return sd_model - def torchao_model(model, op=None, sd_model=None): + def torchao_model(model, op=None, sd_model=None): # pylint: disable=unused-argument q.quantize_(model, fn(), device=devices.device) return model diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 1c6923285..de12021a5 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -4,7 +4,7 @@ import time import torch import safetensors.torch from PIL import Image -from modules import shared, devices, sd_models, errors +from modules import shared, devices, errors from modules.textual_inversion.image_embedding import embedding_from_b64, extract_image_data_embed from modules.files_cache import directory_files, directory_mtime, extension_filter