diff --git a/TODO.md b/TODO.md index ad43e6d06..bf0cecbb4 100644 --- a/TODO.md +++ b/TODO.md @@ -10,9 +10,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates -- Flux IPAdapter: -- Flux NF4: -- GGUF: +- Flux NF4 loader: ## Other @@ -22,15 +20,15 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Code TODO -- python 3.12.4 or higher cause a mess with pydantic -- enable ROCm for windows when available -- enable full VAE mode for resize-latent -- remove duplicate mask params -- fix flux loader for civitai nf4 models -- implement model in-memory caching -- hypertile vae breaks for diffusers when using non-standard sizes -- forcing reloading entire model as loading transformers only leads to massive memory usage -- lora-direct with bnb -- make lora for quantized flux -- control script process -- monkey-patch for modernui missing tabs.select event +- TODO install: python 3.12.4 or higher cause a mess with pydantic (fixme) +- TODO install: enable ROCm for windows when available (fixme) +- TODO resize image: enable full VAE mode for resize-latent (fixme) +- TODO processing: remove duplicate mask params (fixme) +- TODO flux: fix loader for civitai nf4 models (fixme) +- TODO model loader: implement model in-memory caching (fixme) +- TODO hypertile: vae breaks when using non-standard sizes (fixme) +- TODO model load: force-reloading entire model as loading transformers only leads to massive memory usage (fixme) +- TODO lora load: direct with bnb (fixme) +- TODO: lora make: support quantized flux (fixme) +- TODO control: support scripts via api (fixme) +- TODO modernui: monkey-patch for missing tabs.select event (fixme) diff --git a/installer.py b/installer.py index 07917986f..55f23c27f 100644 --- a/installer.py +++ b/installer.py @@ -430,7 +430,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): if args.quick: return log.info(f'Python: version={platform.python_version()} platform={platform.system()} bin="{sys.executable}" venv="{sys.prefix}"') - if int(sys.version_info.major) == 3 and int(sys.version_info.minor) == 12 and int(sys.version_info.micro) > 3: # TODO python 3.12.4 or higher cause a mess with pydantic + if int(sys.version_info.major) == 3 and int(sys.version_info.minor) == 12 and int(sys.version_info.micro) > 3: # TODO install: python 3.12.4 or higher cause a mess with pydantic log.error(f"Python version incompatible: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.12.3 or lower") if reason is not None: log.error(reason) @@ -552,7 +552,7 @@ def install_rocm_zluda(): log.info(msg) torch_command = '' if sys.platform == "win32": - # TODO enable ROCm for windows when available + # TODO install: enable ROCm for windows when available if args.device_id is not None: if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: diff --git a/modules/api/control.py b/modules/api/control.py index 345930341..411f71ff8 100644 --- a/modules/api/control.py +++ b/modules/api/control.py @@ -159,7 +159,7 @@ class APIControl(): output_images = [] output_processed = [] output_info = '' - # TODO control script process + # TODO control: support scripts via api # init script args, call scripts.script_control.run, call scripts.script_control.after run.control_set({ 'do_not_save_grid': not req.save_images, 'do_not_save_samples': not req.save_images, **self.prepare_ip_adapter(req) }) run.control_set(getattr(req, "extra", {})) diff --git a/modules/consistory/consistory_unet_sdxl.py b/modules/consistory/consistory_unet_sdxl.py index 4dd9b42d2..940b4ba01 100644 --- a/modules/consistory/consistory_unet_sdxl.py +++ b/modules/consistory/consistory_unet_sdxl.py @@ -916,7 +916,6 @@ class ConsistorySDXLUNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditio # 1. time timesteps = timestep if not torch.is_tensor(timesteps): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can # This would be a good case for the `match` statement (Python 3.10+) is_mps = sample.device.type == "mps" if isinstance(timestep, float): diff --git a/modules/control/run.py b/modules/control/run.py index efe9c0b8b..8cecb93af 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -311,7 +311,7 @@ def control_run(state: str = '', # processing.process_init(p) resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0 - # TODO monkey-patch for modernui missing tabs.select event + # TODO modernui: monkey-patch for missing tabs.select event if selected_scale_tab_before == 0 and resize_name_before != 'None' and scale_by_before != 1 and inputs is not None and len(inputs) > 0: shared.log.debug('Control: override resize mode=before') selected_scale_tab_before = 1 diff --git a/modules/ctrlx/__init__.py b/modules/ctrlx/__init__.py index 87ff52f6c..07d06aeac 100644 --- a/modules/ctrlx/__init__.py +++ b/modules/ctrlx/__init__.py @@ -136,7 +136,7 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, # TODO: Support prompt_2 and negative_prompt_2 + prompt: Union[str, List[str]] = None, structure_prompt: Optional[Union[str, List[str]]] = None, appearance_prompt: Optional[Union[str, List[str]]] = None, structure_image: Optional[PipelineImageInput] = None, @@ -180,7 +180,6 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 callback_on_step_end_tensor_inputs: List[str] = ["latents"], **kwargs, ): - # TODO: Add function argument documentation callback = kwargs.pop("callback", None) callback_steps = kwargs.pop("callback_steps", None) @@ -205,7 +204,7 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 target_size = target_size or (height, width) # 1. Check inputs. Raise error if not correct - self.check_inputs( # TODO: Custom check_inputs for our method + self.check_inputs( prompt, None, # prompt_2 height, @@ -425,7 +424,7 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 # 7.2 Optionally get guidance scale embedding timestep_cond = None - if self.unet.config.time_cond_proj_dim is not None: # TODO: Make guidance scale embedding work with batch_order + if self.unet.config.time_cond_proj_dim is not None: guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) timestep_cond = self.get_guidance_scale_embedding( guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim @@ -457,7 +456,6 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 register_attr(self, t=t.item(), do_control=True, batch_order=batch_order) - # TODO: For now, assume we are doing classifier-free guidance, support no CF-guidance later latent_model_input = self.scheduler.scale_model_input(latents, t) structure_latent_model_input = self.scheduler.scale_model_input(structure_latents, t) appearance_latent_model_input = self.scheduler.scale_model_input(appearance_latents, t) @@ -563,7 +561,7 @@ class CtrlXStableDiffusionXLPipeline(StableDiffusionXLPipeline): # diffusers==0 # Self-recurrence for _ in range(self_recurrence_schedule[i]): if hasattr(self.scheduler, "_step_index"): # For fancier schedulers - self.scheduler._step_index -= 1 # TODO: Does this actually work? + self.scheduler._step_index -= 1 t_prev = 0 if i + 1 >= num_inference_steps else timesteps[i + 1] latents = noise_t2t(self.scheduler, t_prev, t, latents) diff --git a/modules/face/instantid_model.py b/modules/face/instantid_model.py index 51a4d7850..543b39ded 100644 --- a/modules/face/instantid_model.py +++ b/modules/face/instantid_model.py @@ -344,7 +344,6 @@ class IPAttnProcessor(nn.Module): return hidden_states def _memory_efficient_attention_xformers(self, query, key, value, attention_mask): - # TODO attention_mask query = query.contiguous() key = key.contiguous() value = value.contiguous() diff --git a/modules/face/photomaker_model.py b/modules/face/photomaker_model.py index b62fe73b8..3595c6a36 100644 --- a/modules/face/photomaker_model.py +++ b/modules/face/photomaker_model.py @@ -244,7 +244,7 @@ class PhotoMakerStableDiffusionXLPipeline(StableDiffusionXLPipeline): prompt_embeds_list = [] prompts = [prompt, prompt_2] for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): - input_ids = tokenizer.encode(prompt) # TODO: batch encode + input_ids = tokenizer.encode(prompt) clean_index = 0 clean_input_ids = [] class_token_index = [] @@ -296,7 +296,7 @@ class PhotoMakerStableDiffusionXLPipeline(StableDiffusionXLPipeline): prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) - class_tokens_mask = class_tokens_mask.to(device=device) # TODO: ignoring two-prompt case + class_tokens_mask = class_tokens_mask.to(device=device) return prompt_embeds, pooled_prompt_embeds, class_tokens_mask @@ -332,7 +332,7 @@ class PhotoMakerStableDiffusionXLPipeline(StableDiffusionXLPipeline): callback_steps: int = 1, # Added parameters (for PhotoMaker) input_id_images: PipelineImageInput = None, - start_merge_step: int = 0, # TODO: change to `style_strength_ratio` in the future + start_merge_step: int = 0, class_tokens_mask: Optional[torch.LongTensor] = None, prompt_embeds_text_only: Optional[torch.FloatTensor] = None, pooled_prompt_embeds_text_only: Optional[torch.FloatTensor] = None, @@ -410,7 +410,7 @@ class PhotoMakerStableDiffusionXLPipeline(StableDiffusionXLPipeline): ( prompt_embeds_text_only, negative_prompt_embeds, - pooled_prompt_embeds_text_only, # TODO: replace the pooled_prompt_embeds with text only prompt + pooled_prompt_embeds_text_only, negative_pooled_prompt_embeds, ) = self.encode_prompt( prompt=prompt_text_only, @@ -431,7 +431,7 @@ class PhotoMakerStableDiffusionXLPipeline(StableDiffusionXLPipeline): if not isinstance(input_id_images[0], torch.Tensor): id_pixel_values = self.id_image_processor(input_id_images, return_tensors="pt").pixel_values - id_pixel_values = id_pixel_values.unsqueeze(0).to(device=device, dtype=dtype) # TODO: multiple prompts + id_pixel_values = id_pixel_values.unsqueeze(0).to(device=device, dtype=dtype) # 6. Get the update text embedding with the stacked ID embedding prompt_embeds = self.id_encoder(id_pixel_values, prompt_embeds, class_tokens_mask) diff --git a/modules/ggml/gguf_tensor.py b/modules/ggml/gguf_tensor.py index 4bc9117cb..8b2f608ac 100644 --- a/modules/ggml/gguf_tensor.py +++ b/modules/ggml/gguf_tensor.py @@ -131,7 +131,6 @@ class GGMLTensor(torch.Tensor): if self._ggml_quantization_type in TORCH_COMPATIBLE_QTYPES: return self.quantized_data.to(self.compute_dtype) elif self._ggml_quantization_type in DEQUANTIZE_FUNCTIONS: - # TODO(ryand): Look into how the dtype param is intended to be used. return dequantize( data=self.quantized_data, qtype=self._ggml_quantization_type, oshape=self.tensor_shape, dtype=None ).to(self.compute_dtype) diff --git a/modules/hidiffusion/hidiffusion.py b/modules/hidiffusion/hidiffusion.py index 7874f03af..d6f68eb15 100644 --- a/modules/hidiffusion/hidiffusion.py +++ b/modules/hidiffusion/hidiffusion.py @@ -234,7 +234,7 @@ def make_diffusers_transformer_block(block_class: Type[torch.nn.Module]) -> Type norm_hidden_states = self.norm2(hidden_states) norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp if self._chunk_size is not None: - ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # pylint: disable=undefined-variable # TODO hidiffusion undefined + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # pylint: disable=undefined-variable else: ff_output = self.ff(norm_hidden_states) if self.use_ada_layer_norm_zero: @@ -308,7 +308,7 @@ def make_diffusers_cross_attn_down_block(block_class: Type[torch.nn.Module]) -> self.T1 = int(self.max_timestep * self.T1_ratio) output_states = () - _scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 # TODO hidiffusion unused + _scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 blocks = list(zip(self.resnets, self.attentions)) @@ -407,7 +407,7 @@ def make_diffusers_cross_attn_up_block(block_class: Type[torch.nn.Module]) -> Ty encoder_attention_mask: Optional[torch.FloatTensor] = None, ) -> torch.FloatTensor: - def fix_scale(first, second): # TODO hidiffusion breaks hidden_scale.shape on 3rd generate with sdxl + def fix_scale(first, second): if (first.shape[-1] != second.shape[-1] or first.shape[-2] != second.shape[-2]): rescale = min(second.shape[-2] / first.shape[-2], second.shape[-1] / first.shape[-1]) # log.debug(f"HiDiffusion rescale: {hidden_states.shape} => {res_hidden_states_tuple[0].shape} scale={rescale}") diff --git a/modules/images_resize.py b/modules/images_resize.py index 5b7c816f8..362be79ee 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -11,7 +11,7 @@ def resize_image(resize_mode: int, im: Image.Image, width: int, height: int, ups def latent(im, w, h, upscaler): from modules.processing_vae import vae_encode, vae_decode import torch - latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO enable full VAE mode for resize-latent + latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO resize image: enable full VAE mode for resize-latent latents = torch.nn.functional.interpolate(latents, size=(int(h // 8), int(w // 8)), mode=upscaler["mode"], antialias=upscaler["antialias"]) im = vae_decode(latents, shared.sd_model, output_type='pil', full_quality=False)[0] return im diff --git a/modules/instantir/aggregator.py b/modules/instantir/aggregator.py index fd6151003..1950f9efa 100644 --- a/modules/instantir/aggregator.py +++ b/modules/instantir/aggregator.py @@ -823,7 +823,6 @@ class Aggregator(ModelMixin, ConfigMixin, FromOriginalModelMixin): # 1. time timesteps = timestep if not torch.is_tensor(timesteps): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can # This would be a good case for the `match` statement (Python 3.10+) is_mps = sample.device.type == "mps" if isinstance(timestep, float): diff --git a/modules/instantir/ip_adapter/attention_processor.py b/modules/instantir/ip_adapter/attention_processor.py index ed6cf755f..68dce4281 100644 --- a/modules/instantir/ip_adapter/attention_processor.py +++ b/modules/instantir/ip_adapter/attention_processor.py @@ -390,7 +390,6 @@ class AttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -496,7 +495,6 @@ class split_AttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -640,7 +638,6 @@ class sep_split_AttnProcessor2_0(torch.nn.Module): value_1 = value_1.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_0 = F.scaled_dot_product_attention( query_0, key_0, value_0, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -654,7 +651,6 @@ class sep_split_AttnProcessor2_0(torch.nn.Module): ) hidden_states_0 = hidden_states_0 + ref_scale * _hidden_states_0 * 10 - # TODO: drop this cross-attn _hidden_states_1 = F.scaled_dot_product_attention( query_1, key_0, value_0, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -771,7 +767,6 @@ class AdditiveKV_AttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -883,7 +878,6 @@ class TA_AdditiveKV_AttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -1018,7 +1012,6 @@ class IPAttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -1034,7 +1027,6 @@ class IPAttnProcessor2_0(torch.nn.Module): ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 ip_hidden_states = F.scaled_dot_product_attention( query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False ) @@ -1161,7 +1153,6 @@ class TA_IPAttnProcessor2_0(torch.nn.Module): value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -1181,7 +1172,6 @@ class TA_IPAttnProcessor2_0(torch.nn.Module): ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 ip_hidden_states = F.scaled_dot_product_attention( query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False ) @@ -1337,7 +1327,6 @@ class CNAttnProcessor2_0: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index c7deb2530..58cd065bb 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -182,7 +182,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: make lora for quantized flux + # TODO: lora make support quantized flux # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: # for name, module in shared.sd_model.transformer.named_modules(): # if "norm" in name and "linear" not in name: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index df7778ead..fc90ddd2d 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -416,7 +416,7 @@ def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if deactivate: updown *= -1 if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: - try: # TODO lora-direct with bnb + try: # TODO lora load: direct with bnb weight = bnb.functional.dequantize_4bit(self.weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) new_weight = weight.to(devices.device) + updown.to(devices.device) self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) diff --git a/modules/meissonic/pipeline_img2img.py b/modules/meissonic/pipeline_img2img.py index f26af123d..13e5c3717 100644 --- a/modules/meissonic/pipeline_img2img.py +++ b/modules/meissonic/pipeline_img2img.py @@ -56,9 +56,6 @@ class Img2ImgPipeline(DiffusionPipeline): model_cpu_offload_seq = "text_encoder->transformer->vqvae" - # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before - # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter - # off the meta device. There should be a way to fix this instead of just not offloading it _exclude_from_cpu_offload = ["vqvae"] def __init__( diff --git a/modules/meissonic/pipeline_inpaint.py b/modules/meissonic/pipeline_inpaint.py index 994846fba..d405afa53 100644 --- a/modules/meissonic/pipeline_inpaint.py +++ b/modules/meissonic/pipeline_inpaint.py @@ -53,9 +53,6 @@ class InpaintPipeline(DiffusionPipeline): model_cpu_offload_seq = "text_encoder->transformer->vqvae" - # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before - # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter - # off the meta device. There should be a way to fix this instead of just not offloading it _exclude_from_cpu_offload = ["vqvae"] def __init__( diff --git a/modules/meissonic/transformer.py b/modules/meissonic/transformer.py index 64f91baa2..43e77ddc7 100644 --- a/modules/meissonic/transformer.py +++ b/modules/meissonic/transformer.py @@ -341,11 +341,6 @@ class FusedFluxAttnProcessor2_0: key = torch.cat([encoder_hidden_states_key_proj, key], dim=2) value = torch.cat([encoder_hidden_states_value_proj, value], dim=2) - # if image_rotary_emb is not None: # TODO broken import - # from .embeddings import apply_rotary_emb - # query = apply_rotary_emb(query, image_rotary_emb) - # key = apply_rotary_emb(key, image_rotary_emb) - hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False) hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) hidden_states = hidden_states.to(query.dtype) diff --git a/modules/model_flux.py b/modules/model_flux.py index 0a13a4d46..f2286866e 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -213,7 +213,7 @@ def load_transformer(file_path): # triggered by opts.sd_unet change _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer - elif 'nf4' in quant: # TODO fix flux loader for civitai nf4 models + elif 'nf4' in quant: # TODO flux: fix loader for civitai nf4 models from modules.model_flux_nf4 import load_flux_nf4 _transformer, _text_encoder_2 = load_flux_nf4(file_path) if _transformer is not None: @@ -342,7 +342,7 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch shared.log.debug(f'Load model: type=FLUX cls={cls.__name__} preloaded={list(kwargs)} revision={diffusers_load_config.get("revision", None)}') for c in kwargs: if getattr(kwargs[c], 'quantization_method', None) is not None or getattr(kwargs[c], 'gguf', None) is not None: - shared.log.debug(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} quant={getattr(kwargs[c], 'quantization_method', None) or getattr(kwargs[c], 'gguf', None)}') + shared.log.debug(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} quant={getattr(kwargs[c], "quantization_method", None) or getattr(kwargs[c], "gguf", None)}') if kwargs[c].dtype == torch.float32 and devices.dtype != torch.float32: try: kwargs[c] = kwargs[c].to(dtype=devices.dtype) diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 36dfbd9b4..1834c573e 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -1,7 +1,7 @@ import os import diffusers import transformers -from modules import shared, devices, sd_models, sd_unet, model_te, model_quant, model_tools +from modules import shared, devices, sd_models, sd_unet, model_quant, model_tools def load_overrides(kwargs, cache_dir): diff --git a/modules/omnigen/utils.py b/modules/omnigen/utils.py index 5483d6eab..0304e1732 100644 --- a/modules/omnigen/utils.py +++ b/modules/omnigen/utils.py @@ -25,7 +25,6 @@ def update_ema(ema_model, model, decay=0.9999): """ ema_params = dict(ema_model.named_parameters()) for name, param in model.named_parameters(): - # TODO: Consider applying only to params that require_grad to avoid small numerical changes of pos_embed ema_params[name].mul_(decay).add_(param.data, alpha=1 - decay) diff --git a/modules/onnx_impl/pipelines/__init__.py b/modules/onnx_impl/pipelines/__init__.py index ca1ddd2f7..a11b07fc7 100644 --- a/modules/onnx_impl/pipelines/__init__.py +++ b/modules/onnx_impl/pipelines/__init__.py @@ -241,7 +241,7 @@ class OnnxRawPipeline(PipelineBase): for i in range(len(flow)): flow[i] = flow[i].replace("AutoExecutionProvider", shared.opts.onnx_execution_provider) olive_config["input_model"]["config"]["model_path"] = os.path.abspath(os.path.join(in_dir, submodel, "model.onnx")) - olive_config["systems"]["local_system"]["config"]["accelerators"][0]["device"] = "cpu" if shared.opts.onnx_execution_provider == ExecutionProvider.CPU else "gpu" # TODO: npu + olive_config["systems"]["local_system"]["config"]["accelerators"][0]["device"] = "cpu" if shared.opts.onnx_execution_provider == ExecutionProvider.CPU else "gpu" olive_config["systems"]["local_system"]["config"]["accelerators"][0]["execution_providers"] = [shared.opts.onnx_execution_provider] for pass_key in olive_config["passes"]: diff --git a/modules/pag/pipe_sd.py b/modules/pag/pipe_sd.py index 16f9b5319..11f4fb0cf 100644 --- a/modules/pag/pipe_sd.py +++ b/modules/pag/pipe_sd.py @@ -104,7 +104,6 @@ class PAGIdentitySelfAttnProcessor: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -219,7 +218,6 @@ class PAGCFGIdentitySelfAttnProcessor: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/pag/pipe_sdxl.py b/modules/pag/pipe_sdxl.py index 82ae06c07..3a47af3e5 100644 --- a/modules/pag/pipe_sdxl.py +++ b/modules/pag/pipe_sdxl.py @@ -124,7 +124,6 @@ class PAGIdentitySelfAttnProcessor: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -239,7 +238,6 @@ class PAGCFGIdentitySelfAttnProcessor: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/processing_class.py b/modules/processing_class.py index f502e0dcc..97e5f7b6f 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -170,7 +170,7 @@ class StableDiffusionProcessing: self.image_cfg_scale = image_cfg_scale self.scale_by = scale_by self.mask = mask - self.image_mask = mask # TODO remove duplicate mask params + self.image_mask = mask # TODO processing: remove duplicate mask params self.latent_mask = latent_mask self.mask_blur = mask_blur self.inpainting_fill = inpainting_fill diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d43660ca8..aa820aa34 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -330,13 +330,6 @@ def process_refine(p: processing.StableDiffusionProcessing, output): errors.display(e, 'Processing') modelstats.analyze() - """ # TODO decode using refiner - if not shared.state.interrupted and not shared.state.skipped: - refiner_images = processing_vae.vae_decode(latents=refiner_output.images, model=shared.sd_refiner, full_quality=True, width=max(p.width, p.hr_upscale_to_x), height=max(p.height, p.hr_upscale_to_y)) - for refiner_image in refiner_images: - results.append(refiner_image) - """ - if shared.opts.diffusers_offload_mode == "balanced": shared.sd_refiner = sd_models.apply_balanced_offload(shared.sd_refiner) elif shared.opts.diffusers_move_refiner: diff --git a/modules/pulid/eva_clip/hf_model.py b/modules/pulid/eva_clip/hf_model.py index c4b9fd85b..d148bbff2 100644 --- a/modules/pulid/eva_clip/hf_model.py +++ b/modules/pulid/eva_clip/hf_model.py @@ -31,7 +31,6 @@ from .hf_configs import arch_dict def _camel2snake(s): return re.sub(r'(? self.config.num_train_timesteps: diff --git a/modules/schedulers/scheduler_ufogen.py b/modules/schedulers/scheduler_ufogen.py index e03dec3c0..f4d8aee97 100644 --- a/modules/schedulers/scheduler_ufogen.py +++ b/modules/schedulers/scheduler_ufogen.py @@ -310,14 +310,11 @@ class UFOGenScheduler(SchedulerMixin, ConfigMixin): self.num_inference_steps = num_inference_steps self.custom_timesteps = False - # TODO: For now, handle special case when num_inference_steps == 1 separately if num_inference_steps == 1: # Set the timestep schedule to num_train_timesteps - 1 rather than 0 # (that is, the one-step timestep schedule is always trailing rather than leading or linspace) timesteps = np.array([self.config.num_train_timesteps - 1], dtype=np.int64) else: - # TODO: For now, retain the DDPM timestep spacing logic - # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 if self.config.timestep_spacing == "linspace": timesteps = ( np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) @@ -446,8 +443,6 @@ class UFOGenScheduler(SchedulerMixin, ConfigMixin): # Noise is not used on the final timestep of the timestep schedule. # This also means that noise is not used for one-step sampling. if t != self.timesteps[-1]: - # TODO: is this correct? - # Sample prev sample x_{t - 1} ~ q(x_{t - 1} | x_0 = G(x_t, t)) device = model_output.device noise = randn_tensor(model_output.shape, generator=generator, device=device, dtype=model_output.dtype) sqrt_alpha_prod_t_prev = alpha_prod_t_prev**0.5 diff --git a/modules/schedulers/scheduler_vdm.py b/modules/schedulers/scheduler_vdm.py index 543b29ff3..492c30a0c 100644 --- a/modules/schedulers/scheduler_vdm.py +++ b/modules/schedulers/scheduler_vdm.py @@ -147,7 +147,7 @@ class VDMScheduler(SchedulerMixin, ConfigMixin): self.timesteps = torch.from_numpy(self.get_timesteps(len(self))) if num_train_timesteps: alphas_cumprod = self.alphas_cumprod(torch.flip(self.timesteps, dims=(0,))) - alphas = alphas_cumprod[1:] / alphas_cumprod[:-1] # TODO: Might not be exact + alphas = alphas_cumprod[1:] / alphas_cumprod[:-1] self.alphas = torch.cat([alphas_cumprod[:1], alphas]) self.betas = 1 - self.alphas diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index dbf977b8d..69c4163dc 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -112,7 +112,7 @@ def split_attention(layer: nn.Module, tile_size: int=256, min_tile_size: int=128 out = forward(x, *args[1:], **kwargs) return out if x.ndim == 4: # VAE - # TODO hypertile vae breaks for diffusers when using non-standard sizes + # TODO hypertile: vae breaks when using non-standard sizes if nh * nw > 1: x = rearrange(x, "b c (nh h) (nw w) -> (b nh nw) c h w", nh=nh, nw=nw) out = forward(x, *args[1:], **kwargs) diff --git a/modules/sd_models.py b/modules/sd_models.py index 9daccf11f..087e8ecfc 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1496,7 +1496,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', unload_model_weights(op=op) sd_model = None timer = Timer() - # TODO implement model in-memory caching + # TODO model loader: implement model in-memory caching state_dict = get_checkpoint_state_dict(checkpoint_info, timer) if not shared.native else None checkpoint_config = sd_models_config.find_checkpoint_config(state_dict, checkpoint_info) timer.record("config") diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 2ac7949a6..a90ceec27 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -51,16 +51,6 @@ def single_sample_to_image(sample, approximation=None): return Image.new(mode="RGB", size=(512, 512)) if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] - """ - # TODO remove - if shared.native: # [-x,x] to [-5,5] - sample_max = torch.max(sample) - if sample_max > 5: - sample = sample * (5 / sample_max) - sample_min = torch.min(sample) - if sample_min < -5: - sample = sample * (5 / abs(sample_min)) - """ if approximation == 2: # TAESD if shared.opts.live_preview_downscale and (sample.shape[-1] > 128 or sample.shape[-2] > 128): scale = 128 / max(sample.shape[-1], sample.shape[-2]) diff --git a/modules/sd_unet.py b/modules/sd_unet.py index f730bdb74..deb0b24b0 100644 --- a/modules/sd_unet.py +++ b/modules/sd_unet.py @@ -36,7 +36,7 @@ def load_unet(model): model.prior_pipe.text_encoder = prior_text_encoder.to(devices.device, dtype=devices.dtype) elif "Flux" in model.__class__.__name__ or "StableDiffusion3" in model.__class__.__name__: loaded_unet = shared.opts.sd_unet - sd_models.load_diffuser() # TODO forcing reloading entire model as loading transformers only leads to massive memory usage + sd_models.load_diffuser() # TODO model load: force-reloading entire model as loading transformers only leads to massive memory usage """ from modules.model_flux import load_transformer transformer = load_transformer(unet_dict[shared.opts.sd_unet]) diff --git a/modules/segmoe/segmoe_model.py b/modules/segmoe/segmoe_model.py index 4b96527be..a542c1fbb 100644 --- a/modules/segmoe/segmoe_model.py +++ b/modules/segmoe/segmoe_model.py @@ -136,7 +136,7 @@ class SegMoEPipeline: memory_format=torch.channels_last, ) - def to(self, *args, **kwargs): # TODO added no-op to avoid error + def to(self, *args, **kwargs): self.pipe.to(*args, **kwargs) def load_from_scratch(self, config: str, **kwargs) -> None: @@ -202,7 +202,6 @@ class SegMoEPipeline: self.config["down_idx_start"] = self.down_idx_start self.config["down_idx_end"] = self.down_idx_end - # TODO: Add Support for Scheduler Selection self.pipe.scheduler = DDPMScheduler.from_config(self.pipe.scheduler.config) # Load Experts @@ -242,7 +241,6 @@ class SegMoEPipeline: **kwargs, ) - # TODO: Add Support for Scheduler Selection expert.scheduler = DDPMScheduler.from_config( expert.scheduler.config ) diff --git a/modules/xadapter/adapter.py b/modules/xadapter/adapter.py index 69030fae3..4096f71e7 100644 --- a/modules/xadapter/adapter.py +++ b/modules/xadapter/adapter.py @@ -266,8 +266,6 @@ class Adapter_XL(nn.Module): b, c, _, _ = x[-1].shape if t is not None: if not torch.is_tensor(t): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can - # This would be a good case for the `match` statement (Python 3.10+) is_mps = x[0].device.type == "mps" if isinstance(timestep, float): dtype = torch.float32 if is_mps else torch.float64 diff --git a/modules/xadapter/unet_adapter.py b/modules/xadapter/unet_adapter.py index 5022f1847..5890c7749 100644 --- a/modules/xadapter/unet_adapter.py +++ b/modules/xadapter/unet_adapter.py @@ -807,8 +807,6 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) # 1. time timesteps = timestep if not torch.is_tensor(timesteps): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can - # This would be a good case for the `match` statement (Python 3.10+) is_mps = sample.device.type == "mps" if isinstance(timestep, float): dtype = torch.float32 if is_mps else torch.float64 @@ -1012,7 +1010,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) if is_bridge: if up_block_additional_residual[0].shape != sample.shape: - pass # TODO VM patch + pass elif fusion_guidance_scale is not None: sample = sample + fusion_guidance_scale * (up_block_additional_residual.pop(0) - sample) else: @@ -1051,7 +1049,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) ################# bridge usage ################# if is_bridge and len(up_block_additional_residual) > 0: if sample.shape != up_block_additional_residual[0].shape: - pass # TODO VM PATCH + pass elif fusion_guidance_scale is not None: sample = sample + fusion_guidance_scale * (up_block_additional_residual.pop(0) - sample) else: diff --git a/scripts/instantir.py b/scripts/instantir.py index 5eb7d503a..6ab7733fe 100644 --- a/scripts/instantir.py +++ b/scripts/instantir.py @@ -80,7 +80,7 @@ class Script(scripts.Script): devices.torch_gc() def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=arguments-differ, unused-argument - # TODO instantir is a mess to unload + # TODO instantir: a mess to unload """ if self.orig_pipe is None: return processed diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 127466701..c1283e1b6 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -75,7 +75,7 @@ class Script(scripts.Script): if model_name != model_loaded or c != 'StableVideoDiffusionPipeline': shared.opts.sd_model_checkpoint = model_path sd_models.reload_model_weights() - shared.sd_model = shared.sd_model.to(torch.float32) # TODO svd runs in fp32 + shared.sd_model = shared.sd_model.to(torch.float32) # TODO svd: runs in fp32 causing dtype mismatch # set params if override_resolution: