diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index ff6275405..643758b4c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -4,6 +4,9 @@ on: - push - pull_request +permissions: + contents: read + jobs: lint: runs-on: ubuntu-24.04 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5403ef860..103549298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2026-06-14 +## Update for 2026-06-16 -### Highlights for 2026-06-14 +### Highlights for 2026-06-16 *What's New?* - **Ideogram-4** released, Microsoft joins the game with **Lens** and **Anima** made it to release version @@ -16,9 +16,11 @@ And we have new [Home page](https://vladmandic.github.io/sdnext/) with heavily u Plus continued work on modernization of codebase: UI is now fully TypeScript based And we have a new modular LoRA loader, new native Transformers loader and improved 3rd party finetunes support! -[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) +*Note*: This is a major update due to sheer size of the changes: over 400 commits! -### Details for 2026-06-14 +[Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) + +### Details for 2026-06-16 - **Models** - [CircleStone Anima 1.0](https://huggingface.co/circlestone-labs/Anima) in *Base* and *Turbo* (distilled) variants @@ -150,8 +152,21 @@ And we have a new modular LoRA loader, new native Transformers loader and improv - `samplers` ui sigma methods - `xpu` generator on non-cpu - `compel` compatibility with *transformers==5* + - `gallery` open folder + - `seedvr` unload after upscale + - `tinyvae` with anima - `mixture-tiling` fix for non-square images, thanks @QualiaRain - `prompts-from-file` fix metadata handling, thanks @QualiaRain + - `hypertile` correct width/height assignment, thanks @QualiaRain + - custom allowed-paths, thanks @QualiaRain + - bias dtype, thanks @QualiaRain + - `ipadapter` mask accumulation, thanks @QualiaRain + - `lora` no-lora check, thanks @QualiaRain + - control `video` processing, thanks @QualiaRain + - `freescale` correct width/height assignment, thanks @QualiaRain + - noise `lerp` inversion, thanks @QualiaRain + - additional safety checks, thanks @QualiaRain + - `remote vae` shadowing, thanks @QualiaRain ## Update for 2026-05-13 diff --git a/modules/api/api.py b/modules/api/api.py index bb7d78bd6..86c67ae84 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -17,12 +17,12 @@ class Api: self.credentials = {} if shared.cmd_opts.auth: for auth in shared.cmd_opts.auth.split(","): - user, password = auth.split(":") + user, password = auth.split(":", 1) self.credentials[user.replace('"', '').strip()] = password.replace('"', '').strip() if shared.cmd_opts.auth_file: with open(shared.cmd_opts.auth_file, encoding="utf8") as file: for line in file.readlines(): - user, password = line.split(":") + user, password = line.split(":", 1) self.credentials[user.replace('"', '').strip()] = password.replace('"', '').strip() self.router = APIRouter() if shared.cmd_opts.docs: diff --git a/modules/api/gallery.py b/modules/api/gallery.py index b585ad896..5c38ea8a3 100644 --- a/modules/api/gallery.py +++ b/modules/api/gallery.py @@ -3,7 +3,6 @@ import os import time import base64 from urllib.parse import quote, unquote -from fastapi import FastAPI from fastapi.responses import JSONResponse from starlette.websockets import WebSocket, WebSocketState from pydantic import BaseModel, Field # pylint: disable=no-name-in-module @@ -173,7 +172,7 @@ def register_api(api): # register api unique_folders.append(f) if shared.demo is not None and path not in shared.demo.allowed_paths: debug(f'Browser folders allow: {path}') - shared.demo.allowed_paths.append(quote(path)) + shared.demo.allowed_paths.append(path) debug(f'Browser folders: {unique_folders}') return JSONResponse(content=unique_folders) diff --git a/modules/api/generate.py b/modules/api/generate.py index 678292922..783a78523 100644 --- a/modules/api/generate.py +++ b/modules/api/generate.py @@ -70,6 +70,7 @@ class APIGenerate: p.ip_adapter_starts = [] p.ip_adapter_ends = [] p.ip_adapter_images = [] + p.ip_adapter_masks = [] for ipadapter in request.ip_adapter: if not ipadapter.images or len(ipadapter.images) == 0: continue @@ -79,7 +80,6 @@ class APIGenerate: p.ip_adapter_starts.append(ipadapter.start) p.ip_adapter_ends.append(ipadapter.end) p.ip_adapter_images.append([helpers.decode_base64_to_image(x) for x in ipadapter.images]) - p.ip_adapter_masks = [] if ipadapter.masks: p.ip_adapter_masks.append([helpers.decode_base64_to_image(x) for x in ipadapter.masks]) del request.ip_adapter diff --git a/modules/api/process.py b/modules/api/process.py index 9c3925a51..1abf3950b 100644 --- a/modules/api/process.py +++ b/modules/api/process.py @@ -76,7 +76,7 @@ class APIProcess: if processor is None or processor.processor_id != req.model: with self.queue_lock: processor = processors.Processor(req.model) - for k, v in req.params.items(): + for k, v in (req.params or {}).items(): if k not in processors.config[processor.processor_id]['params']: return JSONResponse(status_code=400, content={"error": f"Processor invalid parameter: id={req.model} {k}={v}"}) jobid = shared.state.begin('API-PRE', api=True) @@ -102,7 +102,7 @@ class APIProcess: return JSONResponse(status_code=400, content={"error": f"Mask type not found: id={req.type}"}) image = decode_base64_to_image(req.image) mask = decode_base64_to_image(req.mask) if req.mask else None - for k, v in req.params.items(): + for k, v in (req.params or {}).items(): if not hasattr(masking.opts, k): return JSONResponse(status_code=400, content={"error": f"Mask invalid parameter: {k}={v}"}) else: diff --git a/modules/control/run.py b/modules/control/run.py index fa021b92e..086c7eac5 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -644,7 +644,6 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg debug_log(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}') status = True frame = None - video = None output_filename = None index = 0 frames = 0 @@ -697,7 +696,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg inputs = [Image.fromarray(frame)] # cv2 to pil for i, input_image in enumerate(inputs): # loop per-input, but with early-break if pipe is None: # pipe may have been reset externally - if video is None: + if cap is None: break # non-video: pipeline was consumed, no need to re-process remaining inputs pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, active_units, control_conditioning, control_guidance_start, control_guidance_end, inits) debug_log(f'Control pipeline reinit: class={pipe.__class__.__name__}') @@ -744,7 +743,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg init_image = inits[i % len(inits)] else: init_image = None - if video is not None and index % (video_skip_frames + 1) != 0: + if cap is not None and index % (video_skip_frames + 1) != 0: index += 1 continue index += 1 @@ -806,13 +805,13 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg if processed_image is not None and isinstance(processed_image, Image.Image): output_images.append(processed_image) - if is_generator and frame is not None and video is not None: + if is_generator and frame is not None and cap is not None: image_txt = f'{output_image.width}x{output_image.height}' if output_image is not None else 'None' msg = f'Control output | {index} of {frames} skip {video_skip_frames} | Frame {image_txt}' yield (output_image, blended_image, msg) # result is control_output, proces_output - if video is not None and frame is not None: - status, frame = video.read() + if cap is not None and frame is not None: + status, frame = cap.read() if status: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) debug_log(f'Control: video frame={index} frames={frames} status={status} skip={index % (video_skip_frames + 1)} progress={index/frames:.2f}') diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 8efbce5a5..96175a33b 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -179,7 +179,7 @@ def api_list_models(model_type: str | None = None): model_list += list(predefined_qwen) if model_type == 'hunyuandit' or model_type == 'all': model_list += list(predefined_hunyuandit) - if model_type == 'zimage': + if model_type == 'zimage' or model_type == 'all': model_list += list(predefined_zimage) model_list += sorted(find_models()) return model_list diff --git a/modules/control/units/lite_model.py b/modules/control/units/lite_model.py index ef6d444f9..3095d7cfe 100644 --- a/modules/control/units/lite_model.py +++ b/modules/control/units/lite_model.py @@ -171,7 +171,7 @@ class ControlNetLLLite(torch.nn.Module): # pylint: disable=abstract-method mapped_block, mapped_number = map_down_lllite_to_unet[int(block)] b = model.down_blocks[mapped_block].attentions[int(mapped_number)].transformer_blocks[int(block_number)] elif root == 'output': - pass # not implemented + continue # not implemented else: b = model.mid_block.attentions[0].transformer_blocks[int(block_number)] b = getattr(b, attn_name, None) diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index 1eb8a69bd..22c6019fc 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -122,7 +122,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): log.warning(msg) yield msg return - if loaded_lora() == "": + if not loaded_lora(): msg = "LoRA extract: no LoRA detected" log.warning(msg) yield msg @@ -141,8 +141,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): with rp.Progress(rp.TextColumn('[cyan]LoRA extract'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=console) as progress: if 'te' in modules and getattr(shared.sd_model, 'text_encoder', None) is not None: - modules = shared.sd_model.text_encoder.named_modules() - task = progress.add_task(description="te1 decompose", total=len(list(modules))) + task = progress.add_task(description="te1 decompose", total=len(list(shared.sd_model.text_encoder.named_modules()))) for name, module in shared.sd_model.text_encoder.named_modules(): progress.update(task, advance=1) weights_backup = getattr(module, "network_weights_backup", None) @@ -157,8 +156,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): t1 = time.time() if 'te' in modules and getattr(shared.sd_model, 'text_encoder_2', None) is not None: - modules = shared.sd_model.text_encoder_2.named_modules() - task = progress.add_task(description="te2 decompose", total=len(list(modules))) + task = progress.add_task(description="te2 decompose", total=len(list(shared.sd_model.text_encoder_2.named_modules()))) for name, module in shared.sd_model.text_encoder_2.named_modules(): progress.update(task, advance=1) weights_backup = getattr(module, "network_weights_backup", None) @@ -172,8 +170,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): t2 = time.time() if 'unet' in modules and getattr(shared.sd_model, 'unet', None) is not None: - modules = shared.sd_model.unet.named_modules() - task = progress.add_task(description="unet decompose", total=len(list(modules))) + task = progress.add_task(description="unet decompose", total=len(list(shared.sd_model.unet.named_modules()))) for name, module in shared.sd_model.unet.named_modules(): progress.update(task, advance=1) weights_backup = getattr(module, "network_weights_backup", None) diff --git a/modules/lora/network.py b/modules/lora/network.py index 576f5831c..5926cd4fa 100644 --- a/modules/lora/network.py +++ b/modules/lora/network.py @@ -166,6 +166,7 @@ class NetworkModule: self.network_key = weights.network_key self.sd_key = weights.sd_key self.sd_module = weights.sd_module + self.shape = None if hasattr(self.sd_module, 'weight'): if hasattr(self.sd_module, "sdnq_dequantizer"): self.shape = self.sd_module.sdnq_dequantizer.original_shape @@ -176,7 +177,7 @@ class NetworkModule: self.alpha = weights.w["alpha"].item() if "alpha" in weights.w else None self.scale = weights.w["scale"].item() if "scale" in weights.w else None self.dora_scale = weights.w.get("dora_scale", None) - self.dora_norm_dims = len(self.shape) - 1 + self.dora_norm_dims = (len(self.shape) - 1) if self.shape is not None else None def multiplier(self): unet_multiplier = 3 * [self.network.unet_multiplier] if not isinstance(self.network.unet_multiplier, list) else self.network.unet_multiplier diff --git a/modules/openai/test.py b/modules/openai/test.py index 8c37104bb..9c9d662b2 100644 --- a/modules/openai/test.py +++ b/modules/openai/test.py @@ -20,7 +20,13 @@ model = AutoModelForCausalLM.from_pretrained( dtype=torch.bfloat16, trust_remote_code=True, attn_implementation="sdpa", + # attn_implementation="eager", + # attn_implementation="flash_attention_2", ) +model.config.use_flash_attention = True + +logger.log.info("OpenAI: eval model...") +model.eval() tokenizer = AutoTokenizer.from_pretrained( "Qwen/Qwen3-0.6B", trust_remote_code=True diff --git a/modules/postprocess/seedvr_model.py b/modules/postprocess/seedvr_model.py index fb7c94c79..9e1e5aebe 100644 --- a/modules/postprocess/seedvr_model.py +++ b/modules/postprocess/seedvr_model.py @@ -47,7 +47,10 @@ class UpscalerSeedVR(Upscaler): self.model.dit.dtype = devices.dtype self.model.vae_encode = self.vae_encode self.model.vae_decode = self.vae_decode - self.model.model_step = generation.generation_step + # Patch generation_loop's generation_step() with our wrapper; stash the original once + # so reloads don't re-wrap the wrapper itself (infinite recursion). + if not hasattr(generation, "generation_step_original"): + generation.generation_step_original = generation.generation_step generation.generation_step = self.model_step self.model._internal_dict = { 'dit': self.model.dit, @@ -119,6 +122,7 @@ class UpscalerSeedVR(Upscaler): return samples def model_step(self, *args, **kwargs): + from modules.seedvr.src.core import generation from modules.seedvr.src.optimization import memory_manager self.model.vae = self.model.vae.to(device="cpu") self.model.dit = self.model.dit.to(device=self.device) @@ -126,7 +130,7 @@ class UpscalerSeedVR(Upscaler): log.debug(f'Upscaler inference: args={len(args)} kwargs={list(kwargs.keys())}') memory_manager.preinitialize_rope_cache(self.model) with devices.inference_context(): - result = self.model.model_step(*args, **kwargs) + result = generation.generation_step_original(*args, **kwargs) self.model.dit = self.model.dit.to(device="cpu") devices.torch_gc() return result diff --git a/modules/processing_vae.py b/modules/processing_vae.py index b793140b5..a889b1748 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -246,13 +246,21 @@ def vae_postprocess(tensor, model, output_type='np'): if hasattr(model, 'video_processor'): if tensor.ndim == 6 and tensor.shape[1] == 1: tensor = tensor.squeeze(0) - images = model.video_processor.postprocess_video(tensor, output_type='pil') + try: + images = model.video_processor.postprocess_video(tensor, output_type='pil') + except Exception as e: + log.warning(f'VAE postprocess: type=video {e}') + images = tensor if isinstance(images, list) and len(images) > 0 and isinstance(images[0], list): images = [frame for batch in images for frame in batch] elif hasattr(model, 'image_processor'): if tensor.ndim == 5 and tensor.shape[1] == 3: # Qwen Image tensor = tensor[:, :, 0] - images = model.image_processor.postprocess(tensor, output_type=output_type) + try: + images = model.image_processor.postprocess(tensor, output_type=output_type) + except Exception as e: + log.warning(f'VAE postprocess: type=image {e}') + images = tensor elif hasattr(model, "vqgan"): images = tensor.permute(0, 2, 3, 1).cpu().float().numpy() if output_type == "pil": @@ -263,6 +271,12 @@ def vae_postprocess(tensor, model, output_type='np'): if tensor.ndim == 5 and tensor.shape[1] == 3: # Qwen Image tensor = tensor[:, :, 0] images = model.image_processor.postprocess(tensor, output_type=output_type) + if torch.is_tensor(images): # failed to postprocess, do naive conversion + images = images.permute(0, 2, 3, 1).cpu().float().numpy() + if images.min() < 0 or images.max() > 1: + images = (images - images.min()) / (images.max() - images.min()) # naive normalization + if output_type == "pil": + images = model.numpy_to_pil(images) else: images = tensor if isinstance(tensor, list) or isinstance(tensor, np.ndarray) else [tensor] except Exception as e: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index f87d7ae25..045d46a28 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -187,6 +187,7 @@ def guess_by_diffusers(fn, current_guess): cls = index.get('_class_name', None) if isinstance(cls, list): cls = cls[-1] + pipeline = None if cls is not None: pipeline = getattr(diffusers, cls, None) if pipeline is None: diff --git a/modules/sd_hijack_accelerate.py b/modules/sd_hijack_accelerate.py index 0467c536b..45e892d1e 100644 --- a/modules/sd_hijack_accelerate.py +++ b/modules/sd_hijack_accelerate.py @@ -84,7 +84,7 @@ def torch_conv_forward(self, input, weight, bias): # pylint: disable=redefined-b if self.padding_mode != 'zeros': return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode), weight, bias, self.stride, _pair(0), self.dilation, self.groups) # pylint: disable=protected-access if weight.dtype != bias.dtype: - bias.to(weight.dtype) + bias = bias.to(weight.dtype) return F.conv2d(input, weight, bias, self.stride, self.padding, self.dilation, self.groups) def hijack_torch_conv(): diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index b072df7ad..eead36266 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -259,8 +259,8 @@ def set_resolution(p, hr=False): if hr: x = getattr(p, 'hr_upscale_to_x', 0) y = getattr(p, 'hr_upscale_to_y', 0) - width = y if y > 0 else p.width - height = x if x > 0 else p.height + width = x if x > 0 else p.width + height = y if y > 0 else p.height else: width = p.width height = p.height diff --git a/modules/sd_hijack_te.py b/modules/sd_hijack_te.py index e947ec4f2..c1163eabb 100644 --- a/modules/sd_hijack_te.py +++ b/modules/sd_hijack_te.py @@ -8,7 +8,7 @@ def hijack_encode_prompt(*args, **kwargs): jobid = shared.state.begin('TE Encode') t0 = time.time() if 'max_sequence_length' in kwargs and kwargs['max_sequence_length'] is not None: - kwargs['max_sequence_length'] = max(kwargs['max_sequence_length'], os.environ.get('MAX_SEQUENCE_LENGTH', 256)) + kwargs['max_sequence_length'] = max(kwargs['max_sequence_length'], int(os.environ.get('MAX_SEQUENCE_LENGTH', 256))) res = None try: args_copy = list(args) diff --git a/modules/ui_common.py b/modules/ui_common.py index b190046fe..6e4201824 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -264,6 +264,8 @@ def save_files(js_data, files, html_info, index): def open_folder(result_gallery, gallery_index = 0): try: + if gallery_index >= len(result_gallery): + gallery_index = 0 folder = os.path.dirname(result_gallery[gallery_index]['name']) except Exception: folder = shared.opts.outdir_samples diff --git a/modules/video_models/video_prompt.py b/modules/video_models/video_prompt.py index bbb29c409..963813e9d 100644 --- a/modules/video_models/video_prompt.py +++ b/modules/video_models/video_prompt.py @@ -1,7 +1,7 @@ from modules import shared, extra_networks, ui_video_vlm -def prepare_prompt(p, init_image, prompt:str, vlm_enhance:bool, vlm_model:str, vlm_system_prompt:str): +def prepare_prompts(p, init_image, prompt:str, vlm_enhance:bool, vlm_model:str, vlm_system_prompt:str): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) shared.prompt_styles.apply_styles_to_extra(p) @@ -18,4 +18,7 @@ def prepare_prompt(p, init_image, prompt:str, vlm_enhance:bool, vlm_model:str, v ) if new_prompt is not None and len(new_prompt) > 0: prompt = new_prompt - return prompt + + p.styles = [] + p.task_args['prompt'] = p.prompt + p.task_args['negative_prompt'] = p.negative_prompt diff --git a/modules/video_models/video_run.py b/modules/video_models/video_run.py index b23fd98e3..569220a45 100644 --- a/modules/video_models/video_run.py +++ b/modules/video_models/video_run.py @@ -95,12 +95,11 @@ def generate(*args, **kwargs): shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) devices.torch_gc(force=True, reason='video') - prompt = video_prompt.prepare_prompt(p, init_image, prompt, vlm_enhance, vlm_model, vlm_system_prompt) # set args + video_prompt.prepare_prompts(p, init_image, prompt, vlm_enhance, vlm_model, vlm_system_prompt) processing.fix_seed(p) video_vae.set_vae_params(p) - video_utils.set_prompt(p) p.task_args['num_inference_steps'] = p.steps p.task_args['width'] = p.width p.task_args['height'] = p.height diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 054486648..18ae1249d 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -30,15 +30,6 @@ def check_av(): return av -def set_prompt(p): - p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) - p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) - shared.prompt_styles.apply_styles_to_extra(p) - p.styles = [] - p.task_args['prompt'] = p.prompt - p.task_args['negative_prompt'] = p.negative_prompt - - def hijack_encode_image(*args, **kwargs): t0 = time.time() try: