From c66342e2e630f3c01e242dac2ea2813ec68f6c15 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 19 Aug 2026 18:03:39 +0200 Subject: [PATCH] fallback unpack-latents, log client auth, update login page Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 10 +++++-- TODO.md | 46 +++++++++++++++----------------- modules/api/api.py | 14 +++++++++- modules/processing_callbacks.py | 17 ++++++++---- modules/processing_correction.py | 5 +++- modules/processing_vae.py | 6 +++-- modules/sd_hijack_triton.py | 2 +- pipelines/model_krea2.py | 36 ++++++++++++++----------- ui/js/login.js | 28 ++++++++++--------- 9 files changed, 100 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7200c1281..73acbf7fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Main focus is improving video workflows which also brings full support for new [ *What else?* - [Detailer.next](https://vladmandic.github.io/sdnext-docs/Detailer) with new support for *vision-language models* and *per-class prompts* -- New [group offload](https://vladmandic.github.io/sdnext-docs/Offload/#group) option which is more aggressive thant the default balanced offload +- New [group offload](https://vladmandic.github.io/sdnext-docs/Offload/#group) option which is more aggressive than the default balanced offload - Extended model support for [Nunchaku-Lite](https://github.com/rootonchair/nunchaku-lite) engine - A lot of [SDNQ](https://github.com/Disty0/sdnq) *quantization and attention* optimizations and features @@ -41,8 +41,10 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - [SDNQ](https://github.com/Disty0/sdnq) is now a separate package and no longer part of sdnext repo installed and used internally by sd.next, but also supported by diffusers natively and sdnq development brings a lot of new optimizations, in both quantization and attention mechanisms -- **Server** +- **Compute** + - torch-rocm for windows switch to *whl-multi-arch* distribution - nunchaku-lite support for `torch==2.13` +- **Server** - update handlers for all authenticated workflows - update handlers for all hf-based progress bars - offload options take effect immediately without restart/reload @@ -52,6 +54,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - add settings -> model load -> *offload state dict* option reduces memory spikes during model load at the cost of disk i/o and slower load times - use `GRADIO_TEMP_DIR` env variable for temp folder if set + - update ui login form - **Video** - reorganized *video* tab - better support for video codeces and formats @@ -62,6 +65,7 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa *note*: video api uses async workflow where you submit request and then later download the result - authentication for websocket connection - allowed path validation for endpoints that get/put files + - log auth methods - **Other** - Krea2: add *settings -> model options -> krea2 dense masking* may provide significant speed-up on some gpus, disabled by default @@ -96,6 +100,8 @@ Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automa - gguf: transformer loader - scripts: mixture-of-diffusers and mixture-tiling update to use igwn-segments - api: process + - api: auth via remote-ip + - krea2: fallback to base pipeline/transformer for nunchaku-lite ## Update for 2026-08-07 diff --git a/TODO.md b/TODO.md index aadd26252..81adad48d 100644 --- a/TODO.md +++ b/TODO.md @@ -2,24 +2,33 @@ ## Short-term -- Test API remote -- Create pre-quant for ltx -- Create pre-quant for minimax-turbo -- Update ltx wiki -- Productize benchmark tool -- Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322 +- Update LTX wiki, @CalamitousFelicitousness +- Productize benchmark tool, @CalamitousFelicitousness +- Nunchaku-Lite Krea2 errors, @vladmandic +- Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322, @vladmandic +- Lora: new handler, @CalamitousFelicitousness +- Control tab verify overrides handling, @vladmandic +- Create pre-quant for LTX-2.5 +- Create pre-quant for MiniMax-H3-Turbo ## Features +### Roadmap + +- Video upscaling: nvidia-vfx, ltx-upscaler, etc. +- Video capabilities to processing tab, add RIFE, upscaling (once available) +- Object clear remover for Kanvas: [Object clear](https://huggingface.co/jixin0101/ObjectClear) +- OpenAI API interface for image generation +- Lightweight scheduler/queue manager +- Distraction-free UI mode with prompt-only, chat-based interface +- Revisit transformer caching for modular pipelines +- Revisit guidance for modular pipelines +- Implement modular for some image models +- Video models: support finetunes + ### Assigned -- Chat-based interface, @vladmandic -- Control tab verify overrides handling, @vladmandic -- Cloud providers, @CalamitousFelicitousness -- Lora: new handler, @CalamitousFelicitousness -- Processing -> Video capabilities, @vladmandic - - `RIFE` in processing -- - [Object clear](https://huggingface.co/jixin0101/ObjectClear) remover for Kanvas, @vladmandic +- Support cloud providers, @CalamitousFelicitousness ### Unassigned @@ -33,17 +42,6 @@ - Integrate natural language image search - [ImageDB](https://github.com/vladmandic/imagedb) -### Roadmap - -- Object clear remover for Kanvas -- OpenAI API interface for image generation -- Lightweight scheduler/queue manager -- Distraction-free UI mode with prompt-only -- Revisit transformer caching for modular pipelines -- Revisit guidance for modular pipelines -- Implement modular for some image models -- Video models: support finetunes - ### OnHold - [nVidia-VFX](https://pypi.org/project/nvidia-vfx/): not compatible with latest nVidia drivers diff --git a/modules/api/api.py b/modules/api/api.py index 954a0dc87..1410481df 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -11,6 +11,7 @@ from modules.api import models, endpoints, script, helpers, server, generate, pr errors.install() +auth_map = [] class Api: @@ -186,6 +187,13 @@ class Api: self.app.add_api_route(f'{shared.opts.subpath}{path}', endpoint=fn, **kwargs) self.app.add_api_route(path, endpoint=fn, **kwargs) + def add_auth(self, host: str, user: str, method: str): + msg = f"ip={host} user={user} method={method}" + if msg in auth_map: + return + auth_map.append(msg) + log.debug(f'Client auth: {msg}') + def auth( self, request: Request, # pylint: disable=unused-argument @@ -194,16 +202,20 @@ class Api: access_token_unsecure: Optional[str] = Cookie(default=None, alias="access-token-unsecure"), ): if not self.credentials: + self.add_auth(host=request.client.host, user=credentials.username if credentials else None, method="none") return True if (credentials is not None) and (credentials.username in self.credentials): if compare_digest(credentials.password, self.credentials[credentials.username]): # client user + encoded password + self.add_auth(host=request.client.host, user=credentials.username if credentials else None, method="digest") return True if hasattr(self.app, 'tokens') and (self.app.tokens is not None): # client sends token as password if credentials.password in self.app.tokens.keys(): + self.add_auth(host=request.client.host, user=credentials.username if credentials else None, method="token") return True cookie_token = access_token or access_token_unsecure if cookie_token and hasattr(self.app, 'tokens') and (self.app.tokens is not None): # client sets cookie with token if cookie_token in self.app.tokens.keys(): + self.add_auth(host=request.client.host, user=None, method="cookie") return True log.error(f'API authentication: user="{credentials.username if credentials else None}"') raise HTTPException(status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": "Basic"}) @@ -212,7 +224,7 @@ class Api: """Log a new browser session with client IP, authenticated user, and user-agent string.""" token = req.cookies.get("access-token") or req.cookies.get("access-token-unsecure") user = self.app.tokens.get(token) if hasattr(self.app, 'tokens') else None - log.info(f'Browser session: user={user} client={req.client.host} agent={agent}') + log.info(f'Client session: user={user} client={req.client.host} agent={agent}') return {} def launch(self): diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 4fa2eebc3..48ac6e75b 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -139,11 +139,18 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict | No else: width = getattr(p, 'width', 1024) height = getattr(p, 'height', 1024) - shared.state.current_latent = pipe._unpack_latents(latents, height, width, pipe.vae_scale_factor) # pylint: disable=protected-access - if current_noise_pred is not None: - shared.state.current_noise_pred = pipe._unpack_latents(current_noise_pred, height, width, pipe.vae_scale_factor) # pylint: disable=protected-access - else: - shared.state.current_noise_pred = current_noise_pred + try: + shared.state.current_latent = pipe._unpack_latents(latents, height, width, pipe.vae_scale_factor) # pylint: disable=protected-access + if current_noise_pred is not None: + shared.state.current_noise_pred = pipe._unpack_latents(current_noise_pred, height, width, pipe.vae_scale_factor) # pylint: disable=protected-access + else: + shared.state.current_noise_pred = current_noise_pred + except Exception: + shared.state.current_latent = pipe._unpack_latents(latents, height, width) # pylint: disable=protected-access # pythoning ask-for-forgiveness if method does not support vae_scale_factor + if current_noise_pred is not None: + shared.state.current_noise_pred = pipe._unpack_latents(current_noise_pred, height, width) # pylint: disable=protected-access # pythoning ask-for-forgiveness if method does not support vae_scale_factor + else: + shared.state.current_noise_pred = current_noise_pred elif hasattr(pipe, "_unpatchify_latents"): # FLUX.2 - unpack [B, seq, patch_ch] to [B, ch, H, W] vae_scale = getattr(pipe, 'vae_scale_factor', 8) if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5) and p.is_hr_pass: diff --git a/modules/processing_correction.py b/modules/processing_correction.py index 9477960dd..da24f3fcd 100644 --- a/modules/processing_correction.py +++ b/modules/processing_correction.py @@ -155,7 +155,10 @@ def _unpack_latents(latents, pipe, p): height = getattr(p, 'height', 1024) if hasattr(pipe, '_unpack_latents') and hasattr(pipe, 'vae_scale_factor'): # Flux 1 / Bria: use pipeline's own unpack method - unpacked = pipe._unpack_latents(latents, height, width, vae_scale) # pylint: disable=protected-access + try: + unpacked = pipe._unpack_latents(latents, height, width, vae_scale) # pylint: disable=protected-access + except Exception: + unpacked = pipe._unpack_latents(latents, height, width) # pylint: disable=protected-access # pythoning ask-for-forgiveness if method does not support vae_scale_factor return unpacked, 'flux1' if hasattr(pipe, '_unpatchify_latents'): # Flux 2: manual reshape [B, seq_len, patch_channels] -> [B, C, H, W] diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 03c1161d2..bd80c4d1f 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -330,8 +330,10 @@ def vae_decode(latents, model, output_type='np', vae_type='Full', width=None, he latents = model._unpack_latents(latents.unsqueeze(0), latent_num_frames, height // 32, width // 32, model.transformer_spatial_patch_size, model.transformer_temporal_patch_size) # pylint: disable=protected-access latents = model._denormalize_latents(latents, model.vae.latents_mean, model.vae.latents_std, model.vae.config.scaling_factor) # pylint: disable=protected-access elif hasattr(model, '_unpack_latents') and hasattr(model, "vae_scale_factor") and width is not None and height is not None and latents.ndim == 3: # FLUX - latents = model._unpack_latents(latents, height, width, model.vae_scale_factor) # pylint: disable=protected-access - + try: + latents = model._unpack_latents(latents, height, width, model.vae_scale_factor) # pylint: disable=protected-access + except Exception: + latents = model._unpack_latents(latents, height, width) # pylint: disable=protected-access # pythoning ask-for-forgiveness if method does not support vae_scale_factor if latents.ndim == 3: # lost a batch dim in hires latents = latents.unsqueeze(0) if latents.shape[-1] <= 4: # not a latent, likely an image diff --git a/modules/sd_hijack_triton.py b/modules/sd_hijack_triton.py index 3a9ff84a6..da4190298 100644 --- a/modules/sd_hijack_triton.py +++ b/modules/sd_hijack_triton.py @@ -198,7 +198,7 @@ def install(): from triton import knobs from triton.runtime.autotuner import Autotuner except Exception as e: - log.debug(f'Kernel autotune: reporting unavailable: {e}') + log.debug(f'Kernel autotune: {e}') return try: knobs.autotuning.listener = make_autotune_listener(getattr(knobs.autotuning, 'listener', None)) diff --git a/pipelines/model_krea2.py b/pipelines/model_krea2.py index 2c2ccf52c..fe9a66b51 100644 --- a/pipelines/model_krea2.py +++ b/pipelines/model_krea2.py @@ -13,21 +13,25 @@ def load_krea2(checkpoint_info, diffusers_load_config=None): load_args, _ = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) log.debug(f'Load model: type=Krea2 repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') - from pipelines.krea2.transformer_krea2 import Krea2Transformer2DModel - from pipelines.krea2.pipeline_krea2 import Krea2Pipeline, Krea2Img2ImgPipeline - from pipelines.krea2.pipeline_krea2_inpaint import Krea2InpaintPipeline from pipelines.krea2 import KREA2_SPEC - diffusers.Krea2Transformer2DModel = Krea2Transformer2DModel - diffusers.Krea2Pipeline = Krea2Pipeline - diffusers.Krea2Img2ImgPipeline = Krea2Img2ImgPipeline - diffusers.Krea2InpaintPipeline = Krea2InpaintPipeline - generic.set_pipeline('Krea2', Krea2Pipeline) - # One class per task so get_diffusers_task defaults to text2image and set_diffuser_pipe switches - # to the img2img variant cleanly (matches the Chroma/Qwen per-task-class pattern). - from diffusers.pipelines import auto_pipeline - auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING['krea2'] = Krea2Pipeline - auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING['krea2'] = Krea2Img2ImgPipeline - auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING['krea2'] = Krea2InpaintPipeline + + if 'nunchaku-lite' in repo_id.lower(): + pass # nunchaku-lite works only with diffusers pipeline/transformer, but diffusers do not support img2img/inpaint + else: + from pipelines.krea2.transformer_krea2 import Krea2Transformer2DModel + from pipelines.krea2.pipeline_krea2 import Krea2Pipeline, Krea2Img2ImgPipeline + from pipelines.krea2.pipeline_krea2_inpaint import Krea2InpaintPipeline + diffusers.Krea2Transformer2DModel = Krea2Transformer2DModel + diffusers.Krea2Pipeline = Krea2Pipeline + diffusers.Krea2Img2ImgPipeline = Krea2Img2ImgPipeline + diffusers.Krea2InpaintPipeline = Krea2InpaintPipeline + from diffusers.pipelines import auto_pipeline + auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING['krea2'] = diffusers.Krea2Pipeline + auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING['krea2'] = Krea2Img2ImgPipeline + auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING['krea2'] = Krea2InpaintPipeline + + generic.set_pipeline('Krea2', diffusers.Krea2Pipeline) + if repo_id is None or repo_id.lower() == 'none': return None @@ -35,10 +39,10 @@ def load_krea2(checkpoint_info, diffusers_load_config=None): # (in=12) are below the int8 GEMM's minimum K; `last` is the output projection; `tmlp`/`tproj` # produce the global per-block modulation, too int8-sensitive to quantize (its error compounds # across blocks and steps). All are tiny next to the 28 blocks, so the memory cost is small. - transformer = generic.load_transformer(repo_id, cls_name=Krea2Transformer2DModel, load_config=diffusers_load_config, native_spec=KREA2_SPEC, modules_to_not_convert=['first', 'last', 'projector', 'tmlp', 'tproj']) + transformer = generic.load_transformer(repo_id, cls_name=diffusers.Krea2Transformer2DModel, load_config=diffusers_load_config, native_spec=KREA2_SPEC, modules_to_not_convert=['first', 'last', 'projector', 'tmlp', 'tproj']) text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3VLModel, load_config=diffusers_load_config) - pipe = Krea2Pipeline.from_pretrained( + pipe = diffusers.Krea2Pipeline.from_pretrained( repo_id, cache_dir=shared.opts.diffusers_dir, transformer=transformer, diff --git a/ui/js/login.js b/ui/js/login.js index 0dc438325..41026433a 100644 --- a/ui/js/login.js +++ b/ui/js/login.js @@ -12,11 +12,13 @@ const loginCSS = ` const loginHTML = `
-

Login

- - - - +

SD.Next Login

+ + + + + +
@@ -30,25 +32,27 @@ function forceLogin() { form.style.cssText = loginCSS; form.innerHTML = loginHTML; document.body.appendChild(form); - const username = form.querySelector('#loginUsername'); - const password = form.querySelector('#loginPassword'); + const status = form.querySelector('#loginStatus'); form.addEventListener('submit', (event) => { event.preventDefault(); + const formData = new FormData(form); - formData.append('username', username.value); - formData.append('password', password.value); console.warn('login', location.href, formData); + fetch(`${location.href}login`, { method: 'POST', body: formData, }) .then(async (res) => { - const json = await res.json(); let txt = ''; - if (res.status === 200) txt = 'login verified'; - else txt = `${res.status}: ${res.statusText} - ${json.detail}`; + if (res.status === 200) { + txt = 'login verified'; + } else { + const json = await res.json().catch(() => ({})); + txt = `${res.status}: ${res.statusText}${json.detail ? ' - ' + json.detail : ''}`; + } status.textContent = txt; console.log('login', txt); if (res.status === 200) location.reload();