Merge branch 'dev' into master

.gitignore

@@ -50,6 +50,7 @@ package-lock.json
 /outputs/**/*
 /models/**/*
 /tmp
+/benchmarks
 .*/
 cache
 tunableop_results*.csv

.gitmodules

@@ -12,3 +12,6 @@
 [submodule "extensions-builtin/sdnext-kanvas"]
   path = extensions-builtin/sdnext-kanvas
   url = https://github.com/vladmandic/sdnext-kanvas
+[submodule "extensions-builtin/sdnq"]
+  path = extensions-builtin/sdnq
+  url = https://github.com/Disty0/sdnq

CHANGELOG.md

@@ -1,5 +1,118 @@
 # Change Log for SD.Next
 
+## Highlights for 2026-08-21
+
+Time for a new release, this is a larger one!
+Main focus is improving video workflows which also brings full support for new [MiniMax H3](https://vladmandic.github.io/sdnext-docs/MiniMax) and [LTXVideo-2.5](https://vladmandic.github.io/sdnext-docs/LTX)  
+and improves general video processing with flexible video upscaling, updated interpolation, etc.
+
+*What else?*
+- [Detailer.next](https://vladmandic.github.io/sdnext-docs/Detailer) with new support for *vision-language models* and *per-class prompts*
+- New [group offload](https://vladmandic.github.io/sdnext-docs/Offload/#group) option which is more aggressive than the default balanced offload
+- Extended model support for [Nunchaku-Lite](https://github.com/rootonchair/nunchaku-lite) engine
+- A lot of [SDNQ](https://github.com/Disty0/sdnq) *quantization and attention* optimizations and features
+
+Plus quite a lot more, see full [changelog](https://github.com/vladmandic/automatic/blob/dev/CHANGELOG.md) for details!
+
+[Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic)  
+
+## Details for 2026-08-21
+
+- **Models**
+  - [MiniMax H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) available in *base* and *ref* variants  
+    MiniMax-H3 is an amazing video model, but absolutely massive at 33B transformer with 32B qwen3-vl text-encoder  
+    with support for *t2v, i2v, fl2v, ref2v, t2i, i2i* workflows  
+    see [MiniMax docs page](https://vladmandic.github.io/sdnext-docs/MiniMax) for details and usage instructions  
+  - [Lightricks LTX-2.5](https://huggingface.co/Lightricks/LTX-2.5) available in *distilled* and *dev* variants  
+    LTX-2.5 is a 22B transformer with 12B gemma-4 text-encoder  
+    with support for *t2v, i2v, cond2v* workflows  
+    see [LTX docs page](https://vladmandic.github.io/sdnext-docs/LTX) for details and usage instructions  
+    *note*: LTX-2.5 is a [gated model](https://vladmandic.github.io/sdnext-docs/Gated/)  
+  - [Nunchaku Lite](https://github.com/rootonchair/nunchaku-lite) now includes [Krea 2 Turbo](https://huggingface.co/lite-infer/krea-2-turbo-nunchaku-lite-int4_r32-bnb4-text-encoder) and [Flux.2-Klein](https://huggingface.co/lite-infer/flux.2-klein-4b-nunchaku-lite-int4_r32-bnb4-text-encoder) 4B and 9B  
+    Nunchaku-lite engine offers significant speed up through heavily optimized quants and kernels for nvidia gpus  
+- **Detailer**: Pretty much *detailer.next* :)  
+  Detailer detection models were traditionally *YOLO* models, but now we can also use:  
+  - [Facebook-SAM3](https://huggingface.co/facebook/sam3) hybrid promptable concept segmentation and detection network
+  - [Qwen3-VL](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) vision-language autoregressive foundation models, in *2B, 4B, and 8B* variants
+  - [Florence-2](https://huggingface.co/microsoft/Florence-2-large) lightweight multi-task vision sequence-to-sequence models, in *base and large* variants
+  - [Grounding-DINO](https://huggingface.co/IDEA-Research/grounding-dino-base) open-vocabulary object detection models, in *tiny and base* variants  
+
+  additionally, detailer can now use per-class prompts, thanks @kirtasshh  
+  see [Detailer wiki page](https://vladmandic.github.io/sdnext-docs/Detailer) for details and usage instructions
+- [SDNQ](https://github.com/Disty0/sdnq) is now a separate package and no longer part of sdnext repo  
+  installed and used internally by sd.next, but also supported by diffusers natively  
+  and sdnq development brings a lot of new optimizations, in both quantization and attention mechanisms  
+- **Compute**
+  - torch-rocm for windows switch to *whl-multi-arch* distribution  
+  - nunchaku-lite support for `torch==2.13`
+- **Server**
+  - update handlers for all authenticated workflows
+  - update handlers for all hf-based progress bars
+  - offload options take effect immediately without restart/reload
+  - log long torch autotune operations
+  - utilize `torch.accelerator` where available
+  - add `SD_DIFFUSERS_DEBUG` and `SD_TRANSFORMERS_DEBUG` env variables to trace diffusers and transformers internal operations  
+  - add settings -> model load -> *offload state dict* option  
+    reduces memory spikes during model load at the cost of disk i/o and slower load times  
+  - use `GRADIO_TEMP_DIR` env variable for temp folder if set
+  - update ui login form
+- **Video**
+  - reorganized *video* tab
+  - better support for video codeces and formats
+  - add flexible video upscaling as video generation step  
+    see [Video wiki page](https://vladmandic.github.io/sdnext-docs/Video) for details and recommended upscaling models  
+  - improved interpolation
+  - add *generate forever* button
+- **Upscalers**
+  - update *spandrel* integration  
+    moving forward, spandrel engine will be main upscaling engine for sdnext  
+    when downloading any upscaling models manually, place them in `models/Spandrel` folder  
+  - update *chainner* integration
+  - add several low-latency upscalers that are better suited for video upscaling  
+- **API**
+  - full support for video generation using api  
+    new endpoints: `/sdapi/v1/video`, `/sdapi/v1/video/models`, `/sdapi/v1/video/file`  
+    *note*: video api uses async workflow where you submit request and then later download the result  
+  - authentication for websocket connection
+  - allowed path validation for endpoints that get/put files
+  - log auth methods
+- **Other**
+  - Krea2: add *settings -> model options -> krea2 dense masking*
+    may provide significant speed-up on some gpus, disabled by default  
+  - AR display ratio on manual resolution change
+- **Wiki**
+  - major updates: *Offload, Detailer, Video, MiniMax, LTX*
+  - minor updates: *Intel-ARC, FramePack, Parameters, API, Installation, SDNQ-Quantization, CLI-Arguments*
+- **Removed**
+  - remove DirectML support  
+    latest release was over 2 years ago and is not compatible with modern frameworks  
+- **Fixes**
+  - hf: init hf env variables before gradio load
+  - lora: skip init and rebuild offload state
+  - lora: keep network multiplier on change
+  - auth: improve handling of hf auth
+  - load: improve pipeline detection for non-cached models
+  - offload: cleanup alt offload codepaths
+  - offload: text encoders no longer take the denoiser profile on modular pipelines
+  - offload: components entered through encode or decode are detected by structure rather than by name
+  - offload: group offload honors the never-offload and model-type exclusion settings
+  - settings: offload settings grouped into shared overrides and per-mode sections
+  - log: hf progress bars
+  - ltx: send the guidance stack and cross-timestep on every 2.x call path
+  - ltx: distilled variants no longer force dynamic shifting on, which remapped their sigma schedule
+  - ltx: sampler shift now reaches flow-match schedulers
+  - ltx: reload the latent upsampler when the model or its repo changes
+  - video: take the audio sample rate from the loaded vocoder
+  - video: keep the shared text encoder out of the registry rows
+  - video: use generic loader methods
+  - log: processing stats reporting
+  - metadata: image metadata handle correct image index
+  - gguf: transformer loader
+  - scripts: mixture-of-diffusers and mixture-tiling update to use igwn-segments
+  - api: process
+  - api: auth via remote-ip
+  - krea2: fallback to base pipeline/transformer for nunchaku-lite
+
 ## Update for 2026-08-07
 
 ### Highlights for 2026-08-07
@@ -192,7 +305,7 @@ Originating as a service-pack update with a handful of fixes and quality-of-life
   - **First-Last-Frame** (FLF2V) support for Wan-2.2-I2V and LTX
   - add option: *compute settings -> force dtype on load*  
     use to force model components to override loading with desired dtype regardless of component config  
-  - add option: *backend settings -> force sychronize*  
+  - add option: *backend settings -> force synchronize*  
     enabled by default, disable to speed up processing but may cause image corruptions, especially during preview
   - add option: *model loading -> attempt to load incomplete model*  
     disabled by default, attempts to load model by mapping it to known model even if some components are missing  
@@ -329,7 +442,7 @@ And we have a new modular LoRA loader, new native Transformers loader and improv
   - **Samplers** reorganized into clear sampler categories
   - **Gallery** add clear cache button to folder menu
   - **Finetunes** improved support for loading model finetunes  
-    this also includes detecting compatibility and falbacks  
+    this also includes detecting compatibility and fallbacks  
   - **UV** much updated `--uv` support for fast installs  
     now also supports global `uv` if present in the system  
   - **Attention Dispatcher** new attention backends dispatcher  

TODO.md

@@ -1,49 +1,58 @@
 # TODO
 
-## Issues
+## Short-term
 
-- Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322
+- Update LTX wiki, @CalamitousFelicitousness
+- Productize benchmark tool, @CalamitousFelicitousness
+- Inpaint: https://discord.com/channels/1101998836328697867/1130536562422186044/1506850651035144322, @vladmandic
+- Lora: new handler, @CalamitousFelicitousness
+- Control tab verify overrides handling, @vladmandic
+- Create pre-quant for LTX-2.5
+- Create pre-quant for MiniMax-H3-Turbo
 
 ## Features
 
+### Roadmap
+
+- Video upscaling: nvidia-vfx, ltx-upscaler, etc.
+- Video capabilities to processing tab, add RIFE, upscaling (once available)
+- Object clear remover for Kanvas: [Object clear](https://huggingface.co/jixin0101/ObjectClear)
+- OpenAI API interface for image generation
+- Lightweight scheduler/queue manager
+- Distraction-free UI mode with prompt-only, chat-based interface
+- Revisit transformer caching for modular pipelines
+- Revisit guidance for modular pipelines
+- Implement modular for some image models
+- Video models: support finetunes
+
 ### Assigned
 
-- Chat-based interface, @vladmandic
-- Control tab verify overrides handling, @vladmandic
-- Cloud providers, @CalamitousFelicitousness
-- Video processing add/verify full API support, @CalamitousFelicitousness
-- Lora: new handler, @CalamitousFelicitousness
-- Processing -> Video capabilities, @vladmandic
-  - `RIFE` in processing
+- Support cloud providers, @CalamitousFelicitousness
 
 ### Unassigned
 
-- [Object clear](https://huggingface.co/jixin0101/ObjectClear) remover for Kanvas
-- [MiniMax H3](https://github.com/huggingface/diffusers/pull/14355)
+- Incorporate [prompting guides](https://github.com/CalamitousFelicitousness/ai-prompting-guides)
 - Video models: add to Reference
-- Video models: support custom entries, finetunes
 - UI Lite vs Expert mode
 - Auto handle scheduler `prediction_type`
-- Cache models in memory
 - JSON image metadata
 - Expand custom VAE support
 - Refactor: remove obsolete code:
-  - Remove `directml`
   - Remove `olive-ai`
 - Integrate natural language image search
   - [ImageDB](https://github.com/vladmandic/imagedb)
-- Unify *huggingface* and *diffusers* model folders
 
 ### OnHold
 
 - [nVidia-VFX](https://pypi.org/project/nvidia-vfx/): not compatible with latest nVidia drivers
 - [QuantFunc](https://huggingface.co/QuantFunc/Klein-9B-Series): once its released as sdk
-- [nVidia LocateAnything](https://huggingface.co/nvidia/LocateAnything-3B) detection for Detailer, once its compatible with transformers==5
 - LoRA add OMI format support for SD35/FLUX.1
 - Remote Text-Encoder support, sidelined for the moment
 - Multi-user support
 - Settings profile manager
 - TensorRT acceleration
+- Cache models in memory
+- Unify *huggingface* and *diffusers* model folders
 
 ### Modular
 

cli/api-control.py

@@ -2,6 +2,7 @@
 # example: api-control.py --prompt "anime girl" --control "Canny:Canny:1.0:0.1:0.9:/home/vlado/generative/Samples/anime1.jpg,None:Depth:0.9:0.0:1.0:/home/vlado/generative/Samples/anime1.jpg" --hires --detailer --output /tmp/anime.jpg
 import os
 import io
+import sys
 import time
 import base64
 import logging
@@ -10,6 +11,7 @@ import requests
 import urllib3
 from PIL import Image
 
+
 sd_url = os.environ.get('SDAPI_URL', "http://127.0.0.1:7860")
 sd_username = os.environ.get('SDAPI_USR', None)
 sd_password = os.environ.get('SDAPI_PWD', None)
@@ -38,6 +40,25 @@ def post(endpoint: str, dct: dict | None = None):
         return req.json()
 
 
+def display(image: Image.Image):
+    """
+    # pip install sixel
+    from sixel import converter
+    buf = io.BytesIO()
+    image.save(buf, format="PNG")
+    buf.seek(0)
+    conv = converter.SixelConverter(buf, f8bit=False, ncolor=256, fast=False)
+    conv.write(sys.stdout)
+    """
+    try:
+        import subprocess
+        buf = io.BytesIO()
+        image.save(buf, format="PNG")
+        subprocess.run(["timg", "--pixelation", "sixel", "-"], input=buf.getvalue())
+    except Exception:
+        pass
+
+
 def encode(f):
     image = Image.open(f)
     if image.mode == 'RGBA':
@@ -144,6 +165,7 @@ def generate(args): # pylint: disable=redefined-outer-name
             if output:
                 image.save(output)
                 log.info(f'image saved: size={image.size} filename={output}')
+            display(image)
 
     if 'images' in data:
         get_image(data['images'], args.output)

cli/api-video.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# python cli/api-video.py --prompt "a paper boat drifting down a rain gutter" --frames 17 --steps 8 --output /tmp/video.mp4
+import os
+import time
+import base64
+import logging
+import argparse
+import threading
+import requests
+import urllib3
+
+sd_url = os.environ.get('SDAPI_URL', "http://127.0.0.1:7860")
+sd_username = os.environ.get('SDAPI_USR', None)
+sd_password = os.environ.get('SDAPI_PWD', None)
+options = {
+    "send_video": True,
+    "send_thumbnail": False,
+}
+
+logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s')
+log = logging.getLogger(__name__)
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def auth():
+    if sd_username is not None and sd_password is not None:
+        return requests.auth.HTTPBasicAuth(sd_username, sd_password)
+    return None
+
+
+def get(endpoint: str, params: dict | None = None, timeout: int = 60):
+    req = requests.get(f'{sd_url}{endpoint}', params=params, timeout=timeout, verify=False, auth=auth())
+    if req.status_code != 200:
+        return { 'error': req.status_code, 'reason': req.reason, 'url': req.url }
+    return req.json()
+
+
+def post(endpoint: str, dct: dict | None = None, timeout: int = 3600):
+    req = requests.post(f'{sd_url}{endpoint}', json=dct, timeout=timeout, verify=False, auth=auth())
+    if req.status_code != 200:
+        res = { 'error': req.status_code, 'reason': req.reason, 'url': req.url }
+        try:
+            res['detail'] = req.json().get('detail', None)
+        except Exception:
+            pass
+        return res
+    return req.json()
+
+
+def encode(f: str):
+    with open(f, 'rb') as file:
+        return base64.b64encode(file.read()).decode()
+
+
+def list_models():
+    data = get('/sdapi/v1/video/models')
+    if isinstance(data, dict) and 'error' in data:
+        log.error(f'video models: {data}')
+        return
+    for item in data:
+        loaded = ' loaded=true' if item.get('loaded') else ''
+        log.info(f'engine="{item["engine"]}" model="{item["name"]}" mode={item["mode"]}{loaded}')
+    log.info(f'video models: {len(data)}')
+
+
+def watch_progress(stop_event: threading.Event):
+    while not stop_event.is_set():
+        status = get('/sdapi/v1/progress', params={ 'skip_current_image': True })
+        if 'error' not in status:
+            state = status.get('state') or {}
+            log.info(f'progress={status.get("progress", 0):.2f} eta={status.get("eta_relative", 0):.1f} step={state.get("sampling_step", 0)}/{state.get("sampling_steps", 0)} info="{status.get("textinfo") or ""}"')
+        stop_event.wait(5)
+
+
+def save_output(data: dict, output: str):
+    if data.get('video'):
+        with open(output, 'wb') as f:
+            f.write(base64.b64decode(data['video']))
+        log.info(f'video saved: filename={output}')
+    elif data.get('still') and data.get('frames'):
+        with open(output, 'wb') as f:
+            f.write(base64.b64decode(data['frames'][0]))
+        log.info(f'still saved: filename={output}')
+    elif data.get('video_path'):
+        req = requests.get(f'{sd_url}/sdapi/v1/video/file', params={ 'file': data['video_path'] }, timeout=300, verify=False, auth=auth())
+        if req.status_code == 200:
+            with open(output, 'wb') as f:
+                f.write(req.content)
+            log.info(f'video fetched: filename={output} size={len(req.content)}')
+        else:
+            log.error(f'video fetch failed: code={req.status_code} reason={req.reason}')
+    else:
+        log.warning('no video output received')
+
+
+def generate(args): # pylint: disable=redefined-outer-name
+    t0 = time.time()
+    if args.engine:
+        options['engine'] = args.engine
+    if args.model:
+        options['model'] = args.model
+    options['prompt'] = args.prompt
+    options['negative_prompt'] = args.negative
+    options['width'] = int(args.width)
+    options['height'] = int(args.height)
+    options['frames'] = int(args.frames)
+    options['steps'] = int(args.steps)
+    options['seed'] = int(args.seed)
+    options['sampler_name'] = args.sampler
+    options['mp4_fps'] = int(args.fps)
+    options['mp4_interpolate'] = int(args.interpolate)
+    options['audio'] = bool(args.audio)
+    if args.init:
+        options['init_image'] = encode(args.init)
+    if args.last:
+        options['last_image'] = encode(args.last)
+    if args.reference:
+        options['references'] = [encode(f) for f in args.reference]
+    stop_event = threading.Event()
+    if args.progress:
+        threading.Thread(target=watch_progress, args=(stop_event,), daemon=True).start()
+    data = post('/sdapi/v1/video', options, timeout=int(args.timeout))
+    stop_event.set()
+    t1 = time.time()
+    if 'error' in data:
+        log.error(f'generate failed: {data}')
+        return
+    log.info(f'video received: frames={data.get("frames_count")} fps={data.get("fps")} duration={data.get("duration")} audio={data.get("has_audio")} still={data.get("still")} path={data.get("video_path")} time={t1-t0:.2f}')
+    if args.output:
+        save_output(data, args.output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description = 'api-video')
+    parser.add_argument('--list', action='store_true', help='list video engines and models')
+    parser.add_argument('--engine', required=False, default=None, help='video engine; omit with --model to use the loaded checkpoint')
+    parser.add_argument('--model', required=False, default=None, help='video model name within the engine')
+    parser.add_argument('--prompt', required=False, default='', help='prompt text')
+    parser.add_argument('--negative', required=False, default='', help='negative prompt text')
+    parser.add_argument('--width', required=False, default=832, help='video width')
+    parser.add_argument('--height', required=False, default=480, help='video height')
+    parser.add_argument('--frames', required=False, default=17, help='number of frames; 1 for a still image')
+    parser.add_argument('--steps', required=False, default=20, help='number of steps')
+    parser.add_argument('--seed', required=False, default=-1, help='initial seed')
+    parser.add_argument('--sampler', required=False, default='Default', help='sampler name')
+    parser.add_argument('--fps', required=False, default=24, help='frames per second')
+    parser.add_argument('--interpolate', required=False, default=0, help='rife interpolation passes')
+    parser.add_argument('--audio', action=argparse.BooleanOptionalAction, default=True, help='generate audio on supported models')
+    parser.add_argument('--init', required=False, default=None, help='init image file')
+    parser.add_argument('--last', required=False, default=None, help='last frame image file')
+    parser.add_argument('--reference', required=False, default=None, action='append', help='reference image file for reference workflows; repeat in the order the model should read them')
+    parser.add_argument('--output', required=False, default=None, help='output video file')
+    parser.add_argument('--progress', action='store_true', help='poll and log progress during generation')
+    parser.add_argument('--timeout', required=False, default=3600, help='request timeout in seconds')
+    args = parser.parse_args()
+    log.info(f'api-video: {args}')
+    if args.list:
+        list_models()
+    else:
+        generate(args)

cli/hf-info.py

@@ -199,7 +199,7 @@ def discover_components(model_index: dict[str, Any] | None, files_map: dict[str,
     components: dict[str, Any] = {
         "mains": [],
         "text_encoders": [],
-        "ae": None,
+        "ae": [],
     }
 
     if isinstance(model_index, dict):
@@ -210,8 +210,8 @@ def discover_components(model_index: dict[str, Any] | None, files_map: dict[str,
         text_keys = sorted([k for k in keys if re.fullmatch(r"text_encoder(_\d+)?", k or "")])
         components["text_encoders"] = text_keys
 
-        if "vae" in keys:
-            components["ae"] = "vae"
+        vae_keys = sorted([k for k in keys if re.fullmatch(r".*vae?", k or "")])
+        components["ae"] = vae_keys
 
     top_dirs = {f.split("/", 1)[0] for f in files_map if "/" in f}
 
@@ -221,8 +221,8 @@ def discover_components(model_index: dict[str, Any] | None, files_map: dict[str,
     if not components["text_encoders"]:
         components["text_encoders"] = sorted([d for d in top_dirs if re.fullmatch(r"text_encoder|mllm(_\d+)?", d or "")])
 
-    if components["ae"] is None and "vae" in top_dirs:
-        components["ae"] = "vae"
+    if not components["ae"]:
+        components["ae"] = sorted([d for d in top_dirs if re.fullmatch(r".*vae?", d or "")])
 
     return components
 
@@ -635,7 +635,7 @@ def search(repo_id: str) -> int:
 
     main_components = components["mains"]
     text_components = components["text_encoders"]
-    ae_component = components["ae"]
+    ae_components = components["ae"]
 
     main_files: list[str] = []
     for main_component in main_components:
@@ -643,7 +643,9 @@ def search(repo_id: str) -> int:
     te_files: list[str] = []
     for te_component in text_components:
         te_files.extend(component_weight_files(te_component, files_map))
-    ae_files = component_weight_files(ae_component, files_map)
+    ae_files: list[str] = []
+    for ae_component in ae_components:
+        ae_files.extend(component_weight_files(ae_component, files_map))
 
     fs = hf.HfFileSystem(token=token)
 
@@ -674,8 +676,12 @@ def search(repo_id: str) -> int:
         arch = arch_from_config(cfg, component_type="te")
         te_arches.append(arch if arch is not None else te_component)
 
-    ae_cfg = component_config(ae_component, repo_id, token)
-    ae_arch = arch_from_config(ae_cfg, component_type="ae")
+    ae_arches: list[str] = []
+    for ae_component in ae_components:
+        cfg = component_config(ae_component, repo_id, token)
+        arch = arch_from_config(cfg, component_type="ae")
+        ae_arches.append(arch if arch is not None else ae_component)
+
     model_class = class_from_model_index(model_index)
     if model_class is None:
         first_main_class = next((c for c in main_component_classes if isinstance(c, str) and c.strip()), None)
@@ -707,10 +713,14 @@ def search(repo_id: str) -> int:
         "dit": ", ".join(main_dit_entries) if len(main_dit_entries) > 0 else None,
         "dit_params": model_params_raw,
         "dit_size": model_size_raw,
+        "dit_size_gb": round(model_size_raw / (1024**3), 2) if isinstance(model_size_raw, int) else None,
         "te": ", ".join(te_arches) if len(te_arches) > 0 else None,
         "te_params": te_params_raw,
         "te_size": te_size_raw,
-        "ae": ae_arch,
+        "te_size_gb": round(te_size_raw / (1024**3), 2) if isinstance(te_size_raw, int) else None,
+        "ae": ", ".join(ae_arches) if len(ae_arches) > 0 else None,
+        "ae_size": ae_size_raw,
+        "ae_size_gb": round(ae_size_raw / (1024**3), 2) if isinstance(ae_size_raw, int) else None,
         "downloads": downloads_int,
         "tags": tags,
     }

cli/sdnq-attention-benchmark.py

@@ -2,7 +2,7 @@
 """
 Benchmark and validate SDNQ attention and weight dequantization on the local GPU.
 
-The attention section runs the kernel from modules/sdnq/kernels/triton_atten.py directly and
+The attention section runs the kernel from sdnq/kernels/triton_atten.py directly and
 compares speed and numerical error against torch scaled_dot_product_attention and
 sageattention when installed. Verifies mask, causal, GQA, cross-attention and padding code
 paths, probes float8 hardware support and the torch.compile input prep, and prints
@@ -47,6 +47,7 @@ import json
 import math
 import time
 import signal
+import inspect
 import logging
 import argparse
 import tempfile
@@ -133,6 +134,8 @@ block_attention_specs = {
     "atten int8 smooth": dict(matmul_dtype="auto", pv_matmul_dtype="auto", smooth_k=True),
     "atten int8 hadamard": dict(matmul_dtype="auto", pv_matmul_dtype="auto", use_hadamard=True),
     "atten full": dict(matmul_dtype="auto", pv_matmul_dtype="int8", smooth_k=True, use_hadamard=True),
+    "atten pv accum": dict(matmul_dtype="auto", pv_matmul_dtype="disabled", use_fp16_accum=True), # the sage-style unsafe mode
+    "atten fp16 accum": dict(matmul_dtype="float16", pv_matmul_dtype="float16", use_fp16_accum=True), # the scaled overflow-proof mode
     "sage": "sage", # external baselines, resolved to the sage wrappers in build_bench_block
     "sage fp16 accum": "sagefp16",
 }
@@ -149,6 +152,8 @@ block_configs = [
     ("int8-mm-smooth", dict(weights_dtype="int8"), True, "atten int8 smooth"),
     ("int8-mm-hadamard", dict(weights_dtype="int8"), True, "atten int8 hadamard"),
     ("int8-mm-atten-full", dict(weights_dtype="int8"), True, "atten full"),
+    ("int8-mm-pvaccum", dict(weights_dtype="int8"), True, "atten pv accum"),
+    ("int8-mm-fp16accum", dict(weights_dtype="int8"), True, "atten fp16 accum"),
     ("int8-mm-sage", dict(weights_dtype="int8"), True, "sage"),
     ("int8-mm-sagefp16", dict(weights_dtype="int8"), True, "sage fp16 accum"),
     ("int6", dict(weights_dtype="int6"), False, "sdpa"),
@@ -161,7 +166,8 @@ block_configs = [
 ]
 
 # benchmark configs: id, label, kwargs for sdnq_triton_atten (None = external baseline);
-# fp8 configs run only on gpus where the float8 probe passes
+# fp8 configs run only on gpus where the float8 probe passes, accum configs only where the
+# installed sdnq has the use_fp16_accum kwarg
 bench_configs = [
     ("base", "torch sdpa", None),
     ("sage", "sageattention", None), # label resolved to the dispatched kernel by sage_kernel_label
@@ -173,15 +179,22 @@ bench_configs = [
     ("hadamard", "sdnq int8 qk + hadamard", dict(matmul_dtype="auto", pv_matmul_dtype="auto", use_hadamard=True)),
     ("smooth_hadamard", "sdnq int8 qk + smooth + hadamard", dict(matmul_dtype="auto", pv_matmul_dtype="auto", smooth_k=True, use_hadamard=True)),
     ("fp16pv", "sdnq int8 qk + fp16 pv", dict(matmul_dtype="auto", pv_matmul_dtype="float16")),
+    ("fp16pv-accum", "sdnq int8 qk + fp16 pv, scaled fp16 accum", dict(matmul_dtype="auto", pv_matmul_dtype="float16", use_fp16_accum=True)),
     ("int8pv", "sdnq int8 qk + int8 pv", dict(matmul_dtype="auto", pv_matmul_dtype="int8")),
     ("fp8pv", "sdnq int8 qk + fp8 pv", dict(matmul_dtype="auto", pv_matmul_dtype="float8_e4m3fn")),
+    ("pvaccum", "sdnq int8 qk + unquantized pv, fp16 accum (unsafe)", dict(matmul_dtype="auto", pv_matmul_dtype="disabled", use_fp16_accum=True)),
     ("full", "sdnq int8 qk + smooth + hadamard + int8 pv", dict(matmul_dtype="auto", pv_matmul_dtype="int8", smooth_k=True, use_hadamard=True)),
     ("fp16qk", "sdnq fp16 qk", dict(matmul_dtype="float16", pv_matmul_dtype="auto")),
+    ("fp16full", "sdnq fp16 qk + fp16 pv", dict(matmul_dtype="float16", pv_matmul_dtype="float16")),
+    ("fp16full-accum", "sdnq fp16 qk + fp16 pv, scaled fp16 accum", dict(matmul_dtype="float16", pv_matmul_dtype="float16", use_fp16_accum=True)),
     ("fp8qk", "sdnq fp8 qk", dict(matmul_dtype="float8_e4m3fn", pv_matmul_dtype="auto")),
     ("fp8full", "sdnq fp8 qk + fp8 pv", dict(matmul_dtype="float8_e4m3fn", pv_matmul_dtype="float8_e4m3fn")),
 ]
-# external baselines are compared against but never starred or recommended as sdnq configs
+# external baselines are compared against but never starred or recommended as sdnq configs;
+# the unsafe accum mode is measured and displayed under the same rule, since its overflow
+# tail lives outside what mean error can see
 external_config_ids = ("base", "sage", "sagefp16", "amdflash")
+unsafe_config_ids = ("pvaccum",)
 # every preset runs the full config list (availability gates still apply per config); only
 # hard technical exclusions live here, never runtime trims. sd15: compiling hadamard with
 # a non pow2 head dim currently hangs torch inductor
@@ -255,6 +268,7 @@ atten_settings = [
     ("sdnq_attention_pv_matmul_type", "PV MatMul type"),
     ("sdnq_attention_smooth_k", "Use Smooth K"),
     ("sdnq_attention_use_hadamard", "Use Hadamard"),
+    ("sdnq_attention_use_fp16_accum", "Use FP16 Accumulation"),
     ("sdnq_attention_hadamard_group_size", "Hadamard Group Size"),
 ]
 
@@ -275,8 +289,9 @@ def parse_cli():
     parser.add_argument("--skip-bench", action="store_true", help="skip benchmarks, run checks and the fp8 and compile probes only")
     parser.add_argument("--dtype", type=str, default="auto", choices=["auto", "bf16", "fp16"], help="tensor dtype for benchmarks; auto uses the dtype the webui selected for this gpu (default: %(default)s)")
     parser.add_argument("--config-timeout", type=int, default=300, help="best effort: abort a config whose compile plus first call exceeds this many seconds, 0 disables; cannot interrupt native-level hangs (default: %(default)s)")
-    parser.add_argument("--save", type=str, default=None, help="write a plain-text copy of all tables and notes to this file; keeps colors and live progress on the terminal, unlike piping through tee")
-    parser.add_argument("--json", type=str, default=None, help="write structured results (environment, probes, per-shape and dequant timings, recommendations) to this file")
+    parser.add_argument("--save", type=str, default="auto", help="plain-text copy of all tables and notes; 'auto' (default) names it <gpu>-t<torch>-<date>.txt in the output directory, 'none' disables, anything else is used as the path")
+    parser.add_argument("--json", type=str, default="auto", help="structured results (environment, probes, per-shape and dequant timings, recommendations); 'auto' (default) names it <gpu>-t<torch>-<date>.json in the output directory, 'none' disables, anything else is used as the path")
+    parser.add_argument("--outdir", type=str, default=None, help="directory for auto-named outputs (default: $SDNQ_BENCH_DIR, or benchmarks/ under the sdnext root)")
     args = parser.parse_args()
     sys.argv = sys.argv[:1] # sdnext parses argv again on import and rejects unknown arguments
     return args
@@ -358,7 +373,7 @@ def load_sdnext():
         with capture_console_output() as startup_log:
             from modules import shared as shared_module
             from modules import devices as devices_module
-            from modules.sdnq.kernels.triton_atten import sdnq_triton_atten as atten
+            from sdnq.kernels.triton_atten import sdnq_triton_atten as atten
     except BaseException as e: # pylint: disable=broad-exception-caught # SystemExit is not an Exception: the bootstrap exits on a failed torch or library import
         if isinstance(e, KeyboardInterrupt):
             raise
@@ -381,6 +396,14 @@ def load_sdnext():
     # importing modules.shared installs the configured sdp override hijacks in this process;
     # restore stock sdpa so baselines and references measure torch itself
     torch.nn.functional.scaled_dot_product_attention = stock_sdpa
+    # the triton autotune hijack draws its own rich progress bar on the logger console during
+    # sweeps; a second live display on this tty tramples the benchmark's live tables (stale
+    # border lines left in scrollback), so disarm the bar and keep the hijack's bookkeeping
+    try:
+        from modules import sd_hijack_triton
+        sd_hijack_triton.start_progress = lambda name, total: (None, None)
+    except Exception:
+        pass
     shared = shared_module
     devices = devices_module
     sdnq_triton_atten = atten
@@ -390,6 +413,43 @@ def load_sdnext():
     return True
 
 
+def atten_supports_fp16_accum():
+    # accum rows need the use_fp16_accum kwarg; skip them on older sdnq builds instead of
+    # failing every row with a TypeError. the kernel entry is wrapped by an inference-context
+    # decorator, so unwrap before reading the signature
+    if sdnq_triton_atten is None:
+        return False
+    try:
+        return "use_fp16_accum" in inspect.signature(inspect.unwrap(sdnq_triton_atten)).parameters
+    except (TypeError, ValueError):
+        return False
+
+
+def triton_mm_supports_fp16_accum():
+    try:
+        from sdnq.kernels import triton_mm, triton_scaled_mm
+        return hasattr(triton_mm, "USE_FP16_ACCUM") and hasattr(triton_scaled_mm, "USE_FP16_ACCUM")
+    except Exception:
+        return False
+
+
+@contextmanager
+def triton_mm_fp16_accum():
+    # SDNQ_TRITON_MM_USE_FP16_ACCUM is read once at import into module globals that the
+    # kernel wrappers reread per call (they are triton_op custom ops, so the read happens at
+    # runtime even under torch.compile), and the flag is part of the autotune key, so both
+    # variants cache side by side. sdnq_triton_mm holds its own imported copy of the global,
+    # so both module namespaces get the flip
+    from sdnq.kernels import triton_mm, triton_scaled_mm
+    saved = (triton_mm.USE_FP16_ACCUM, triton_scaled_mm.USE_FP16_ACCUM)
+    triton_mm.USE_FP16_ACCUM = True
+    triton_scaled_mm.USE_FP16_ACCUM = True
+    try:
+        yield
+    finally:
+        triton_mm.USE_FP16_ACCUM, triton_scaled_mm.USE_FP16_ACCUM = saved
+
+
 def sage_attention():
     # mirror the backend selection from modules/attention.py: sm86 needs the cuda backend
     try:
@@ -727,7 +787,7 @@ def fp8_compile_gate_flag():
     # False on gpus where sdnq upcasts e4m3 storage to the scale dtype before the compiled
     # dequant, because triton cannot convert e4m3 there; absent on builds without the gate
     try:
-        from modules.sdnq import kernel_wrappers as sdnq_kernel_wrappers
+        from sdnq import kernel_wrappers as sdnq_kernel_wrappers
         return getattr(sdnq_kernel_wrappers, "is_fp8_compile_supported", None)
     except Exception:
         return None
@@ -743,13 +803,13 @@ def fp8_failure_is_capability(detail):
 def print_environment(fp8_result, prep_status, prep_detail, weight_dequant_result=None):
     device = torch.device(torch_device)
     capability = torch_device_module.get_device_capability(device)
-    # backend runtime versions (cuda/cudnn/driver, hip, ipex, openvino, directml) so a shared
+    # backend runtime versions (cuda/cudnn/driver, hip, ipex, openvino) so a shared
     # report identifies the stack without inferring it from the torch version string
     try:
         gpu_info = devices.get_gpu_info() or {}
     except Exception:
         gpu_info = {}
-    runtime_versions = {key: gpu_info[key] for key in ("cuda", "hip", "cudnn", "driver", "ipex", "openvino", "directml") if gpu_info.get(key)}
+    runtime_versions = {key: gpu_info[key] for key in ("cuda", "hip", "cudnn", "driver", "ipex", "openvino") if gpu_info.get(key)}
     runtime_line = f"python: {sys.version.split()[0]} ({sys.platform})  backend: {getattr(devices, 'backend', 'unknown')}"
     if runtime_versions:
         runtime_line += "  " + "  ".join(f"{key}: {value}" for key, value in runtime_versions.items())
@@ -774,9 +834,9 @@ def print_environment(fp8_result, prep_status, prep_detail, weight_dequant_resul
     else:
         lines.append(f"compiled input prep: [red]failing, every sdnq attention call errors at generation[/red] [dim]({escape(prep_detail)})[/dim]")
         if prep_status == "failing_dynamic":
-            lines.append("  fix, verified on this machine: set SDNQ_COMPILE_KWARGS='{\"dynamic\": false}' (recompiles per shape); or install msvc build tools; or disable Compute Settings -> SDNQ -> Dequantize using torch.compile")
+            lines.append("  fix, verified on this machine: set SDNQ_COMPILE_KWARGS='{\"dynamic\": false}' (recompiles per shape); or install msvc build tools; or set SDNQ_USE_TORCH_COMPILE=0")
         else:
-            lines.append("  fix: install a host c++ compiler (msvc build tools on windows), or disable Compute Settings -> SDNQ -> Dequantize using torch.compile")
+            lines.append("  fix: install a host c++ compiler (msvc build tools on windows), or set SDNQ_USE_TORCH_COMPILE=0")
     if weight_dequant_result is not None:
         gate_flag = fp8_compile_gate_flag()
         e4m3_ok, e4m3_detail = weight_dequant_result["float8_e4m3fn"]
@@ -788,10 +848,12 @@ def print_environment(fp8_result, prep_status, prep_detail, weight_dequant_resul
             if gate_flag is False:
                 lines.append(f"  sdnq upcasts e4m3 weights to {dtype_label()} before the compiled dequant, so generation is safe and the fp8 rows below are measured that way (SDNQ_ALLOW_FP8_COMPILE overrides)")
             elif gate_flag is None:
-                lines.append("  [red]this sdnq build has no fp8 compile gate: loading fp8 storage weights with Dequantize using torch.compile enabled fails at generation[/red]")
+                lines.append("  [red]this sdnq build has no fp8 compile gate: loading fp8 storage weights with sdnq torch.compile active fails at generation (SDNQ_USE_TORCH_COMPILE=0 avoids it)[/red]")
         e5m2_verdict = "[green]supported[/green]" if e5m2_ok else f"[red]fails[/red] [dim]({escape(e5m2_detail)})[/dim]"
         lines.append(f"compiled weight dequant, float8_e5m2 storage: {e5m2_verdict}")
-    overrides = [f"{key}={value}" for key, value in os.environ.items() if key.startswith("SDNQ_TRITON_ATTEN") or key.startswith("SDNQ_ALLOW_FP8") or key.startswith("SDNQ_COMPILE")]
+    if not atten_supports_fp16_accum():
+        lines.append("fp16 accumulation kwarg: [yellow]absent in this sdnq build, accum rows skipped[/yellow]")
+    overrides = [f"{key}={value}" for key, value in os.environ.items() if key.startswith("SDNQ_TRITON_ATTEN") or key.startswith("SDNQ_TRITON_MM") or key.startswith("SDNQ_ALLOW_FP8") or key.startswith("SDNQ_COMPILE")]
     if overrides:
         lines.append(f"env overrides: {' '.join(overrides)}")
     emit(Panel("\n".join(lines), title="environment", box=ROUNDED_BOX))
@@ -807,12 +869,44 @@ def print_environment(fp8_result, prep_status, prep_detail, weight_dequant_resul
         dtype=dtype_label(),
         **runtime_versions,
         fp8_attention_matmul=fp8_result["qk"][0] if fp8_result is not None else None,
+        atten_fp16_accum=atten_supports_fp16_accum(),
+        triton_mm_fp16_accum=os.environ.get("SDNQ_TRITON_MM_USE_FP16_ACCUM", None),
         compiled_input_prep=prep_status,
         fp8_compile_gate=fp8_compile_gate_flag(),
         weight_dequant_compile={name: dict(ok=ok, detail=detail) for name, (ok, detail) in (weight_dequant_result or {}).items()},
     )
 
 
+def resolve_output_paths(args):
+    # auto outputs follow the archive convention <gpu>-t<torch major.minor>-<mondd>, one
+    # stem shared by the txt/json pair, suffixed -2, -3... when the pair already exists
+    args.save = None if str(args.save).strip().lower() in {"none", ""} else args.save
+    args.json = None if str(args.json).strip().lower() in {"none", ""} else args.json
+    if args.save != "auto" and args.json != "auto":
+        return
+    try:
+        gpu = torch_device_module.get_device_name(torch.device(torch_device))
+    except Exception:
+        gpu = "gpu"
+    drop = {"nvidia", "geforce", "rtx", "gtx", "amd", "radeon", "rx", "intel", "arc", "graphics", "apple", "laptop", "gpu"}
+    tokens = ["".join(ch for ch in token if ch.isalnum()) for token in gpu.lower().replace("(r)", " ").replace("(tm)", " ").split()]
+    tokens = [token for token in tokens if token and token not in drop]
+    gpu_id = "".join(tokens) or "gpu"
+    torch_digits = "".join(str(torch.__version__).split("+", maxsplit=1)[0].split(".")[:2])
+    stem = f"{gpu_id}-t{torch_digits}-{time.strftime('%b%d').lower()}"
+    outdir = args.outdir or os.environ.get("SDNQ_BENCH_DIR", None) or os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "benchmarks")
+    outdir = os.path.expanduser(outdir)
+    os.makedirs(outdir, exist_ok=True)
+    candidate, counter = stem, 1
+    while os.path.exists(os.path.join(outdir, f"{candidate}.txt")) or os.path.exists(os.path.join(outdir, f"{candidate}.json")):
+        counter += 1
+        candidate = f"{stem}-{counter}"
+    if args.save == "auto":
+        args.save = os.path.join(outdir, f"{candidate}.txt")
+    if args.json == "auto":
+        args.json = os.path.join(outdir, f"{candidate}.json")
+
+
 def print_banner(selected, sections, args):
     lines = [
         "measures sdnq attention and weight dequantization speed and accuracy on this gpu and recommends values for [cyan]Compute Settings -> SDNQ / SDNQ Attention[/cyan]",
@@ -839,13 +933,13 @@ def probe_fp8():
     # toggling torch._dynamo.config.disable: torch 2.13+ raises "found no compiled frames"
     # for fullgraph-compiled functions called inside a disable window
     try:
-        from modules.sdnq import kernel_wrappers as sdnq_kernel_wrappers
+        from sdnq import kernel_wrappers as sdnq_kernel_wrappers
         is_fp8_mm_supported = getattr(sdnq_kernel_wrappers, "is_fp8_mm_supported", True)
     except Exception:
         is_fp8_mm_supported = True
     if not is_fp8_mm_supported:
         return dict(qk=(False, "FP8 matmul is not supported in this architecture"), pv=(False, "FP8 matmul is not supported in this architecture"))
-    from modules.sdnq.kernels import triton_atten as atten_module
+    from sdnq.kernels import triton_atten as atten_module
     q, k, v = make_qkv(1, 2, 256, 64, structured=False)
     result = {}
     compiled_prep = atten_module.get_attn_inputs
@@ -872,7 +966,8 @@ def probe_compiled_prep():
     # host c++ compiler (msvc on windows) fails every sdnq attention call at generation.
     # must run before any other sdnq_triton_atten call: a prior eager run masks the
     # cold-start failure the webui hits
-    if not shared.opts.sdnq_dequantize_compile:
+    from sdnq.common import use_torch_compile
+    if not use_torch_compile:
         return "disabled", None
     q, k, v = make_qkv(1, 2, 256, 64, structured=False)
     with console.status("probing compiled input prep (compiles on first run, cached afterwards)"):
@@ -884,7 +979,7 @@ def probe_compiled_prep():
             torch._dynamo.reset() # pylint: disable=protected-access # drop failed compile state
             detail = f"{type(e).__name__}: {error_summary(e, 120)}"
     # check whether the dynamic=false workaround holds
-    from modules.sdnq.kernels import triton_atten as atten_module
+    from sdnq.kernels import triton_atten as atten_module
     compiled_prep = atten_module.get_attn_inputs
     inner = getattr(compiled_prep, "_torchdynamo_orig_callable", None)
     if inner is None:
@@ -948,8 +1043,8 @@ def make_block_master():
 
 
 def build_bench_block(master_sd, weights_cfg, use_mm, attention_spec):
-    from modules.sdnq import SDNQConfig
-    from modules.sdnq.quantizer import apply_sdnq_to_module
+    from sdnq import SDNQConfig
+    from sdnq.quantizer import apply_sdnq_to_module
     hidden, heads, mlp_dim = block_geometry["hidden"], block_geometry["heads"], block_geometry["mlp_dim"]
     block = BenchBlock(hidden, heads, mlp_dim, device=torch_device, dtype=bench_dtype)
     block.load_state_dict(master_sd)
@@ -990,8 +1085,8 @@ def make_quantized_linear(weight, weights_dtype, group_size=0, use_quantized_mat
     # quantize through the same entry point model loading uses, so forward benches measure
     # the production wrapper classes and dequantizer configuration; returns the quantize wall
     # time, a one-shot measurement of what on-the-fly quantization pays per layer at load
-    from modules.sdnq import SDNQConfig
-    from modules.sdnq.quantizer import sdnq_quantize_layer
+    from sdnq import SDNQConfig
+    from sdnq.quantizer import sdnq_quantize_layer
     out_features, in_features = weight.shape
     linear = torch.nn.Linear(in_features, out_features, bias=False, device=device, dtype=bench_dtype)
     with torch.no_grad():
@@ -1058,7 +1153,7 @@ def get_compiled_dequantize_weight():
         for limit_name in ("recompile_limit", "cache_size_limit", "accumulated_recompile_limit", "accumulated_cache_size_limit"):
             if hasattr(torch._dynamo.config, limit_name): # pylint: disable=protected-access
                 setattr(torch._dynamo.config, limit_name, max(8192, getattr(torch._dynamo.config, limit_name) or 0)) # pylint: disable=protected-access
-        from modules.sdnq.dequantizer import dequantize_weight
+        from sdnq.dequantizer import dequantize_weight
         compiled_dequantize_weight = torch.compile(dequantize_weight, fullgraph=True, dynamic=False)
     return compiled_dequantize_weight
 
@@ -1120,6 +1215,21 @@ def run_correctness():
     ko[:, :, :2, :] *= 1000.0
     checks.append(("stress: 1000x outlier keys", (qs, ko, vs), {}, {}))
     checks.append(("stress: fp16 100x activations", ((qs * 100.0).to(torch.float16), (ks * 100.0).to(torch.float16), (vs * 100.0).to(torch.float16)), {}, {}))
+    # value-side stress: the existing rows perturb k (the qk-quant failure axis); the unsafe
+    # pv accumulation (the accum flag with an unquantized pv, the sage-style fast mode) fails
+    # on v instead. unnormalized softmax weights are <= 1 each until the epilogue normalize,
+    # so near-uniform scores with same-sign values push one 16-wide kv-block dot past the
+    # fp16 max: those rows pass by overflowing, while the scaled paths must stay finite
+    vo = vs.clone()
+    vo[:, :, :2, :] *= 1000.0
+    checks.append(("stress: 1000x outlier values", (qs, ks, vo), {}, {}))
+    qu = qs * 1e-3 # near-zero scores: every unnormalized softmax weight ~1
+    vu = vs.abs() * 10000.0 # same-sign, below the fp16 cast limit
+    checks.append(("stress: uniform attention 10000x values", (qu, ks, vu), {}, {}))
+    if atten_supports_fp16_accum():
+        checks.append(("stress: uniform 10000x values, scaled fp16 accum", (qu, ks, vu), {}, dict(matmul_dtype="float16", pv_matmul_dtype="float16", use_fp16_accum=True)))
+        checks.append(("stress: uniform 10000x values, unsafe pv accum", (qu, ks, vu), {}, dict(pv_matmul_dtype="disabled", use_fp16_accum=True)))
+        checks.append(("stress: bf16 100000x values, unsafe pv accum", (qs.to(torch.bfloat16), ks.to(torch.bfloat16), vs.to(torch.bfloat16) * 100000.0), {}, dict(pv_matmul_dtype="disabled", use_fp16_accum=True)))
 
     table = Table(box=box.SIMPLE_HEAVY)
     table.add_column("code path")
@@ -1127,14 +1237,14 @@ def run_correctness():
     table.add_column("int8 error", justify="right")
     table.add_column("int8 max token", justify="right")
     table.add_column("result", justify="center")
-    panel = Panel(table, title="kernel correctness", subtitle="[dim]small shapes, vs fp32 sdpa reference; stress rows pass on finite output[/dim]", box=ROUNDED_BOX, expand=False)
+    panel = Panel(table, title="kernel correctness", subtitle="[dim]small shapes, vs fp32 sdpa reference; stress rows pass on finite output, unsafe accum rows by overflowing[/dim]", box=ROUNDED_BOX, expand=False)
     failed = []
     details = {}
     # checks target kernel behavior, so the input prep runs eager via a module-global swap.
     # do not toggle torch._dynamo.config.disable for this: newer torch raises "found no
     # compiled frames" when a fullgraph-compiled function is called inside a disable window,
     # failing every check and poisoning the first compiled call afterwards
-    from modules.sdnq.kernels import triton_atten as atten_module
+    from sdnq.kernels import triton_atten as atten_module
     compiled_prep = atten_module.get_attn_inputs
     inner_prep = getattr(compiled_prep, "_torchdynamo_orig_callable", None)
     if inner_prep is not None:
@@ -1157,17 +1267,24 @@ def run_correctness():
                     err_plain = rel_err(plain, ref)
                     err_quant = rel_err(quant, ref)
                     max_err = max_token_err(quant, ref)
-                    finite = bool(torch.isfinite(plain).all().item() and torch.isfinite(quant).all().item())
-                    if name.startswith("stress:"):
+                    finite_quant = bool(torch.isfinite(quant).all().item())
+                    finite = bool(torch.isfinite(plain).all().item()) and finite_quant
+                    expect_overflow = name.endswith("unsafe pv accum")
+                    if expect_overflow:
+                        ok = not finite_quant # the row demonstrates the overflow; finite output means the demonstration failed, not that the mode is safe
+                    elif name.startswith("stress:"):
                         ok = finite
                     else:
                         ok = finite and err_plain < 0.01 and err_quant < 0.2
                     if not ok:
                         failed.append(name)
                     details[name] = dict(kernel_err=err_plain, int8_err=err_quant, max_token_err=max_err, finite=finite, ok=ok)
-                    verdict = "[green]pass[/green]" if ok else "[red]fail[/red]"
-                    if not finite:
+                    if expect_overflow:
+                        verdict = "[yellow]overflows (as designed)[/yellow]" if ok else "[red]did not overflow[/red]"
+                    elif not finite:
                         verdict = "[red]non-finite[/red]"
+                    else:
+                        verdict = "[green]pass[/green]" if ok else "[red]fail[/red]"
                     table.add_row(name, err_cell(err_plain), err_cell(err_quant), err_cell(max_err), verdict)
                 except Exception as e:
                     failed.append(name)
@@ -1194,9 +1311,9 @@ def run_correctness():
 
 def make_prep_fn(q, k, v, attn_mask, kwargs, is_causal=False, enable_gqa=False):
     # mirror sdnq_triton_atten's prep call so the prep column measures the same code path
-    from modules.sdnq.kernels import triton_atten as atten_module
-    from modules.sdnq.quant_utils import get_hadamard, get_hadamard_group_size
-    from modules.sdnq.utils import next_power_of_2
+    from sdnq.kernels import triton_atten as atten_module
+    from sdnq.quant_utils import get_hadamard, get_hadamard_group_size
+    from sdnq.utils import next_power_of_2
     matmul_dtype = kwargs.get("matmul_dtype", "int8")
     do_quantize = kwargs.get("do_quantize", True)
     hadamard_group_size = kwargs.get("hadamard_group_size", 256)
@@ -1261,6 +1378,8 @@ def bench_shape(preset, iters, warmup, position=None, config_timeout=300, fp8_re
             continue
         if config_id == "fp8full" and not (fp8_result and fp8_result["qk"][0] and fp8_result["pv"][0]):
             continue
+        if kwargs is not None and kwargs.get("use_fp16_accum") and not atten_supports_fp16_accum():
+            continue
         selected_configs.append((config_id, config_label(config_id, label), kwargs))
 
     def make_table():
@@ -1377,7 +1496,7 @@ def bench_shape(preset, iters, warmup, position=None, config_timeout=300, fp8_re
 
 
 def bench_dequant_shape(shape_label, out_features, in_features, iters, warmup, position=None, config_timeout=300, selected_dtypes=None):
-    from modules.sdnq.common import check_torch_compile
+    from sdnq.common import check_torch_compile
     compile_on = check_torch_compile()
     dtype_configs = [(dtype_id, label, cfg) for dtype_id, label, cfg in dequant_dtype_configs if selected_dtypes is None or dtype_id in selected_dtypes]
 
@@ -1487,10 +1606,10 @@ def bench_dequant_shape(shape_label, out_features, in_features, iters, warmup, p
 
             # eager-mode forward: the dequantizer's __call__ resolves dequantize_weight_compiled
             # as a module global at call time, so pointing it at the eager function for the
-            # bench matches the webui with Dequantize using torch.compile off. do not toggle
+            # bench matches the webui with sdnq torch.compile off (SDNQ_USE_TORCH_COMPILE=0). do not toggle
             # torch._dynamo.config.disable instead: code objects called during a disable window
             # keep their skip marking and never compile again in this process
-            from modules.sdnq import dequantizer as dequantizer_module
+            from sdnq import dequantizer as dequantizer_module
             saved_compiled_fn = dequantizer_module.dequantize_weight_compiled
             try:
                 phase("timing linear forward, eager dequant")
@@ -1679,7 +1798,12 @@ def bench_float_mm_alternatives(shape_label, out_features, in_features, plain_re
     panel = Panel(table, title=f"float weights, explicit MatMul type: {shape_label} {dtype_label()}", subtitle="[dim]quantized matmul with the MatMul type set explicitly; dequant path and enabled rows repeated dimmed; vs dequant above x1.00 = faster than the dequant-path forward[/dim]", box=ROUNDED_BOX, expand=False)
 
     results = {}
-    runs = [(dtype_id, label, cfg, mm_dtype) for dtype_id, label, cfg in dtype_configs for mm_dtype in float_mm_alternative_dtypes]
+    # the accum variant repeats the float16 mm with the triton fp16-accum globals flipped
+    # in-process; the env var is not a dropdown value, so its rows never enter candidacy
+    mm_variants = [(mm_dtype, False) for mm_dtype in float_mm_alternative_dtypes]
+    if triton_mm_supports_fp16_accum() and "float16" in float_mm_alternative_dtypes:
+        mm_variants.append(("float16", True))
+    runs = [(dtype_id, label, cfg, mm_dtype, accum) for dtype_id, label, cfg in dtype_configs for mm_dtype, accum in mm_variants]
     progress, task = live_progress()
     with Live(Group(panel, progress), console=console, refresh_per_second=4) as live:
         weight = make_source_weight(out_features, in_features)
@@ -1688,8 +1812,8 @@ def bench_float_mm_alternatives(shape_label, out_features, in_features, plain_re
         x = torch.randn(dequant_forward_tokens, in_features, device=torch_device, dtype=bench_dtype, generator=generator)
         ref_out = fp32_linear_reference(x, weight_fp32)
         added_plain = set()
-        for index, (dtype_id, label, cfg, mm_dtype) in enumerate(runs, start=1):
-            row_label = f"{label} + {mm_dtype} mm"
+        for index, (dtype_id, label, cfg, mm_dtype, accum) in enumerate(runs, start=1):
+            row_label = f"{label} + {mm_dtype} mm" + (", fp16 accum" if accum else "")
             def phase(step, current_label=row_label, current_index=index):
                 progress.update(task, description=f"float mm {current_index}/{len(runs)}  {current_label}: {step}")
             plain = plain_results.get(dtype_id) or {}
@@ -1702,8 +1826,8 @@ def bench_float_mm_alternatives(shape_label, out_features, in_features, plain_re
                     table.add_row(f"[dim]{label} + enabled ({plain.get('mm_dtype')})[/dim]", "-", f"[dim]{plain['mm_ms']:8.3f} ms[/dim]", f"[dim]{plain['mm_err']:.5f}[/dim]" if plain.get("mm_err") else "-", f"[dim]{speedup_cell(fwd_ms, plain['mm_ms'])}[/dim]" if fwd_ms else "-")
                 elif plain.get("mm_error"):
                     table.add_row(f"[dim]{label} + enabled ({plain.get('mm_dtype') or 'float8_e4m3fn'})[/dim]", "-", failure_text(RuntimeError(plain["mm_error"])), "-", "-")
-            entry = dict(quant_s=None, mm_ms=None, mm_err=None, mm_dtype=None)
-            results[f"{dtype_id}+{mm_dtype}"] = entry
+            entry = dict(quant_s=None, mm_ms=None, mm_err=None, mm_dtype=None, fp16_accum=accum)
+            results[f"{dtype_id}+{mm_dtype}" + ("+accum" if accum else "")] = entry
             progress.reset(task)
             phase("quantizing")
             try:
@@ -1714,7 +1838,10 @@ def bench_float_mm_alternatives(shape_label, out_features, in_features, plain_re
                 table.add_row(row_label, "-", failure_text(e), "-", "-")
                 continue
 
-            def mm_fn(current=layer):
+            def mm_fn(current=layer, use_accum=accum):
+                if use_accum:
+                    with triton_mm_fp16_accum():
+                        return current(x)
                 return current(x)
 
             try:
@@ -1754,10 +1881,10 @@ def bench_float_mm_alternatives(shape_label, out_features, in_features, plain_re
 # never defined and the row needs SDNQ_USE_TRITON_MM=0.
 
 mm_swap_targets = [
-    ("modules.sdnq.layers.linear.linear_int8", "int_scaled_mm_func"),
-    ("modules.sdnq.layers.linear.linear_uint8", "int_scaled_mm_func"),
-    ("modules.sdnq.layers.linear.linear_fp16", "fp_scaled_mm_func"),
-    ("modules.sdnq.layers.linear.linear_fp8", "fp8_scaled_mm_func"),
+    ("sdnq.layers.linear.linear_int8", "int_scaled_mm_func"),
+    ("sdnq.layers.linear.linear_uint8", "int_scaled_mm_func"),
+    ("sdnq.layers.linear.linear_fp16", "fp_scaled_mm_func"),
+    ("sdnq.layers.linear.linear_fp8", "fp8_scaled_mm_func"),
 ]
 
 
@@ -1774,7 +1901,7 @@ def mm_backend_bindings():
         if func is not None:
             bound[(module_path, attr)] = func
     try:
-        from modules.sdnq.kernels.triton_scaled_mm import sdnq_scaled_mm
+        from sdnq.kernels.triton_scaled_mm import sdnq_scaled_mm
     except Exception as e:
         return {}, {"triton": f"triton scaled mm unavailable: {error_summary(e, 120)}"}
 
@@ -2284,8 +2411,8 @@ def fp32_conv_reference(x, weight_fp32, padding):
 
 
 def make_quantized_conv(weight, weights_dtype, use_quantized_matmul=False):
-    from modules.sdnq import SDNQConfig
-    from modules.sdnq.quantizer import sdnq_quantize_layer
+    from sdnq import SDNQConfig
+    from sdnq.quantizer import sdnq_quantize_layer
     out_channels, in_channels, kh, kw = weight.shape
     conv = torch.nn.Conv2d(in_channels, out_channels, (kh, kw), padding=(kh // 2, kw // 2), bias=False, device=torch_device, dtype=bench_dtype)
     with torch.no_grad():
@@ -2407,10 +2534,16 @@ def bench_block_geometry(iters, warmup, config_timeout=300, selected=None):
     # component speedups and errors do not compose multiplicatively
     configs = [c for c in block_configs if selected is None or c[0] in selected]
     sage_missing = {"sage": sage_attention() is None, "sage fp16 accum": sage_attention_fp16_accum() is None}
-    skipped = [config_id for config_id, _w, _mm, spec in configs if sage_missing.get(spec, False)]
+    accum_missing = not atten_supports_fp16_accum()
+    def spec_unavailable(spec):
+        if sage_missing.get(spec, False):
+            return True
+        spec_kwargs = block_attention_specs.get(spec)
+        return accum_missing and isinstance(spec_kwargs, dict) and bool(spec_kwargs.get("use_fp16_accum"))
+    skipped = [config_id for config_id, _w, _mm, spec in configs if spec_unavailable(spec)]
     if skipped:
         configs = [c for c in configs if c[0] not in skipped]
-        emit(f"[dim]block: skipping {', '.join(skipped)}, sageattention (or this accumulation mode) is unavailable here[/dim]")
+        emit(f"[dim]block: skipping {', '.join(skipped)}, sageattention or the sdnq accumulation kwarg is unavailable here[/dim]")
     if not configs:
         return {}
     hidden, heads, mlp_dim, tokens = block_geometry["hidden"], block_geometry["heads"], block_geometry["mlp_dim"], block_geometry["tokens"]
@@ -2514,7 +2647,7 @@ def bench_block_geometry(iters, warmup, config_timeout=300, selected=None):
     if current_id and results.get(current_id, {}).get("ms"):
         notes.append(f"current config runs the {results[current_id]['label']} row for int8-quantized models")
     if any(entry.get("ms") for config_id, entry in results.items() if config_id.endswith("sagefp16")):
-        notes.append("sage fp16 accum: benchmark-only, no sdnext setting reaches it (sage pins fp32 accum on sm86)")
+        notes.append("sage fp16 accum: benchmark-only for sage (sdnext pins its fp32 accum); the equivalent sdnq mode is the FP16 Accumulation checkbox with pv matmul disabled, same overflow tail")
     if notes:
         emit(Panel("\n".join(notes), title="block notes", box=ROUNDED_BOX))
     return results
@@ -2528,7 +2661,7 @@ def measured(results, config_id):
 
 def best_config(results):
     # lowest error among rows within 5% of the fastest sdnq time
-    candidates = [(config_id, entry["ms"], entry["err"]) for config_id, entry in results.items() if entry.get("ms") is not None and config_id not in external_config_ids]
+    candidates = [(config_id, entry["ms"], entry["err"]) for config_id, entry in results.items() if entry.get("ms") is not None and config_id not in external_config_ids and config_id not in unsafe_config_ids]
     if not candidates:
         return None
     fastest = min(ms for _config_id, ms, _err in candidates)
@@ -2548,6 +2681,7 @@ def config_settings(kwargs):
         "pv": "disabled" if pv == "auto" else pv,
         "smooth": bool(kwargs.get("smooth_k", False)),
         "hadamard": bool(kwargs.get("use_hadamard", False)),
+        "accum": bool(kwargs.get("use_fp16_accum", False)),
     }
 
 
@@ -2563,6 +2697,8 @@ def select_attention_config(results):
         settings = config_settings(kwargs)
         if settings is None:
             continue
+        if settings["accum"] and settings["pv"] == "disabled":
+            continue # the unsafe accumulation combo is never a candidate; the accum row cites it directly
         ms, err = measured(results, config_id)
         if ms is None:
             continue
@@ -2690,11 +2826,11 @@ def build_recommendations(all_results, fp8_result, prep_status, block_results=No
     by_settings = {}
     for entry in pool + capped:
         s = entry["settings"]
-        by_settings[(s["matmul"], s["pv"], s["smooth"], s["hadamard"])] = entry
+        by_settings[(s["matmul"], s["pv"], s["smooth"], s["hadamard"], s["accum"])] = entry
 
     def sibling(entry, **overrides):
         s = dict(entry["settings"], **overrides)
-        other = by_settings.get((s["matmul"], s["pv"], s["smooth"], s["hadamard"]))
+        other = by_settings.get((s["matmul"], s["pv"], s["smooth"], s["hadamard"], s["accum"]))
         return None if other is entry else other
 
     def block_buyback_survives(block_config_id):
@@ -2757,6 +2893,35 @@ def build_recommendations(all_results, fp8_result, prep_status, block_results=No
     else:
         rows.append(("PV MatMul type", current("sdnq_attention_pv_matmul_type"), "disabled", "qk quantization is not recommended above; pv on an unquantized qk path was not measured"))
 
+    # the accumulation flag only changes fp16 dots: scaled and overflow-proof on quantized
+    # float16 types, inert on int8 types, and the sage-style unsafe mode on an unquantized
+    # pv. only the scaled combos are candidates; the unsafe one is cited, never recommended
+    accum_key = "sdnq_attention_use_fp16_accum"
+    if atten_supports_fp16_accum():
+        accum_current = current(accum_key) if hasattr(shared.opts, accum_key) else "-"
+        pvaccum_ms, _pvaccum_err = measured(results, "pvaccum")
+        unsafe_note = "with an unquantized pv the flag is the sage-style unsafe accumulation instead (the stress rows demonstrate its overflow)"
+        if pvaccum_ms and int8_ms:
+            unsafe_note += f", measured x{int8_ms / pvaccum_ms:.2f} vs int8 qk"
+        if use_quantized and chosen["settings"]["accum"]:
+            without = sibling(chosen, accum=False)
+            accum_reason = f"part of the selected config ({chosen['label']})"
+            if without:
+                accum_reason = f"scaled fp16 accumulation measured x{without['ms'] / chosen['ms']:.2f} over the same stack without it"
+                if chosen["err"] and without["err"]:
+                    accum_reason += f", error {chosen['err']:.5f} vs {without['err']:.5f}"
+            accum_reason += "; safe here because every fp16 dot in the stack is quantized and pre-scaled"
+            rows.append(("Use FP16 Accumulation", accum_current, "True", accum_reason))
+        elif qk_inconclusive:
+            rows.append(("Use FP16 Accumulation", accum_current, accum_current, "the qk verdict above is inconclusive; the accumulation flag follows it"))
+        else:
+            best_accum = best_in_subset([entry for entry in pool + capped if entry["settings"]["accum"]])
+            if chosen is not None and best_accum is not None:
+                accum_reason = f"the strongest scaled-accum config ({best_accum['label']}) measured x{chosen['ms'] / best_accum['ms']:.2f} the speed of the selected config"
+            else:
+                accum_reason = "no scaled-accum config was measured at this shape"
+            rows.append(("Use FP16 Accumulation", accum_current, "False", f"{accum_reason}; {unsafe_note}"))
+
     def toggle_row(setting_key, ui_name, on, buyback_reason, subset, block_config_id, caveat=None):
         # a toggle the selected config carries keeps the buyback evidence as its reason;
         # one it omits cites the strongest variant that carried it, plus the block-scope
@@ -2848,7 +3013,7 @@ def build_recommendations(all_results, fp8_result, prep_status, block_results=No
     if prep_status == "failing_dynamic":
         notes.append("[red]generation fails here: compile cannot build the dynamic-shape prep. fix: set SDNQ_COMPILE_KWARGS='{\"dynamic\": false}' (verified on this machine) or install msvc build tools; numbers above use that workaround[/red]")
     elif prep_status == "failing":
-        notes.append("[red]generation fails here: torch compile is broken. fix: disable Dequantize using torch.compile or install msvc build tools; numbers above use eager prep[/red]")
+        notes.append("[red]generation fails here: torch compile is broken. fix: set SDNQ_USE_TORCH_COMPILE=0 or install msvc build tools; numbers above use eager prep[/red]")
     if not fp8_result["qk"][0]:
         if fp8_failure_is_capability(fp8_result["qk"][1]):
             notes.append("[red]float8_e4m3fn unsupported on this gpu: selecting it in either dropdown fails generation[/red]")
@@ -2858,8 +3023,11 @@ def build_recommendations(all_results, fp8_result, prep_status, block_results=No
     if sage_ms and int8_ms:
         notes.append(f"vs sage at {reference}: sdnq int8 {int8_ms:.2f} ms, sage {sage_ms:.2f} ms; sdnq also covers masks, gqa, causal")
     sagefp16_ms, _sagefp16_err = measured(results, "sagefp16")
-    if sagefp16_ms:
-        notes.append("sage fp16 accum: benchmark-only, no sdnext setting reaches it (sage pins fp32 accum on sm86)")
+    pvaccum_ms, _pvaccum_err = measured(results, "pvaccum")
+    if sagefp16_ms and pvaccum_ms:
+        notes.append(f"sage fp16 accum {sagefp16_ms:.2f} ms vs sdnq unquantized-pv accum {pvaccum_ms:.2f} ms: the same accumulation mode with the same overflow tail; sdnext reaches the sdnq one via SDNQ Attention use FP16 Accumulation with PV MatMul disabled, sage itself stays pinned to fp32 accum")
+    elif sagefp16_ms:
+        notes.append("sage fp16 accum: benchmark-only for sage (sdnext pins its fp32 accum on sm86); the equivalent sdnq mode is the unquantized-pv accum config")
     # compare flash against the config the verdict above actually recommends
     amdflash_ms, _amdflash_err = measured(results, "amdflash")
     if amdflash_ms:
@@ -2931,10 +3099,12 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
     mode_ids = {cfg["weights_dtype"]: dtype_id for dtype_id, _label, cfg in dequant_dtype_configs}
     mm_id = mode_ids.get(weights_mode, "int8")
 
-    # the compile toggle is output-neutral (dequant drift is checked separately), so its verdict
-    # is a symmetric faster/slower test (margin 1.0) at the layer-forward scope the option gates,
-    # voted across every measured shape; the standalone kernel can lose to eager on small layers
-    # from launch overhead alone, which is why it never decides this row
+    # the compile toggle became automatic: sdnq compiles whenever triton is available and
+    # SDNQ_USE_TORCH_COMPILE is the only override, so the verdict is a note addressed to the
+    # env var rather than a settings row. it stays a symmetric faster/slower test (margin 1.0)
+    # at the layer-forward scope compile gates, voted across every measured shape; the
+    # standalone kernel can lose to eager on small layers from launch overhead alone, which
+    # is why it never decides the verdict
     compile_id = mm_id if entry(mm_id).get("fwd_eager_ms") and entry(mm_id).get("fwd_compiled_ms") else "int8"
     fwd_votes = []
     for shape_label in dequant_results:
@@ -2944,35 +3114,33 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
             test = speed_verdict(fwd_compiled, fwd_eager, sigma=pair_sigma(vote_row, "fwd_compiled_ms", vote_row, "fwd_eager_ms"), margin=1.0)
             fwd_votes.append((fwd_eager / fwd_compiled, test))
     int8_eager, int8_compiled = entry("int8").get("eager_ms"), entry("int8").get("compiled_ms")
+    try:
+        from sdnq.common import use_torch_compile as compile_active
+    except Exception:
+        compile_active = None
+    compile_origin = "SDNQ_USE_TORCH_COMPILE" if os.environ.get("SDNQ_USE_TORCH_COMPILE", None) is not None else "automatic with triton"
+    compile_state = {True: "on", False: "off"}.get(compile_active, "unknown")
+    compile_note = None
     if fwd_votes:
         ratios = sorted(ratio for ratio, _test in fwd_votes)
         span = f"x{ratios[0]:.2f}" if len(ratios) == 1 else f"x{ratios[0]:.2f}-x{ratios[-1]:.2f}"
         scope_text = f"{compile_id} layer forward measured {span} compiled vs eager across {len(fwd_votes)} shape{'s' if len(fwd_votes) > 1 else ''}"
         kinds = {test for _ratio, test in fwd_votes}
         if "faster" in kinds and "not_faster" not in kinds:
-            compile_choice, compile_reason = "True", scope_text
+            verdict_text = "keep it on" if compile_active in {True, None} else "[yellow]compiled is faster here, drop SDNQ_USE_TORCH_COMPILE=0[/yellow]"
         elif "not_faster" in kinds and "faster" not in kinds:
-            compile_choice, compile_reason = "False", scope_text
+            verdict_text = "SDNQ_USE_TORCH_COMPILE=0 would help on this gpu" if compile_active in {True, None} else "keeping it off matches the measurement"
         elif "faster" in kinds:
             n_faster = sum(1 for _ratio, test in fwd_votes if test == "faster")
             n_slower = sum(1 for _ratio, test in fwd_votes if test == "not_faster")
-            compile_choice = current("sdnq_dequantize_compile")
-            compile_reason = f"{scope_text}; split verdict ({n_faster} faster, {n_slower} slower), keeping the current setting"
+            verdict_text = f"split verdict ({n_faster} faster, {n_slower} slower)"
         else:
-            compile_choice = current("sdnq_dequantize_compile")
-            compile_reason = f"{scope_text}, within this run's noise; keeping the current setting"
-        rows.append(("Dequantize using torch.compile", current("sdnq_dequantize_compile"), compile_choice, compile_reason))
+            verdict_text = "within this run's noise"
+        compile_note = f"dequant torch.compile ({compile_state}, {compile_origin}): {scope_text}; {verdict_text}"
     elif int8_eager and int8_compiled:
         compile_test = speed_verdict(int8_compiled, int8_eager, sigma=pair_sigma(entry("int8"), "compiled_ms", entry("int8"), "eager_ms"), margin=1.0)
-        compile_reason = f"int8 dequant kernel measured {ratio_text(int8_eager, int8_compiled)} compiled vs eager, no layer forward data"
-        if compile_test == "inconclusive":
-            compile_choice = current("sdnq_dequantize_compile")
-            compile_reason += "; within this run's noise, keeping the current setting"
-        else:
-            compile_choice = str(compile_test == "faster")
-        rows.append(("Dequantize using torch.compile", current("sdnq_dequantize_compile"), compile_choice, compile_reason))
-    elif int8_eager:
-        rows.append(("Dequantize using torch.compile", current("sdnq_dequantize_compile"), current("sdnq_dequantize_compile"), "compiled int8 dequant unavailable, keeping the current value"))
+        verdict_text = {"faster": "keep it on", "not_faster": "SDNQ_USE_TORCH_COMPILE=0 would help on this gpu", "inconclusive": "within this run's noise"}[compile_test]
+        compile_note = f"dequant torch.compile ({compile_state}, {compile_origin}): int8 dequant kernel measured {ratio_text(int8_eager, int8_compiled)} compiled vs eager, no layer forward data; {verdict_text}"
 
     # judge quantized matmul on the dtype the current config would quantize with, falling back to int8;
     # speed never wins alone: the faster path must also hold output error within recommend_error_cap
@@ -3020,6 +3188,9 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
             mm_reason += f"; enabled resolves to {best_resolved} for {mm_id} weights"
         else:
             mm_reason += f"; enabled (float8_e4m3fn) failed on this gpu for {mm_id} weights"
+        accum_alt = float_mm.get(f"{mm_id}+float16+accum") or {}
+        if accum_alt.get("mm_ms"):
+            mm_reason += f"; float16 mm with SDNQ_TRITON_MM_USE_FP16_ACCUM=1 measured {accum_alt['mm_ms']:.3f} ms (env var, not a dropdown value)"
         if mm_test == "inconclusive" and err_ok:
             mm_choice = current("sdnq_quantize_matmul_mode")
             mm_reason += "; too close to the margin to call at this run's noise, keeping the current setting"
@@ -3156,6 +3327,8 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
         emit("[green]current settings already match the recommendations[/green]")
 
     notes = []
+    if compile_note:
+        notes.append(compile_note)
     speedups = []
     for dtype_id, label, _cfg in dequant_dtype_configs:
         text = ratio_text(entry(dtype_id).get("eager_ms"), entry(dtype_id).get("compiled_ms"))
@@ -3382,6 +3555,7 @@ def main():
     if unknown:
         console.print(f"[red]unknown shape preset(s): {', '.join(unknown)}; available: {', '.join(shape_presets)}[/red]")
         sys.exit(1)
+    resolve_output_paths(args)
     global pending_outputs # pylint: disable=global-statement
     pending_outputs = dict(save=args.save, json=args.json, args=args, sections=sections, selected=selected)
     print_banner(selected, sections, args)
@@ -3403,13 +3577,13 @@ def main():
         if free_vram_gb() < 2.0:
             emit(f"[yellow]skipping dequant benchmarks: needs about 2 gb free vram, {free_vram_gb():.1f} gb available[/yellow]")
         else:
-            from modules.sdnq.common import check_torch_compile
+            from sdnq.common import check_torch_compile
             if not check_torch_compile():
                 # the module-level compiled dequant is a passthrough with the option off; swap in
                 # a real compiled variant so the compiled fwd rows measure what enabling it gives
-                from modules.sdnq import dequantizer as dequantizer_module
+                from sdnq import dequantizer as dequantizer_module
                 dequantizer_module.dequantize_weight_compiled = get_compiled_dequantize_weight()
-                emit("[yellow]Dequantize using torch.compile is off in the current config: compiled fwd rows are measured with a tool-compiled dequant, matching the webui after enabling it[/yellow]")
+                emit("[yellow]sdnq torch.compile is off (SDNQ_USE_TORCH_COMPILE=0): compiled fwd rows are measured with a tool-compiled dequant, matching the webui with it back on[/yellow]")
             for index, (shape_label, out_features, in_features) in enumerate(dequant_shapes, start=1):
                 dequant_results[shape_label] = bench_dequant_shape(shape_label, out_features, in_features, args.iters, args.warmup, position=(index, len(dequant_shapes)), config_timeout=args.config_timeout, selected_dtypes=selected_dtypes)
                 torch_device_module.empty_cache()
@@ -3446,13 +3620,13 @@ def main():
     if "attention" in sections:
         # bench the prep mode the advice points to: compiled, static workaround, or eager
         if prep_status == "failing_dynamic":
-            from modules.sdnq.kernels import triton_atten as atten_module
+            from sdnq.kernels import triton_atten as atten_module
             inner = getattr(atten_module.get_attn_inputs, "_torchdynamo_orig_callable", None)
             atten_module.get_attn_inputs = torch.compile(inner, fullgraph=True, dynamic=False)
             emit("[yellow]dynamic-shape compile is broken here: benchmarking with the dynamic=false workaround applied, numbers match the webui after setting SDNQ_COMPILE_KWARGS='{\"dynamic\": false}'[/yellow]")
         elif prep_status == "failing":
-            emit("[yellow]torch compile is broken here: benchmarking with eager input prep, numbers match the webui after disabling Dequantize using torch.compile[/yellow]")
-            from modules.sdnq.kernels import triton_atten as atten_module
+            emit("[yellow]torch compile is broken here: benchmarking with eager input prep, numbers match the webui after setting SDNQ_USE_TORCH_COMPILE=0[/yellow]")
+            from sdnq.kernels import triton_atten as atten_module
             inner = getattr(atten_module.get_attn_inputs, "_torchdynamo_orig_callable", None)
             if inner is not None: # swap in the eager prep; a disable toggle raises on torch 2.13+
                 atten_module.get_attn_inputs = inner

data/previews.json

@@ -39,5 +39,6 @@
   "vladmandic--Qwen-Lightning-Edit": "Qwen-Lightning.jpg",
   "vladmandic--Qwen-Lightning": "Qwen-Lightning.jpg",
   "Wan-AI--Wan2.1-T2V-14B-Diffusers": "Wan-AI--Wan2.1-T2V-14B-Diffusers.jpg",
-  "Wan-AI--Wan2.2-T2V-A14B-Diffusers": "Wan-AI--Wan2.2-T2V-A14B-Diffusers.jpg"
+  "Wan-AI--Wan2.2-T2V-A14B-Diffusers": "Wan-AI--Wan2.2-T2V-A14B-Diffusers.jpg",
+  "OzzyGT--MiniMax_H3_sdnq_dynamic_4bit": "MiniMaxAI--MiniMax-H3.jpg"
 }

data/reference-base.json

@@ -374,6 +374,23 @@
     "size": 75.64,
     "date": "2025 September"
   },
+  "MiniMaxAI MiniMax-H3": {
+    "path": "MiniMaxAI/MiniMax-H3",
+    "preview": "MiniMaxAI--MiniMax-H3.jpg",
+    "desc": "MiniMax-H3 generates video with synchronized stereo audio in a single denoising pass through a 33B single-stream transformer with a Qwen3-VL conditioner. In image tabs the model runs in experimental still mode, keeping the first frame of a minimal generation.",
+    "extras": "sampler: Default",
+    "size": 134.12,
+    "date": "2026 August"
+  },
+  "MiniMaxAI MiniMax-H3 Ref2VA": {
+    "path": "MiniMaxAI/MiniMax-H3",
+    "subfolder": "ref2va",
+    "preview": "MiniMaxAI--MiniMax-H3.jpg",
+    "desc": "The omni-reference variant of MiniMax-H3, sharing one repository with the base model as a separate checkpoint partition. Video with synchronized stereo audio is conditioned on reference images for identity and appearance, with reference rows held clean while video rows denoise.",
+    "extras": "sampler: Default",
+    "size": 134.12,
+    "date": "2026 August"
+  },
   "Freepik F-Lite": {
     "path": "Freepik/F-Lite",
     "preview": "Freepik--F-Lite.jpg",

data/reference-nunchaku.json

@@ -274,5 +274,26 @@
     "desc": "Nunchaku-Lite quantization using precompiled Kernels",
     "size": 16.81,
     "date": "2026 July"
+  },
+  "Krea 2 Turbo Nunchaku-Lite": {
+    "path": "lite-infer/krea-2-turbo-nunchaku-lite-int4_r32-bnb4-text-encoder",
+    "preview": "CalamitousFelicitousness--Krea-2-Turbo-Diffusers.jpg",
+    "desc": "Nunchaku-Lite quantization using precompiled Kernels",
+    "size": 10.32,
+    "date": "2026 August"
+  },
+  "BFL FLUX.2 Klein Base 4B Nunchaku-Lite": {
+    "path": "lite-infer/flux.2-klein-4b-nunchaku-lite-int4_r32-bnb4-text-encoder",
+    "preview": "black-forest-labs--FLUX.2-klein-base-4B.jpg",
+    "desc": "Nunchaku-Lite quantization using precompiled Kernels",
+    "size": 4.87,
+    "date": "2026 August"
+  },
+  "BFL FLUX.2 Klein Base 9B Nunchaku-Lite": {
+    "path": "lite-infer/flux.2-klein-9b-nunchaku-lite-int4_r32-bnb4-text-encoder",
+    "preview": "black-forest-labs--FLUX.2-klein-base-9B.jpg",
+    "desc": "Nunchaku-Lite quantization using precompiled Kernels",
+    "size": 10.92,
+    "date": "2026 August"
   }
 }

data/reference-quantized.json

@@ -79,6 +79,40 @@
     "date": "2025 October",
     "size": 23.53
   },
+  "MiniMaxAI MiniMax-H3 sdnq-uint4": {
+    "path": "OzzyGT/MiniMax_H3_sdnq_dynamic_4bit",
+    "preview": "MiniMaxAI--MiniMax-H3.jpg",
+    "desc": "Quantization of MiniMaxAI/MiniMax-H3 using SDNQ: dynamic 4-bit uint. Video with synchronized audio; in image tabs the model runs in experimental still mode.",
+    "extras": "sampler: Default",
+    "size": 64.80,
+    "date": "2026 August"
+  },
+  "MiniMaxAI MiniMax-H3 sdnq-uint4 Ref2VA": {
+    "path": "OzzyGT/MiniMax_H3_sdnq_dynamic_4bit",
+    "preview": "MiniMaxAI--MiniMax-H3.jpg",
+    "subfolder": "ref2va",
+    "desc": "Quantization of MiniMaxAI/MiniMax-H3 using SDNQ: dynamic 4-bit uint. Video with synchronized audio; in image tabs the model runs in experimental still mode.",
+    "extras": "sampler: Default",
+    "size": 64.80,
+    "date": "2026 August"
+  },
+  "MiniMaxAI MiniMax-H3 Pruned sdnq-uint4": {
+    "path": "OzzyGT/MiniMax_H3_sdnq_4bit_pruned",
+    "preview": "OzzyGT--MiniMax_H3_sdnq_4bit_pruned.jpg",
+    "desc": "Quantization of MiniMaxAI/MiniMax-H3 using SDNQ: dynamic 4-bit uint. Video with synchronized audio; in image tabs the model runs in experimental still mode.",
+    "extras": "sampler: Default",
+    "size": 23.70,
+    "date": "2026 August"
+  },
+  "MiniMaxAI MiniMax-H3 Pruned sdnq-uint4 Ref2VA": {
+    "path": "OzzyGT/MiniMax_H3_sdnq_4bit_pruned",
+    "preview": "OzzyGT--MiniMax_H3_sdnq_4bit_pruned.jpg",
+    "subfolder": "ref2va",
+    "desc": "Quantization of MiniMaxAI/MiniMax-H3 using SDNQ: dynamic 4-bit uint. Video with synchronized audio; in image tabs the model runs in experimental still mode.",
+    "extras": "sampler: Default",
+    "size": 23.70,
+    "date": "2026 August"
+  },
   "Z-Image-Turbo sdnq-svd-uint4": {
     "path": "Disty0/Z-Image-Turbo-SDNQ-uint4-svd-r32",
     "preview": "Disty0--Z-Image-Turbo-SDNQ-uint4-svd-r32.jpg",

installer.py

@@ -54,7 +54,6 @@ args = Dot({
     'skip_requirements': False,
     'skip_git': False,
     'skip_torch': False,
-    'use_directml': False,
     'use_ipex': False,
     'use_cuda': False,
     'use_rocm': False,
@@ -67,6 +66,7 @@ args = Dot({
     'uv': False,
 })
 git_commit = "unknown"
+sdnq_commit = "unknown"
 diffusers_commit = "unknown"
 transformers_commit = "unknown"
 restart_required = False
@@ -478,8 +478,8 @@ def get_platform():
             release = platform.release()
         return {
             'arch': platform.machine(),
-            'cpu': f'{platform.processor()}',
-            'system': platform.system(),
+            'cpu': f'"{platform.processor()}"',
+            'system': f'"{platform.system()}"',
             'release': release,
             'python': platform.python_version(),
             'locale': locale.getlocale(),
@@ -549,13 +549,43 @@ def check_python(supported_minors=None, experimental_minors=None, reason=None):
     ts('python', t_start)
 
 
+# register sdnq package from github submodule
+def register_sdnq(skip=False, devices=None, shared=None):
+    if not skip:
+        t_start = time.time()
+        fn = os.path.join('extensions-builtin', 'sdnq', 'src', 'sdnq', '__init__.py')
+        name = "sdnq"
+        spec = importlib.util.spec_from_file_location(name, fn)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[name] = module
+        spec.loader.exec_module(module) # this is where actual import happens
+        import sdnq # pylint: disable=unused-import # test import
+        ts('sdnq', t_start)
+    if devices is not None:
+        import sdnq
+        sdnq.sdnext.devices = devices
+        sdnq.quantizer.devices = devices
+        sdnq.dequantizer.devices = devices
+        sdnq.quant_utils.devices = devices
+        sdnq.kernel_wrappers.devices = devices
+    if shared is not None:
+        import sdnq
+        sdnq.sdnext.shared = shared
+        sdnq.quantizer.shared = shared
+        sdnq.dequantizer.shared = shared
+        sdnq.quant_utils.shared = shared
+        sdnq.kernel_wrappers.shared = shared
+        sdnq.common.shared = shared
+        sdnq.loader.shared = shared
+
+
 # check diffusers version
 def check_diffusers():
     t_start = time.time()
     if args.skip_all:
         return
-    target_commit = "6f2010e8bbe61fd2a81a659b858e298edcba8fab" # diffusers commit hash == 0.40.0.dev0 == 08-04-2026
-    # if args.use_rocm or args.use_zluda or args.use_directml:
+    target_commit = "d5baa4fb548294f47dbca49890abd4b291204c60" # diffusers commit hash == 0.40.0.dev0 == 08-15-2026
+    # if args.use_rocm or args.use_zluda:
     #     sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
     pkg = package_spec('diffusers')
     parts = pkg.version.split('.') if pkg is not None else []
@@ -585,36 +615,21 @@ def check_transformers():
     # target_commit = '753d61104116eefc8ffc977327b441ee0c8d599f' # transformers commit hash == 4.57.6
     # target_commit = "cf8572d34e39818e42dbf220701fbd3eb5b5a82a" # transformers commit hash == 5.14.0.dev0 == 08-04-2026
     target_commit = "b70d02fc724d04c916832ca4ead03ff05e8fb1ee" # transformers commit hash == 5.13.0.dev0 == 07-03-2026
-    if args.use_directml:
-        target_transformers = '4.52.4'
-        target_tokenizers = '0.21.4'
-    else:
-        # target_transformers = '4.57.6'
-        target_transformers = None
-        target_tokenizers = '0.22.2'
-    if target_transformers is not None:
-        # Pinned release version (e.g. DirectML)
-        if args.reinstall or (pkg_transformers is None) or ((pkg_transformers.version != target_transformers) or (pkg_tokenizers is None) or ((pkg_tokenizers.version != target_tokenizers) and (not args.experimental))):
-            if pkg_transformers is None:
-                log.info(f'Install: package="transformers" version={target_transformers}')
-            else:
-                log.info(f'Update: package="transformers" current={pkg_transformers.version} target={target_transformers}')
-            pip('uninstall --yes transformers', ignore=True, quiet=True)
-            pip(f'install tokenizers=={target_tokenizers}', ignore=False, quiet=True)
-            pip(f'install transformers=={target_transformers}', ignore=False, quiet=True)
-    else:
-        # Git commit-pinned version
-        current = package_commit(pkg_transformers)
-        if args.reinstall or (pkg_transformers is None) or (pkg_transformers.version.startswith('4')) or (current != target_commit):
-            if pkg_transformers is None:
-                log.info(f'Install: package="transformers" commit={target_commit}')
-            else:
-                log.info(f'Update: package="transformers" current={pkg_transformers.version} commit={current} target={target_commit}')
-            pip('uninstall --yes transformers', ignore=True, quiet=True)
-            pip(f'install tokenizers=={target_tokenizers}', ignore=False, quiet=True)
-            pip(f'install git+https://github.com/huggingface/transformers@{target_commit}', ignore=False, quiet=True)
-            global transformers_commit # pylint: disable=global-statement
-            transformers_commit = target_commit
+    target_tokenizers = '0.22.2'
+    # Git commit-pinned version
+    current = package_commit(pkg_transformers)
+    if args.reinstall or (pkg_transformers is None) or (pkg_transformers.version.startswith('4')) or (current != target_commit):
+        if pkg_transformers is None:
+            log.info(f'Install: package="transformers" commit={target_commit}')
+        else:
+            log.info(f'Update: package="transformers" current={pkg_transformers.version} commit={current} target={target_commit}')
+        pip('uninstall --yes transformers', ignore=True, quiet=True)
+        pip(f'install tokenizers=={target_tokenizers}', ignore=False, quiet=True)
+        pip(f'install git+https://github.com/huggingface/transformers@{target_commit}', ignore=False, quiet=True)
+        global transformers_commit # pylint: disable=global-statement
+        transformers_commit = target_commit
+    if args.reinstall or (pkg_tokenizers is None) or (pkg_tokenizers.version != target_tokenizers):
+        pip(f'install tokenizers=={target_tokenizers}', ignore=False, quiet=True)
     ts('transformers', t_start)
 
 
@@ -684,7 +699,7 @@ def install_rocm_zluda():
 
     if sys.platform == "win32" and (not args.use_zluda) and (device is not None) and (device.therock is not None) and not installed("rocm"):
         check_python(supported_minors=[11, 12, 13], reason='ROCm-Windows: python==3.11/3.12/3.13 required')
-        install(f"rocm[devel,libraries] --index-url https://rocm.nightlies.amd.com/{device.therock}")
+        install("rocm-sdk-devel --index-url https://rocm.nightlies.amd.com/whl-multi-arch")
         rocm.refresh()
 
     msg = f'ROCm: version={rocm.version}'
@@ -718,9 +733,16 @@ def install_rocm_zluda():
         else: # TODO rocm: switch to pytorch source when it becomes available
             if device is None:
                 log.error('ROCm: no agent found - make sure that graphics driver is installed and up to date')
-            if isinstance(rocm.environment, rocm.PythonPackageEnvironment):
+            if device is not None and device.therock is not None:
                 check_python(supported_minors=[11, 12, 13], reason='ROCm-Windows: python==3.11/3.12/3.13 required')
-                torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://rocm.nightlies.amd.com/{device.therock}')
+                # Extract device-specific package family from therock path (e.g., 'amd-torch-device-gfx1030' from 'whl-multi-arch/amd-torch-device-gfx1030')
+                torch_family = device.therock.rsplit('/', 1)[-1]
+                torchvision_family = torch_family.replace('amd-torch-device-', 'amd-torchvision-device-')
+                # Use device-specific index for torch/torchvision, with root index as fallback for torchaudio and other packages
+                torch_command = os.environ.get('TORCH_COMMAND', f'{torch_family} {torchvision_family} torchaudio --index-url https://rocm.nightlies.amd.com/{device.therock} --extra-index-url https://rocm.nightlies.amd.com/whl-multi-arch')
+            elif isinstance(rocm.environment, rocm.PythonPackageEnvironment):
+                check_python(supported_minors=[11, 12, 13], reason='ROCm-Windows: python==3.11/3.12/3.13 required')
+                torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision torchaudio --index-url https://rocm.nightlies.amd.com/whl-multi-arch')
             else:
                 check_python(supported_minors=[12], reason='ROCm-Windows: preview python==3.12 required')
                 # torch 2.8.0a0 is the last version with rocm 6.4 support
@@ -728,33 +750,45 @@ def install_rocm_zluda():
 
     else: # linux
         #check_python(supported_minors=[10, 11, 12, 13, 14], reason='ROCm backend requires a Python version between 3.10 and 3.13')
+        rocm_major, rocm_minor = (int(x) for x in rocm.version.split('.')) if rocm.version is not None else (0, 0)
         if args.use_nightly:
-            if rocm.version is None or float(rocm.version) >= 7.2: # assume the latest if version check fails
+            if rocm.version is None or (rocm_major > 7 or (rocm_major == 7 and rocm_minor >= 2)): # assume the latest if version check fails
                 torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm7.2')
             else: # oldest rocm version on nightly is 7.1
                 torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm7.1')
         else:
-            if rocm.version is None or float(rocm.version) >= 7.2: # assume the latest if version check fails
+            if rocm.version is None or rocm_major > 7: # assume the latest if version check fails
                 torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.13.0+rocm7.2 torchvision==0.28.0+rocm7.2 --index-url https://download.pytorch.org/whl/rocm7.2')
-            elif rocm.version == "7.1":
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.13.0+rocm7.1 torchvision==0.28.0+rocm7.1 --index-url https://download.pytorch.org/whl/rocm7.1')
-            elif rocm.version == "7.0":
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.10.0+rocm7.0 torchvision==0.25.0+rocm7.0 --index-url https://download.pytorch.org/whl/rocm7.0')
-            elif rocm.version == "6.4":
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.9.1+rocm6.4 torchvision==0.24.1+rocm6.4 --index-url https://download.pytorch.org/whl/rocm6.4')
-            elif rocm.version == "6.3":
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.9.1+rocm6.3 torchvision==0.24.1+rocm6.3 --index-url https://download.pytorch.org/whl/rocm6.3')
-            elif rocm.version == "6.2":
-                # use rocm 6.2.4 instead of 6.2 as torch==2.7.1+rocm6.2 doesn't exists
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+rocm6.2.4 torchvision==0.22.1+rocm6.2.4 --index-url https://download.pytorch.org/whl/rocm6.2.4')
-            elif rocm.version == "6.1":
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+rocm6.1 torchvision==0.21.0+rocm6.1 --index-url https://download.pytorch.org/whl/rocm6.1')
             else:
-                # lock to 2.4.1 instead of 2.5.1 for performance reasons there are no support for torch 2.6 for rocm 6.0
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1+rocm6.0 torchvision==0.19.1+rocm6.0 --index-url https://download.pytorch.org/whl/rocm6.0')
-                if float(rocm.version) < 6.0:
-                    log.warning(f"ROCm: unsupported version={rocm.version}")
-                    log.warning("ROCm: minimum supported version=6.0")
+                match rocm_major:
+                    case 7:
+                        if rocm_minor >= 2: # latest supported rocm 7.x version is 7.2
+                            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.13.0+rocm7.2 torchvision==0.28.0+rocm7.2 --index-url https://download.pytorch.org/whl/rocm7.2')
+                        else:
+                            match rocm_minor:
+                                case 1:
+                                    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.13.0+rocm7.1 torchvision==0.28.0+rocm7.1 --index-url https://download.pytorch.org/whl/rocm7.1')
+                                case _:
+                                    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.10.0+rocm7.0 torchvision==0.25.0+rocm7.0 --index-url https://download.pytorch.org/whl/rocm7.0')
+                    case 6:
+                        if rocm_minor >= 4: # latest supported rocm 6.x version is 6.4
+                            torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.9.1+rocm6.4 torchvision==0.24.1+rocm6.4 --index-url https://download.pytorch.org/whl/rocm6.4')
+                        else:
+                            match rocm_minor:
+                                case 3:
+                                    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.9.1+rocm6.3 torchvision==0.24.1+rocm6.3 --index-url https://download.pytorch.org/whl/rocm6.3')
+                                case 2:
+                                    # use rocm 6.2.4 instead of 6.2 as torch==2.7.1+rocm6.2 doesn't exists
+                                    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+rocm6.2.4 torchvision==0.22.1+rocm6.2.4 --index-url https://download.pytorch.org/whl/rocm6.2.4')
+                                case 1:
+                                    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+rocm6.1 torchvision==0.21.0+rocm6.1 --index-url https://download.pytorch.org/whl/rocm6.1')
+                                case _:
+                                    torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1+rocm6.0 torchvision==0.19.1+rocm6.0 --index-url https://download.pytorch.org/whl/rocm6.0')
+                    case _:
+                        # lock to 2.4.1 instead of 2.5.1 for performance reasons there are no support for torch 2.6 for rocm 6.0
+                        torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1+rocm6.0 torchvision==0.19.1+rocm6.0 --index-url https://download.pytorch.org/whl/rocm6.0')
+                        log.warning(f"ROCm: unsupported version={rocm.version}")
+                        log.warning("ROCm: minimum supported version=6.0")
 
     if device is None or os.environ.get("HSA_OVERRIDE_GFX_VERSION", None) is not None:
         log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped: device={device} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}')
@@ -881,20 +915,17 @@ def check_torch():
     if args.profile:
         pr = cProfile.Profile()
         pr.enable()
-    allow_cuda = not (args.use_rocm or args.use_directml or args.use_ipex or args.use_openvino)
-    allow_rocm = not (args.use_cuda or args.use_directml or args.use_ipex or args.use_openvino)
-    allow_ipex = not (args.use_cuda or args.use_rocm or args.use_directml or args.use_openvino)
-    allow_directml = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_openvino)
-    allow_openvino = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_directml)
-    log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} directml={args.use_directml} openvino={args.use_openvino} zluda={args.use_zluda}')
-    # log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}')
+    allow_cuda = not (args.use_rocm or args.use_ipex or args.use_openvino)
+    allow_rocm = not (args.use_cuda or args.use_ipex or args.use_openvino)
+    allow_ipex = not (args.use_cuda or args.use_rocm or args.use_openvino)
+    allow_openvino = not (args.use_cuda or args.use_rocm or args.use_ipex)
+    log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} openvino={args.use_openvino} zluda={args.use_zluda}')
+    # log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} openvino={allow_openvino}')
     torch_command = os.environ.get('TORCH_COMMAND', '')
 
     if sys.platform != 'win32':
         if args.use_zluda:
             log.error('ZLUDA is only supported on Windows')
-        if args.use_directml:
-            log.error('DirectML is only supported on Windows')
 
     if torch_command != '':
         is_cuda_available = False
@@ -924,15 +955,8 @@ def check_torch():
         elif is_ipex_available:
             torch_command = install_ipex()
         else:
-            machine = platform.machine()
             if sys.platform == 'darwin':
                 torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision')
-            elif allow_directml and args.use_directml and ('arm' not in machine and 'aarch' not in machine):
-                log.info('DirectML: selected')
-                torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.4.1 torchvision torch-directml==0.2.4.dev240913')
-                if 'torch' in torch_command and not args.version:
-                    install(torch_command, 'torch torchvision')
-                install('onnxruntime-directml', 'onnxruntime-directml', ignore=True)
             else:
                 log.warning('Torch: CPU-only version installed')
                 torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision')
@@ -1010,7 +1034,6 @@ def check_torch():
                     torch_info.set(type='rocm', hip=torch.version.hip)
                 else:
                     log.warning('Torch backend: cannot detect type')
-                log.info(f"Torch backend: {torch_info}")
                 for device in [torch.cuda.device(i) for i in range(torch.cuda.device_count())]:
                     props = torch.cuda.get_device_properties(device)
                     gpu = {
@@ -1023,21 +1046,18 @@ def check_torch():
                     log.info(f'Torch detected: {gpu}')
             except Exception as e:
                 log.error(f'Torch: type=cuda/rocm {e}')
-
-        if args.use_directml and allow_directml:
+        if hasattr(torch, "accelerator") and torch.accelerator.is_available():
             try:
-                import torch_directml # pylint: disable=import-error
-                dml_ver = package_version("torch-directml")
-                log.warning(f'Torch backend: DirectML ({dml_ver})')
-                log.warning('DirectML: end-of-life')
-                for i in range(0, torch_directml.device_count()):
-                    gpu = {
-                        'gpu': torch_directml.device_name(i),
-                    }
-                    gpu_info.append(gpu)
-                    log.info(f'Torch detected: {gpu}')
+                _index = torch.accelerator.current_device_index()
+                _count = torch.accelerator.device_count()
+                _current = torch.accelerator.current_accelerator()
+                torch_info.set(accelerator=str(_current))
             except Exception as e:
-                log.warning(f"Torch: type=directml {e}")
+                log.error(f'Torch: type=accelerator {e}')
+                torch_info.set(accelerator=False)
+        else:
+            torch_info.set(accelerator=False)
+        log.info(f"Torch backend: {torch_info}")
 
     except Exception as e:
         log.error(f'Torch cannot load: {e}')
@@ -1366,7 +1386,6 @@ def install_requirements():
 # set environment variables controlling the behavior of various libraries
 def set_environment():
     log.debug('Setting environment tuning')
-    os.environ.setdefault('SDNQ_REGISTER_DIFFUSERS', '1')
     os.environ.setdefault('ACCELERATE', 'True')
     os.environ.setdefault('ATTN_PRECISION', 'fp16')
     os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100')
@@ -1404,6 +1423,14 @@ def set_environment():
     os.environ.setdefault('UV_INDEX_STRATEGY', 'unsafe-any-match')
     os.environ.setdefault('UV_NO_BUILD_ISOLATION', '1')
     os.environ.setdefault('UVICORN_TIMEOUT_KEEP_ALIVE', '60')
+    # duplicate here since hf_init cannot be called before loader
+    os.environ.setdefault('HF_HUB_DISABLE_EXPERIMENTAL_WARNING', '1')
+    os.environ.setdefault('HF_HUB_DISABLE_IMPLICIT_TOKEN', '1')
+    os.environ.setdefault('HF_HUB_DISABLE_SYMLINKS_WARNING', '1')
+    os.environ.setdefault('HF_HUB_DISABLE_TELEMETRY', '1')
+    os.environ.setdefault('HF_HUB_VERBOSITY', 'warning')
+    os.environ.setdefault('HF_HUB_DOWNLOAD_TIMEOUT', '60')
+    os.environ.setdefault('HF_HUB_ETAG_TIMEOUT', '10')
     allocator = f'garbage_collection_threshold:{opts.get("torch_gc_threshold", 80)/100:0.2f},max_split_size_mb:512'
     if opts.get("torch_malloc", "native") == 'cudaMallocAsync':
         allocator += ',backend:cudaMallocAsync'

launch.py

@@ -155,13 +155,12 @@ def run_extension_installer(ext_dir): # compatibility function
 
 
 def get_memory_stats(detailed:bool=False):
-    from modules.memstats import ram_stats, memory_stats
+    from modules.memstats import ram_stats, memory_stats, model_stats
     if not detailed:
         res = ram_stats()
         return f'{res["used"]}/{res["total"]}'
     else:
-        res = memory_stats()
-        return res
+        return { **memory_stats(), 'model': model_stats(as_gb=True) } # fresh dict: memory_stats returns a module global that the per-generation log also prints
 
 
 def clean_server():

modules/api/api.py

@@ -1,15 +1,17 @@
 import os
 from threading import Lock
 from secrets import compare_digest
-from fastapi import FastAPI, APIRouter, Depends, Request
+from typing import Optional
+from fastapi import FastAPI, APIRouter, Depends, Request, Cookie
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
 from fastapi.exceptions import HTTPException
 from modules import errors, shared, paths
 from modules.logger import log
-from modules.api import models, endpoints, script, helpers, server, generate, process, control, docs, gpu
+from modules.api import models, endpoints, script, helpers, server, generate, process, control, video, docs, gpu
 
 
 errors.install()
+auth_map = []
 
 
 class Api:
@@ -33,13 +35,16 @@ class Api:
         self.generate = generate.APIGenerate(queue_lock)
         self.process = process.APIProcess(queue_lock)
         self.control = control.APIControl(queue_lock)
+        self.video = video.APIVideo(queue_lock)
         # compatibility api
         self.text2imgapi = self.generate.post_text2img
         self.img2imgapi = self.generate.post_img2img
 
     def register(self):
         # fetch js/css
-        self.add_api_route("/js", server.get_js, methods=["GET"], auth=False)
+        self.add_api_route("/js", server.get_js, methods=["GET"], auth=False, tags=["Base"])
+        self.add_api_route("/manifest", server.get_manifest, methods=["GET"], auth=False, tags=["Base"])
+        self.add_api_route("/icon", server.get_icon, methods=["GET"], auth=False, tags=["Base"])
 
         # server api
         self.add_api_route("/sdapi/v1/motd", server.get_motd, methods=["GET"], response_model=str, tags=["Server"])
@@ -66,6 +71,7 @@ class Api:
         self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img, tags=["Generation"])
         self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img, tags=["Generation"])
         self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl, tags=["Generation"])
+        self.add_api_route("/sdapi/v1/video", self.video.post_video, methods=["POST"], response_model=video.ResVideo, tags=["Generation"])
         self.add_api_route("/sdapi/v1/process", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage, tags=["Processing"])
         self.add_api_route("/sdapi/v1/extra-single-image", self.process.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage, tags=["Processing"])
         self.add_api_route("/sdapi/v1/process-batch", self.process.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch, tags=["Processing"])
@@ -102,9 +108,11 @@ class Api:
         self.add_api_route("/sdapi/v1/extra-network-detail", endpoints.get_extra_network_detail, methods=["GET"], response_model=models.ItemExtraNetworkFull, tags=["Enumerators"])
         self.add_api_route("/sdapi/v1/extra-network-details", endpoints.get_extra_network_details, methods=["GET"], response_model=models.ResExtraNetworkDetails, tags=["Enumerators"])
         self.add_api_route("/sdapi/v1/unets", endpoints.get_unets, methods=["GET"], response_model=list[models.ItemUNet], tags=["Enumerators"])
+        self.add_api_route("/sdapi/v1/video/models", self.video.get_video_models, methods=["GET"], response_model=list[video.ItemVideoModel], tags=["Enumerators"])
 
         # functional api
         self.add_api_route("/sdapi/v1/file", endpoints.get_file, methods=["GET"], tags=["Functional"])
+        self.add_api_route("/sdapi/v1/video/file", self.video.get_video_file, methods=["GET"], tags=["Functional"])
         self.add_api_route("/sdapi/v1/delete-image", endpoints.get_deleteimage, methods=["DELETE"], tags=["Functional"])
         self.add_api_route("/sdapi/v1/delete-file", endpoints.get_deletefile, methods=["DELETE"], tags=["Functional"])
         self.add_api_route("/sdapi/v1/png-info", endpoints.get_pnginfo, methods=["GET"], response_model=models.ResImageInfo, tags=["Functional"])
@@ -173,29 +181,50 @@ class Api:
     def add_api_route(self, path: str, fn, auth: bool = True, **kwargs):
         if auth and self.credentials:
             deps = list(kwargs.get('dependencies', []))
-            deps.append(Depends(self.auth))
+            deps.append(Depends(self.auth, use_cache=True))
             kwargs['dependencies'] = deps
         if shared.opts.subpath is not None and len(shared.opts.subpath) > 0:
             self.app.add_api_route(f'{shared.opts.subpath}{path}', endpoint=fn, **kwargs)
         self.app.add_api_route(path, endpoint=fn, **kwargs)
 
-    def auth(self, credentials: HTTPBasicCredentials = Depends(HTTPBasic())):
+    def add_auth(self, host: str, user: str, method: str):
+        msg = f"ip={host} user={user} method={method}"
+        if msg in auth_map:
+            return
+        auth_map.append(msg)
+        log.debug(f'Client auth: {msg}')
+
+    def auth(
+            self,
+            request: Request, # pylint: disable=unused-argument
+            credentials: Optional[HTTPBasicCredentials] = Depends(HTTPBasic(auto_error=False)),
+            access_token: Optional[str] = Cookie(default=None, alias="access_token"),  # Change alias to your cookie name
+            access_token_unsecure: Optional[str] = Cookie(default=None, alias="access-token-unsecure"),
+        ):
         if not self.credentials:
+            self.add_auth(host=request.client.host, user=credentials.username if credentials else None, method="none")
             return True
-        if credentials.username in self.credentials:
-            if compare_digest(credentials.password, self.credentials[credentials.username]):
+        if (credentials is not None) and (credentials.username in self.credentials):
+            if compare_digest(credentials.password, self.credentials[credentials.username]): # client user + encoded password
+                self.add_auth(host=request.client.host, user=credentials.username if credentials else None, method="digest")
                 return True
-            if hasattr(self.app, 'tokens') and (self.app.tokens is not None):
+            if hasattr(self.app, 'tokens') and (self.app.tokens is not None): # client sends token as password
                 if credentials.password in self.app.tokens.keys():
+                    self.add_auth(host=request.client.host, user=credentials.username if credentials else None, method="token")
                     return True
-        log.error(f'API authentication: user="{credentials.username}"')
+        cookie_token = access_token or access_token_unsecure
+        if cookie_token and hasattr(self.app, 'tokens') and (self.app.tokens is not None): # client sets cookie with token
+            if cookie_token in self.app.tokens.keys():
+                self.add_auth(host=request.client.host, user=None, method="cookie")
+                return True
+        log.error(f'API authentication: user="{credentials.username if credentials else None}"')
         raise HTTPException(status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": "Basic"})
 
     def get_session_start(self, req: Request, agent: str | None = None):
         """Log a new browser session with client IP, authenticated user, and user-agent string."""
         token = req.cookies.get("access-token") or req.cookies.get("access-token-unsecure")
         user = self.app.tokens.get(token) if hasattr(self.app, 'tokens') else None
-        log.info(f'Browser session: user={user} client={req.client.host} agent={agent}')
+        log.info(f'Client session: user={user} client={req.client.host} agent={agent}')
         return {}
 
     def launch(self):

modules/api/caption.py

@@ -263,7 +263,7 @@ def validate_image(image_b64: str):
     return image.convert('RGB')
 
 
-def build_clip_overrides(req) -> dict:
+def build_clip_overrides(req) -> dict | None:
     """Build clip_interrogator overrides dict from request fields."""
     overrides = {}
     for key in ('max_length', 'chunk_size', 'min_flavors', 'max_flavors', 'flavor_count', 'num_beams'):
@@ -304,7 +304,7 @@ def do_openclip(image, req):
     return caption, get_top_item(results[0]), get_top_item(results[1]), get_top_item(results[2]), get_top_item(results[3]), get_top_item(results[4])
 
 
-def build_vqa_kwargs(req) -> dict:
+def build_vqa_kwargs(req) -> dict | None:
     """Build generation kwargs dict from VQA request fields."""
     kwargs = {}
     for key in ('max_tokens', 'temperature', 'top_k', 'top_p', 'num_beams', 'do_sample', 'keep_thinking', 'keep_prefill'):
@@ -377,7 +377,7 @@ def do_analyze(image, req):
     return answer, annotated_b64
 
 
-def parse_tagger_scores(tags: str) -> dict:
+def parse_tagger_scores(tags: str) -> dict | None:
     """Parse confidence scores from tagger output string."""
     scores = {}
     for item in tags.split(', '):

modules/api/control.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 from threading import Lock
 from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
 from modules import errors, shared, processing_helpers
@@ -44,24 +44,29 @@ class ItemXYZ(BaseModel):
     include_text: bool = Field(title="Include text", default=False, description="Show generation parameters as text overlay")
 
 
-ReqControl = models.create_model_from_signature(
-    func = run.control_run,
-    model_name = "StableDiffusionProcessingControl",
-    additional_fields = [
-        {"key": "sampler_name", "type": str, "default": "Default"},
-        {"key": "script_name", "type": Optional[str], "default": None},
-        {"key": "script_args", "type": list, "default": []},
-        {"key": "send_images", "type": bool, "default": True},
-        {"key": "save_images", "type": bool, "default": False},
-        {"key": "alwayson_scripts", "type": dict, "default": {}},
-        {"key": "ip_adapter", "type": Optional[list[models.ItemIPAdapter]], "default": None, "exclude": True},
-        {"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True},
-        {"key": "control", "type": Optional[list[ItemControl]], "default": [], "exclude": True},
-        {"key": "xyz", "type": Optional[ItemXYZ], "default": None, "exclude": True},
-        {"key": "extra", "type": Optional[dict], "default": {}, "exclude": True},
-        {"key": "init_control", "type": Optional[list], "default": None, "exclude": True},
-    ]
-)
+
+if TYPE_CHECKING:
+    class ReqControl(BaseModel): # tell static type checkers that ReqControl is a BaseModel subclass
+        pass
+else:
+    ReqControl = models.create_model_from_signature(
+        func = run.control_run,
+        model_name = "StableDiffusionProcessingControl",
+        additional_fields = [
+            {"key": "sampler_name", "type": str, "default": "Default"},
+            {"key": "script_name", "type": Optional[str], "default": None},
+            {"key": "script_args", "type": list, "default": []},
+            {"key": "send_images", "type": bool, "default": True},
+            {"key": "save_images", "type": bool, "default": False},
+            {"key": "alwayson_scripts", "type": dict, "default": {}},
+            {"key": "ip_adapter", "type": Optional[list[models.ItemIPAdapter]], "default": None, "exclude": True},
+            {"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True},
+            {"key": "control", "type": Optional[list[ItemControl]], "default": [], "exclude": True},
+            {"key": "xyz", "type": Optional[ItemXYZ], "default": None, "exclude": True},
+            {"key": "extra", "type": Optional[dict], "default": {}, "exclude": True},
+            {"key": "init_control", "type": Optional[list], "default": None, "exclude": True},
+        ]
+    )
 if not hasattr(ReqControl, "__config__"):
     ReqControl.__config__ = models.DummyConfig
 
@@ -92,6 +97,10 @@ class APIControl:
         args.pop('inpaint_full_res', None)
         args.pop('inpaint_full_res_padding', None)
         args.pop('inpainting_mask_invert', None)
+        args.pop('xyz', None)
+        for k, v in args.copy().items():
+            if v is None:
+                args.pop(k, None)
         args['override_script_name'] = args.pop('script_name', None)
         args['override_script_args'] = args.pop('script_args', None)
         return args

modules/api/endpoints.py

@@ -15,9 +15,9 @@ def _format_tags(raw_tags):
 
 def get_samplers():
     from modules import sd_samplers_diffusers
-    all_samplers = []
+    all_samplers = [{'name': 'Default', 'options': {}}] # restores the model's own scheduler; first to match the ui dropdown order
     for k, v in sd_samplers_diffusers.config.items():
-        if k in ['All', 'Default', 'Res4Lyf']:
+        if k in ['All', 'Default', 'Res4Lyf']: # remaining keys are shared config templates, not samplers
             continue
         all_samplers.append({'name': k, 'options': v})
     return all_samplers

modules/api/gallery.py

@@ -1,9 +1,12 @@
 import io
 import os
+from pathlib import Path
 import time
 import base64
+from secrets import compare_digest
 from urllib.parse import quote, unquote
 from fastapi.responses import JSONResponse
+from fastapi.exceptions import HTTPException
 from starlette.websockets import WebSocket, WebSocketState
 from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
 from PIL import Image
@@ -138,6 +141,25 @@ def register_api(api): # register api
             log.error(f'Gallery image failed: file="{filepath}" | Error: {e}')
             return {}
 
+    def ws_authenticated(ws: WebSocket):
+        if not api.credentials and not getattr(shared.cmd_opts, 'auth_file', None):
+            return True
+        token = ws.cookies.get('access-token') or ws.cookies.get('access-token-unsecure')
+        if token and hasattr(api.app, 'tokens') and (api.app.tokens is not None) and token in (api.app.tokens):
+            return True
+        auth_header = ws.headers.get('authorization')
+        if auth_header and auth_header.lower().startswith('basic '):
+            try:
+                payload = base64.b64decode(auth_header.split(' ', 1)[1]).decode('utf-8')
+                username, password = payload.split(':', 1)
+            except Exception:
+                return False
+            if username in api.credentials and compare_digest(password, api.credentials[username]):
+                return True
+            if hasattr(api.app, 'tokens') and (api.app.tokens is not None) and (password in api.app.tokens):
+                return True
+        return False
+
     # @app.get('/sdapi/v1/browser/folders', response_model=list[dict])
     def get_folders():
         def make_folder(path, label=None):
@@ -190,6 +212,9 @@ def register_api(api): # register api
     async def get_thumb(file: str, exif: bool = False):
         try:
             decoded = unquote(file).replace('%3A', ':')
+            allowed_dirs = shared.demo.allowed_paths
+            if not any(Path(folder).absolute() in Path(file).absolute().parents for folder in allowed_dirs):
+                raise HTTPException(status_code=403, detail=f"file {file}: must be in one of allowed directories")
             if decoded.lower().endswith('.mp4'):
                 return JSONResponse(content=get_video_thumbnail(decoded))
             else:
@@ -203,6 +228,9 @@ def register_api(api): # register api
     async def ht_files(folder: str):
         try:
             t0 = time.time()
+            allowed_dirs = shared.demo.allowed_paths
+            if not any(Path(folder).absolute() in Path(folder).absolute().parents for folder in allowed_dirs):
+                raise HTTPException(status_code=403, detail=f"folder {folder}: must be in one of allowed directories")
             files = files_cache.directory_files(folder, recursive=True)
             lines = []
             for f in files:
@@ -223,6 +251,10 @@ def register_api(api): # register api
 
     @api.app.websocket("/sdapi/v1/browser/files")
     async def ws_files(ws: WebSocket):
+        if not ws_authenticated(ws):
+            log.error(f'WS unauthorized: client={ws.client.host}')
+            await ws.close(code=1008)
+            return
         try:
             await manager.connect(ws)
             folder = await ws.receive_text()
@@ -242,5 +274,5 @@ def register_api(api): # register api
             t1 = time.time()
             log.debug(f'Gallery: type=ws folder="{folder}" files={numFiles} time={t1-t0:.3f}')
         except Exception as e:
-            debug(f'Browser WS error: {e}')
+            debug(f'WS error: {e}')
         manager.disconnect(ws)

modules/api/generate.py

@@ -1,6 +1,6 @@
 from threading import Lock
 from fastapi.responses import JSONResponse
-from modules import errors, shared, scripts_manager, ui
+from modules import errors, shared, scripts_manager
 from modules.api import models, script, helpers
 from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images
 from modules.paths import resolve_output_path
@@ -90,7 +90,7 @@ class APIGenerate:
         script_runner = scripts_manager.scripts_txt2img
         if not script_runner.scripts:
             script_runner.initialize_scripts(False)
-            ui.create_ui(None)
+            # ui.create_ui(None)
         if not self.default_script_arg_txt2img:
             self.default_script_arg_txt2img = script.init_default_script_args(script_runner)
         selectable_scripts, selectable_script_idx = script.get_selectable_script(txt2imgreq.script_name, script_runner)
@@ -141,7 +141,7 @@ class APIGenerate:
         script_runner = scripts_manager.scripts_img2img
         if not script_runner.scripts:
             script_runner.initialize_scripts(True)
-            ui.create_ui(None)
+            # ui.create_ui(None)
         if not self.default_script_arg_img2img:
             self.default_script_arg_img2img = script.init_default_script_args(script_runner)
         selectable_scripts, selectable_script_idx = script.get_selectable_script(img2imgreq.script_name, script_runner)

modules/api/helpers.py

@@ -1,4 +1,5 @@
 import io
+import os
 import base64
 from PIL import Image, PngImagePlugin
 import piexif
@@ -90,6 +91,24 @@ def encode_pil_to_base64(image):
     return b64
 
 
+MAX_B64_BYTES = 256 * 1024 * 1024 # base64 expands ~4/3 and the response is built in memory; larger artifacts are fetched by path instead
+
+
+def encode_file_to_base64(fn: str, max_bytes: int = MAX_B64_BYTES) -> str | None:
+    try:
+        if fn is None or not os.path.isfile(fn):
+            return None
+        size = os.path.getsize(fn)
+        if size > max_bytes:
+            log.warning(f'API cannot encode file: fn="{fn}" size={size} max={max_bytes}')
+            return None
+        with open(fn, 'rb') as f:
+            return base64.b64encode(f.read()).decode('ascii')
+    except Exception as e:
+        log.warning(f'API cannot encode file: fn="{fn}" {e}')
+        return None
+
+
 def upscaler_to_index(name: str):
     try:
         return [x.name.lower() for x in shared.sd_upscalers].index(name.lower())

modules/api/middleware.py

@@ -144,7 +144,7 @@ def setup_middleware(app: FastAPI, cmd_opts):
 
         if not isinstance(e, HTTPException) and err['error'] != 'TypeError': # do not print backtrace on known httpexceptions
             errors.display(e, 'HTTP API', [anyio, fastapi, uvicorn, starlette])
-        elif err['code'] in [404, 401, 400]:
+        elif err['code'] in [404, 401, 400, 403]:
             pass
         else:
             log.debug(e, exc_info=True) # print stack trace

modules/api/mime.py

@@ -10,3 +10,6 @@ def register():
     mimetypes.add_type('image/webp', '.webp')
     mimetypes.add_type('image/jxl', '.jxl')
     mimetypes.add_type('font/ttf', '.ttf')
+    mimetypes.add_type('video/mp4', '.mp4')
+    mimetypes.add_type('video/webm', '.webm')
+    mimetypes.add_type('video/x-matroska', '.mkv')

modules/api/models.py

@@ -62,8 +62,8 @@ class PydanticModelGenerator:
         additional_fields: list[dict[str, Any]] | None = None,
         exclude_fields: list | None = None,
     ):
-        if exclude_fields is None:
-            exclude_fields = []
+        additional_fields = additional_fields or []
+        exclude_fields = exclude_fields or []
         def field_type_generator(_k, v):
             field_type = v.annotation
             return Optional[field_type]
@@ -552,6 +552,7 @@ class ResEmbeddings(BaseModel):
 class ResMemory(BaseModel):
     ram: dict = Field(title="RAM", description="System memory stats")
     cuda: dict = Field(title="CUDA", description="nVidia CUDA memory stats")
+    model: dict = Field(default={}, title="Model", description="Loaded model bytes per component and device")
 
 class ResScripts(BaseModel):
     txt2img: list[str] = Field(title="Txt2img", description="Titles of scripts (txt2img)")

modules/api/process.py

@@ -1,6 +1,7 @@
 from threading import Lock
 from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
 from fastapi.responses import JSONResponse
+from fastapi.exceptions import HTTPException
 from modules.api.helpers import decode_base64_to_image, encode_pil_to_base64
 from modules import errors, shared, postprocessing
 from modules.api import models, helpers
@@ -190,6 +191,7 @@ class APIProcess:
                 strength=req.detailer_strength if req.detailer_strength is not None else 0.3,
                 resolution=req.detailer_resolution if req.detailer_resolution is not None else 1024,
                 seed=req.seed if req.seed is not None else -1,
+                classes=req.detailer_classes if req.detailer_classes is not None else None,
                 overrides=overrides,
             )
 
@@ -281,7 +283,12 @@ class APIProcess:
         reqDict['image'] = helpers.decode_base64_to_image(reqDict['image'])
         with self.queue_lock:
             result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", video="", save_output=False, script_args=script_args, **reqDict)
-        return models.ResProcessImage(image=helpers.encode_pil_to_base64(result[0][0]), html_info=result[1])
+        if len(result) != 4:
+            raise HTTPException(status_code=500, detail="Invalid processing response")
+        outputs, _video, info, _params = result
+        if outputs is None or len(outputs) == 0:
+            raise HTTPException(status_code=500, detail="Invalid processing response")
+        return models.ResProcessImage(image=helpers.encode_pil_to_base64(outputs[0]), html_info=info)
 
     def extras_batch_images_api(self, req: models.ReqProcessBatch):
         """Upscale or postprocess a batch of images using the configured upscaler pipeline."""
@@ -290,4 +297,9 @@ class APIProcess:
         image_folder = [helpers.decode_base64_to_image(x.data) for x in image_list]
         with self.queue_lock:
             result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", video="", save_output=False, script_args=script_args, **reqDict)
-        return models.ResProcessBatch(images=list(map(helpers.encode_pil_to_base64, result[0])), html_info=result[1])
+        if len(result) != 4:
+            raise HTTPException(status_code=500, detail="Invalid processing response")
+        outputs, _video, info, _params = result
+        if outputs is None or len(outputs) == 0:
+            raise HTTPException(status_code=500, detail="Invalid processing response")
+        return models.ResProcessBatch(images=list(map(helpers.encode_pil_to_base64, outputs)), html_info=info)

modules/api/script.py

@@ -73,6 +73,8 @@ def init_default_script_args(script_runner):
             if script.ui(script.is_img2img):
                 ui_default_values = []
                 for elem in script.ui(script.is_img2img):
+                    if elem is None:
+                        continue
                     ui_default_values.append(elem.value)
                 script_args[script.args_from:script.args_to] = ui_default_values
     return script_args

modules/api/server.py

@@ -45,6 +45,16 @@ def get_js(request: Request):
 def get_version():
     return installer.get_version()
 
+def get_icon():
+    icon_path = os.path.join(shared.script_path, "ui", "assets", "favicon.png")
+    return FileResponse(icon_path, media_type="image/png")
+
+def get_manifest():
+    from modules import paths
+    manifest_path = os.path.join(paths.script_path, "ui", "manifest", "manifest.json")
+    log.debug(f"API manifest={manifest_path}")
+    return FileResponse(manifest_path, media_type="application/json")
+
 def get_motd():
     import requests
     motd = ""
@@ -187,4 +197,5 @@ def get_memory():
             cuda = { 'error': 'unavailable' }
     except Exception as err:
         cuda = { 'error': f'{err}' }
-    return models.ResMemory(ram = ram, cuda = cuda)
+    from modules import memstats
+    return models.ResMemory(ram = ram, cuda = cuda, model = memstats.model_stats())

modules/api/upload.py

@@ -32,15 +32,24 @@ class ResUpload(BaseModel):
 
 
 def check_file(filename, path, overwrite):
-    namegen = FilenameGenerator()
+    from modules import shared
     if len(path) > 0 and (os.path.isabs(path) or not os.path.isdir(path)):
         raise HTTPException(status_code=400, detail="Invalid path")
-    fn = os.path.join(path, filename)
+    fn = os.path.basename(filename)
+    if not fn or fn in {'.', '..'}:
+        raise HTTPException(status_code=400, detail="Invalid filename")
+    namegen = FilenameGenerator()
+    fn = os.path.join(path, fn)
     fn = namegen.sanitize(fn)
+    if os.path.isabs(fn):
+        raise HTTPException(status_code=400, detail="Absolute paths are not allowed")
     if Path(fn).parent == Path('.'): # just filename, no path
         fn = os.path.join(tempfile.gettempdir(), fn)
     else:
         fn = os.path.join(paths.data_path, fn)
+    allowed_dirs = shared.demo.allowed_paths
+    if not any(Path(folder).absolute() in Path(fn).absolute().parents for folder in allowed_dirs):
+        raise HTTPException(status_code=403, detail=f"file {fn}: must be in one of allowed directories")
     if os.path.exists(fn) and len(overwrite) == 0:
         raise HTTPException(status_code=400, detail="File exists")
     return fn

modules/api/validate.py

@@ -12,6 +12,7 @@ request_cost = {
     "/sdapi/v1/txt2img": 5,
     "/sdapi/v1/img2img": 5,
     "/sdapi/v1/control": 5,
+    "/sdapi/v1/video": 5,
 }
 log_cost = {
     "/.well-known/appspecific/com.chrome.devtools.json": -1,

modules/api/video.py

@@ -0,0 +1,265 @@
+from types import SimpleNamespace
+from threading import Lock
+from pydantic import BaseModel, Field # pylint: disable=no-name-in-module
+from fastapi.exceptions import HTTPException
+from modules import errors, shared, scripts_manager, ui
+from modules.api import script, helpers
+from modules.paths import resolve_output_path
+from modules.video_models import models_def, video_load, video_run
+
+
+errors.install()
+
+
+class ReqVideo(BaseModel):
+    engine: str | None = Field(default=None, title="Engine", description="Video engine family; omit together with model to use the currently loaded checkpoint")
+    model: str | None = Field(default=None, title="Model", description="Video model name within the engine; see GET /sdapi/v1/video/models")
+    prompt: str = Field(default="", title="Prompt", description="Text prompt")
+    negative_prompt: str = Field(default="", title="Negative prompt", description="Negative text prompt")
+    styles: list[str] = Field(default=[], title="Styles", description="Prompt style names to apply")
+    width: int = Field(default=832, ge=64, le=4096, title="Width", description="Output width; snapped to the model canvas multiple")
+    height: int = Field(default=480, ge=64, le=4096, title="Height", description="Output height; snapped to the model canvas multiple")
+    frames: int = Field(default=17, ge=1, le=1024, title="Frames", description="Number of frames; 1 produces a single still image on workflow models")
+    steps: int = Field(default=50, ge=1, le=200, title="Steps", description="Number of inference steps")
+    sampler_name: str = Field(default="Default", title="Sampler", description="Sampler name; Default keeps the model scheduler")
+    sampler_shift: float = Field(default=-1.0, title="Sampler shift", description="Scheduler flow shift; -1 keeps the model default")
+    dynamic_shift: bool = Field(default=False, title="Dynamic shift", description="Enable dynamic scheduler shifting")
+    seed: int = Field(default=-1, title="Seed", description="Generation seed; -1 for random")
+    guidance_scale: float = Field(default=-1.0, title="Guidance scale", description="CFG scale; -1 keeps the model default")
+    guidance_true: float = Field(default=-1.0, title="True guidance", description="True CFG scale; -1 keeps the model default")
+    init_image: str | None = Field(default=None, title="Init image", description="Base64 or data URI for the first-frame image; an upload reference resolves only where an extension provides the upload store")
+    init_strength: float = Field(default=0.8, ge=0.0, le=1.0, title="Init strength", description="Denoising strength for the init image")
+    last_image: str | None = Field(default=None, title="Last image", description="Base64 or data URI for the last-frame image; an upload reference resolves only where an extension provides the upload store")
+    references: list[str] = Field(default=[], title="References", description="Reference images for a reference workflow, in the order the model reads them; base64 or data URIs, or upload references where an extension provides the upload store. Images only: the video core also conditions on video and audio references, which this endpoint cannot carry. At most 9, each within a 1:4 to 4:1 aspect ratio. Rejected on models that do not condition on references")
+    vae_type: str = Field(default="Default", title="VAE type", description="Decode variant: Default, Tiny, Remote, or Upscale")
+    vae_tile_frames: int = Field(default=16, ge=1, le=64, title="VAE tile frames", description="Frames per VAE decode tile")
+    audio: bool = Field(default=True, title="Audio", description="Generate audio on models that support it")
+    mp4_fps: int = Field(default=24, ge=1, le=60, title="FPS", description="Frames per second of the saved video")
+    mp4_interpolate: int = Field(default=0, ge=0, le=10, title="Interpolation", description="RIFE interpolation passes between frames")
+    mp4_codec: str = Field(default="libx264", title="Codec", description="Video codec; none skips video encoding")
+    mp4_ext: str = Field(default="mp4", title="Container", description="Container extension; the muxer is inferred from it")
+    mp4_opt: str = Field(default="crf=16", title="Codec options", description="Encoder options as key=value pairs separated by : or ,")
+    mp4_video: bool = Field(default=True, title="Save video", description="Write the video container to disk")
+    mp4_frames: bool = Field(default=False, title="Save frames", description="Write individual frame images to disk")
+    mp4_sf: bool = Field(default=False, title="Save safetensors", description="Write raw frames as a safetensors file")
+    mp4_thumb: bool = Field(default=True, title="Save thumbnail", description="Write a thumbnail image next to the video")
+    override_settings: dict = Field(default={}, title="Override settings", description="Setting overrides applied for this generation only")
+    script_args: list = Field(default=[], title="Script args", description="Positional arguments for a selectable script")
+    alwayson_scripts: dict = Field(default={}, title="Always-on scripts", description="Per-script argument overrides, keyed by script name")
+    send_video: bool = Field(default=True, title="Send video", description="Return the video base64-encoded in the response")
+    send_frames: bool = Field(default=False, title="Send frames", description="Return every frame base64-encoded in the response")
+    send_thumbnail: bool = Field(default=True, title="Send thumbnail", description="Return the thumbnail base64-encoded in the response")
+    extra: dict | None = Field(default={}, exclude=True, title="Extra", description="Extra attributes set on the processing object")
+
+
+class ResVideo(BaseModel):
+    video: str | None = Field(default=None, title="Video", description="Base64-encoded video file; empty when not requested, above the size cap, or in still mode")
+    video_path: str | None = Field(default=None, title="Video path", description="Server path of the saved video; fetch via GET /sdapi/v1/video/file")
+    thumbnail: str | None = Field(default=None, title="Thumbnail", description="Base64-encoded thumbnail image")
+    thumbnail_path: str | None = Field(default=None, title="Thumbnail path", description="Server path of the saved thumbnail")
+    frames: list[str] = Field(default=[], title="Frames", description="Base64-encoded frames; always populated in still mode")
+    frames_count: int = Field(default=0, title="Frame count", description="Number of frames written, after interpolation")
+    fps: float = Field(default=0.0, title="FPS", description="Effective frames per second of the saved video")
+    duration: float = Field(default=0.0, title="Duration", description="Video duration in seconds")
+    has_audio: bool = Field(default=False, title="Has audio", description="Whether the video carries an audio track")
+    still: bool = Field(default=False, title="Still", description="Single-frame result; the product is in frames and no video was written")
+    params: dict = Field(default={}, title="Parameters", description="Echo of the request parameters used for generation")
+    info: str = Field(default="", title="Info", description="Generation info string with seed, sampler, and pipeline details")
+
+
+class ItemVideoModel(BaseModel):
+    engine: str = Field(title="Engine", description="Video engine family")
+    name: str = Field(title="Name", description="Model name; pass together with engine to select it")
+    repo: str = Field(default="", title="Repo", description="Model repository or path")
+    url: str = Field(default="", title="URL", description="Model information page")
+    mode: str = Field(title="Mode", description="Input mode: workflow, t2v, i2v, flf2v, vace, animate, condition, or unknown; condition models accept conditioning the generic path does not wire and run as text to video here")
+    workflow: str | None = Field(default=None, title="Workflow", description="Modular workflow name when the model dispatches on inputs; ref2va conditions on references and ignores the keyframe images")
+    base: bool = Field(default=False, title="Base", description="Also listed in the base checkpoint dropdown")
+    loaded: bool = Field(default=False, title="Loaded", description="Currently loaded through the video registry")
+
+
+def model_mode(m: models_def.Model) -> str:
+    return models_def.dispatch_mode(m)
+
+
+class APIVideo:
+    def __init__(self, queue_lock: Lock):
+        self.queue_lock = queue_lock
+        self.default_script_arg_video = []
+
+    def prepare_scripts(self, p_stub, req: ReqVideo):
+        script_runner = scripts_manager.scripts_video
+        if not script_runner.scripts:
+            script_runner.initialize_scripts(is_img2img=False, is_control=False, is_video=True)
+            ui.create_ui(None)
+        if not self.default_script_arg_video:
+            self.default_script_arg_video = script.init_default_script_args(script_runner)
+        script_args = script.init_script_args(p_stub, req, self.default_script_arg_video, None, None, script_runner)
+        return script_runner, script_args
+
+    def sanitize_b64(self, req: ReqVideo):
+        def sanitize_str(args: list):
+            for idx in range(0, len(args)):
+                if isinstance(args[idx], str) and len(args[idx]) >= 1000:
+                    args[idx] = f"<str {len(args[idx])}>"
+        for name in ('init_image', 'last_image'):
+            val = getattr(req, name, None)
+            if isinstance(val, str) and len(val) >= 1000:
+                setattr(req, name, f"<str {len(val)}>")
+        if req.references:
+            sanitize_str(req.references)
+        if req.script_args:
+            sanitize_str(req.script_args)
+        if req.alwayson_scripts:
+            for script_obj in req.alwayson_scripts.values():
+                if script_obj and "args" in script_obj and script_obj["args"]:
+                    sanitize_str(script_obj["args"])
+
+    def post_video(self, req: ReqVideo):
+        """Generate a video, or a single still frame, using a video model.
+
+        Omit `engine` and `model` to drive the currently loaded checkpoint when it is
+        video-capable; this covers models loaded from local folders that have no registry
+        entry. Pass both names to select a registry model, which is loaded on demand;
+        `GET /sdapi/v1/video/models` enumerates the valid pairs.
+
+        `frames` of 1 on a workflow model produces a single still image returned in `frames`.
+        Disk outputs are controlled by `mp4_video`, `mp4_frames`, `mp4_sf`, and `mp4_thumb`;
+        response payloads are controlled independently by `send_video`, `send_frames`, and
+        `send_thumbnail`. Artifacts above the base64 size cap return `video` empty with
+        `video_path` set; fetch those via `GET /sdapi/v1/video/file`.
+
+        `init_image` and `last_image` accept base64 data or data URIs. An `upload:` reference
+        resolves only where an extension registers an upload store; without one it is rejected.
+        Models whose workflow is `ref2va` condition on `references` instead: an ordered list of
+        images the prompt addresses as `<Picture 1>`, `<Picture 2>` and so on, following list
+        order. A single reference may also be passed as `init_image`. Reference images do not
+        set the output canvas, and `last_image` is ignored. The workflow also conditions on video
+        and audio references, addressed as `<Video i>` and `<Audio i>`, but they decode from files
+        rather than from the wire, so this endpoint carries images alone.
+
+        Progress is reported on `GET /sdapi/v1/progress`; `POST /sdapi/v1/interrupt` cancels.
+        Switching checkpoints via `override_settings` is not supported here; use
+        `POST /sdapi/v1/checkpoint` before generating.
+        """
+        try:
+            selected, needs_load = video_run.resolve_model(req.engine, req.model)
+        except video_run.VideoError as e:
+            raise HTTPException(status_code=e.code, detail=str(e)) from e
+        sampler_name = helpers.validate_sampler_name(req.sampler_name)
+        init_image = helpers.decode_base64_to_image(req.init_image) if req.init_image else None
+        last_image = helpers.decode_base64_to_image(req.last_image) if req.last_image else None
+        references = [helpers.decode_base64_to_image(x) for x in (req.references or [])]
+        overrides = dict(req.override_settings or {})
+        for key in ('sd_model_checkpoint', 'sd_model_refiner'):
+            if key in overrides:
+                raise HTTPException(status_code=400, detail=f"{key} override is not supported here: switch models via POST /sdapi/v1/checkpoint before generating")
+        p_stub = SimpleNamespace(per_script_args={})
+        script_runner, script_args = self.prepare_scripts(p_stub, req)
+        extra = getattr(req, 'extra', {}) or {}
+
+        with self.queue_lock:
+            jobid = shared.state.begin('API-VID', api=True)
+            try:
+                res = video_run.run(
+                    selected,
+                    prompt=req.prompt,
+                    negative=req.negative_prompt,
+                    styles=req.styles,
+                    width=req.width,
+                    height=req.height,
+                    frames=req.frames,
+                    steps=req.steps,
+                    sampler_name=sampler_name,
+                    sampler_shift=req.sampler_shift,
+                    dynamic_shift=req.dynamic_shift,
+                    seed=req.seed,
+                    guidance_scale=req.guidance_scale,
+                    guidance_true=req.guidance_true,
+                    init_image=init_image,
+                    init_strength=req.init_strength,
+                    last_image=last_image,
+                    references=references,
+                    vae_type=req.vae_type,
+                    vae_tile_frames=req.vae_tile_frames,
+                    audio=req.audio,
+                    mp4_fps=req.mp4_fps,
+                    mp4_interpolate=req.mp4_interpolate,
+                    mp4_codec=req.mp4_codec,
+                    mp4_ext=req.mp4_ext,
+                    mp4_opt=req.mp4_opt,
+                    mp4_video=req.mp4_video,
+                    mp4_frames=req.mp4_frames,
+                    mp4_sf=req.mp4_sf,
+                    mp4_thumb=req.mp4_thumb,
+                    override_settings=overrides,
+                    engine=req.engine,
+                    scripts=script_runner,
+                    script_args=script_args,
+                    per_script_args=p_stub.per_script_args,
+                    extra_p=extra,
+                    needs_load=needs_load,
+                )
+            except video_run.VideoError as e:
+                raise HTTPException(status_code=e.code, detail=str(e)) from e
+            finally:
+                shared.state.end(jobid, api=False)
+
+        send_frames = req.send_frames or res.still # a still request has no other product to return
+        b64_frames = list(map(helpers.encode_pil_to_base64, res.images)) if send_frames else []
+        video_b64 = helpers.encode_file_to_base64(res.video_path) if req.send_video and res.video_path else None
+        thumb_b64 = helpers.encode_file_to_base64(res.thumb_path) if req.send_thumbnail and res.thumb_path else None
+        duration = round(res.num_frames / res.fps, 3) if res.fps > 0 else 0.0
+        self.sanitize_b64(req)
+        params = {k: v for k, v in vars(req).items() if k != 'extra'}
+        return ResVideo(
+            video=video_b64,
+            video_path=res.video_path,
+            thumbnail=thumb_b64,
+            thumbnail_path=res.thumb_path,
+            frames=b64_frames,
+            frames_count=res.num_frames,
+            fps=res.fps,
+            duration=duration,
+            has_audio=res.has_audio,
+            still=res.still,
+            params=params,
+            info=res.processed.info,
+        )
+
+    def get_video_models(self, engine: str | None = None):
+        """List video engines and models; optionally filter by engine."""
+        items = []
+        for family, rows in models_def.models.items():
+            if engine is not None and family.lower() != engine.lower():
+                continue
+            for m in rows:
+                if not models_def.is_model(m):
+                    continue
+                items.append(ItemVideoModel(
+                    engine=family,
+                    name=m.name,
+                    repo=m.repo or '',
+                    url=m.url or '',
+                    mode=model_mode(m),
+                    workflow=m.workflow,
+                    base=m.base,
+                    loaded=(m.name == video_load.loaded_model),
+                ))
+        return items
+
+    def get_video_file(self, file: str):
+        """Serve a video artifact produced by this endpoint; the path must resolve inside the video output directory."""
+        import mimetypes
+        from pathlib import Path
+        from starlette.responses import FileResponse
+        if not file or not file.strip():
+            raise HTTPException(status_code=400, detail="file path is required")
+        root = Path(resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video)).resolve()
+        target = Path(file).resolve()
+        if root not in target.parents:
+            raise HTTPException(status_code=403, detail=f"file {file}: must be inside the video output directory")
+        if not target.is_file():
+            raise HTTPException(status_code=404, detail=f"file not found: {file}")
+        media_type = mimetypes.guess_type(target.name)[0] or 'application/octet-stream'
+        return FileResponse(str(target), media_type=media_type, filename=target.name)

modules/attention.py

@@ -20,10 +20,10 @@ def set_dynamic_attention():
 def set_sdnq_attention():
     try:
         from modules import shared
-        from modules.sdnq.kernels.triton_atten import sdnq_triton_atten
+        from sdnq.kernels.triton_atten import sdnq_triton_atten
         sdpa_pre_sdnq_atten = torch.nn.functional.scaled_dot_product_attention
         @wraps(sdpa_pre_sdnq_atten)
-        def sdpa_sdnq_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
+        def sdpa_sdnq_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.Tensor:
             if (
                 query.device.type != "cpu"
                 and (query.shape[-2] >= 32 and key.shape[-2] >= 32)
@@ -38,6 +38,7 @@ def set_sdnq_attention():
                     smooth_k=shared.opts.sdnq_attention_smooth_k,
                     use_hadamard=shared.opts.sdnq_attention_use_hadamard,
                     hadamard_group_size=shared.opts.sdnq_attention_hadamard_group_size,
+                    use_fp16_accum=shared.opts.sdnq_attention_use_fp16_accum,
                 )
             else:
                 if enable_gqa:
@@ -45,7 +46,7 @@ def set_sdnq_attention():
                 return sdpa_pre_sdnq_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
         torch.nn.functional.scaled_dot_product_attention = sdpa_sdnq_atten
         torch_info.set(attention='sdnq')
-        log.debug(f'Torch attention: type="SDNQ attention" matmul={shared.opts.sdnq_attention_matmul_type}:{shared.opts.sdnq_attention_pv_matmul_type} smooth={shared.opts.sdnq_attention_smooth_k} hadamard={shared.opts.sdnq_attention_use_hadamard}')
+        log.debug(f'Torch attention: type="SDNQ attention" matmul={shared.opts.sdnq_attention_matmul_type}:{shared.opts.sdnq_attention_pv_matmul_type} smooth={shared.opts.sdnq_attention_smooth_k} hadamard={shared.opts.sdnq_attention_use_hadamard} fp16_accum={shared.opts.sdnq_attention_use_fp16_accum}')
     except Exception as err:
         log.error(f'Torch attention: type="SDNQ attention" {err}')
 
@@ -57,7 +58,7 @@ def set_triton_flash_attention(backend: str):
 
             sdpa_pre_triton_flash_atten = torch.nn.functional.scaled_dot_product_attention
             @wraps(sdpa_pre_triton_flash_atten)
-            def sdpa_triton_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
+            def sdpa_triton_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.Tensor:
                 use_triton = (
                     query.shape[-1] <= 128
                     and attn_mask is None
@@ -98,7 +99,7 @@ def set_flex_attention():
 
         sdpa_pre_flex_atten = torch.nn.functional.scaled_dot_product_attention
         @wraps(sdpa_pre_flex_atten)
-        def sdpa_flex_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor: # pylint: disable=unused-argument
+        def sdpa_flex_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.Tensor: # pylint: disable=unused-argument
             score_mod = None
             block_mask = None
             if attn_mask is not None:
@@ -140,7 +141,7 @@ def set_ck_flash_attention(backend: str, device: torch.device):
 
         sdpa_pre_flash_atten = torch.nn.functional.scaled_dot_product_attention
         @wraps(sdpa_pre_flash_atten)
-        def sdpa_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
+        def sdpa_flash_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.Tensor:
             use_flash = (
                 query.shape[-1] <= 128
                 and attn_mask is None
@@ -215,7 +216,7 @@ def set_sage_attention(backend: str, device: torch.device):
 
         sdpa_pre_sage_atten = torch.nn.functional.scaled_dot_product_attention
         @wraps(sdpa_pre_sage_atten)
-        def sdpa_sage_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.FloatTensor:
+        def sdpa_sage_atten(query: torch.FloatTensor, key: torch.FloatTensor, value: torch.FloatTensor, attn_mask: torch.Tensor | None = None, dropout_p: float = 0.0, is_causal: bool = False, scale: float | None = None, enable_gqa: bool = False, **kwargs) -> torch.Tensor:
             use_sage = (
                 query.shape[-1] in {128, 96, 64}
                 and attn_mask is None

modules/call_queue.py

@@ -7,20 +7,41 @@ from modules import shared, progress, errors, timer
 from modules.logger import log
 
 
-queue_lock = threading.Lock()
-debug = os.environ.get('SD_QUEUE_DEBUG', None) is not None
+_queue_lock = threading.Lock() # internal
+_queue_debug = os.environ.get('SD_QUEUE_DEBUG', None) is not None
+
+
+class Queue:
+    def __enter__(self):
+        _queue_lock.acquire()
+        if _queue_debug:
+            fn = f'{sys._getframe(3).f_code.co_name}:{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
+            log.debug(f'Queue: lock state={_queue_lock.locked()} fn={fn}')
+        return _queue_lock
+
+    def __exit__(self, exc_type, exc_val, exc_tb): # pylint: disable=unused-argument
+        if _queue_lock.locked():
+            _queue_lock.release()
+        if _queue_debug:
+            fn = f'{sys._getframe(3).f_code.co_name}:{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
+            log.debug(f'Queue: unlock state={_queue_lock.locked()} fn={fn}')
+        return _queue_lock
+
+
+queue_lock = Queue() # public lock for external use
 
 
 def get_lock():
-    if debug:
-        fn = f'{sys._getframe(3).f_code.co_name}:{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
-        log.debug(f'Queue: fn={fn} lock={queue_lock.locked()}')
     return queue_lock
 
 
+def is_locked():
+    return _queue_lock.locked()
+
+
 def wrap_queued_call(func):
     def f(*args, **kwargs):
-        with get_lock():
+        with Queue():
             res = func(*args, **kwargs)
         return res
     return f
@@ -35,7 +56,7 @@ def wrap_gradio_gpu_call(func, extra_outputs=None, name=None):
             progress.add_task_to_queue(id_task)
         else:
             id_task = None
-        with get_lock():
+        with Queue():
             progress.start_task(id_task)
             try:
                 res = func(*args, **kwargs)

modules/caption/toriigate.py

@@ -57,7 +57,7 @@ Give some info about objects on background, describe the location (if seen). The
 /or if there are no characters
 "main content" : "long and detailed description of main content of image that might be the main focus if characters are missing",
 /
-"background" : "Detailed descritpion of background and it's content",
+"background" : "Detailed description of background and it's content",
 "image_effects" : "If there are some visual effects like fisheye distortion, chromatic aberration, glitches, messy drawing or anything else - write about it. If it's just a general anime art - omit this field."
 "texts" : "Speech bubbles, bars, marks, signs etc. with texts if present, else None",
 "atmosphere" : "...",
@@ -72,7 +72,7 @@ In special cases you can add extra keys.
 You need to think here and compare peoples/creatures that you see on the picture  with given popular tags, or descriptions, or your memories for each characters to determine who is who.
 If no characters are listed in input - just write here "No named characters"
 # 2. Key details
-Here you need to write about the key details on image, prefere using regular text.
+Here you need to write about the key details on image, prefer using regular text.
 # 3. Structured description
 ## General
 Write about general composition, content of image, background and all things that are not related to characters directly.
@@ -85,13 +85,13 @@ Mention image effect, style, camera angle
 </format>
 In general stick to shorter descriptions.
 """,
-"json_comic": """Use json-style caption to describe to comin, stick to following structure:
+"json_comic": """Use json-style caption to describe the comic, stick to following structure:
 {
-"comic_format": "menation the format, for example Comic of N frames",
+"comic_format": "mention the format, for example Comic of N frames",
 "1st_frame": "Main description of the content for fist frame",
 "2nd_frame": "Same for the second",
 ...
-"Nth_ftame": "...",
+"Nth_frame": "...",
 "character_1": "Describe the characters in comic",
 ...
 "character_N": "Separate description for each",
@@ -105,15 +105,15 @@ You need to think here and compare peoples/creatures that you see on the picture
 # 2. Key details
 Here you need to determine key details on comic and list them.
 # 3. Comic format
-In this section come up with the description of comic format, how many pages there are, horisontal/vertical orientation and other things. Optionally you can list main characters here.
+In this section come up with the description of comic format, how many pages there are, horizontal/vertical orientation and other things. Optionally you can list main characters here.
 # 4. Details for each frame
 ## 4.1 Frame 1 (position)
-Description for each frame, includding characters, objects, interactions, texts/speech bubbles and other things. Be detailed but not overdoo.
+Description for each frame, including characters, objects, interactions, texts/speech bubbles and other things. Be detailed but not overdoo.
 ## 4.2 Frame 2 (position)
 Same for each frame.
 ...
 # 5. Extra comment
-Here you should write general desciption and some other info about the image.
+Here you should write general description and some other info about the image.
 </format>
 """,
 "min_structured_json": """
@@ -127,7 +127,7 @@ Use json-style caption for given image with following structure:
 "texts" : "Speech bubbles, bars, marks, signs etc. with texts if present, else None",
 "watermarks" : "If present",
 }
-Prefere shorter description and tags.
+Prefer shorter description and tags.
 """,
 "chroma-style": """Your task is to describe the picture in very detail using a structure of 4 parts.
 ### 1. Regular Summary:

modules/caption/vqa_detection.py

@@ -288,7 +288,7 @@ def calculate_eye_position(face_bbox: dict) -> tuple:
     return (eye_x, eye_y)
 
 
-def draw_bounding_boxes(image: Image.Image, detections: list, points: list | None = None) -> Image.Image:
+def draw_bounding_boxes(image: Image.Image, detections: list, points: list | None = None) -> Image.Image | None:
     """
     Draw bounding boxes and/or points on an image.
 

modules/caption/waifudiffusion.py

@@ -270,7 +270,8 @@ class WaifuDiffusionTagger:
         character_count = 0
         rating_count = 0
 
-        for i, (tag_name, prob) in enumerate(zip(self.tags, probs, strict=False)):
+        tags = self.tags or []
+        for i, (tag_name, prob) in enumerate(zip(tags, probs, strict=False)):
             category = self.tag_categories[i]
             tag_lower = tag_name.lower()
 

modules/cmd_args.py

@@ -37,7 +37,6 @@ def add_compute_args(p):
     p.add_argument("--use-rocm", default=env_flag("SD_USEROCM", False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s")
     p.add_argument('--use-zluda', default=env_flag("SD_USEZLUDA", False), action='store_true', help="Force use ZLUDA, AMD GPUs only, default: %(default)s")
     p.add_argument("--use-openvino", default=env_flag("SD_USEOPENVINO", False), action='store_true', help="Use Intel OpenVINO backend, default: %(default)s")
-    p.add_argument('--use-directml', default=env_flag("SD_USEDIRECTML", False), action='store_true', help="Use DirectML if no compatible GPU is detected, default: %(default)s")
     p.add_argument("--use-xformers", default=env_flag("SD_USEXFORMERS", False), action='store_true', help="Force use xFormers cross-optimization, default: %(default)s")
     p.add_argument("--use-nightly", default=env_flag("SD_USENIGHTLY", False), action='store_true', help="Force use nightly torch builds, default: %(default)s")
     p.add_argument("--no-half", default=env_flag("SD_NOHALF", False), action='store_true', help="Do not switch the model to 16-bit float, default: %(default)s")

modules/control/proc/mediapipe_face_util.py

@@ -1,6 +1,6 @@
 from typing import Mapping
 import numpy as np
-from modules.shared import log
+from modules.logger import log
 
 try:
     import mediapipe as mp

modules/control/processor.py

@@ -242,7 +242,8 @@ def preprocess_image(
             p.init_images = [init_image] * len(active_model)
         if hasattr(shared.sd_model, 'controlnet') and 'control_image' in p.task_args and len(p.task_args['control_image']) > 1 and (shared.sd_model.__class__.__name__ == 'StableDiffusionXLControlNetUnionPipeline'): # special case for controlnet-union
             p.task_args['control_image'] = [[x] for x in p.task_args['control_image']]
-            p.task_args['control_mode'] = [[x] for x in p.task_args['control_mode']]
+            control_mode = p.task_args.get('control_mode') or []
+            p.task_args['control_mode'] = [[x] for x in control_mode]
 
     # determine txt2img, img2img, inpaint pipeline
     if unit_type == 'reference' and has_models: # special case

modules/control/run.py

@@ -1,5 +1,6 @@
 import os
 import sys
+from functools import partial
 import cv2
 from PIL import Image
 from modules.logger import log
@@ -336,6 +337,12 @@ def control_process(p: StableDiffusionProcessingControl,
     return output, info, script_run
 
 
+def generate(*args, **kwargs):
+    generate_call = partial(control_run, *args, **kwargs)
+    # return [], '', '', 'Queued'
+    return generate_call()
+
+
 def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                 units: list[unit.Unit] | None = None, inputs: list[Image.Image] | None = None, inits: list[Image.Image] | None = None, mask: Image.Image = None, unit_type: str | None = None, is_generator: bool = True,
                 input_type: int = 0,
@@ -346,7 +353,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                 guidance_name: str = 'Default', guidance_scale: float = 6.0, guidance_rescale: float = 0.0, guidance_start: float = 0.0, guidance_stop: float = 1.0,
                 cfg_scale: float = 6.0, clip_skip: float = 1.0, cfg_image: float = 6.0, cfg_rescale: float = 0.7, cfg_true: float = 0.0, cfg_adaptive: float = 0.5, cfg_end: float = 1.0,
                 vae_type: str = 'Full', tiling: bool = False, hidiffusion: bool = False,
-                detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024,
+                detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024,  detailer_classes: str = '',
                 hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, hdr_sharpen: float = 0, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 0.95,
                 hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundary: float = 1.0, hdr_color_picker: str | None = None, hdr_tint_ratio: float = 0, hdr_apply_hires: bool = True,
                 grading_brightness: float = 0.0, grading_contrast: float = 0.0, grading_saturation: float = 0.0, grading_hue: float = 0.0,
@@ -368,7 +375,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                 override_script_name: str | None = None, override_script_args = None, extra: dict | None = None,
                 *input_script_args,
                 # API-only params (keyword-only, not wired to Gradio)
-                detailer_segmentation: bool | None = None, detailer_include_detections: bool | None = None, detailer_merge: bool | None = None, detailer_sort: bool | None = None, detailer_classes: str | None = None,
+                detailer_segmentation: bool | None = None, detailer_include_detections: bool | None = None, detailer_merge: bool | None = None, detailer_sort: bool | None = None,
                 detailer_conf: float | None = None, detailer_iou: float | None = None, detailer_max: int | None = None,
                 detailer_min_size: float | None = None, detailer_max_size: float | None = None,
                 detailer_blur: int | None = None, detailer_padding: int | None = None,
@@ -823,8 +830,8 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
 
             debug_log(f'Control: pipeline units={len(active_model)} process={len(active_process)} outputs={len(output_images)}')
     except Exception as e:
-        log.error(f'Control: type={unit_type} units={len(active_model)} {e}')
-        errors.display(e, 'Control')
+        log.error(f'Generate: {e}')
+        errors.display(e, 'Generate')
 
     if len(output_images) == 0:
         output_images = None

modules/detailer/detailer.py

@@ -36,28 +36,64 @@ class Detailer():
             offload: bool | None = None,
             p = None,
         ) -> list[DetailerResult]:
+        jobid = shared.state.begin('Detect')
         if 'LocateAnything' in name:
             from modules.detailer import locateanything
-            return locateanything.predict(self, model, image, device=device, mask=mask, offload=offload, p=p)
-
-        from modules.detailer import yolo
-        return yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p)
+            results = locateanything.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Qwen3-VL' in name:
+            from modules.detailer import qwen
+            results = qwen.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Florence-2' in name:
+            from modules.detailer import florence
+            results = florence.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Grounding-DINO' in name:
+            from modules.detailer import dino
+            results = dino.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Rex-Omni' in name:
+            from modules.detailer import rexomni
+            results = rexomni.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        elif 'Facebook-SAM3' in name:
+            from modules.detailer import sam
+            results = sam.predict(self, name, image, device=device, mask=mask, offload=offload, p=p)
+        else:
+            from modules.detailer import yolo
+            results = yolo.predict(self, model, image, imgsz=imgsz, half=half, device=device, agnostic=agnostic, retina=retina, mask=mask, augment=augment, offload=offload, p=p)
+        shared.state.end(jobid)
+        return results
 
     def enumerate(self):
         from modules.detailer import list_models
         return list_models(self)
 
     def load(self, model_name: str | None = None):
+        jobid = shared.state.begin('Load detailer')
         if 'LocateAnything' in model_name:
             from modules.detailer import locateanything
-            return locateanything.load(model_name=model_name)
-
-        from modules.detailer import yolo
-        return yolo.load(self, model_name=model_name)
+            model_name, model = locateanything.load(self, model_name=model_name)
+        elif 'Qwen3-VL' in model_name:
+            from modules.detailer import qwen
+            model_name, model = qwen.load(self, model_name=model_name)
+        elif 'Florence-2' in model_name:
+            from modules.detailer import florence
+            model_name, model = florence.load(self, model_name=model_name)
+        elif 'Grounding-DINO' in model_name:
+            from modules.detailer import dino
+            model_name, model = dino.load(self, model_name=model_name)
+        elif 'Rex-Omni' in model_name:
+            from modules.detailer import rexomni
+            model_name, model = rexomni.load(self, model_name=model_name)
+        elif 'Facebook-SAM3' in model_name:
+            from modules.detailer import sam
+            model_name, model = sam.load(self, model_name=model_name)
+        else:
+            from modules.detailer import yolo
+            model_name, model = yolo.load(self, model_name=model_name)
+        shared.state.end(jobid)
+        return model_name, model
 
     def merge(self, items: list[DetailerResult]) -> list[DetailerResult]:
         if items is None or len(items) == 0:
-            return None
+            return []
         box=[min(item.box[0] for item in items), min(item.box[1] for item in items), max(item.box[2] for item in items), max(item.box[3] for item in items)]
         mask = Image.new('L', items[0].mask.size, 0)
         for item in items:
@@ -74,14 +110,38 @@ class Detailer():
         )
         return [merged]
 
-    def draw_masks(self, image: Image.Image, items: list[DetailerResult], p=None) -> Image.Image:
+    def filter(self, items: list[DetailerResult], image: Image.Image, p: processing.StableDiffusionProcessing = None) -> list[DetailerResult]:
+        if items is None or len(items) == 0:
+            return []
+        if p is not None:
+            min_conf = detailer_opt(p, 'detailer_conf')
+            max_detected = detailer_opt(p, 'detailer_max')
+            filtered = [item for item in items if item.score >= min_conf]
+            opt_min = detailer_opt(p, 'detailer_min_size') or 0
+            opt_max = detailer_opt(p, 'detailer_max_size') or 1
+            for item in filtered.copy():
+                w, h = item.box[2] - item.box[0], item.box[3] - item.box[1]
+                x_size, y_size = w/image.width, h/image.height
+                min_size = opt_min if 0 <= opt_min <= 1 else 0
+                max_size = opt_max if 0 < opt_max <= 1 else 1
+                if not ((x_size >= min_size) and (y_size >= min_size) and (x_size <= max_size) and (y_size <= max_size)):
+                    filtered.remove(item)
+            filtered = sorted(filtered, key=lambda x: x.score, reverse=True)
+            filtered = filtered[:max_detected]
+        else:
+            filtered = items
+        if len(filtered) != len(items):
+            log.debug(f'Detailer: items={len(items)} filtered={len(filtered)}')
+        return filtered
+
+    def draw_masks(self, image: Image.Image, items: list[DetailerResult], p=None) -> Image.Image | np.ndarray:
         if not isinstance(image, Image.Image):
             image = Image.fromarray(image)
         image = image.convert('RGBA')
         size = min(image.width, image.height) // 32
         font = images.get_font(size)
         color = (0, 190, 190)
-        log.debug(f'Detailer: draw={items}')
+        # log.debug(f'Detailer: draw={items}')
         for i, item in enumerate(items):
             if detailer_opt(p, 'detailer_segmentation') and item.mask is not None:
                 mask = item.mask.convert('L')
@@ -175,6 +235,7 @@ class Detailer():
             if image is None:
                 image = Image.fromarray(np_image)
             items = self.predict(name, model, image, p=p)
+            items = self.filter(items, image, p=p)
 
             if len(items) == 0:
                 log.info(f'Detailer: model="{name}" no items detected')
@@ -319,7 +380,7 @@ class Detailer():
             np_images.append(annotated) # save debug image with boxes
         return np_images
 
-    def make_processing(self, image, prompt='', negative='', steps=10, strength=0.3, resolution=1024, seed=-1, overrides=None):
+    def make_processing(self, image, prompt='', negative='', steps=10, strength=0.3, resolution=1024, seed=-1, overrides=None, classes=''):
         """Build a synthetic Img2Img processing object to run restore() standalone, with no base generation pass.
 
         The primary params map to the detailer_* fields restore() reads directly. overrides is an optional
@@ -351,6 +412,7 @@ class Detailer():
             detailer_steps=steps,
             detailer_strength=strength,
             detailer_resolution=resolution,
+            detailer_classes=classes,
         )
         for attr, val in (overrides or {}).items():
             if val is not None:
@@ -376,11 +438,11 @@ class Detailer():
             return gr.update(visible=False), gr.update(visible=True, value=value), gr.update(visible=False)
 
     def ui(self, tab: str):
-        def ui_settings_change(merge, detailers, text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg):
+        def ui_settings_change(merge, detailers, text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg): # pylint: disable=unused-argument
             shared.opts.detailer_merge = merge
             shared.opts.detailer_models = detailers
             shared.opts.detailer_args = text if not self.ui_mode else ''
-            shared.opts.detailer_classes = classes
+            # shared.opts.detailer_classes = classes
             shared.opts.detailer_padding = padding
             shared.opts.detailer_blur = blur
             shared.opts.detailer_conf = min_confidence
@@ -395,7 +457,7 @@ class Detailer():
             shared.opts.detailer_segmentation = seg
             # shared.opts.detailer_resolution = resolution
             shared.opts.save(silent=True)
-            log.debug(f'Detailer settings: models={detailers} classes={classes} strength={strength} conf={min_confidence} max={max_detected} iou={iou} size={min_size}-{max_size} padding={padding} steps={steps} resolution={resolution} save={save} sort={sort} seg={seg}')
+            # log.debug(f'Detailer settings: models={detailers} classes={classes} strength={strength} conf={min_confidence} max={max_detected} iou={iou} size={min_size}-{max_size} padding={padding} steps={steps} resolution={resolution} save={save} sort={sort} seg={seg}')
             if not self.ui_mode:
                 log.debug(f'Detailer expert: {text}')
 
@@ -415,7 +477,7 @@ class Detailer():
                 ui_mode = ui_components.ToolButton(value=ui_symbols.view, elem_id=f'{tab}_yolo_models_list')
                 ui_mode.click(fn=self.change_mode, inputs=[detailers, detailers_text], outputs=[detailers, detailers_text, refresh_btn])
             with gr.Row():
-                classes = gr.Textbox(label="Detailer classes", placeholder="Classes", elem_id=f"{tab}_detailer_classes")
+                classes = gr.Textbox(label="Detailer classes or instructions", placeholder="List of classes or human instructions", elem_id=f"{tab}_detailer_classes")
             if tab == 'extras': # Process tab is standalone, there is no base prompt to fall back to
                 prompt_placeholder = 'detailer prompt, leave empty for none'
                 negative_placeholder = 'detailer negative prompt, leave empty for none'
@@ -480,5 +542,5 @@ class Detailer():
             sort.change(fn=ui_settings_change, inputs=[merge, detailers, detailers_text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg], outputs=[])
             seg.change(fn=ui_settings_change, inputs=[merge, detailers, detailers_text, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps, renoise_value, renoise_end, resolution, save, sort, seg], outputs=[])
             if tab == 'extras':
-                return enabled, prompt, negative, steps, strength, resolution, sampler_block
-            return enabled, prompt, negative, steps, strength, resolution
+                return enabled, prompt, negative, steps, strength, resolution, classes, sampler_block
+            return enabled, prompt, negative, steps, strength, resolution, classes

modules/detailer/dino.py

@@ -0,0 +1,125 @@
+import time
+import re
+import torch
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, sd_models, model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def format_grounding_dino_prompt(prompt: str) -> str:
+    # Formats user input into Grounding DINO query format. Grounding DINO requires lowercase text separated by periods and ending with a period.
+    clean_prompt = prompt.strip().lower()
+    if not clean_prompt or clean_prompt == "detect and locate all objects":
+        return "object."
+    items = [i.strip() for i in re.split(r'[,.]', clean_prompt) if i.strip()]
+    if not items:
+        return "object."
+    return ". ".join(items) + "."
+
+
+def load(self, model_name: str | None = None) -> tuple[str | None, transformers.AutoModelForZeroShotObjectDetection | object]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = 'IDEA-Research/' + model_name.lower() if '/' not in model_name else model_name
+    sd_models.hf_auth_check(repo_id)
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d'])
+    model = transformers.AutoModelForZeroShotObjectDetection.from_pretrained(**load_kwargs, **quant_args)
+    model = model.eval()
+    model.processor = transformers.AutoProcessor.from_pretrained(**load_kwargs)
+
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(data: dict, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    w, h = image.size
+    boxes = data.get("boxes", [])
+    scores = data.get("scores", [])
+    labels = data.get("labels", [])
+    for box, score, label in zip(boxes, scores, labels):
+        if len(box) == 4:
+            if isinstance(box, torch.Tensor):
+                box = box.tolist()
+            xmin, ymin, xmax, ymax = map(int, box)
+            box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+            mask, cropped = get_mask(box, image, include_mask=include_mask)
+            result = DetailerResult(box=box,
+                                    label=str(label),
+                                    score=float(score),
+                                    cls=-1,
+                                    mask=mask,
+                                    item=cropped
+                                   )
+            # log.trace(f'Detailer box: {result}')
+            results.append(result)
+    response = f"Grounding DINO detected {len(results)} objects."
+    return response, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    text_input = format_grounding_dino_prompt(prompt)
+    threshold = detailer_opt(p, 'detailer_conf')
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload} threshold={threshold}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    with devices.llm_context():
+        inputs = model.processor(
+            images=image,
+            text=text_input,
+            return_tensors="pt"
+        ).to(model.device, dtype=devices.dtype)
+        with torch.autocast(device_type=model.device.type, dtype=devices.dtype):
+            outputs = model(**inputs)
+
+    parsed_output = model.processor.post_process_grounded_object_detection(
+        outputs=outputs,
+        input_ids=inputs.input_ids,
+        threshold=threshold,
+        text_threshold=threshold,
+        target_sizes=[(image.height, image.width)]
+    )[0]
+
+    t1 = time.time()
+    response, results = parse(parsed_output, image, include_mask=mask)
+
+    log.debug(f'Detailer: name="{name}" response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/florence.py

@@ -0,0 +1,172 @@
+import time
+import json
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, sd_models, model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def select_florence_task(prompt: str) -> tuple[str, str]:
+    # Analyzes a user prompt and automatically determines whether to use <OPEN_VOCABULARY_DETECTION>, <CAPTION_TO_PHRASE_GROUNDING>, or default <OD>
+    clean_prompt = prompt.strip()
+    if not clean_prompt or clean_prompt == "Detect and locate all objects":
+        return "<OD>", "<OD>"
+    """
+    import re
+    items = [i.strip() for i in clean_prompt.split(",") if i.strip()]
+    is_class_list = len(items) > 1 or all(len(item.split()) <= 2 for item in items)
+    descriptive_keywords = re.search(r'\b(a|an|the|with|wearing|in|on|next to|holding|under|near)\b', clean_prompt, re.IGNORECASE)
+    if is_class_list and not descriptive_keywords:
+        task = "<OPEN_VOCABULARY_DETECTION>"
+        # formatted = f"{task}{', '.join(items)}"
+        formatted = f"{task}{'. '.join(items)}"
+    else:
+        task = "<CAPTION_TO_PHRASE_GROUNDING>"
+        formatted = f"{task}{clean_prompt}"
+    """
+    task = "<CAPTION_TO_PHRASE_GROUNDING>"
+    formatted = f"{task}{clean_prompt}"
+    return task, formatted
+
+
+def load(self, model_name: str | None = None) -> tuple[str | None, transformers.AutoModelForCausalLM | object]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = 'florence-community/' + model_name if '/' not in model_name else model_name
+    sd_models.hf_auth_check(repo_id)
+
+    orig_get_imports = transformers.dynamic_module_utils.get_imports
+
+    def hijack_get_imports(f):
+        R = orig_get_imports(f)
+        if "flash_attn" in R:
+            R.remove("flash_attn")  # flash_attn is optional
+        return R
+
+    transformers.dynamic_module_utils.get_imports = hijack_get_imports
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+        'trust_remote_code': True,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d'])
+    model = transformers.Florence2ForConditionalGeneration.from_pretrained(
+        **load_kwargs,
+        **quant_args,
+        attn_implementation="sdpa"
+    )
+    model = model.eval()
+    model.processor = transformers.AutoProcessor.from_pretrained(**load_kwargs, max_pixels=1024*1024)
+
+    transformers.dynamic_module_utils.get_imports = orig_get_imports
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(data: dict | str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    response = ""
+    w, h = image.size
+    try:
+        parsed_data = {}
+        if isinstance(data, str):
+            clean = data.strip()
+            if clean.startswith("```json"):
+                clean = clean[7:]
+            if clean.endswith("```"):
+                clean = clean[:-3]
+            parsed_data = json.loads(clean.strip())
+        elif isinstance(data, dict):
+            parsed_data = data
+        detection_data = None
+        for key in ["<OPEN_VOCABULARY_DETECTION>", "<CAPTION_TO_PHRASE_GROUNDING>", "<OD>"]:
+            if key in parsed_data:
+                detection_data = parsed_data[key]
+                response = f"task '{key}' executed successfully."
+                break
+        if detection_data:
+            bboxes = detection_data.get("bboxes", [])
+            # Fix key mismatch: Florence-2 returns 'bboxes_labels' for OPEN_VOCABULARY_DETECTION and CAPTION_TO_PHRASE_GROUNDING
+            labels = detection_data.get("labels") or detection_data.get("bboxes_labels") or ["object"] * len(bboxes)
+
+            for box, label in zip(bboxes, labels):
+                if len(box) == 4:
+                    xmin, ymin, xmax, ymax = map(int, box)
+                    box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+                    mask, cropped = get_mask(box, image, include_mask=include_mask)
+                    result = DetailerResult(box=box, label=label, score=1.0, cls=-1, mask=mask, item=cropped)
+                    log.trace(f'Detailer box: {result}')
+                    results.append(result)
+    except Exception as err:
+        log.error(f'Detailer: failed to parse detection output: {err}')
+        log.error(f'Detailer: raw output: {data}')
+
+    return response, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+
+    # Dynamic task routing (<OD>, <OPEN_VOCABULARY_DETECTION>, or <CAPTION_TO_PHRASE_GROUNDING>)
+    task_prompt, text_input = select_florence_task(prompt)
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    with devices.llm_context():
+        inputs = model.processor(
+            text=text_input,
+            images=image,
+            return_tensors="pt"
+        ).to(model.device, dtype=devices.dtype)
+        generated_ids = model.generate(
+            input_ids=inputs["input_ids"],
+            pixel_values=inputs["pixel_values"],
+            max_new_tokens=1024,
+            do_sample=False,
+            num_beams=3, # beams=3 improves grounding recall on small objects
+            early_stopping=False,
+        )
+    generated_text = model.processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+    parsed_output = model.processor.post_process_generation(
+        generated_text,
+        task=task_prompt,
+        image_size=(image.width, image.height)
+    )
+    t1 = time.time()
+    response, results = parse(parsed_output, image, include_mask=mask)
+
+    token_count = generated_ids.shape[1] if hasattr(generated_ids, 'shape') else 0
+    log.debug(f'Detailer: name="{name}" tokens={token_count} response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/helper.py

@@ -1,6 +1,6 @@
 import os
 import re
-from PIL import Image
+from PIL import Image, ImageDraw
 from modules.logger import log
 
 
@@ -85,6 +85,16 @@ def assign_prompts(text: str, items: list) -> list[str]:
     return resolved
 
 
+def get_mask(box: list[int], image: Image.Image, include_mask: bool = True) -> tuple[Image.Image | None, Image.Image]:
+    cropped = image.crop(box)
+    if not include_mask:
+        return None, cropped
+    mask = Image.new('L', image.size, 0)
+    draw_mask = ImageDraw.Draw(mask)
+    draw_mask.rectangle(box, fill="white", outline=None, width=0)
+    return mask, cropped
+
+
 class DetailerResult:
     def __init__(self, cls: int, label: str, score: float, box: list[int], mask: Image.Image = None, item: Image.Image = None, width = 0, height = 0, args = None):
         if args is None:
@@ -95,9 +105,9 @@ class DetailerResult:
         self.box = box
         self.mask = mask
         self.item = item
-        self.width = width
-        self.height = height
+        self.width = width if width > 0 else box[2] - box[0]
+        self.height = height if height > 0 else box[3] - box[1]
         self.args = args
 
     def __str__(self):
-        return f'DetailerResult(cls={self.cls} label={self.label} score={self.score:.2f} box={self.box} size={self.width}x{self.height} args={self.args})'
+        return f'DetailerResult(cls={self.cls} label="{self.label}" score={self.score:.2f} box={self.box} size={self.width}x{self.height} args={self.args})'

modules/detailer/locateanything.py

@@ -14,7 +14,7 @@ def dependencies():
     install('decord')
 
 
-def load(model_name: str | None = None):
+def load(self, model_name: str | None = None):
     import transformers
     global tokenizer, processor # pylint: disable=global-statement
     load_kwargs = {
@@ -30,6 +30,7 @@ def load(model_name: str | None = None):
     if shared.opts.detailer_unload:
         model.to(devices.cpu)
     log.info(f'Detailer model="{model_name}" cls={model.__class__.__name__} loaded')
+    self.models[model_name] = model
     return model_name, model
 
 
@@ -44,16 +45,16 @@ def predict(
     ) -> list[DetailerResult]:
     if offload is None:
         offload = shared.opts.detailer_unload
-    log.info(f'Detailer cls="{model.__class__.__name__}" image={image} device={device} mask={mask} offload={offload}')
     result = []
     if isinstance(model, str):
         cached = self.models.get(model, None)
         if cached is None:
-            _, model = self.load(model)
+            _, model = load(self, model)
         else:
             model = cached
     if model is None:
         return result
+    log.info(f'Detailer cls="{model.__class__.__name__}" image={image} device={device} mask={mask} offload={offload}')
     model = model.to(device)
     prompt = detailer_opt(p, 'detailer_classes') or ''
     log.debug(f'Detailer prompt="{prompt}"')

modules/detailer/models.py

@@ -9,5 +9,14 @@ detailer_models = [ # <https://huggingface.co/vladmandic/yolo-detailers/tree/mai
     'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-eyes-seg.pt',
     'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-face-1024-seg-8n.pt',
     'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/anzhc-head-seg-8n.pt',
-    # 'nvidia-LocateAnything-3B',
+    'Qwen3-VL-2B-Instruct',
+    'Qwen3-VL-4B-Instruct',
+    'Qwen3-VL-8B-Instruct',
+    'Florence-2-base-ft',
+    'Florence-2-large-ft',
+    'Grounding-DINO-tiny',
+    'Grounding-DINO-base',
+    'Facebook-SAM3',
+    # 'Rex-Omni', # not compatible with transformers==5
+    # 'nvidia-LocateAnything-3B', # not compatible with transformers==5
 ]

modules/detailer/qwen.py

@@ -0,0 +1,191 @@
+import time
+import json
+import transformers
+from pydantic import BaseModel, Field
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, sd_models, model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+class BoundingBoxItem(BaseModel):
+    label: str = Field(..., description="Label of the detected object")
+    confidence: float = Field(default=1.0, ge=0.0, le=1.0, description="Confidence score between 0.0 and 1.0")
+    box_2d: list[int] = Field(..., min_items=4, max_items=4, description="Bounding box coordinates in 0-1000 normalized format: [ymin, xmin, ymax, xmax]")
+
+
+class ObjectDetectionOutput(BaseModel):
+    response: str = Field(..., description="Reasoning summary: state which requested objects were identified and their location before outputting coordinates.")
+    objects: list[BoundingBoxItem]
+
+
+def template(prompt: str, schema: str, min_confidence: float) -> list[dict]:
+    def confidence() -> str:
+        conf = max(0.0, min(1.0, float(min_confidence)))
+        if conf >= 0.85:
+            instruction = "Detect only obvious, fully visible target objects."
+        elif conf >= 0.65:
+            instruction = "Detect clear targets, ignoring faint or ambiguous cases."
+        elif conf >= 0.45:
+            instruction = "Detect all distinct targets, including partially covered ones."
+        elif conf >= 0.25:
+            instruction = "Detect candidate targets, including small, blurry, or occluded ones."
+        else:
+            instruction = "Detect all possible target candidates, background objects, or fragments."
+        return instruction
+
+    instructions = (
+        "You are an expert vision assistant for object detection.\n"
+        f"INSTRUCTIONS:\n"
+        f"1. Scan the image carefully for EACH requested class.\n"
+        f"2. Add a BoundingBoxItem to 'objects' for EVERY instance found.\n"
+        f"3. In the 'response' field, explicitly list which target classes were found and which were missing.\n"
+        f"4. {confidence()}\n"
+        f"5. You MUST respond strictly with a valid JSON object matching this schema: \n```json\n{schema}\n```\n\n"
+        "6. Do not include any Markdown text outside of the JSON string. Add any text explanations or clarifications inside the 'response' field of the JSON object.\n"
+        "7. Bounding box coordinates must be in 0-1000 normalized format: [ymin, xmin, ymax, xmax]."
+    )
+    return [
+        { "role": "system", "content": instructions },
+        { "role": "user",
+          "content": [
+            {"type": "image"}, # injected later using processor
+            {"type": "text", "text": prompt},
+          ],
+        },
+    ]
+
+
+def load(self, model_name: str | None = None) -> tuple[str | None, transformers.Qwen3VLForConditionalGeneration | object]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = 'Qwen/' + model_name if not model_name.startswith('Qwen/') else model_name
+    sd_models.hf_auth_check(repo_id)
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d'])
+    model = transformers.Qwen3VLForConditionalGeneration.from_pretrained(**load_kwargs, **quant_args, attn_implementation="sdpa")
+    model = model.eval()
+    model.processor = transformers.Qwen3VLProcessor.from_pretrained(**load_kwargs)
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(data: str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    response = ''
+    w, h = image.size
+    try:
+        clean = data.strip()
+        if clean.startswith("```json"):
+            clean = clean[7:]
+        if clean.endswith("```"):
+            clean = clean[:-3]
+        parsed = json.loads(clean.strip())
+        response = parsed.get("response", "")
+        objects = parsed.get("objects", [])
+        for item in objects:
+            box_2d = item.get("box_2d", [])
+            label = item.get("label", "")
+            confidence = float(item.get("confidence", 1.0))
+            if len(box_2d) == 4:
+                xmin, ymin, xmax, ymax = map(int, box_2d)
+                xmin, ymin = int((xmin / 1000.0) * w), int((ymin / 1000.0) * h)
+                xmax, ymax = int((xmax / 1000.0) * w), int((ymax / 1000.0) * h)
+                box = (xmin, ymin, xmax, ymax)
+                mask, cropped = get_mask(box, image, include_mask)
+                result = DetailerResult(box=box,
+                                        label=label,
+                                        score=confidence,
+                                        cls=-1,
+                                        mask=mask,
+                                        item=cropped
+                                       )
+                # log.trace(f'Detailer box: {result}')
+                results.append(result)
+    except Exception as err:
+        log.error(f'Detailer: failed to parse object detection output: {err}')
+        log.error(f'Detailer: raw output: {data}')
+    return response, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    if not prompt:
+        prompt = 'Detect and locate all objects'
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{prompt}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    schema = json.dumps(ObjectDetectionOutput.model_json_schema(), indent=2)
+    messages = template(prompt=prompt, schema=schema, min_confidence=detailer_opt(p, 'detailer_conf'))
+
+    with devices.llm_context():
+        text = model.processor.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        inputs = model.processor(
+            text=[text],
+            images=[image],
+            padding=True,
+            return_tensors="pt",
+            min_pixels=128 * 28 * 28,
+            max_pixels=1280 * 28 * 28,  # Force 1 MP cap
+        )
+        inputs.pop("token_type_ids", None)
+        inputs = inputs.to(model.device)
+        eos_id = model.processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
+        if eos_id is None or isinstance(eos_id, list):
+            eos_id = model.processor.tokenizer.eos_token_id
+        pad_id = model.processor.tokenizer.pad_token_id if model.processor.tokenizer.pad_token_id is not None else eos_id
+        generated_ids = model.generate(
+            **inputs,
+            max_new_tokens=2048,
+            do_sample=False,         # Deterministic decoding keeps bbox integer tokens strict
+            temperature=None,        # Forces argmax token selection
+            repetition_penalty=1.03, # Breaks coordinate repetition loops without distorting valid coordinates
+            no_repeat_ngram_size=0,  # MUST be 0/None—setting this > 0 corrupts valid repeated bbox coordinates
+            eos_token_id=eos_id,     # Prevent premature EOS token stopping
+            pad_token_id=pad_id,     # Prevent premature EOS token stopping
+        )
+
+    prompt_len = inputs["input_ids"].shape[1]
+    output_tokens = generated_ids[0][prompt_len:]
+    output_text = model.processor.tokenizer.decode(output_tokens, skip_special_tokens=True)
+    t1 = time.time()
+    response, results = parse(output_text, image, include_mask=mask)
+
+    log.debug(f'Detailer: name="{name}" tokens={output_tokens.shape[0]} response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/rexomni.py

@@ -0,0 +1,158 @@
+import time
+import re
+import torch
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_offload_aux, sd_models,model_quant
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def format_rex_prompt(prompt: str) -> str:
+    clean_prompt = prompt.strip()
+    if not clean_prompt or clean_prompt.lower() == "detect and locate all objects":
+        return "<|grounding|>Locate all objects in the image."
+    if clean_prompt.startswith("<|") and "|>" in clean_prompt:
+        return clean_prompt
+    return f"<|grounding|>{clean_prompt}"
+
+
+def load(self, model_name: str | None = None) -> tuple[str | None, transformers.AutoModelForCausalLM | object]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+
+    repo_id = 'IDEA-Research/' + model_name if '/' not in model_name else model_name
+    sd_models.hf_auth_check(repo_id)
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+        'trust_remote_code': True,
+    }
+    quant_args = model_quant.create_config(module='LLM', modules_to_not_convert=['conv1d', 'linear_attn.conv1d', 'embed_tokens', 'lm_head'])
+    model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(
+        **load_kwargs,
+        **quant_args,
+        attn_implementation="sdpa"
+    )
+    model = model.eval()
+    model.processor = transformers.AutoProcessor.from_pretrained(
+        repo_id,
+        cache_dir=shared.opts.hfcache_dir,
+        trust_remote_code=True
+    )
+
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def parse(raw_output: str, image: Image.Image, include_mask: bool = True) -> tuple[str, list[DetailerResult]]:
+    results = []
+    w, h = image.size
+    bboxes = []
+    labels = []
+    box_pattern = re.compile(r'(?:<box>|\(|\[)\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:</box>|\)|\])\s*([^<\n,]+)?')
+    matches = box_pattern.findall(raw_output)
+    for match in matches:
+        if len(match) >= 4:
+            coords = [int(match[i]) for i in range(4)]
+            label = match[4].strip() if len(match) > 4 and match[4].strip() else "object"
+            ymin, xmin, ymax, xmax = coords
+            if max(coords) <= 1000:
+                xmin = int((xmin / 1000.0) * w)
+                ymin = int((ymin / 1000.0) * h)
+                xmax = int((xmax / 1000.0) * w)
+                ymax = int((ymax / 1000.0) * h)
+            bboxes.append((xmin, ymin, xmax, ymax))
+            labels.append(label)
+    for box, label in zip(bboxes, labels):
+        if len(box) == 4:
+            xmin, ymin, xmax, ymax = map(int, box)
+            box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+            if box[2] > box[0] and box[3] > box[1]:
+                mask, cropped = get_mask(box, image, include_mask=include_mask)
+                result = DetailerResult(box=box, label=label, score=1.0, cls=-1, mask=mask, item=cropped)
+                log.trace(f'Detailer box: {result}')
+                results.append(result)
+    return raw_output, results
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    text_input = format_rex_prompt(prompt)
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "text", "text": text_input},
+            ],
+        }
+    ]
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{text_input}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    with devices.llm_context():
+        # Use processor directly with images and text to automatically build image tokens and grid thw
+        text = model.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        inputs = model.processor(
+            text=[text],
+            images=[image],
+            padding=True,
+            return_tensors="pt",
+        )
+        target_device = model.device
+        prepared_inputs = {}
+        for k, v in inputs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.to(device=target_device)
+                if torch.is_floating_point(v):
+                    v = v.to(dtype=devices.dtype)
+                prepared_inputs[k] = v
+            else:
+                prepared_inputs[k] = v
+        generated_ids = model.generate(
+            **prepared_inputs,
+            max_new_tokens=1024,
+            do_sample=False,
+            use_cache=True,
+        )
+
+    input_len = prepared_inputs["input_ids"].shape[1]
+    generated_ids_trimmed = generated_ids[:, input_len:]
+    generated_text = model.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=False)[0]
+
+    t1 = time.time()
+    response, results = parse(generated_text, image, include_mask=mask)
+
+    token_count = generated_ids.shape[1]
+    log.debug(f'Detailer: name="{name}" tokens={token_count} response="{response}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/sam.py

@@ -0,0 +1,103 @@
+import time
+import transformers
+from PIL import Image
+from modules import shared, devices, sd_models, sd_offload_aux
+from modules.detailer import DetailerResult, detailer_opt, get_mask
+from modules.logger import log
+
+
+def load(self, model_name: str | None = None) -> tuple[str | None, transformers.Sam3Model | object]: # pylint: disable=unused-argument
+    cached = sd_offload_aux.get_aux_model(model_name)
+    if cached is not None:
+        return model_name, cached
+    repo_id = model_name.lower().replace('-', '/')
+    sd_models.hf_auth_check(repo_id, force=True)
+    load_kwargs = {
+        'pretrained_model_name_or_path': repo_id,
+        'cache_dir': shared.opts.hfcache_dir,
+        'torch_dtype': devices.dtype,
+    }
+    model = transformers.Sam3Model.from_pretrained(**load_kwargs)
+    model = model.eval()
+    model.processor = transformers.Sam3Processor.from_pretrained(**load_kwargs)
+    sd_offload_aux.register_aux(model_name, model)
+    if shared.opts.detailer_unload:
+        sd_offload_aux.offload_aux(model_name)
+    log.info(f'Load: type=Detailer name="{model_name}" cls="{model.__class__.__name__}" processor="{model.processor.__class__.__name__}"')
+    return model_name, model
+
+
+def predict(
+    self,
+    name: str,
+    image: Image.Image,
+    device = devices.device,
+    mask: bool = True,
+    offload: bool | None = None,
+    p = None,
+) -> list[DetailerResult]:
+    if offload is None:
+        offload = shared.opts.detailer_unload
+    if image is None:
+        return []
+    cached = sd_offload_aux.get_aux_model(name)
+    if cached is None:
+        name, model = load(self, name)
+    else:
+        model = cached
+    if model is None:
+        return []
+    prompt = detailer_opt(p, 'detailer_classes') or ''
+    if not prompt:
+        prompt = 'object'
+
+    log.debug(f'Detailer: name="{name}" cls={model.__class__.__name__} prompt="{prompt}" image={image.size} device={device} mask={mask} offload={offload}')
+    sd_offload_aux.move_aux_to_gpu(name)
+
+    t0 = time.time()
+    results = []
+
+    with devices.llm_context():
+        inputs = model.processor(images=image, text=prompt, return_tensors="pt")
+        inputs = inputs.to(model.device)
+        outputs = model(**inputs)
+        target_sizes = inputs.get("original_sizes").tolist() if "original_sizes" in inputs else [image.size[::-1]]
+        results_list = model.processor.post_process_instance_segmentation(
+            outputs,
+            threshold=detailer_opt(p, 'detailer_conf') or 0.3,
+            mask_threshold=0.5,
+            target_sizes=target_sizes,
+        )
+
+    w, h = image.size
+    if results_list and len(results_list) > 0:
+        res = results_list[0]
+        boxes = res.get("boxes", [])
+        scores = res.get("scores", [])
+        labels = res.get("labels", [])
+        masks = res.get("masks", []) if mask else [None] * len(boxes)
+        for i, box_tensor in enumerate(boxes):
+            box_coords = box_tensor.tolist()
+            xmin, ymin, xmax, ymax = map(int, box_coords)
+            box = (max(0, xmin), max(0, ymin), min(w, xmax), min(h, ymax))
+            score = float(scores[i].item()) if i < len(scores) else 1.0
+            label = str(labels[i].item()) if i < len(labels) else prompt
+            masked, cropped = get_mask(box, image, include_mask=mask)
+            if mask and detailer_opt(p, 'detailer_segmentation') and (i < len(masks) and masks[i] is not None):
+                masked = Image.fromarray(masks[i].detach().cpu().numpy().astype('uint8') * 255)
+            cropped = image.crop(box)
+            result = DetailerResult(
+                box=box,
+                label=label,
+                score=score,
+                cls=-1,
+                mask=masked,
+                item=cropped
+            )
+            results.append(result)
+
+    t1 = time.time()
+    log.debug(f'Detailer: name="{name}" items={len(results)} time={t1-t0:.3f}')
+
+    sd_offload_aux.offload_aux(name)
+    return results

modules/detailer/yolo.py

@@ -2,8 +2,8 @@ from typing import TYPE_CHECKING
 import os
 import threading
 import numpy as np
-from PIL import Image, ImageDraw
-from modules.detailer import DetailerResult, detailer_opt
+from PIL import Image
+from modules.detailer import DetailerResult, detailer_opt, get_mask
 from modules.logger import log
 from modules import shared, devices
 
@@ -138,33 +138,19 @@ def predict(
             if len(desired) > 0 and label.lower() not in desired:
                 continue
             box = box.tolist()
-            w, h = box[2] - box[0], box[3] - box[1]
-            x_size, y_size = w/image.width, h/image.height
-            opt_min = detailer_opt(p, 'detailer_min_size') or 0
-            opt_max = detailer_opt(p, 'detailer_max_size') or 1
-            min_size = opt_min if 0 <= opt_min <= 1 else 0
-            max_size = opt_max if 0 < opt_max <= 1 else 1
-            if x_size >= min_size and y_size >=min_size and x_size <= max_size and y_size <= max_size:
-                if mask:
-                    if detailer_opt(p, 'detailer_segmentation') and seg is not None:
-                        masked = seg
-                    else:
-                        masked = Image.new('L', image.size, 0)
-                        draw = ImageDraw.Draw(masked)
-                        draw.rectangle(box, fill="white", outline=None, width=0)
-                    cropped = image.crop(box)
-                    res = DetailerResult(
-                        cls=cls,
-                        label=label,
-                        score=round(score, 2),
-                        box=box,
-                        mask=masked,
-                        item=cropped,
-                        width=w,
-                        height=h,
-                        args=args,
-                    )
-                    result.append(res)
+            masked, cropped = get_mask(box, image, include_mask=mask)
+            if detailer_opt(p, 'detailer_segmentation') and seg is not None:
+                masked = seg
+            res = DetailerResult(
+                cls=cls,
+                label=label,
+                score=round(score, 2),
+                box=box,
+                mask=masked,
+                item=cropped,
+                args=args,
+            )
+            result.append(res)
             if len(result) >= (detailer_opt(p, 'detailer_max') or 2):
                 break
     return result

modules/devices.py

@@ -1,4 +1,5 @@
 import os
+import gc
 import sys
 import time
 import contextlib
@@ -79,8 +80,6 @@ def get_backend(shared_cmd_opts):
     args = shared_cmd_opts
     if args.use_openvino:
         name = 'openvino'
-    elif args.use_directml:
-        name = 'directml'
     elif has_xpu():
         name = 'ipex'
     elif has_zluda():
@@ -131,11 +130,6 @@ def get_gpu_info():
                     'devices': devices,
                     'openvino': get_package_version("openvino"),
                 }
-            elif backend == 'directml':
-                return {
-                    'device': f'{torch.cuda.get_device_name(torch.cuda.current_device())} n={torch.cuda.device_count()}',
-                    'directml': get_package_version("torch-directml"),
-                }
             else:
                 return {}
         except Exception:
@@ -150,7 +144,10 @@ def get_gpu_info():
                 }
             elif backend == 'cuda' or backend == 'zluda':
                 return {
-                    'device': f'{torch.cuda.get_device_name(torch.cuda.current_device())} n={torch.cuda.device_count()} arch={torch.cuda.get_arch_list()[-1]} capability={torch.cuda.get_device_capability(device)}',
+                    'device': f'"{torch.cuda.get_device_name(torch.cuda.current_device())}"',
+                    'count': torch.cuda.device_count(),
+                    'arch': torch.cuda.get_arch_list()[-1],
+                    'capability': torch.cuda.get_device_capability(torch.cuda.current_device()),
                     'cuda': torch.version.cuda,
                     'cudnn': torch.backends.cudnn.version(),
                     'driver': get_driver(),
@@ -176,10 +173,6 @@ def get_cuda_device_string():
         if cmd_opts.device_id is not None:
             return f"xpu:{cmd_opts.device_id}"
         return "xpu"
-    elif backend == 'directml' and torch.dml.is_available():
-        if cmd_opts.device_id is not None:
-            return f"privateuseone:{cmd_opts.device_id}"
-        return torch.dml.get_device_string(torch.dml.default_device().index)
     else:
         if cmd_opts.device_id is not None:
             return f"cuda:{cmd_opts.device_id}"
@@ -189,7 +182,7 @@ def get_cuda_device_string():
 def get_optimal_device_name():
     if backend == 'openvino':
         return "cpu"
-    if cuda_ok or backend == 'directml':
+    if cuda_ok:
         return get_cuda_device_string()
     if has_mps() and backend != 'openvino':
         return "mps"
@@ -205,18 +198,14 @@ def torch_gc(force: bool = False, fast: bool = False, reason: str | None = None)
         mem_dict = memstats.memory_stats()
         gpu_dict = mem_dict.get('gpu', {})
         ram_dict = mem_dict.get('ram', {})
-        oom = gpu_dict.get('oom', 0)
         ram = ram_dict.get('used', 0)
-        if backend == "directml":
-            gpu = torch.cuda.memory_allocated() / (1 << 30)
-        else:
-            gpu = gpu_dict.get('used', 0)
+        oom = gpu_dict.get('oom', 0)
+        gpu = gpu_dict.get('used', 0)
         used_gpu = round(100 * gpu / gpu_dict.get('total', 1)) if gpu_dict.get('total', 1) > 1 else 0
         used_ram = round(100 * ram / ram_dict.get('total', 1)) if ram_dict.get('total', 1) > 1 else 0
         return gpu, used_gpu, ram, used_ram, oom
 
     global previous_oom # pylint: disable=global-statement
-    import gc
     from modules import timer, memstats
     from modules.shared import cmd_opts
 
@@ -242,10 +231,16 @@ def torch_gc(force: bool = False, fast: bool = False, reason: str | None = None)
     if force:
         # actual gc
         collected = gc.collect() if not fast else 0 # python gc
+        if collected > 0:
+            gc.collect() # deal with weakref cycles
         try:
             if hasattr(torch, "accelerator") and torch.accelerator.is_available(): # torch >= 2.6
-                torch.accelerator.synchronize()
-                torch.accelerator.empty_cache()
+                if hasattr(torch.accelerator, "synchronize"):
+                    torch.accelerator.synchronize()
+                if hasattr(torch.accelerator, "empty_cache"):
+                    torch.accelerator.empty_cache()
+                if hasattr(torch.accelerator, "empty_host_cache"):
+                    torch.accelerator.empty_host_cache()
                 if torch.cuda.is_available() and hasattr(torch.cuda, "ipc_collect"):
                     torch.cuda.ipc_collect()
                 elif hasattr(torch, "xpu") and hasattr(torch.xpu, "ipc_collect"):
@@ -253,7 +248,8 @@ def torch_gc(force: bool = False, fast: bool = False, reason: str | None = None)
             elif torch.cuda.is_available(): # Fallback for older PyTorch versions
                 torch.cuda.synchronize()
                 torch.cuda.empty_cache()
-                torch.cuda.ipc_collect()
+                if hasattr(torch.cuda, "ipc_collect"):
+                    torch.cuda.ipc_collect()
             elif hasattr(torch, "xpu") and torch.xpu.is_available():
                 torch.xpu.synchronize()
                 torch.xpu.empty_cache()
@@ -374,7 +370,7 @@ def test_bf16():
     if bf16_ok is not None:
         return bf16_ok
     if opts.cuda_dtype != 'BF16': # don't override if the user sets it
-        if sys.platform == "darwin" or backend in {'directml', 'cpu'}: # override
+        if sys.platform == "darwin" or backend == 'cpu': # override
             bf16_ok = False
             return bf16_ok
         elif backend == 'openvino':
@@ -456,8 +452,6 @@ def test_triton(early: bool = False):
     t1 = time.time()
     fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
     log.debug(f'Triton: pass={triton_ok} version={triton_version} fn={fn} time={t1-t0:.2f}')
-    if not triton_ok and opts is not None:
-        opts.sdnq_dequantize_compile = False
     return triton_ok
 
 
@@ -556,7 +550,7 @@ def set_sdpa_params():
             log.debug(f'Torch attention installed: flashattn={flash} sageattention={sage}')
 
         from diffusers.models import attention_dispatch as a
-        log.debug(f'Torch attention available: flash={a._CAN_USE_FLASH_ATTN} flash3={a._CAN_USE_FLASH_ATTN_3} aiter={a._CAN_USE_AITER_ATTN} sage={a._CAN_USE_SAGE_ATTN} flex={a._CAN_USE_FLEX_ATTN} npu={a._CAN_USE_NPU_ATTN} xla={a._CAN_USE_XLA_ATTN} xformers={a._CAN_USE_XFORMERS_ATTN} kernels={a.is_kernels_available()} sdnq=True') # pylint: disable=protected-access
+        log.debug(f'Torch attention available: flash={a._CAN_USE_FLASH_ATTN} flash3={a._CAN_USE_FLASH_ATTN_3} sage={a._CAN_USE_SAGE_ATTN} flex={a._CAN_USE_FLEX_ATTN} npu={a._CAN_USE_NPU_ATTN} xla={a._CAN_USE_XLA_ATTN} xformers={a._CAN_USE_XFORMERS_ATTN} kernels={a.is_kernels_available()} sdnq=True') # pylint: disable=protected-access
 
     except Exception as e:
         log.warning(f'Torch SDPA: {e}')
@@ -678,8 +672,6 @@ def randn_without_seed(shape):
 def autocast(disable=False):
     if disable or dtype == torch.float32:
         return contextlib.nullcontext()
-    if backend == 'directml':
-        return torch.dml.amp.autocast(dtype)
     if cuda_ok:
         return torch.autocast("cuda")
     else:
@@ -689,8 +681,6 @@ def autocast(disable=False):
 def without_autocast(disable=False):
     if disable:
         return contextlib.nullcontext()
-    if backend == 'directml':
-        return torch.dml.amp.autocast(enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext() # pylint: disable=unexpected-keyword-arg
     if cuda_ok:
         return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext()
     else:
@@ -774,7 +764,7 @@ def llm_context():
         yield
 
 
-def torch_reset() -> bool:
+def torch_reset() -> None:
     """
     Resets PyTorch execution graph, flushes VRAM caches, and syncs streams.
     """

modules/dml/Generator.py

@@ -1,6 +0,0 @@
-import torch
-
-
-class Generator(torch.Generator):
-    def __init__(self, device: torch.device | None = None):
-        super().__init__("cpu")

modules/dml/__init__.py

@@ -1,116 +0,0 @@
-import platform
-from typing import NamedTuple
-from collections.abc import Callable
-import torch
-from modules.logger import log
-from modules.sd_hijack_utils import CondFunc
-
-
-memory_providers = ["None", "atiadlxx (AMD only)"]
-default_memory_provider = "None"
-if platform.system() == "Windows":
-    memory_providers.append("Performance Counter")
-    default_memory_provider = "Performance Counter"
-do_nothing = lambda: None # pylint: disable=unnecessary-lambda-assignment
-do_nothing_with_self = lambda self: None # pylint: disable=unnecessary-lambda-assignment
-
-
-def _set_memory_provider():
-    from modules.shared import opts, cmd_opts
-    if opts.directml_memory_provider == "Performance Counter":
-        from .backend import pdh_mem_get_info
-        from .memory import MemoryProvider
-        torch.dml.mem_get_info = pdh_mem_get_info
-        if torch.dml.memory_provider is not None:
-            del torch.dml.memory_provider
-        torch.dml.memory_provider = MemoryProvider()
-    elif opts.directml_memory_provider == "atiadlxx (AMD only)":
-        device_name = torch.dml.get_device_name(cmd_opts.device_id)
-        if "AMD" not in device_name and "Radeon" not in device_name:
-            log.warning(f"Memory stats provider is changed to None because the current device is not AMDGPU. Current Device: {device_name}")
-            opts.directml_memory_provider = "None"
-            _set_memory_provider()
-            return
-        from .backend import amd_mem_get_info
-        torch.dml.mem_get_info = amd_mem_get_info
-    else:
-        from .backend import mem_get_info
-        torch.dml.mem_get_info = mem_get_info
-    torch.cuda.mem_get_info = torch.dml.mem_get_info
-
-
-def directml_init():
-    try:
-        from modules.dml.backend import DirectML # pylint: disable=ungrouped-imports
-        # Alternative of torch.cuda for DirectML.
-        torch.dml = DirectML
-
-        torch.cuda.is_available = lambda: False
-        torch.cuda.device = torch.dml.device
-        torch.cuda.device_count = torch.dml.device_count
-        torch.cuda.current_device = torch.dml.current_device
-        torch.cuda.get_device_name = torch.dml.get_device_name
-        torch.cuda.get_device_properties = torch.dml.get_device_properties
-
-        torch.cuda.empty_cache = do_nothing
-        torch.cuda.ipc_collect = do_nothing
-        torch.cuda.memory_stats = torch.dml.memory_stats
-        torch.cuda.mem_get_info = torch.dml.mem_get_info
-        torch.cuda.memory_allocated = torch.dml.memory_allocated
-        torch.cuda.max_memory_allocated = torch.dml.max_memory_allocated
-        torch.cuda.reset_peak_memory_stats = torch.dml.reset_peak_memory_stats
-        torch.cuda.utilization = lambda: 0
-
-        torch.Tensor.directml = lambda self: self.to(torch.dml.current_device())
-    except Exception as e:
-        log.error(f'DirectML initialization failed: {e}')
-        return False, e
-    return True, None
-
-
-def directml_do_hijack():
-    import modules.dml.hijack # pylint: disable=unused-import
-    from modules.devices import device
-
-    CondFunc('torch.Generator',
-        lambda orig_func, device = None: orig_func("cpu"),
-        lambda orig_func, device = None: True)
-
-    if not torch.dml.has_float64_support(device):
-        torch.Tensor.__str__ = do_nothing_with_self
-        CondFunc('torch.from_numpy',
-            lambda orig_func, *args, **kwargs: orig_func(args[0].astype('float32')),
-            lambda *args, **kwargs: args[1].dtype == float)
-
-    _set_memory_provider()
-
-
-class OverrideItem(NamedTuple):
-    value: str
-    condition: Callable | None
-    message: str | None
-
-
-opts_override_table = {
-    "diffusers_generator_device": OverrideItem("CPU", None, "DirectML does not support torch Generator API"),
-}
-
-
-def directml_override_opts():
-    from modules import shared
-
-    if shared.cmd_opts.experimental:
-        return
-
-    count = 0
-    for key in opts_override_table:
-        item = opts_override_table[key]
-        if getattr(shared.opts, key) != item.value and (item.condition is None or item.condition(shared.opts)):
-            count += 1
-            setattr(shared.opts, key, item.value)
-            log.warning(f'Overriding: {key}={item.value} {item.message if item.message is not None else ""}')
-
-    if count > 0:
-        log.info(f'Options override: count={count}. If you want to keep them from overriding, run with --experimental argument.')
-
-    _set_memory_provider()

modules/dml/amp/__init__.py

@@ -1 +0,0 @@
-from .autocast_mode import autocast

modules/dml/amp/autocast_mode.py

@@ -1,66 +0,0 @@
-import importlib
-from typing import Any
-import torch
-
-
-ops = ["torch.Tensor.__matmul__", "torch.addbmm", "torch.addmm", "torch.addmv", "torch.addr", "torch.baddbmm", "torch.bmm", "torch.chain_matmul", "torch.linalg.multi_dot", "torch.nn.functional.conv1d", "torch.nn.functional.conv2d", "torch.nn.functional.conv3d", "torch.nn.functional.conv_transpose1d", "torch.nn.functional.conv_transpose2d", "torch.nn.functional.conv_transpose3d", "torch.nn.GRUCell", "torch.nn.functional.linear", "torch.nn.LSTMCell", "torch.matmul", "torch.mm", "torch.mv", "torch.prelu", "torch.nn.RNNCell", "torch.embedding"]
-supported_cast_pairs = {
-    torch.float16: (torch.float32,),
-    torch.float32: (torch.float16,),
-}
-
-
-def forward(op, args: tuple, kwargs: dict):
-    if not torch.dml.is_autocast_enabled:
-        return op(*args, **kwargs)
-    args = list(map(cast, args))
-    for kwarg in kwargs:
-        kwargs[kwarg] = cast(kwargs[kwarg])
-    return op(*args, **kwargs)
-
-
-def cast(tensor: torch.Tensor):
-    if not torch.is_tensor(tensor):
-        return tensor
-    dtype: torch.dtype = tensor.dtype
-    if dtype not in supported_cast_pairs or (torch.dml.autocast_gpu_dtype != dtype and torch.dml.autocast_gpu_dtype not in supported_cast_pairs[dtype]):
-        return tensor
-    return tensor.type(torch.dml.autocast_gpu_dtype)
-
-
-def cond(op: str):
-    if isinstance(op, str):
-        func_path = op.split('.')
-        for i in range(len(func_path)-1, -1, -1):
-            try:
-                resolved_obj = importlib.import_module('.'.join(func_path[:i]))
-                break
-            except ImportError:
-                pass
-        for attr_name in func_path[i:-1]:
-            resolved_obj = getattr(resolved_obj, attr_name)
-        op = getattr(resolved_obj, func_path[-1])
-        setattr(resolved_obj, func_path[-1], lambda *args, **kwargs: forward(op, args, kwargs))
-
-
-for o in ops:
-    cond(o)
-
-
-class autocast:
-    prev: bool
-
-    fast_dtype: torch.dtype = torch.float16
-    prev_fast_dtype: torch.dtype
-    def __init__(self, dtype: torch.dtype | None = torch.float16):
-        self.fast_dtype = dtype
-
-    def __enter__(self):
-        self.prev = torch.dml.is_autocast_enabled
-        self.prev_fast_dtype = torch.dml.autocast_gpu_dtype
-        torch.dml.is_autocast_enabled = True
-        torch.dml.autocast_gpu_dtype = self.fast_dtype
-
-    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any):
-        torch.dml.is_autocast_enabled = self.prev
-        torch.dml.autocast_gpu_dtype = self.prev_fast_dtype

modules/dml/backend.py

@@ -1,80 +0,0 @@
-# pylint: disable=no-member,no-self-argument,no-method-argument
-from collections.abc import Callable
-import torch
-import torch_directml # pylint: disable=import-error
-import modules.dml.amp as amp
-from .utils import rDevice, get_device
-from .device import Device
-from .Generator import Generator
-from .device_properties import DeviceProperties
-
-
-def amd_mem_get_info(device: rDevice | None=None) -> tuple[int, int]:
-    from .memory_amd import AMDMemoryProvider
-    return AMDMemoryProvider.mem_get_info(get_device(device).index)
-
-
-def pdh_mem_get_info(device: rDevice | None=None) -> tuple[int, int]:
-    mem_info = DirectML.memory_provider.get_memory(get_device(device).index)
-    return (mem_info["total_committed"] - mem_info["dedicated_usage"], mem_info["total_committed"])
-
-
-def mem_get_info(device: rDevice | None=None) -> tuple[int, int]: # pylint: disable=unused-argument
-    return (8589934592, 8589934592)
-
-
-class DirectML:
-    amp = amp
-    device = Device
-    Generator = Generator
-
-    context_device: torch.device | None = None
-
-    is_autocast_enabled = False
-    autocast_gpu_dtype = torch.float16
-
-    memory_provider = None
-
-    def is_available() -> bool:
-        return torch_directml.is_available()
-
-    def is_directml_device(device: torch.device) -> bool:
-        return device.type == "privateuseone"
-
-    def has_float64_support(device: rDevice | None=None) -> bool:
-        return torch_directml.has_float64_support(get_device(device).index)
-
-    def device_count() -> int:
-        return torch_directml.device_count()
-
-    def current_device() -> torch.device:
-        return DirectML.context_device or DirectML.default_device()
-
-    def default_device() -> torch.device:
-        return torch_directml.device(torch_directml.default_device())
-
-    def get_device_string(device: rDevice | None=None) -> str:
-        return f"privateuseone:{get_device(device).index}"
-
-    def get_device_name(device: rDevice | None=None) -> str:
-        return torch_directml.device_name(get_device(device).index)
-
-    def get_device_properties(device: rDevice | None=None) -> DeviceProperties:
-        return DeviceProperties(get_device(device))
-
-    def memory_stats(device: rDevice | None=None):
-        return {
-            "num_ooms": 0,
-            "num_alloc_retries": 0,
-        }
-
-    mem_get_info: Callable = mem_get_info
-
-    def memory_allocated(device: rDevice | None=None) -> int:
-        return sum(torch_directml.gpu_memory(get_device(device).index)) * (1 << 20)
-
-    def max_memory_allocated(device: rDevice | None=None):
-        return DirectML.memory_allocated(device) # DirectML does not empty GPU memory
-
-    def reset_peak_memory_stats(device: rDevice | None=None):
-        return

modules/dml/device.py

@@ -1,16 +0,0 @@
-import torch
-from .utils import rDevice, get_device
-
-
-class Device:
-    idx: int
-
-    def __enter__(self, device: rDevice | None=None):
-        torch.dml.context_device = get_device(device)
-        self.idx = torch.dml.context_device.index
-
-    def __init__(self, device: rDevice | None=None) -> torch.device: # pylint: disable=return-in-init
-        self.idx = get_device(device).index
-
-    def __exit__(self, t, v, tb):
-        torch.dml.context_device = None

modules/dml/device_properties.py

@@ -1,20 +0,0 @@
-import torch
-
-
-class DeviceProperties:
-    type: str = "directml"
-    name: str
-    major: int = 0
-    minor: int = 0
-    total_memory: int
-    multi_processor_count: int = 1
-
-    def __init__(self, device: torch.device):
-        self.name = torch.dml.get_device_name(device)
-        self.total_memory = torch.dml.mem_get_info(device)[0]
-
-    def __str__(self):
-        return f"DeviceProperties(name='{self.name}', total_memory='{self.total_memory}')"
-
-    def __repr__(self):
-        return f"DeviceProperties(name='{self.name}', total_memory='{self.total_memory}')"

modules/dml/hijack/__init__.py

@@ -1,4 +0,0 @@
-import modules.dml.hijack.torch
-import modules.dml.hijack.realesrgan_model
-import modules.dml.hijack.transformers
-import modules.dml.hijack.tomesd

modules/dml/hijack/realesrgan_model.py

@@ -1,67 +0,0 @@
-import math
-import torch
-from modules.postprocess.realesrgan_model_arch import RealESRGANer
-from modules.logger import log
-
-
-# DML Solution: Some of contents of output tensor turn to 0 after Extended Slices. Move it to cpu.
-def tile_process(self):
-    batch, channel, height, width = self.img.shape
-    output_height = height * self.scale
-    output_width = width * self.scale
-    output_shape = (batch, channel, output_height, output_width)
-
-    # start with black image
-    self.output = self.img.new_zeros(output_shape)
-    tiles_x = math.ceil(width / self.tile_size)
-    tiles_y = math.ceil(height / self.tile_size)
-
-    # loop over all tiles
-    for y in range(tiles_y):
-        for x in range(tiles_x):
-            # extract tile from input image
-            ofs_x = x * self.tile_size
-            ofs_y = y * self.tile_size
-            # input tile area on total image
-            input_start_x = ofs_x
-            input_end_x = min(ofs_x + self.tile_size, width)
-            input_start_y = ofs_y
-            input_end_y = min(ofs_y + self.tile_size, height)
-
-            # input tile area on total image with padding
-            input_start_x_pad = max(input_start_x - self.tile_pad, 0)
-            input_end_x_pad = min(input_end_x + self.tile_pad, width)
-            input_start_y_pad = max(input_start_y - self.tile_pad, 0)
-            input_end_y_pad = min(input_end_y + self.tile_pad, height)
-
-            # input tile dimensions
-            input_tile_width = input_end_x - input_start_x
-            input_tile_height = input_end_y - input_start_y
-            _tile_idx = y * tiles_x + x + 1
-            input_tile = self.img[0:self.img.shape[0], 0:self.img.shape[1], input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
-
-            # upscale tile
-            try:
-                with torch.no_grad():
-                    output_tile = self.model(input_tile)
-            except Exception as e:
-                log.error(f'Upscale error: type=R-ESRGAN {e}')
-
-            # output tile area on total image
-            output_start_x = input_start_x * self.scale
-            output_end_x = input_end_x * self.scale
-            output_start_y = input_start_y * self.scale
-            output_end_y = input_end_y * self.scale
-
-            # output tile area without padding
-            output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale
-            output_end_x_tile = output_start_x_tile + input_tile_width * self.scale
-            output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale
-            output_end_y_tile = output_start_y_tile + input_tile_height * self.scale
-
-            self.output = self.output.cpu()
-            # put tile into output image
-            self.output[0:self.output.shape[0], 0:self.output.shape[1], output_start_y:output_end_y, output_start_x:output_end_x] = output_tile.cpu()[0:output_tile.shape[0], 0:output_tile.shape[1], output_start_y_tile:output_end_y_tile, output_start_x_tile:output_end_x_tile]
-            self.output = self.output.to(output_tile.device)
-
-RealESRGANer.tile_process = tile_process

modules/dml/hijack/tomesd.py

@@ -1,26 +0,0 @@
-import torch
-from modules.dml.hijack.utils import catch_nan
-
-
-def make_tome_block(block_class: type[torch.nn.Module]) -> type[torch.nn.Module]:
-    class ToMeBlock(block_class):
-        # Save for unpatching later
-        _parent = block_class
-
-        def _forward(self, x: torch.Tensor, context: torch.Tensor = None) -> torch.Tensor:
-            m_a, m_c, m_m, u_a, u_c, u_m = tomesd.patch.compute_merge(x, self._tome_info)
-
-            # This is where the meat of the computation happens
-            x = u_a(self.attn1(m_a(self.norm1(x)), context=context if self.disable_self_attn else None)) + x
-            x = catch_nan(lambda: (u_c(self.attn2(m_c(self.norm2(x)), context=context)) + x))
-            x = u_m(self.ff(m_m(self.norm3(x)))) + x
-
-            return x
-
-    return ToMeBlock
-
-try:
-    import tomesd
-    tomesd.patch.make_tome_block = make_tome_block
-except Exception:
-    pass

modules/dml/hijack/torch.py

@@ -1,35 +0,0 @@
-import torch
-from modules.sd_hijack_utils import CondFunc
-
-
-CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone')
-# https://github.com/microsoft/DirectML/issues/400
-CondFunc('torch.Tensor.new', lambda orig, self, *args, **kwargs: orig(self.cpu(), *args, **kwargs).to(self.device), lambda orig, self, *args, **kwargs: torch.dml.is_directml_device(self.device))
-
-
-def cuda(self: torch.Tensor):
-    return self.to(torch.dml.current_device())
-torch.Tensor.cuda = cuda
-
-
-# https://github.com/lshqqytiger/stable-diffusion-webui-directml/issues/436
-_pow_ = torch.Tensor.pow_
-def pow_(self: torch.Tensor, *args, **kwargs):
-    if self.dtype == torch.float64:
-        return _pow_(self.cpu(), *args, **kwargs).to(self.device)
-    return _pow_(self, *args, **kwargs)
-torch.Tensor.pow_ = pow_
-
-
-_load = torch.load
-def load(f, map_location = "cpu", *args, **kwargs):
-    if type(map_location) in (str, torch.device,):
-        device = torch.device(map_location)
-        if device.type == "privateuseone":
-            data = _load(f, *args, map_location="cpu", **kwargs)
-            for k in data:
-                for weight in data[k]:
-                    data[k][weight] = data[k][weight].to(device)
-            return data
-    return _load(f, *args, map_location=map_location, **kwargs)
-torch.load = load

modules/dml/hijack/transformers.py

@@ -1,43 +0,0 @@
-import torch
-import transformers.models.clip.modeling_clip
-
-# Copied from transformers.models.bart.modeling_bart._make_causal_mask
-def _make_causal_mask(
-    input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
-):
-    """
-    Make causal mask used for bi-directional self-attention.
-    """
-    bsz, tgt_len = input_ids_shape
-    min = torch.tensor(torch.finfo(dtype).min, device="cpu")
-    mask = torch.full((tgt_len, tgt_len), min, device=device) # https://discord.com/channels/1101998836328697867/1127441997184122920
-    mask_cond = torch.arange(mask.size(-1), device=device)
-    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
-    mask = mask.to(dtype)
-
-    if past_key_values_length > 0:
-        mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
-    return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
-
-def CLIPTextEmbeddings_forward(
-    self: transformers.models.clip.modeling_clip.CLIPTextEmbeddings,
-    input_ids: torch.LongTensor | None = None,
-    position_ids: torch.LongTensor | None = None,
-    inputs_embeds: torch.FloatTensor | None = None,
-) -> torch.Tensor:
-    from modules.devices import dtype
-    seq_length = input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2]
-
-    if position_ids is None:
-        position_ids = self.position_ids[:, :seq_length]
-
-    if inputs_embeds is None:
-        inputs_embeds = self.token_embedding(input_ids).type(dtype) # Type correction.
-
-    position_embeddings = self.position_embedding(position_ids)
-    embeddings = inputs_embeds + position_embeddings
-
-    return embeddings
-
-transformers.models.clip.modeling_clip._make_causal_mask = _make_causal_mask
-transformers.models.clip.modeling_clip.CLIPTextEmbeddings.forward = CLIPTextEmbeddings_forward

modules/dml/hijack/utils.py

@@ -1,19 +0,0 @@
-import torch
-from collections.abc import Callable
-from modules.shared import log, opts
-
-
-def catch_nan(func: Callable[[], torch.Tensor]):
-    if not opts.directml_catch_nan:
-        return func()
-
-    tries = 0
-    tensor = func()
-    while tensor.isnan().sum() != 0 and tries < 10:
-        if tries == 0:
-            log.warning("NaN is produced. Retry with same values...")
-        tries += 1
-        tensor = func()
-    if tensor.isnan().sum() != 0:
-        log.error("Failed to cover NaN.")
-    return tensor

modules/dml/memory.py

@@ -1,31 +0,0 @@
-from os import getpid
-from collections import defaultdict
-from modules.dml.pdh import HQuery, HCounter, expand_wildcard_path
-
-
-class MemoryProvider:
-    hQuery: HQuery
-    hCounters: defaultdict[str, list[HCounter]]
-
-    def __init__(self):
-        self.hQuery = HQuery()
-        self.hCounters = defaultdict(list)
-
-    def get_memory(self, device_id: int) -> dict[str, int]:
-        if len(self.hCounters) == 0:
-            pid = getpid()
-            paths_dedicated = expand_wildcard_path(f"\\GPU Process Memory(pid_{pid}_*_phys_{device_id})\\Dedicated Usage")
-            paths_committed = expand_wildcard_path(f"\\GPU Process Memory(pid_{pid}_*_phys_{device_id})\\Total Committed")
-            for path in paths_dedicated:
-                self.hCounters["dedicated_usage"].append(self.hQuery.add_counter(path))
-            for path in paths_committed:
-                self.hCounters["total_committed"].append(self.hQuery.add_counter(path))
-        self.hQuery.collect_data()
-        result = defaultdict(int)
-        for key in self.hCounters:
-            for hCounter in self.hCounters[key]:
-                result[key] += hCounter.get_formatted_value(int)
-        return dict(result)
-
-    def __del__(self):
-        self.hQuery.close()

modules/dml/memory_amd/__init__.py

@@ -1,10 +0,0 @@
-from .driver.atiadlxx import ATIADLxx
-
-
-class AMDMemoryProvider:
-    driver: ATIADLxx = ATIADLxx()
-
-    @staticmethod
-    def mem_get_info(index):
-        usage = AMDMemoryProvider.driver.get_dedicated_vram_usage(index) * (1 << 20)
-        return (AMDMemoryProvider.driver.iHyperMemorySize - usage, AMDMemoryProvider.driver.iHyperMemorySize)

modules/dml/memory_amd/driver/atiadlxx.py

@@ -1,47 +0,0 @@
-import ctypes as C
-from modules.dml.memory_amd.driver.atiadlxx_apis import ADL2_Main_Control_Create, ADL_Main_Memory_Alloc, ADL2_Adapter_NumberOfAdapters_Get, ADL2_Adapter_AdapterInfo_Get, ADL2_Adapter_MemoryInfo2_Get, ADL2_Adapter_DedicatedVRAMUsage_Get, ADL2_Adapter_VRAMUsage_Get
-from modules.dml.memory_amd.driver.atiadlxx_structures import ADL_CONTEXT_HANDLE, AdapterInfo, LPAdapterInfo, ADLMemoryInfo2
-from modules.dml.memory_amd.driver.atiadlxx_defines import ADL_OK
-
-
-class ATIADLxx:
-    iHyperMemorySize = 0
-
-    def __init__(self):
-        self.context = ADL_CONTEXT_HANDLE()
-        ADL2_Main_Control_Create(ADL_Main_Memory_Alloc, 1, C.byref(self.context))
-        num_adapters = C.c_int(-1)
-        ADL2_Adapter_NumberOfAdapters_Get(self.context, C.byref(num_adapters))
-        AdapterInfoArray = (AdapterInfo * num_adapters.value)()
-        ADL2_Adapter_AdapterInfo_Get(self.context, C.cast(AdapterInfoArray, LPAdapterInfo), C.sizeof(AdapterInfoArray))
-        self.devices = []
-        busNumbers = []
-        for adapter in AdapterInfoArray:
-            if adapter.iBusNumber not in busNumbers: # filter duplicate device
-                self.devices.append(adapter)
-                busNumbers.append(adapter.iBusNumber)
-        self.iHyperMemorySize = self.get_memory_info2(0).iHyperMemorySize
-
-    def get_memory_info2(self, adapterIndex: int) -> ADLMemoryInfo2:
-        info = ADLMemoryInfo2()
-
-        if ADL2_Adapter_MemoryInfo2_Get(self.context, adapterIndex, C.byref(info)) != ADL_OK:
-            raise RuntimeError("ADL2: Failed to get MemoryInfo2")
-
-        return info
-
-    def get_dedicated_vram_usage(self, index: int) -> int:
-        usage = C.c_int(-1)
-
-        if ADL2_Adapter_DedicatedVRAMUsage_Get(self.context, self.devices[index].iAdapterIndex, C.byref(usage)) != ADL_OK:
-            raise RuntimeError("ADL2: Failed to get DedicatedVRAMUsage")
-
-        return usage.value
-
-    def get_vram_usage(self, index: int) -> int:
-        usage = C.c_int(-1)
-
-        if ADL2_Adapter_VRAMUsage_Get(self.context, self.devices[index].iAdapterIndex, C.byref(usage)) != ADL_OK:
-            raise RuntimeError("ADL2: Failed to get VRAMUsage")
-
-        return usage.value

modules/dml/memory_amd/driver/atiadlxx_apis.py

@@ -1,50 +0,0 @@
-import ctypes as C
-from platform import platform
-from modules.dml.memory_amd.driver.atiadlxx_structures import ADL_CONTEXT_HANDLE, LPAdapterInfo, ADLMemoryInfo2
-
-
-if 'Windows' in platform():
-    atiadlxx = C.WinDLL("atiadlxx.dll")
-else:
-    atiadlxx = C.CDLL("libatiadlxx.so") # Not tested on Linux system. But will be supported.
-
-
-ADL_MAIN_MALLOC_CALLBACK = C.CFUNCTYPE(C.c_void_p, C.c_int)
-ADL_MAIN_FREE_CALLBACK = C.CFUNCTYPE(None, C.POINTER(C.c_void_p))
-
-
-@ADL_MAIN_MALLOC_CALLBACK
-def ADL_Main_Memory_Alloc(iSize):
-    return C._malloc(iSize)
-
-
-@ADL_MAIN_FREE_CALLBACK
-def ADL_Main_Memory_Free(lpBuffer):
-    if lpBuffer[0] is not None:
-        C._free(lpBuffer[0])
-        lpBuffer[0] = None
-
-
-ADL2_Main_Control_Create = atiadlxx.ADL2_Main_Control_Create
-ADL2_Main_Control_Create.restype = C.c_int
-ADL2_Main_Control_Create.argtypes = [ADL_MAIN_MALLOC_CALLBACK, C.c_int, ADL_CONTEXT_HANDLE]
-
-ADL2_Adapter_NumberOfAdapters_Get = atiadlxx.ADL2_Adapter_NumberOfAdapters_Get
-ADL2_Adapter_NumberOfAdapters_Get.restype = C.c_int
-ADL2_Adapter_NumberOfAdapters_Get.argtypes = [ADL_CONTEXT_HANDLE, C.POINTER(C.c_int)]
-
-ADL2_Adapter_AdapterInfo_Get = atiadlxx.ADL2_Adapter_AdapterInfo_Get
-ADL2_Adapter_AdapterInfo_Get.restype = C.c_int
-ADL2_Adapter_AdapterInfo_Get.argtypes = [ADL_CONTEXT_HANDLE, LPAdapterInfo, C.c_int]
-
-ADL2_Adapter_MemoryInfo2_Get = atiadlxx.ADL2_Adapter_MemoryInfo2_Get
-ADL2_Adapter_MemoryInfo2_Get.restype = C.c_int
-ADL2_Adapter_MemoryInfo2_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(ADLMemoryInfo2)]
-
-ADL2_Adapter_DedicatedVRAMUsage_Get = atiadlxx.ADL2_Adapter_DedicatedVRAMUsage_Get
-ADL2_Adapter_DedicatedVRAMUsage_Get.restype = C.c_int
-ADL2_Adapter_DedicatedVRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]
-
-ADL2_Adapter_VRAMUsage_Get = atiadlxx.ADL2_Adapter_VRAMUsage_Get
-ADL2_Adapter_VRAMUsage_Get.restype = C.c_int
-ADL2_Adapter_VRAMUsage_Get.argtypes = [ADL_CONTEXT_HANDLE, C.c_int, C.POINTER(C.c_int)]

modules/dml/memory_amd/driver/atiadlxx_defines.py

@@ -1 +0,0 @@
-ADL_OK = 0

modules/dml/memory_amd/driver/atiadlxx_structures.py

@@ -1,90 +0,0 @@
-import ctypes as C
-
-
-class _ADLPMActivity(C.Structure):
-    __slot__ = [
-        'iActivityPercent',
-        'iCurrentBusLanes',
-        'iCurrentBusSpeed',
-        'iCurrentPerformanceLevel',
-        'iEngineClock',
-        'iMaximumBusLanes',
-        'iMemoryClock',
-        'iReserved',
-        'iSize',
-        'iVddc',
-    ]
-_ADLPMActivity._fields_ = [ # pylint: disable=protected-access
-    ('iActivityPercent', C.c_int),
-    ('iCurrentBusLanes', C.c_int),
-    ('iCurrentBusSpeed', C.c_int),
-    ('iCurrentPerformanceLevel', C.c_int),
-    ('iEngineClock', C.c_int),
-    ('iMaximumBusLanes', C.c_int),
-    ('iMemoryClock', C.c_int),
-    ('iReserved', C.c_int),
-    ('iSize', C.c_int),
-    ('iVddc', C.c_int),
-]
-ADLPMActivity = _ADLPMActivity
-
-
-class _ADLMemoryInfo2(C.Structure):
-    __slot__ = [
-        'iHyperMemorySize',
-        'iInvisibleMemorySize',
-        'iMemoryBandwidth',
-        'iMemorySize',
-        'iVisibleMemorySize',
-        'strMemoryType'
-    ]
-_ADLMemoryInfo2._fields_ = [ # pylint: disable=protected-access
-    ('iHyperMemorySize', C.c_longlong),
-    ('iInvisibleMemorySize', C.c_longlong),
-    ('iMemoryBandwidth', C.c_longlong),
-    ('iMemorySize', C.c_longlong),
-    ('iVisibleMemorySize', C.c_longlong),
-    ('strMemoryType', C.c_char * 256)
-]
-ADLMemoryInfo2 = _ADLMemoryInfo2
-
-
-class _AdapterInfo(C.Structure):
-    __slot__ = [
-        'iSize',
-        'iAdapterIndex',
-        'strUDID',
-        'iBusNumber',
-        'iDeviceNumber',
-        'iFunctionNumber',
-        'iVendorID',
-        'strAdapterName',
-        'strDisplayName',
-        'iPresent',
-        'iExist',
-        'strDriverPath',
-        'strDriverPathExt',
-        'strPNPString',
-        'iOSDisplayIndex',
-    ]
-_AdapterInfo._fields_ = [ # pylint: disable=protected-access
-    ('iSize', C.c_int),
-    ('iAdapterIndex', C.c_int),
-    ('strUDID', C.c_char * 256),
-    ('iBusNumber', C.c_int),
-    ('iDeviceNumber', C.c_int),
-    ('iFunctionNumber', C.c_int),
-    ('iVendorID', C.c_int),
-    ('strAdapterName', C.c_char * 256),
-    ('strDisplayName', C.c_char * 256),
-    ('iPresent', C.c_int),
-    ('iExist', C.c_int),
-    ('strDriverPath', C.c_char * 256),
-    ('strDriverPathExt', C.c_char * 256),
-    ('strPNPString', C.c_char * 256),
-    ('iOSDisplayIndex', C.c_int)
-]
-AdapterInfo = _AdapterInfo
-LPAdapterInfo = C.POINTER(_AdapterInfo)
-
-ADL_CONTEXT_HANDLE = C.c_void_p

modules/dml/pdh/__init__.py

@@ -1,90 +0,0 @@
-from ctypes import byref, cast, c_size_t
-from ctypes.wintypes import LPCWSTR, DWORD, WCHAR
-from typing import NamedTuple, TypeVar
-from .apis import PdhExpandWildCardPathW, PdhOpenQueryW, PdhAddEnglishCounterW, PdhCollectQueryData, PdhGetFormattedCounterValue, PdhGetFormattedCounterArrayW, PdhCloseQuery
-from .structures import PDH_HQUERY, PDH_HCOUNTER, PDH_FMT_COUNTERVALUE, PPDH_FMT_COUNTERVALUE_ITEM_W
-from .defines import PDH_FMT_LARGE, PDH_FMT_DOUBLE, PDH_FMT_NOSCALE, PDH_NOEXPANDCOUNTERS, PDH_MORE_DATA, PDH_OK
-from .msvcrt import malloc
-from .errors import PDHError
-
-
-class __InternalAbstraction(NamedTuple):
-    flag: int
-    attr_name: str
-
-
-_type_map = {
-    int: __InternalAbstraction(PDH_FMT_LARGE, "largeValue"),
-    float: __InternalAbstraction(PDH_FMT_DOUBLE, "doubleValue"),
-}
-
-
-def expand_wildcard_path(path: str) -> list[str]:
-    listLength = DWORD(0)
-    if PdhExpandWildCardPathW(None, LPCWSTR(path), None, byref(listLength), PDH_NOEXPANDCOUNTERS) != PDH_MORE_DATA:
-        raise PDHError("Something went wrong.")
-    expanded = (WCHAR * listLength.value)()
-    if PdhExpandWildCardPathW(None, LPCWSTR(path), expanded, byref(listLength), PDH_NOEXPANDCOUNTERS) != PDH_OK:
-        raise PDHError(f"Couldn't expand wildcard path '{path}'")
-    result = []
-    cur = ""
-    for c in expanded:
-        if c == '\0':
-            result.append(cur)
-            cur = ""
-        else:
-            cur += c
-    result.pop()
-    return result
-
-
-T = TypeVar("T", *_type_map.keys())
-
-
-class HCounter(PDH_HCOUNTER):
-    def get_formatted_value(self, typ: T) -> T:
-        if typ not in _type_map:
-            raise PDHError(f"Invalid value type: {typ}")
-        flag, attr_name = _type_map[typ]
-        value = PDH_FMT_COUNTERVALUE()
-        if PdhGetFormattedCounterValue(self, DWORD(flag | PDH_FMT_NOSCALE), None, byref(value)) != PDH_OK:
-            raise PDHError("Couldn't get formatted counter value.")
-        return getattr(value.u, attr_name)
-
-    def get_formatted_dict(self, typ: T) -> dict[str, T]:
-        if typ not in _type_map:
-            raise PDHError(f"Invalid value type: {typ}")
-        flag, attr_name = _type_map[typ]
-        bufferSize = DWORD(0)
-        itemCount = DWORD(0)
-        if PdhGetFormattedCounterArrayW(self, DWORD(flag | PDH_FMT_NOSCALE), byref(bufferSize), byref(itemCount), None) != PDH_MORE_DATA:
-            raise PDHError("Something went wrong.")
-        itemBuffer = cast(malloc(c_size_t(bufferSize.value)), PPDH_FMT_COUNTERVALUE_ITEM_W)
-        if PdhGetFormattedCounterArrayW(self, DWORD(flag | PDH_FMT_NOSCALE), byref(bufferSize), byref(itemCount), itemBuffer) != PDH_OK:
-            raise PDHError("Couldn't get formatted counter array.")
-        result: dict[str, T] = {}
-        for i in range(0, itemCount.value):
-            item = itemBuffer[i]
-            result[item.szName] = getattr(item.FmtValue.u, attr_name)
-        return result
-
-
-class HQuery(PDH_HQUERY):
-    def __init__(self):
-        super().__init__()
-        if PdhOpenQueryW(None, None, byref(self)) != PDH_OK:
-            raise PDHError("Couldn't open PDH query.")
-
-    def add_counter(self, path: str) -> HCounter:
-        hCounter = HCounter()
-        if PdhAddEnglishCounterW(self, LPCWSTR(path), None, byref(hCounter)) != PDH_OK:
-            raise PDHError("Couldn't add counter query.")
-        return hCounter
-
-    def collect_data(self):
-        if PdhCollectQueryData(self) != PDH_OK:
-            raise PDHError("Couldn't collect query data.")
-
-    def close(self):
-        if PdhCloseQuery(self) != PDH_OK:
-            raise PDHError("Couldn't close PDH query.")

modules/dml/pdh/apis.py

@@ -1,37 +0,0 @@
-from ctypes import CDLL, POINTER
-from ctypes.wintypes import LPCWSTR, LPDWORD, DWORD
-from collections.abc import Callable
-from .structures import PDH_HQUERY, PDH_HCOUNTER, PPDH_FMT_COUNTERVALUE, PPDH_FMT_COUNTERVALUE_ITEM_W
-from .defines import PDH_FUNCTION, PZZWSTR, DWORD_PTR
-
-
-pdh = CDLL("pdh.dll")
-
-
-PdhExpandWildCardPathW: Callable = pdh.PdhExpandWildCardPathW
-PdhExpandWildCardPathW.restype = PDH_FUNCTION
-PdhExpandWildCardPathW.argtypes = [LPCWSTR, LPCWSTR, PZZWSTR, LPDWORD, DWORD]
-
-PdhOpenQueryW: Callable = pdh.PdhOpenQueryW
-PdhOpenQueryW.restype = PDH_FUNCTION
-PdhOpenQueryW.argtypes = [LPCWSTR, DWORD_PTR, POINTER(PDH_HQUERY)]
-
-PdhAddEnglishCounterW: Callable = pdh.PdhAddEnglishCounterW
-PdhAddEnglishCounterW.restype = PDH_FUNCTION
-PdhAddEnglishCounterW.argtypes = [PDH_HQUERY, LPCWSTR, DWORD_PTR, POINTER(PDH_HCOUNTER)]
-
-PdhCollectQueryData: Callable = pdh.PdhCollectQueryData
-PdhCollectQueryData.restype = PDH_FUNCTION
-PdhCollectQueryData.argtypes = [PDH_HQUERY]
-
-PdhGetFormattedCounterValue: Callable = pdh.PdhGetFormattedCounterValue
-PdhGetFormattedCounterValue.restype = PDH_FUNCTION
-PdhGetFormattedCounterValue.argtypes = [PDH_HCOUNTER, DWORD, LPDWORD, PPDH_FMT_COUNTERVALUE]
-
-PdhGetFormattedCounterArrayW: Callable = pdh.PdhGetFormattedCounterArrayW
-PdhGetFormattedCounterArrayW.restype = PDH_FUNCTION
-PdhGetFormattedCounterArrayW.argtypes = [PDH_HCOUNTER, DWORD, LPDWORD, LPDWORD, PPDH_FMT_COUNTERVALUE_ITEM_W]
-
-PdhCloseQuery: Callable = pdh.PdhCloseQuery
-PdhCloseQuery.restype = PDH_FUNCTION
-PdhCloseQuery.argtypes = [PDH_HQUERY]

modules/dml/pdh/defines.py

@@ -1,23 +0,0 @@
-from ctypes import c_int, POINTER
-from ctypes.wintypes import DWORD, WCHAR
-
-
-PDH_FUNCTION = c_int
-PDH_OK = 0x00000000
-PDH_MORE_DATA = -2147481646#0x800007D2
-
-DWORD_PTR = POINTER(DWORD)
-PWSTR = POINTER(WCHAR)
-PZZWSTR = POINTER(WCHAR)
-
-PDH_NOEXPANDCOUNTERS = 1
-PDH_NOEXPANDINSTANCES = 2
-PDH_REFRESHCOUNTERS = 4
-
-PDH_FMT_LONG = 0x00000100
-PDH_FMT_DOUBLE = 0x00000200
-PDH_FMT_LARGE = 0x00000400
-
-PDH_FMT_NOSCALE = 0x00001000
-PDH_FMT_1000 = 0x00002000
-PDH_FMT_NOCAP100 = 0x00008000

modules/dml/pdh/errors.py

@@ -1,3 +0,0 @@
-class PDHError(Exception):
-    def __init__(self, message: str):
-        super().__init__(message)

modules/dml/pdh/msvcrt.py

@@ -1,13 +0,0 @@
-from ctypes import CDLL, c_void_p, c_size_t
-
-
-msvcrt = CDLL("msvcrt")
-
-
-malloc = msvcrt.malloc
-malloc.restype = c_void_p
-malloc.argtypes = [c_size_t]
-
-free = msvcrt.free
-free.restype = None
-free.argtypes = [c_void_p]

modules/dml/pdh/structures.py

@@ -1,45 +0,0 @@
-from ctypes import Union, c_double, c_longlong, Structure, POINTER
-from ctypes.wintypes import HANDLE, LONG, LPCSTR, LPCWSTR, DWORD, LPWSTR
-
-
-PDH_HQUERY = HANDLE
-PDH_HCOUNTER = HANDLE
-
-
-class PDH_FMT_COUNTERVALUE_U(Union):
-    _fields_ = [
-        ("longValue", LONG),
-        ("doubleValue", c_double),
-        ("largeValue", c_longlong),
-        ("AnsiStringValue", LPCSTR),
-        ("WideStringValue", LPCWSTR),
-    ]
-
-    longValue: int
-    doubleValue: float
-    largeValue: int
-    AnsiStringValue: LPCSTR
-    WideStringValue: LPCWSTR
-
-
-class PDH_FMT_COUNTERVALUE(Structure):
-    _anonymous_ = ("u",)
-    _fields_ = [
-        ("CStatus", DWORD),
-        ("u", PDH_FMT_COUNTERVALUE_U),
-    ]
-
-    CStatus: DWORD
-    u: PDH_FMT_COUNTERVALUE_U
-PPDH_FMT_COUNTERVALUE = POINTER(PDH_FMT_COUNTERVALUE)
-
-
-class PDH_FMT_COUNTERVALUE_ITEM_W(Structure):
-    _fields_ = [
-        ("szName", LPWSTR),
-        ("FmtValue", PDH_FMT_COUNTERVALUE),
-    ]
-
-    szName: str
-    FmtValue: PDH_FMT_COUNTERVALUE
-PPDH_FMT_COUNTERVALUE_ITEM_W = POINTER(PDH_FMT_COUNTERVALUE_ITEM_W)

modules/dml/utils.py

@@ -1,9 +0,0 @@
-from typing import Union
-import torch
-
-
-rDevice = Union[torch.device, int]
-def get_device(device: rDevice | None=None) -> torch.device:
-    if device is None:
-        device = torch.dml.current_device()
-    return torch.device(device)

modules/extensions.py

@@ -64,6 +64,7 @@ def temp_disable_extensions():
         'multidiffusion-upscaler-for-automatic1111',
         'a1111-sd-webui-lycoris',
         'sd-webui-animatediff',
+        'sdnq',
     ]
     disable_obsolete = [
         'Lora',

modules/files_cache.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Union
 import os
 from collections import UserDict
@@ -8,16 +10,13 @@ from modules.logger import log
 
 do_cache_folders = os.environ.get('SD_NO_CACHE', None) is None
 
-class Directory: # forward declaration
-    ...
-
 FilePathList = list[str]
 FilePathIterator = Iterator[str]
 DirectoryPathList = list[str]
 DirectoryPathIterator = Iterator[str]
-DirectoryList = list[Directory]
-DirectoryIterator = Iterator[Directory]
-DirectoryCollection = dict[str, Directory]
+DirectoryList = list['Directory']
+DirectoryIterator = Iterator['Directory']
+DirectoryCollection = dict[str, 'Directory']
 ExtensionFilter = Callable
 ExtensionList = list[str]
 RecursiveType = Union[bool, Callable]
@@ -35,7 +34,7 @@ def real_path(directory_path: str) -> str | None:
 
 
 @dataclass(frozen=True)
-class Directory(Directory): # pylint: disable=E0102
+class Directory:
     path: str = field(default_factory=str)
     files: FilePathList = field(default_factory=list)
     directories: DirectoryPathList = field(default_factory=list)
@@ -265,7 +264,7 @@ def unique_paths(directory_paths: DirectoryPathList) -> DirectoryPathIterator:
                 yield r
 
 
-def get_directories(*directory_paths: DirectoryPathList, fetch: bool = True, recursive: RecursiveType = True) -> DirectoryCollection:
+def get_directories(*directory_paths: DirectoryPathList, fetch: bool = True, recursive: RecursiveType = True) -> list[Directory]:
     dirs = unique_directories(directory_paths, recursive=recursive)
     return [d for d in (get_directory(p, fetch=fetch) for p in dirs) if d]
 

modules/framepack/create-video.py

@@ -111,7 +111,7 @@ if __name__ == "__main__":
     parser.add_argument('--rescale', type=float, default=0.0, help='cfg rescale')
     parser.add_argument('--distilled', type=float, default=10.0, help='cfg distilled')
     parser.add_argument('--shift', type=float, default=3.0, help='sampler shift')
-    parser.add_argument('--variant', type=str, default='bi-directional', choices=['bi-directional', 'forward-only'], help='model variant')
+    parser.add_argument('--variant', type=str, default='Bi-Directional', choices=['Bi-Directional', 'Forward-Only'], help='model variant')
     args = parser.parse_args()
     log.info(f'api-framepack: {args}')
     generate(args)

modules/framepack/framepack_api.py

@@ -34,7 +34,7 @@ class ReqFramepack(BaseModel):
     mp4_video: bool | None = Field(default=True, title="Save Video", description="Save video")
     mp4_frames: bool | None = Field(default=False, title="Save Frames", description="Save frames for the video")
     mp4_thumb: bool | None = Field(default=True, title="Save Thumbnail", description="Save thumbnail for the video")
-    mp4_opt: str | None = Field(default="crf:16", title="Options", description="Options for the video codec")
+    mp4_opt: str | None = Field(default="crf=16", title="Options", description="Options for the video codec")
     mp4_ext: str | None = Field(default="mp4", title="Format", description="Format for the video")
     mp4_interpolate: int | None = Field(default=0, title="Interpolation", description="Interpolation for the video")
     attention: str | None = Field(default="Default", title="Attention", description="Attention type for the model")

modules/framepack/framepack_install.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import git as gitpython
 from installer import install, git
-from modules.shared import log
+from modules.logger import log
 
 
 def rename(src:str, dst:str):

modules/framepack/framepack_load.py

@@ -5,8 +5,9 @@ from modules.logger import log
 
 
 models = {
-    'bi-directional': 'lllyasviel/FramePackI2V_HY',
-    'forward-only': 'lllyasviel/FramePack_F1_I2V_HY_20250503',
+    'None': None,
+    'Bi-Directional': 'lllyasviel/FramePackI2V_HY',
+    'Forward-Only': 'lllyasviel/FramePack_F1_I2V_HY_20250503',
 }
 default_model = {
     'pipeline': { 'repo': 'hunyuanvideo-community/HunyuanVideo', 'subfolder': '' },
@@ -21,7 +22,7 @@ default_model = {
     'tokenizer_2': { 'repo': 'hunyuanvideo-community/HunyuanVideo', 'subfolder': 'tokenizer_2' },
     'feature_extractor': { 'repo': 'lllyasviel/flux_redux_bfl', 'subfolder': 'feature_extractor' },
     'image_encoder': { 'repo': 'lllyasviel/flux_redux_bfl', 'subfolder': 'image_encoder' },
-    'transformer': { 'repo': models.get('bi-directional'), 'subfolder': '' },
+    'transformer': { 'repo': models.get('Bi-Directional'), 'subfolder': '' },
 }
 model = default_model.copy()
 
@@ -196,7 +197,7 @@ def load_model(variant: str | None = None, pipeline: str | None = None, text_enc
 
         diffusers.loaders.peft._SET_ADAPTER_SCALE_FN_MAPPING['HunyuanVideoTransformer3DModelPacked'] = lambda model_cls, weights: weights # pylint: disable=protected-access
         log.info(f'FramePack load: model={shared.sd_model.__class__.__name__} variant="{variant}" type={shared.sd_model_type} time={t1-t0:.2f}')
-        sd_models.apply_balanced_offload(shared.sd_model)
+        sd_models.set_diffuser_offload(shared.sd_model)
         devices.torch_gc(force=True, reason='load')
 
     except Exception as e:

modules/framepack/framepack_ui.py

@@ -12,13 +12,13 @@ def change_sections(duration, mp4_fps, mp4_interpolate, latent_ws, variant):
     return gr.update(value=f'Target video: {num_frames} frames in {num_sections} sections'), gr.update(lines=max(2, 2*num_sections//3))
 
 
-def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb):
+def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler):
     with gr.Row():
         with gr.Column(variant='compact', elem_id="framepack_settings", elem_classes=['settings-column'], scale=1):
             with gr.Row():
                 generate = gr.Button('Generate', elem_id="framepack_generate_btn", variant='primary', visible=False)
             with gr.Row():
-                variant = gr.Dropdown(label="FP model variant", choices=list(framepack_load.models), value='bi-directional', type='value')
+                variant = gr.Dropdown(label="FP model variant", choices=list(framepack_load.models), value='None', type='value')
             with gr.Row():
                 resolution = gr.Slider(label="FP resolution", minimum=240, maximum=1088, value=640, step=16)
                 duration = gr.Slider(label="FP duration", minimum=1, maximum=120, value=4, step=0.1)
@@ -64,7 +64,7 @@ def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_
                 use_cfgzero = gr.Checkbox(label='FP enable CFGZero', value=False)
                 use_preview = gr.Checkbox(label='FP enable Preview', value=True)
                 attention = gr.Dropdown(label="FP attention", choices=['Default', 'Xformers', 'FlashAttention', 'SageAttention'], value='Default', type='value')
-                vae_type = gr.Dropdown(label="FP VAE", choices=['Full', 'Tiny', 'Remote'], value='Local', type='value')
+                vae_type = gr.Dropdown(label="FP VAE", choices=['Full', 'Tiny', 'Remote'], value='Full', type='value')
 
         with gr.Column(elem_id='framepack-output-column', scale=2) as _column_output:
             with gr.Tabs():
@@ -107,6 +107,7 @@ def create_ui(prompt, negative, styles, _overrides, script_inputs, mp4_fps, mp4_
         shift,
         use_teacache, use_cfgzero, use_preview,
         mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate,
+        mp4_scale, mp4_upscaler,
         attention, vae_type, variant,
         vlm_enhance, vlm_model, vlm_system_prompt,
     ]

modules/framepack/framepack_vae.py

@@ -51,7 +51,7 @@ def vae_decode_tiny(latents):
         taesd = taesd.to(device=devices.device, dtype=devices.dtype)
         latents = latents.transpose(1, 2) # pipe produces NCTHW and tae wants NTCHW
         images = taesd.decode_video(latents, parallel=False, show_progress_bar=False)
-        images = images.transpose(1, 2).mul_(2).sub_(1) # normalize
+        images = images.transpose(1, 2) # taehv decodes to [-1,1] already, which is the range the full vae path returns
         taesd = taesd.to(device=devices.cpu, dtype=devices.dtype)
     return images
 

modules/framepack/framepack_worker.py

@@ -14,7 +14,7 @@ stream = None # AsyncStream
 def get_latent_paddings(mp4_fps, mp4_interpolate, latent_window_size, total_second_length, variant):
     try:
         real_fps = mp4_fps / (mp4_interpolate + 1)
-        is_f1 = variant == 'forward-only'
+        is_f1 = variant == 'Forward-Only'
         if is_f1:
             total_latent_sections = (total_second_length * real_fps) / (latent_window_size * 4)
             total_latent_sections = int(max(round(total_latent_sections), 1))
@@ -42,6 +42,7 @@ def worker(
         shift,
         use_teacache, use_cfgzero, use_preview,
         mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate,
+        mp4_scale, mp4_upscaler,
         vae_type,
         variant,
         metadata: dict | None = None,
@@ -59,7 +60,7 @@ def worker(
     from modules.framepack.pipeline import utils
     from modules.framepack.pipeline import k_diffusion_hunyuan
 
-    is_f1 = variant == 'forward-only'
+    is_f1 = variant == 'Forward-Only'
     total_generated_frames = 0
     total_generated_latent_frames = 0
     latent_paddings = get_latent_paddings(mp4_fps, mp4_interpolate, latent_window_size, total_second_length, variant)
@@ -130,6 +131,7 @@ def worker(
         else:
             end_latent = None
         sd_models.apply_balanced_offload(shared.sd_model)
+        sd_models.offload_ondemand(shared.sd_model, reason='vae encode') # group offload returns the vae through its on-demand placement rather than the balanced seam
         timer.process.add('encode', time.time()-t0)
         shared.state.end(jobid)
         return start_latent, end_latent
@@ -317,6 +319,7 @@ def worker(
                         current_pixels = framepack_vae.vae_decode(real_history_latents[:, :, :section_latent_frames], vae_type=vae_type).cpu()
                         history_pixels = utils.soft_append_bcthw(current_pixels, history_pixels, overlapped_frames)
                 sd_models.apply_balanced_offload(shared.sd_model)
+                sd_models.offload_ondemand(shared.sd_model, reason='vae decode')
                 timer.process.add('vae', time.time()-t_vae)
 
                 if is_last_section:
@@ -345,6 +348,8 @@ def worker(
                 mp4_frames=mp4_frames,
                 mp4_thumb=mp4_thumb,
                 mp4_interpolate=0,
+                upscale_scale=mp4_scale,
+                upscale_upscaler=mp4_upscaler,
                 pbar=pbar,
                 stream=stream,
                 metadata=metadata,
@@ -376,7 +381,8 @@ def worker(
         errors.display(e, 'FramePack')
 
     sd_models.apply_balanced_offload(shared.sd_model)
+    sd_models.offload_ondemand(shared.sd_model, reason='finish')
     stream.output_queue.push(('end', None))
     t1 = time.time()
-    log.info(f'Processed: frames={total_generated_frames} fps={total_generated_frames/(t1-t0):.2f} its={(shared.state.sampling_step)/(t1-t0):.2f} time={t1-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}')
+    log.info(f'Processed: frames={total_generated_frames} fps={total_generated_frames/(t1-t0):.2f} its={(shared.state.sampling_step)/(t1-t0):.3f} time={t1-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}')
     shared.state.end(videojob)

modules/framepack/framepack_wrappers.py

@@ -114,8 +114,12 @@ def unload_model():
     yield gr.update(), gr.update(), 'Model unloaded'
 
 
-def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_weight, vision_weight, prompt, system_prompt, optimized_prompt, section_prompt, negative_prompt, styles, seed, resolution, duration, latent_ws, steps, cfg_scale, cfg_distilled, cfg_rescale, shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, *_args, **_kwargs):
-    variant = variant or 'bi-directional'
+def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_weight, vision_weight, prompt, system_prompt, optimized_prompt, section_prompt, negative_prompt, styles, seed, resolution, duration, latent_ws, steps, cfg_scale, cfg_distilled, cfg_rescale, shift, use_teacache, use_cfgzero, use_preview, mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate, mp4_scale, mp4_upscaler, attention, vae_type, variant, vlm_enhance, vlm_model, vlm_system_prompt, *_args, **_kwargs):
+    variant = variant or 'Bi-Directional'
+    if variant == 'None':
+        log.error('FramePack: no model selected')
+        yield gr.update(), gr.update(), 'No FramePack model selected'
+        return
     if init_image is None:
         init_image = np.zeros((resolution, resolution, 3), dtype=np.uint8)
         mode = 't2v'
@@ -182,6 +186,7 @@ def run_framepack(task_id, _ui_state, init_image, end_image, start_weight, end_w
             shift,
             use_teacache, use_cfgzero, use_preview,
             mp4_fps, mp4_codec, mp4_sf, mp4_video, mp4_frames, mp4_thumb, mp4_opt, mp4_ext, mp4_interpolate,
+            mp4_scale, mp4_upscaler,
             vae_type, variant,
         )
 

modules/gr_tempdir.py

@@ -50,14 +50,6 @@ def check_tmp_file(gradio, filename):
 
 
 def pil_to_temp_file(self, img: Image.Image, dir: str, format="png") -> str: # pylint: disable=redefined-builtin,unused-argument
-    """
-    # original gradio implementation
-    bytes_data = gr.processing_utils.encode_pil_to_bytes(img, format)
-    temp_dir = Path(dir) / self.hash_bytes(bytes_data)
-    temp_dir.mkdir(exist_ok=True, parents=True)
-    filename = str(temp_dir / f"image.{format}")
-    img.save(filename, pnginfo=gr.processing_utils.get_pil_metadata(img))
-    """
     folder = dir
     already_saved_as = getattr(img, 'already_saved_as', None)
     exists = os.path.isfile(already_saved_as) if already_saved_as is not None else False
@@ -118,8 +110,11 @@ def cleanup_tmpdr():
         return
     for root, _dirs, files in os.walk(temp_dir, topdown=False):
         for name in files:
-            _, extension = os.path.splitext(name)
-            if extension not in {".png", ".jpg", ".webp", ".jxl"}:
-                continue
-            filename = os.path.join(root, name)
-            os.remove(filename)
+            try:
+                _, extension = os.path.splitext(name)
+                if extension not in {".png", ".jpg", ".webp", ".jxl"}:
+                    continue
+                filename = os.path.join(root, name)
+                os.remove(filename)
+            except Exception:
+                pass