From 09eeff30c7462ff44a37e2b502ee0fae085ce2b7 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Fri, 14 Aug 2026 23:59:55 +0100 Subject: [PATCH] refactor(settings): reorganize the offload settings by mode The offload section interleaved shared and mode-specific options. The page now reads top down: general options, an Offload Overrides section holding the exclusions shared by balanced and group offload, then one tuning section per mode. Use torch streams becomes Prefetch with streams and Record torch streams becomes Overlap stream transfers, the section headers carry hints, and the hints reflect live behavior, including dropping the model-reload badge from options that apply on change. --- CHANGELOG.md | 1 + modules/ui_definitions.py | 11 ++++++----- ui/locale/locale_en.json | 23 ++++++++++++----------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad0d38dd..3759f65b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ - offload: components entered through encode or decode are detected by structure rather than by name - offload: group offload honors the never-offload and model-type exclusion settings - offload: offload options take effect when changed instead of waiting for a model reload + - settings: offload settings grouped into shared overrides and per-mode sections - hf progress bars - ltx: send the guidance stack and cross-timestep on every 2.x call path - ltx: distilled variants no longer force dynamic shifting on, which remapped their sigma schedule diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index dc356cbfb..5148c75a3 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -138,19 +138,20 @@ def create_settings(cmd_opts): "offload_sep": OptionInfo("

Model Offloading

", "", gr.HTML), "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'group', 'model', 'sequential']}), "diffusers_offload_nonblocking": OptionInfo(False, "Non-blocking move operations"), + "offload_overrides_sep": OptionInfo("

Offload Overrides

", "", gr.HTML), + "models_not_to_offload": OptionInfo("", "Model types not to offload"), + "diffusers_offload_always": OptionInfo(startup_offload_always, "Modules to always offload"), + "diffusers_offload_never": OptionInfo(startup_offload_never, "Modules to never offload"), "offload_balanced_sep": OptionInfo("

Balanced Offload

", "", gr.HTML), "diffusers_offload_pre": OptionInfo(True, "Offload during pre-forward", gr.Checkbox, {"visible": False}), "diffusers_offload_streams": OptionInfo(False, "Offload using streams"), "diffusers_offload_min_gpu_memory": OptionInfo(startup_offload_min_gpu, "Offload low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_gpu_memory": OptionInfo(startup_offload_max_gpu, "Offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }), - "models_not_to_offload": OptionInfo("", "Model types not to offload"), - "diffusers_offload_always": OptionInfo(startup_offload_always, "Modules to always offload"), - "diffusers_offload_never": OptionInfo(startup_offload_never, "Modules to never offload"), "offload_group_sep": OptionInfo("

Group Offload

", "", gr.HTML), "group_offload_type": OptionInfo("leaf_level", "Group offload type", gr.Radio, {"choices": ['leaf_level', 'block_level']}), - "group_offload_stream": OptionInfo(False, "Use torch streams", gr.Checkbox), - 'group_offload_record': OptionInfo(False, "Record torch streams", gr.Checkbox), + "group_offload_stream": OptionInfo(False, "Prefetch with streams", gr.Checkbox), + 'group_offload_record': OptionInfo(False, "Overlap stream transfers", gr.Checkbox), 'group_offload_pin': OptionInfo(True, "Pin offload memory", gr.Checkbox), 'group_offload_blocks': OptionInfo(1, "Offload blocks", gr.Number), "caption_offload_sep": OptionInfo("

Caption Model Offloading

", "", gr.HTML), diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index b216e8c41..8692256c9 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -161,7 +161,7 @@ {"id":"","label":"Benchmark","localized":"","hint":"Run benchmarks"}, {"id":"","label":"Backend Settings","localized":"","hint":"Settings related to compute backends: torch, onnx and olive"}, {"id":"","label":"Beta","localized":"","hint":"","ui":"component-5674"}, - {"id":"","label":"Balanced Offload","localized":"","hint":"","ui":"settings_offload"}, + {"id":"","label":"Balanced Offload","localized":"","hint":"Moves whole components between VRAM and system memory as each one is needed, keeping as much on the GPU as the watermarks below allow.
Suited to models whose individual components fit in VRAM; a single component larger than the card still will not fit.

Applies only when Model offload mode is balanced.","ui":"settings_offload"}, {"id":"","label":"BitsAndBytes","localized":"","hint":"","ui":"settings_quantization"}, {"id":"","label":"Batch count","localized":"","hint":"How many batches of images to create (has no impact on generation performance or VRAM usage)","ui":"txt2img"}, {"id":"","label":"Batch size","localized":"","hint":"How many image to create in a single batch (increases generation performance at cost of higher VRAM usage)","ui":"txt2img"}, @@ -621,8 +621,8 @@ {"id":"get_changelog","label":"Get Changelog","localized":"","hint":"","ui":"system_tab_changelog"}, {"id":"","label":"Generic","localized":"","hint":"","ui":"video"}, {"id":"","label":"Google GenAI","localized":"","hint":"","ui":"settings_model_options"}, - {"id":"","label":"Group Offload","localized":"","hint":"","ui":"settings_offload"}, - {"id":"","label":"Group offload type","localized":"","hint":"Granularity used by group offload.
- leaf_level: offloads at the smallest module level; maximum memory savings, slower
- block_level: offloads groups of transformer blocks (size set by Offload blocks); faster with less savings
Text encoders always offload at leaf level. Small VAEs stay resident on the GPU; VAEs above 1GB rest in system memory and load whole for each encode or decode.

Applies only when Model offload mode is group.

Default is leaf_level.","reload":"model","ui":"settings_offload"}, + {"id":"","label":"Group Offload","localized":"","hint":"Offloads components in groups of layers rather than as a whole, so only the layers in use occupy VRAM.
Lets a single component larger than the card run, at the cost of transferring weights throughout every step.

Applies only when Model offload mode is group.","ui":"settings_offload"}, + {"id":"","label":"Group offload type","localized":"","hint":"Granularity used by group offload.
- leaf_level: offloads at the smallest module level; maximum memory savings, slower
- block_level: offloads groups of transformer blocks (size set by Offload blocks); faster with less savings
This setting applies to the parts of the model that run at every step. Components used once per generation, such as text encoders, always offload at leaf_level. The VAE is handled separately: it waits in system memory and loads as a whole when encoding or decoding.
Anything named in Modules to never offload or Model types not to offload stays in VRAM instead.

Applies only when Model offload mode is group.

Default is leaf_level.","ui":"settings_offload"}, {"id":"","label":"Grid Options","localized":"","hint":"","ui":"settings_saving-images"}, {"id":"","label":"Grids","localized":"","hint":"","ui":"settings_saving-paths"}, {"id":"","label":"Guider","localized":"","hint":"","ui":"txt2img"}, @@ -881,7 +881,7 @@ {"id":"","label":"Models & Networks","localized":"","hint":"View lists of all available models and networks"}, {"id":"","label":"Model Loading","localized":"","hint":"Settings related to how model is loaded"}, {"id":"","label":"Model Options","localized":"","hint":"Settings related to behavior of specific models"}, - {"id":"","label":"Model Offloading","localized":"","hint":"Settings related to model offloading and memory management"}, + {"id":"","label":"Model Offloading","localized":"","hint":"Moving model components between VRAM and system memory so that models larger than the GPU can still run, at the cost of transfer time on every generation.
The settings directly below apply to every mode. Offload Overrides holds exceptions honored by balanced and group offload; each of those modes then has its own tuning section.

Start with Model offload mode; the rest only take effect once a mode that uses them is selected."}, {"id":"","label":"Model Quantization","localized":"","hint":"Settings related to model quantization which is used to reduce memory usage"}, {"id":"","label":"Model Compile","localized":"","hint":"Settings related to different model compilation methods"}, {"id":"","label":"Metadata","localized":"","hint":"Update metadata for all available models"}, @@ -945,9 +945,9 @@ {"id":"","label":"Model load using streams","localized":"","hint":"When loading models attempt stream loading optimized for slow or network storage","ui":"settings_sd"}, {"id":"","label":"Model load model direct to GPU","localized":"","hint":"","ui":"settings_sd"}, {"id":"","label":"Model offload mode","localized":"","hint":"Controls how model components move between VRAM and system RAM to fit larger models on less VRAM.
- none: keeps everything on the GPU; fastest, but only works if the whole model fits in VRAM
- balanced: the recommended default; offloads only when VRAM use crosses a threshold, so it suits almost any GPU (tuned by the watermarks below)
- group: offloads groups of layers via diffusers group offloading; an alternative middle ground with optional stream prefetch
- model: offloads whole components such as the VAE or text encoder when idle; a more compatible fallback when balanced or group are unsupported, with smaller savings
- sequential: offloads layer by layer; the most memory saving but slowest, for when even balanced runs out of memory

Command-line flags override the automatic choice:
- --lowvram: forces sequential
- --medvram: forces balanced with low watermark 0

With no flag, balanced is the automatic default on any GPU, with watermarks set by GPU memory (low / high):
- 12 GB or less: 0 / 0.6
- 12-24 GB: 0.2 / 0.6
- 24 GB or more: 0.2 / 0.8
(or none if no GPU is detected)","reload":"model","ui":"settings_offload"}, - {"id":"","label":"Model types not to offload","localized":"","hint":"Model architectures to skip when offloading, separated by spaces or commas.
Useful for model types that misbehave when offloaded.

Applies only to balanced offload.

Default is empty.","ui":"settings_offload"}, - {"id":"","label":"Modules to always offload","localized":"","hint":"Modules that are always offloaded in balanced mode, separated by spaces, commas, or semicolons, regardless of the watermarks.
Entries match either a class name (T5EncoderModel) or a pipeline component name (text_encoder, text_encoder_2).
A component entry covers every model architecture without naming each encoder class.

Applies only to balanced offload.

Default by GPU memory: the large text encoders (T5, UMT5) are added at roughly 4-12 GB and at 24 GB or more; empty otherwise.","reload":"model","ui":"settings_offload"}, - {"id":"","label":"Modules to never offload","localized":"","hint":"Modules that are never offloaded in balanced mode, separated by spaces, commas, or semicolons, keeping them resident in VRAM.
Entries match either a class name (CLIPTextModel) or a pipeline component name (vae).
This list takes precedence, so a class entry here exempts one model from a component entry in Modules to always offload.

Applies only to balanced offload.

Default by GPU memory: the CLIP text encoders and the VAE are kept resident at 24 GB or more; empty otherwise.","reload":"model","ui":"settings_offload"}, + {"id":"","label":"Model types not to offload","localized":"","hint":"Model architectures to skip when offloading, separated by spaces or commas.
Useful for model types that misbehave when offloaded.

Applies to balanced and group offload.

Default is empty.","ui":"settings_offload"}, + {"id":"","label":"Modules to always offload","localized":"","hint":"Modules that are always offloaded in balanced mode, separated by spaces, commas, or semicolons, regardless of the watermarks.
Entries match either a class name (T5EncoderModel) or a pipeline component name (text_encoder, text_encoder_2).
A component entry covers every model architecture without naming each encoder class.

Applies only to balanced offload, since group offload returns every component it manages to system memory anyway.

Default is empty.","ui":"settings_offload"}, + {"id":"","label":"Modules to never offload","localized":"","hint":"Modules that are never offloaded, separated by spaces, commas, or semicolons, keeping them resident in VRAM.
Entries match either a class name (CLIPTextModel) or a pipeline component name (vae).
This list takes precedence, so a class entry here exempts one model from a component entry in Modules to always offload.

Applies to balanced and group offload.

Default by GPU memory: the CLIP text encoders and the VAE are kept resident at 22 GB or more; empty otherwise.","ui":"settings_offload"}, {"id":"","label":"Model types not to quantize","localized":"","hint":"Model families that quantization always skips, even when it is otherwise enabled. Space or comma separated list of model type codes; when the loaded model matches, none of its components are quantized.

Codes are the short family names shown in the load log, such as sd, sdxl, sd3, f1.

Example: sd sdxl leaves SD and SDXL checkpoints in full precision while other families are still quantized.

Applies to all quantization backends.

Default is empty.","reload":"model","ui":"settings_quantization"}, {"id":"","label":"Modules to not convert","localized":"","hint":"Names of modules to leave unquantized (kept in original precision), separated by spaces, commas, or semicolons.
Useful for layers that are sensitive to quantization, such as gate or projection layers. Example: proj_out, x_embedder.

Some models already exclude sensitive modules by default; entries here extend that built-in list rather than replacing it.

Default is empty.","reload":"model","ui":"settings_quantization"}, {"id":"","label":"Modules dtype dict","localized":"","hint":"Advanced: JSON mapping a quantization type to a list of module names, to quantize specific modules at a different type than the global Quantization type. Example: {\"uint4\": [\"proj_out\"]}.

Some models already assign certain modules a specific type by default; entries here merge with those built-in mappings rather than replacing them.

Default is empty.","reload":"model","ui":"settings_quantization"}, @@ -1046,7 +1046,9 @@ {"id":"","label":"Offload low watermark","localized":"","hint":"Lower VRAM threshold for balanced offload, as a fraction of total GPU memory. While VRAM use stays below this, nothing is offloaded; above it, idle components are moved back to RAM.

Applies only to balanced offload. --lowvram and --medvram set this to 0.

Default by GPU memory: 0 at 12 GB or less, 0.2 above.","ui":"settings_offload"}, {"id":"","label":"Offload GPU high watermark","localized":"","hint":"Upper VRAM threshold for balanced offload, as a fraction of total GPU memory. Sets the most VRAM a single component may use before it is offloaded.

Applies only to balanced offload.

Default by GPU memory: 0.6, rising to 0.8 at 24 GB or more.","ui":"settings_offload"}, {"id":"","label":"Offload CPU high watermark","localized":"","hint":"Upper system-RAM threshold for offloaded weights in balanced offload, as a fraction of total RAM.

Applies only to balanced offload.

Default is 0.9.","ui":"settings_offload"}, - {"id":"","label":"Offload blocks","localized":"","hint":"Number of transformer blocks per offload group when Group offload type is block_level. Larger groups offload less often (faster, more VRAM); smaller groups save more memory.

Applies only to group offload with block_level.

Default is 1.","reload":"model","ui":"settings_offload"}, + {"id":"","label":"Offload Overrides","localized":"","hint":"Exceptions to the behavior chosen by Model offload mode, matched by model architecture or by pipeline component.
Use these to keep a component in VRAM when the mode would offload it, or to offload one the mode would keep.
The module lists match either a class name (CLIPTextModel) or a component name (vae); a component name covers every architecture at once.

Applies to balanced and group offload; Modules to always offload applies to balanced only.","ui":"settings_offload"}, + {"id":"","label":"Overlap stream transfers","localized":"","hint":"Skips a stream synchronization each time a group offloads, letting transfers and compute overlap more tightly.
Slightly faster at the cost of slightly higher VRAM use; correctness is maintained either way.
Has no effect unless Prefetch with streams is enabled.

Applies only to group offload with streams.

Disabled by default.","ui":"settings_offload"}, + {"id":"","label":"Offload blocks","localized":"","hint":"Number of transformer blocks per offload group when Group offload type is block_level. Larger groups offload less often (faster, more VRAM); smaller groups save more memory.

Applies only to group offload with block_level.

Default is 1.","ui":"settings_offload"}, {"id":"","label":"OpenVINO activations mode","localized":"","hint":"","ui":"settings_quantization"}, {"id":"","label":"ONNX Execution Provider","localized":"","hint":"","ui":"settings_backends"}, {"id":"","label":"ONNX allow fallback to CPU","localized":"","hint":"Allow fallback to CPU when selected execution provider failed","ui":"settings_backends"}, @@ -1070,7 +1072,8 @@ {"id":"","label":"OpenBody","localized":"","hint":"","ui":"control"} ], "p": [ - {"id":"","label":"Pin offload memory","localized":"","hint":"Keeps the CPU copy of every stream-offloaded weight in pinned non-pageable memory for the fastest transfers, at a host memory cost equal to the full module size.
When disabled, memory is pinned one group at a time during transfer: slower, but the weights stay pageable and use no extra memory at rest.
Modules larger than half of system memory fall back to per-group pinning automatically.

Applies only to group offload with Use torch streams enabled.

Enabled by default.","reload":"model","ui":"settings_offload"}, + {"id":"","label":"Prefetch with streams","localized":"","hint":"In group offload, uses CUDA streams to prefetch the next group while the current one runs, hiding transfer latency.
Faster, but offloaded weights are then staged in pinned non-pageable host memory: the whole module when Pin offload memory is enabled, one group at a time otherwise.
Components used once per generation, such as text encoders, are exempt and always offload without streams.

Applies only to group offload.

Disabled by default.","ui":"settings_offload"}, + {"id":"","label":"Pin offload memory","localized":"","hint":"Keeps the CPU copy of every stream-offloaded weight in pinned non-pageable memory for the fastest transfers, at a host memory cost equal to the full module size.
When disabled, memory is pinned one group at a time during transfer: slower, but the weights stay pageable and use no extra memory at rest.
A component too large for the system memory free when it loads falls back to block_level without streams; some memory is always left for the rest of the system.

Applies only to group offload with Prefetch with streams enabled.

Enabled by default.","ui":"settings_offload"}, {"id":"extras_nav","label":"Process","localized":"","hint":"Process existing image
Can be used to upscale images, remove backgrounds, obfuscate NSFW content, apply various filters and effects"}, {"id":"txt2img_prompts","label":"Prompts","localized":"","hint":"Image prompt and negative prompt","ui":"txt2img"}, {"id":"txt2img_pause","label":"Pause","localized":"","hint":"Pause processing","ui":"txt2img"}, @@ -1241,7 +1244,6 @@ {"id":"","label":"Rebase","localized":"","hint":"","ui":"tab_update"}, {"id":"","label":"Repos","localized":"","hint":"","ui":"component-8779"}, {"id":"","label":"Refiner model","localized":"","hint":"Refiner model used for second-pass operations","ui":"settings_sd"}, - {"id":"","label":"Record torch streams","localized":"","hint":"Skips a stream synchronization each time a group offloads, letting transfers and compute overlap more tightly.
Slightly faster at the cost of slightly higher VRAM use; correctness is maintained either way.
Has no effect unless Use torch streams is enabled.

Applies only to group offload with streams.

Disabled by default.","reload":"model","ui":"settings_offload"}, {"id":"","label":"Remote VAE image type","localized":"","hint":"","ui":"settings_vae_encoder"}, {"id":"","label":"Remote VAE for encode","localized":"","hint":"","ui":"settings_vae_encoder"}, {"id":"","label":"RAS enabled","localized":"","hint":"","ui":"settings_advanced"}, @@ -1566,7 +1568,6 @@ {"id":"","label":"Username","localized":"","hint":"","ui":"component-8823"}, {"id":"","label":"UNET model","localized":"","hint":"","ui":"settings_sd"}, {"id":"","label":"UNET model secondary","localized":"","hint":"Override for the second transformer of dual-transformer architectures:
- Ideogram 4: the unconditional transformer
- Wan combined stage: the second expert (transformer_2)

Shown only when the loaded model has a second transformer.
Default loads the component from the base model.","ui":"settings_sd"}, - {"id":"","label":"Use torch streams","localized":"","hint":"In group offload, uses CUDA streams to prefetch the next group while the current one runs, hiding transfer latency.
Faster, but offloaded weights are then staged in pinned non-pageable host memory: the whole module when Pin offload memory is enabled, one group at a time otherwise.
Text encoders are exempt and always offload without streams.

Applies only to group offload.

Disabled by default.","reload":"model","ui":"settings_offload"}, {"id":"","label":"Use SVD quantization","localized":"","hint":"Adds a low-rank (SVDQuant) correction on top of SDNQ to recover accuracy lost at low bit widths, at the cost of extra size and compute.
Tuned by SVD rank size and SVD steps.

Disabled by default.","reload":"model","ui":"settings_quantization"}, {"id":"","label":"Use Dynamic quantization","localized":"","hint":"Picks a per-layer weight type automatically instead of one type everywhere. Each layer starts at the Quantization type (the minimum) and steps up to higher precision until its error meets the Dynamic loss threshold.
Protects error-sensitive layers at the cost of a larger model and slower load.

Disabled by default.","reload":"model","ui":"settings_quantization"}, {"id":"","label":"Use Hadamard rotations","localized":"","hint":"Applies a Hadamard rotation before quantizing to spread out weight outliers, which can improve accuracy at low bit widths.
Group size is set by Hadamard group size.

Disabled by default.","reload":"model","ui":"settings_quantization"},