mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
docs(i18n): flag quant and offload settings that need model reload or server restart
This commit is contained in:
CalamitousFelicitousness
+30
-30
@@ -392,9 +392,9 @@
|
||||
{"id":"","label":"Diffusers load using Run:ai streamer","localized":"","hint":"","ui":"settings_sd"},
|
||||
{"id":"","label":"Disable accelerate","localized":"","hint":"","ui":"settings_sd"},
|
||||
{"id":"","label":"Disable T5 text encoder","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"Dynamic loss threshold","localized":"","hint":"Target per-layer quantization error (normalized MSE) for dynamic quantization. Each layer starts at the base <b><i>Quantization type</i></b> and steps up to higher precision until its error falls at or below this value.<br>Lower values keep more layers at higher precision for a larger, more accurate model; higher values let more layers stay at the base type for a smaller one.<br><br>Only takes effect when <b><i>Use Dynamic quantization</i></b> is enabled.<br><br><b>-1</b> auto-selects a threshold from the base type, about <b>1e-4</b> for 8-bit or <b>1e-2</b> for 4-bit.<br><b>0</b> accepts only layers that quantize losslessly, which on a normal model leaves almost everything at full precision.<br><br>Default is <b>-1</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Dequantize using torch.compile","localized":"","hint":"Compiles the dequantization step with <i>torch.compile</i> for faster inference. Requires <i>Triton</i>.<br><br>Changing this needs a full restart to take effect.<br><br>Enabled by default when Triton is available.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Dequantize using full precision","localized":"","hint":"Uses <b>FP32</b> for the dequantization step for better numerical accuracy, at a small speed cost.<br><br>Enabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Dynamic loss threshold","localized":"","hint":"Target per-layer quantization error (normalized MSE) for dynamic quantization. Each layer starts at the base <b><i>Quantization type</i></b> and steps up to higher precision until its error falls at or below this value.<br>Lower values keep more layers at higher precision for a larger, more accurate model; higher values let more layers stay at the base type for a smaller one.<br><br>Only takes effect when <b><i>Use Dynamic quantization</i></b> is enabled.<br><br><b>-1</b> auto-selects a threshold from the base type, about <b>1e-4</b> for 8-bit or <b>1e-2</b> for 4-bit.<br><b>0</b> accepts only layers that quantize losslessly, which on a normal model leaves almost everything at full precision.<br><br>Default is <b>-1</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Dequantize using torch.compile","localized":"","hint":"Compiles the dequantization step with <i>torch.compile</i> for faster inference. Requires <i>Triton</i>.<br><br>Changing this needs a full restart to take effect.<br><br>Enabled by default when Triton is available.","reload":"server","ui":"settings_quantization"},
|
||||
{"id":"","label":"Dequantize using full precision","localized":"","hint":"Uses <b>FP32</b> for the dequantization step for better numerical accuracy, at a small speed cost.<br><br>Enabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Disabled","localized":"","hint":"","ui":"settings_cuda"},
|
||||
{"id":"","label":"Dynamic Attention BMM","localized":"","hint":"Performs attention computation in steps instead of all at once. Slower inference times, but greatly reduced memory usage","ui":"settings_cuda"},
|
||||
{"id":"","label":"Dynamic attention","localized":"","hint":"Adjusts attention computation dynamically per step. Saves VRAM but slows generation.","ui":"settings_cuda"},
|
||||
@@ -621,7 +621,7 @@
|
||||
{"id":"","label":"Generic","localized":"","hint":"","ui":"video"},
|
||||
{"id":"","label":"Google GenAI","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"Group Offload","localized":"","hint":"","ui":"settings_offload"},
|
||||
{"id":"","label":"Group offload type","localized":"","hint":"Granularity used by <b>group</b> offload.<br>- <b>leaf_level</b>: offloads at the smallest module level; maximum memory savings, slower<br>- <b>block_level</b>: offloads groups of transformer blocks (size set by <b><i>Offload blocks</i></b>); faster with less savings, and keeps the VAE resident<br><br>Applies only when <b><i>Model offload mode</i></b> is <b>group</b>.<br><br>Default is <b>leaf_level</b>.","ui":"settings_offload"},
|
||||
{"id":"","label":"Group offload type","localized":"","hint":"Granularity used by <b>group</b> offload.<br>- <b>leaf_level</b>: offloads at the smallest module level; maximum memory savings, slower<br>- <b>block_level</b>: offloads groups of transformer blocks (size set by <b><i>Offload blocks</i></b>); faster with less savings, and keeps the VAE resident<br><br>Applies only when <b><i>Model offload mode</i></b> is <b>group</b>.<br><br>Default is <b>leaf_level</b>.","reload":"model","ui":"settings_offload"},
|
||||
{"id":"","label":"Grid Options","localized":"","hint":"","ui":"settings_saving-images"},
|
||||
{"id":"","label":"Grids","localized":"","hint":"","ui":"settings_saving-paths"},
|
||||
{"id":"","label":"Guider","localized":"","hint":"","ui":"txt2img"},
|
||||
@@ -646,7 +646,7 @@
|
||||
{"id":"","label":"Google Cloud project ID","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"Google Cloud location ID","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"group","localized":"","hint":"","ui":"settings_offload"},
|
||||
{"id":"","label":"Group size","localized":"","hint":"Number of weight elements that share one quantization scale. Smaller groups improve accuracy at a small size and speed cost; larger groups are leaner but coarser.<br><br><b>0</b> auto-selects a group size from the <b><i>Quantization type</i></b>; with <b><i>Use quantized MatMul</i></b> enabled at 6-bit or wider weights, it falls back to one scale per row. <b>-1</b> uses one scale per row (no grouping).<br><br>Default is <b>0</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Group size","localized":"","hint":"Number of weight elements that share one quantization scale. Smaller groups improve accuracy at a small size and speed cost; larger groups are leaner but coarser.<br><br><b>0</b> auto-selects a group size from the <b><i>Quantization type</i></b>; with <b><i>Use quantized MatMul</i></b> enabled at 6-bit or wider weights, it falls back to one scale per row. <b>-1</b> uses one scale per row (no grouping).<br><br>Default is <b>0</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"GC threshold","localized":"","hint":"","ui":"settings_backends"},
|
||||
{"id":"","label":"Grid max rows count","localized":"","hint":"","ui":"settings_saving-images"},
|
||||
{"id":"","label":"Grid max columns count","localized":"","hint":"","ui":"settings_saving-images"},
|
||||
@@ -663,7 +663,7 @@
|
||||
{"id":"","label":"Huggingface","localized":"","hint":"Settings related huggingface access"},
|
||||
{"id":"","label":"HiDream","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"HyperTile","localized":"","hint":"","ui":"settings_advanced"},
|
||||
{"id":"","label":"Hadamard group size","localized":"","hint":"Number of elements that share one Hadamard rotation group when <b><i>Use Hadamard rotations</i></b> is enabled.<br><br>Applies only when <b><i>Use Hadamard rotations</i></b> is enabled.<br><br>Default is <b>128</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Hadamard group size","localized":"","hint":"Number of elements that share one Hadamard rotation group when <b><i>Use Hadamard rotations</i></b> is enabled.<br><br>Applies only when <b><i>Use Hadamard rotations</i></b> is enabled.<br><br>Default is <b>128</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"HiDiffusion","localized":"","hint":"HiDiffusion allows creation of high-resolution images using your standard models without duplicates/distortions and improved performance","ui":"settings_advanced"},
|
||||
{"id":"","label":"Height","localized":"","hint":"Target height of the output image in pixels.<br>For generation, this sets the resolution the model produces. For resize and upscale operations, this is the height the input is fitted to.<br><br>Should be a multiple of 8 for <i>SD1.x</i> and <i>SDXL</i> latents; newer architectures (<i>Flux</i>, <i>SD3</i>, video models) may require higher multiples (16, 32, or 64). Values that don't match are automatically floored to the nearest valid multiple for the loaded model.","ui":"txt2img"},
|
||||
{"id":"","label":"HiRes steps","localized":"","hint":"Number of sampling steps for upscaled picture. If 0, uses same as for original","ui":"txt2img"},
|
||||
@@ -940,13 +940,13 @@
|
||||
{"id":"","label":"Model auto-download on demand","localized":"","hint":"","ui":"settings_sd"},
|
||||
{"id":"","label":"Model load using streams","localized":"","hint":"When loading models attempt stream loading optimized for slow or network storage","ui":"settings_sd"},
|
||||
{"id":"","label":"Model load model direct to GPU","localized":"","hint":"","ui":"settings_sd"},
|
||||
{"id":"","label":"Model offload mode","localized":"","hint":"Controls how model components move between VRAM and system RAM to fit larger models on less VRAM.<br>- <b>none</b>: keeps everything on the GPU; fastest, but only works if the whole model fits in VRAM<br>- <b>balanced</b>: the recommended default; offloads only when VRAM use crosses a threshold, so it suits almost any GPU (tuned by the watermarks below)<br>- <b>group</b>: offloads groups of layers via diffusers group offloading; an alternative middle ground with optional stream prefetch<br>- <b>model</b>: offloads whole components such as the VAE or text encoder when idle; a more compatible fallback when balanced or group are unsupported, with smaller savings<br>- <b>sequential</b>: offloads layer by layer; the most memory saving but slowest, for when even balanced runs out of memory<br><br>Command-line flags override the automatic choice:<br>- <code>--lowvram</code>: forces <b>sequential</b><br>- <code>--medvram</code>: forces <b>balanced</b> with low watermark <b>0</b><br><br>With no flag, <b>balanced</b> is the automatic default on any GPU, with watermarks set by GPU memory (low / high):<br>- 12 GB or less: <b>0</b> / <b>0.6</b><br>- 12-24 GB: <b>0.2</b> / <b>0.6</b><br>- 24 GB or more: <b>0.2</b> / <b>0.8</b><br>(or <b>none</b> if no GPU is detected)","ui":"settings_offload"},
|
||||
{"id":"","label":"Model offload mode","localized":"","hint":"Controls how model components move between VRAM and system RAM to fit larger models on less VRAM.<br>- <b>none</b>: keeps everything on the GPU; fastest, but only works if the whole model fits in VRAM<br>- <b>balanced</b>: the recommended default; offloads only when VRAM use crosses a threshold, so it suits almost any GPU (tuned by the watermarks below)<br>- <b>group</b>: offloads groups of layers via diffusers group offloading; an alternative middle ground with optional stream prefetch<br>- <b>model</b>: offloads whole components such as the VAE or text encoder when idle; a more compatible fallback when balanced or group are unsupported, with smaller savings<br>- <b>sequential</b>: offloads layer by layer; the most memory saving but slowest, for when even balanced runs out of memory<br><br>Command-line flags override the automatic choice:<br>- <code>--lowvram</code>: forces <b>sequential</b><br>- <code>--medvram</code>: forces <b>balanced</b> with low watermark <b>0</b><br><br>With no flag, <b>balanced</b> is the automatic default on any GPU, with watermarks set by GPU memory (low / high):<br>- 12 GB or less: <b>0</b> / <b>0.6</b><br>- 12-24 GB: <b>0.2</b> / <b>0.6</b><br>- 24 GB or more: <b>0.2</b> / <b>0.8</b><br>(or <b>none</b> if no GPU is detected)","reload":"model","ui":"settings_offload"},
|
||||
{"id":"","label":"Model types not to offload","localized":"","hint":"Model architectures to skip when offloading, separated by spaces or commas.<br>Useful for model types that misbehave when offloaded.<br><br>Applies only to <b>balanced</b> offload.<br><br>Default is empty.","ui":"settings_offload"},
|
||||
{"id":"","label":"Modules to always offload","localized":"","hint":"Module names that are always offloaded in <b>balanced</b> mode, separated by spaces, commas, or semicolons, regardless of the watermarks.<br><br>Applies only to <b>balanced</b> offload.<br><br>Default by GPU memory: the large text encoders (<i>T5</i>, <i>UMT5</i>) are added at roughly 4-12 GB and at 24 GB or more; empty otherwise.","ui":"settings_offload"},
|
||||
{"id":"","label":"Modules to never offload","localized":"","hint":"Module names that are never offloaded in <b>balanced</b> mode, separated by spaces, commas, or semicolons, keeping them resident in VRAM.<br><br>Applies only to <b>balanced</b> offload.<br><br>Default by GPU memory: the CLIP text encoders and the VAE are kept resident at 24 GB or more; empty otherwise.","ui":"settings_offload"},
|
||||
{"id":"","label":"Model types not to quantize","localized":"","hint":"","ui":"settings_quantization"},
|
||||
{"id":"","label":"Modules to not convert","localized":"","hint":"Names of modules to leave unquantized (kept in original precision), separated by spaces, commas, or semicolons.<br>Useful for layers that are sensitive to quantization, such as gate or projection layers. Example: <code>proj_out, x_embedder</code>.<br><br>Some models already exclude sensitive modules by default; entries here extend that built-in list rather than replacing it.<br><br>Default is empty.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Modules dtype dict","localized":"","hint":"Advanced: JSON mapping a quantization type to a list of module names, to quantize specific modules at a different type than the global <b><i>Quantization type</i></b>. Example: <code>{\"uint4\": [\"proj_out\"]}</code>.<br><br>Some models already assign certain modules a specific type by default; entries here merge with those built-in mappings rather than replacing them.<br><br>Default is empty.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Modules to not convert","localized":"","hint":"Names of modules to leave unquantized (kept in original precision), separated by spaces, commas, or semicolons.<br>Useful for layers that are sensitive to quantization, such as gate or projection layers. Example: <code>proj_out, x_embedder</code>.<br><br>Some models already exclude sensitive modules by default; entries here extend that built-in list rather than replacing it.<br><br>Default is empty.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Modules dtype dict","localized":"","hint":"Advanced: JSON mapping a quantization type to a list of module names, to quantize specific modules at a different type than the global <b><i>Quantization type</i></b>. Example: <code>{\"uint4\": [\"proj_out\"]}</code>.<br><br>Some models already assign certain modules a specific type by default; entries here merge with those built-in mappings rather than replacing them.<br><br>Default is empty.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Math","localized":"","hint":"","ui":"settings_cuda"},
|
||||
{"id":"","label":"Memory limit","localized":"","hint":"","ui":"settings_backends"},
|
||||
{"id":"","label":"migraphx","localized":"","hint":"","ui":"settings_compile"},
|
||||
@@ -997,7 +997,7 @@
|
||||
{"id":"","label":"Near threshold","localized":"","hint":"","ui":"control"},
|
||||
{"id":"","label":"Noise scale","localized":"","hint":"","ui":"video"},
|
||||
{"id":"","label":"Note","localized":"","hint":"","ui":"component-8823"},
|
||||
{"id":"","label":"Non-blocking move operations","localized":"","hint":"Uses non-blocking transfers when moving weights between GPU and RAM, letting copies overlap with other work.<br>Can be faster, but may be unstable on some platforms.<br><br>Disabled by default.","ui":"settings_offload"},
|
||||
{"id":"","label":"Non-blocking move operations","localized":"","hint":"Uses non-blocking transfers when moving weights between GPU and RAM, letting copies overlap with other work.<br>Can be faster, but may be unstable on some platforms.<br><br>Disabled by default.","reload":"model","ui":"settings_offload"},
|
||||
{"id":"","label":"Nunchaku attention","localized":"","hint":"Replaces default attention with Nunchaku's custom FP16 attention kernel for faster inference on consumer NVIDIA GPUs.<br>Might provide performance improvement on GPUs which have higher FP16 tensor cores throughput than BF16.<br><br>Currently only affects <i>Flux</i>-based models (<i>Dev</i>, <i>Schnell</i>, <i>Kontext</i>, <i>Fill</i>, <i>Depth</i>, etc.). Has no effect on <i>Qwen</i>, <i>SDXL</i>, <i>Sana</i>, or other architectures.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Nunchaku offloading","localized":"","hint":"Enables Nunchaku's own per-block CPU offloading with asynchronous CUDA streams to reduce VRAM usage.<br>Uses a ping-pong buffer strategy: while one transformer block computes on GPU, the next block preloads from CPU in the background, hiding most of the transfer latency.<br><br>Can reduce VRAM usage at the cost of slower inference.<br>This replaces SD.Next's pipeline offloading for the transformer component.<br><br>Only useful on low-VRAM GPUs. If your GPU has enough memory to hold the quantized model (16+ GB), keep this disabled for maximum speed.<br>Supports <i>Flux</i> and <i>Qwen</i> models. Not supported for <i>SDXL</i> where this setting is ignored.<br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"native","localized":"","hint":"","ui":"settings_text_encoder"},
|
||||
@@ -1042,7 +1042,7 @@
|
||||
{"id":"","label":"Offload low watermark","localized":"","hint":"Lower VRAM threshold for <b>balanced</b> offload, as a fraction of total GPU memory. While VRAM use stays below this, nothing is offloaded; above it, idle components are moved back to RAM.<br><br>Applies only to <b>balanced</b> offload. <code>--lowvram</code> and <code>--medvram</code> set this to <b>0</b>.<br><br>Default by GPU memory: <b>0</b> at 12 GB or less, <b>0.2</b> above.","ui":"settings_offload"},
|
||||
{"id":"","label":"Offload GPU high watermark","localized":"","hint":"Upper VRAM threshold for <b>balanced</b> offload, as a fraction of total GPU memory. Sets the most VRAM a single component may use before it is offloaded.<br><br>Applies only to <b>balanced</b> offload.<br><br>Default by GPU memory: <b>0.6</b>, rising to <b>0.8</b> at 24 GB or more.","ui":"settings_offload"},
|
||||
{"id":"","label":"Offload CPU high watermark","localized":"","hint":"Upper system-RAM threshold for offloaded weights in <b>balanced</b> offload, as a fraction of total RAM.<br><br>Applies only to <b>balanced</b> offload.<br><br>Default is <b>0.9</b>.","ui":"settings_offload"},
|
||||
{"id":"","label":"Offload blocks","localized":"","hint":"Number of transformer blocks per offload group when <b><i>Group offload type</i></b> is <b>block_level</b>. Larger groups offload less often (faster, more VRAM); smaller groups save more memory.<br><br>Applies only to <b>group</b> offload with <b>block_level</b>.<br><br>Default is <b>1</b>.","ui":"settings_offload"},
|
||||
{"id":"","label":"Offload blocks","localized":"","hint":"Number of transformer blocks per offload group when <b><i>Group offload type</i></b> is <b>block_level</b>. Larger groups offload less often (faster, more VRAM); smaller groups save more memory.<br><br>Applies only to <b>group</b> offload with <b>block_level</b>.<br><br>Default is <b>1</b>.","reload":"model","ui":"settings_offload"},
|
||||
{"id":"","label":"OpenVINO activations mode","localized":"","hint":"","ui":"settings_quantization"},
|
||||
{"id":"","label":"ONNX Execution Provider","localized":"","hint":"","ui":"settings_backends"},
|
||||
{"id":"","label":"ONNX allow fallback to CPU","localized":"","hint":"Allow fallback to CPU when selected execution provider failed","ui":"settings_backends"},
|
||||
@@ -1153,15 +1153,15 @@
|
||||
{"id":"","label":"Qwen layered","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"Quicksettings","localized":"","hint":"","ui":"settings_ui"},
|
||||
{"id":"","label":"Qwen layered number of layers","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"Quantization enabled","localized":"","hint":"Selects which model components SDNQ quantizes: <b>Model</b> (the diffusion transformer or UNet), <b>TE</b> (pipeline text encoders), <b>LLM</b> (standalone LLM/VLM tools like captioning and prompt enhance), <b>Control</b> (ControlNet), or <b>VAE</b>.<br>Quantization lowers VRAM use and can speed up inference at some accuracy cost.<br><br>An empty selection disables SDNQ.<br><br>Default is empty.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization mode","localized":"","hint":"When SDNQ quantizes the model.<br>- <b>auto</b>: quantizes during load when possible, otherwise after load<br>- <b>pre</b>: quantizes layer by layer as the model loads, lowering peak memory<br>- <b>post</b>: quantizes after the full model is loaded<br><br>Default is <b>auto</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization type","localized":"","hint":"Weight data type SDNQ quantizes to. Lower bit widths (e.g. <b>uint4</b>) shrink the model and can speed up inference; higher bit widths (e.g. <b>int8</b>) keep more accuracy.<br>Integer types are widely supported; float types (<b>float8_e4m3fn</b>, etc.) only accelerate on newer GPUs.<br><br>Applies to the components selected in <b><i>Quantization enabled</i></b>.<br><br>Default is <b>int8</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantized MatMul type","localized":"","hint":"Compute data type for quantized matrix multiplication, used when <b><i>Use quantized MatMul</i></b> is enabled.<br><b>auto</b> picks <b>int8</b> for integer weights or <b>float8_e4m3fn</b> / <b>float16</b> for float weights.<br><br>Default is <b>auto</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization type for Text Encoders","localized":"","hint":"Weight data type for text encoder quantization, separate from the main <b><i>Quantization type</i></b>.<br><b>Same as model</b> uses the main <b><i>Quantization type</i></b>.<br><br>Applies to both <b>TE</b> (pipeline text encoders) and <b>LLM</b> (standalone tools like captioning and prompt enhance) when selected in <b><i>Quantization enabled</i></b>.<br><br>Default is <b>Same as model</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantized MatMul type for Text Encoders","localized":"","hint":"Quantized matrix multiplication type for text encoders and standalone LLM tools (captioning, prompt enhance), separate from the main <b><i>Quantized MatMul type</i></b>.<br><b>Same as model</b> uses the main setting.<br><br>Default is <b>Same as model</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantize convolutional layers","localized":"","hint":"Also quantizes convolutional layers, such as those in UNet models like <i>SDXL</i>.<br>Saves more memory, but convolutions are often more sensitive to quantization.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantize embedding layers","localized":"","hint":"Also quantizes embedding layers in text models.<br>Saves extra memory at some risk to quality.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantize using GPU","localized":"","hint":"Runs the quantization computation on the GPU instead of the CPU, which is much faster but uses VRAM during model load.<br>Disabling it keeps the computation on the CPU when load-time VRAM is limited.<br><br>Enabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization enabled","localized":"","hint":"Selects which model components SDNQ quantizes: <b>Model</b> (the diffusion transformer or UNet), <b>TE</b> (pipeline text encoders), <b>LLM</b> (standalone LLM/VLM tools like captioning and prompt enhance), <b>Control</b> (ControlNet), or <b>VAE</b>.<br>Quantization lowers VRAM use and can speed up inference at some accuracy cost.<br><br>An empty selection disables SDNQ.<br><br>Default is empty.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization mode","localized":"","hint":"When SDNQ quantizes the model.<br>- <b>auto</b>: quantizes during load when possible, otherwise after load<br>- <b>pre</b>: quantizes layer by layer as the model loads, lowering peak memory<br>- <b>post</b>: quantizes after the full model is loaded<br><br>Default is <b>auto</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization type","localized":"","hint":"Weight data type SDNQ quantizes to. Lower bit widths (e.g. <b>uint4</b>) shrink the model and can speed up inference; higher bit widths (e.g. <b>int8</b>) keep more accuracy.<br>Integer types are widely supported; float types (<b>float8_e4m3fn</b>, etc.) only accelerate on newer GPUs.<br><br>Applies to the components selected in <b><i>Quantization enabled</i></b>.<br><br>Default is <b>int8</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantized MatMul type","localized":"","hint":"Compute data type for quantized matrix multiplication, used when <b><i>Use quantized MatMul</i></b> is enabled.<br><b>auto</b> picks <b>int8</b> for integer weights or <b>float8_e4m3fn</b> / <b>float16</b> for float weights.<br><br>Default is <b>auto</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization type for Text Encoders","localized":"","hint":"Weight data type for text encoder quantization, separate from the main <b><i>Quantization type</i></b>.<br><b>Same as model</b> uses the main <b><i>Quantization type</i></b>.<br><br>Applies to both <b>TE</b> (pipeline text encoders) and <b>LLM</b> (standalone tools like captioning and prompt enhance) when selected in <b><i>Quantization enabled</i></b>.<br><br>Default is <b>Same as model</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantized MatMul type for Text Encoders","localized":"","hint":"Quantized matrix multiplication type for text encoders and standalone LLM tools (captioning, prompt enhance), separate from the main <b><i>Quantized MatMul type</i></b>.<br><b>Same as model</b> uses the main setting.<br><br>Default is <b>Same as model</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantize convolutional layers","localized":"","hint":"Also quantizes convolutional layers, such as those in UNet models like <i>SDXL</i>.<br>Saves more memory, but convolutions are often more sensitive to quantization.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantize embedding layers","localized":"","hint":"Also quantizes embedding layers in text models.<br>Saves extra memory at some risk to quality.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantize using GPU","localized":"","hint":"Runs the quantization computation on the GPU instead of the CPU, which is much faster but uses VRAM during model load.<br>Disabling it keeps the computation on the CPU when load-time VRAM is limited.<br><br>Enabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization weights type","localized":"","hint":"","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quantization activations type","localized":"","hint":"","ui":"settings_quantization"},
|
||||
{"id":"","label":"Quicksettings list","localized":"","hint":"List of setting names, separated by commas, for settings that should go to the quick access bar at the top instead the setting tab","ui":"settings_ui"}
|
||||
@@ -1237,7 +1237,7 @@
|
||||
{"id":"","label":"Rebase","localized":"","hint":"","ui":"tab_update"},
|
||||
{"id":"","label":"Repos","localized":"","hint":"","ui":"component-8779"},
|
||||
{"id":"","label":"Refiner model","localized":"","hint":"Refiner model used for second-pass operations","ui":"settings_sd"},
|
||||
{"id":"","label":"Record torch streams","localized":"","hint":"Records CUDA stream usage during <b>group</b> offload so reused buffers stay correct when <b><i>Use torch streams</i></b> is enabled.<br><br>Applies only to <b>group</b> offload with streams.<br><br>Disabled by default.","ui":"settings_offload"},
|
||||
{"id":"","label":"Record torch streams","localized":"","hint":"Records CUDA stream usage during <b>group</b> offload so reused buffers stay correct when <b><i>Use torch streams</i></b> is enabled.<br><br>Applies only to <b>group</b> offload with streams.<br><br>Disabled by default.","reload":"model","ui":"settings_offload"},
|
||||
{"id":"","label":"Remote VAE image type","localized":"","hint":"","ui":"settings_vae_encoder"},
|
||||
{"id":"","label":"Remote VAE for encode","localized":"","hint":"","ui":"settings_vae_encoder"},
|
||||
{"id":"","label":"RAS enabled","localized":"","hint":"","ui":"settings_advanced"},
|
||||
@@ -1377,9 +1377,9 @@
|
||||
{"id":"","label":"Search Changelog","localized":"","hint":"","ui":"system_tab_changelog"},
|
||||
{"id":"","label":"Stage boundary ratio","localized":"","hint":"","ui":"settings_model_options"},
|
||||
{"id":"","label":"sequential","localized":"","hint":"","ui":"settings_offload"},
|
||||
{"id":"","label":"SVD rank size","localized":"","hint":"Rank of the low-rank correction added by <b><i>Use SVD quantization</i></b>. Higher ranks recover more accuracy but add parameters and compute.<br><br>Applies only when <b><i>Use SVD quantization</i></b> is enabled.<br><br>Default is <b>32</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"SVD steps","localized":"","hint":"Number of iterations used to estimate the low-rank correction for <b><i>Use SVD quantization</i></b>. More steps refine the estimate at the cost of longer quantization.<br><br>Applies only when <b><i>Use SVD quantization</i></b> is enabled.<br><br>Default is <b>8</b>.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Shuffle weights in post mode","localized":"","hint":"In <b>post</b> quantization mode, processes model components through the GPU one at a time, moving each off the GPU before the next, to limit peak VRAM during quantization.<br><br>Applies only when <b><i>Quantization mode</i></b> is <b>post</b>.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"SVD rank size","localized":"","hint":"Rank of the low-rank correction added by <b><i>Use SVD quantization</i></b>. Higher ranks recover more accuracy but add parameters and compute.<br><br>Applies only when <b><i>Use SVD quantization</i></b> is enabled.<br><br>Default is <b>32</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"SVD steps","localized":"","hint":"Number of iterations used to estimate the low-rank correction for <b><i>Use SVD quantization</i></b>. More steps refine the estimate at the cost of longer quantization.<br><br>Applies only when <b><i>Use SVD quantization</i></b> is enabled.<br><br>Default is <b>8</b>.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Shuffle weights in post mode","localized":"","hint":"In <b>post</b> quantization mode, processes model components through the GPU one at a time, moving each off the GPU before the next, to limit peak VRAM during quantization.<br><br>Applies only when <b><i>Quantization mode</i></b> is <b>post</b>.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"SDXL: Use weighted pooled embeds","localized":"","hint":"","ui":"settings_text_encoder"},
|
||||
{"id":"","label":"Sana: Use complex human instructions","localized":"","hint":"","ui":"settings_text_encoder"},
|
||||
{"id":"","label":"Scaled-Dot-Product","localized":"","hint":"Memory optimization. Non-Deterministic unless SDP memory attention is disabled.","ui":"settings_cuda"},
|
||||
@@ -1561,12 +1561,12 @@
|
||||
{"id":"","label":"Use spaces","localized":"","hint":"Replace underscores with spaces in tag output.<br>Some prompt systems prefer spaces between words (e.g., 'long hair') while others use underscores (e.g., 'long_hair').","ui":"caption"},
|
||||
{"id":"","label":"Username","localized":"","hint":"","ui":"component-8823"},
|
||||
{"id":"","label":"UNET model","localized":"","hint":"","ui":"settings_sd"},
|
||||
{"id":"","label":"Use torch streams","localized":"","hint":"In <b>group</b> offload, uses CUDA streams to prefetch the next group while the current one runs, hiding transfer latency.<br>Faster, but uses more VRAM.<br><br>Applies only to <b>group</b> offload.<br><br>Disabled by default.","ui":"settings_offload"},
|
||||
{"id":"","label":"Use SVD quantization","localized":"","hint":"Adds a low-rank (SVDQuant) correction on top of SDNQ to recover accuracy lost at low bit widths, at the cost of extra size and compute.<br>Tuned by <b><i>SVD rank size</i></b> and <b><i>SVD steps</i></b>.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use Dynamic quantization","localized":"","hint":"Picks a per-layer weight type automatically instead of one type everywhere. Each layer starts at the <b><i>Quantization type</i></b> (the minimum) and steps up to higher precision until its error meets the <b><i>Dynamic loss threshold</i></b>.<br>Protects error-sensitive layers at the cost of a larger model and slower load.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use Hadamard rotations","localized":"","hint":"Applies a Hadamard rotation before quantizing to spread out weight outliers, which can improve accuracy at low bit widths.<br>Group size is set by <b><i>Hadamard group size</i></b>.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use quantized MatMul","localized":"","hint":"Runs matrix multiplications in <b>int8</b> or <b>fp8</b> instead of <b>bf16</b> / <b>fp16</b>, which can speed up inference on supported hardware.<br>Compute type is set by <b><i>Quantized MatMul type</i></b>.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use quantized MatMul with conv","localized":"","hint":"Runs quantized matrix multiplication on convolutional layers, such as those in UNets like <i>SDXL</i>. This is the convolution counterpart of <b><i>Use quantized MatMul</i></b> and is controlled independently of it.<br><br>Only affects conv layers that are quantized, so it needs <b><i>Quantize convolutional layers</i></b>.<br><br>Disabled by default.","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use torch streams","localized":"","hint":"In <b>group</b> offload, uses CUDA streams to prefetch the next group while the current one runs, hiding transfer latency.<br>Faster, but uses more VRAM.<br><br>Applies only to <b>group</b> offload.<br><br>Disabled by default.","reload":"model","ui":"settings_offload"},
|
||||
{"id":"","label":"Use SVD quantization","localized":"","hint":"Adds a low-rank (SVDQuant) correction on top of SDNQ to recover accuracy lost at low bit widths, at the cost of extra size and compute.<br>Tuned by <b><i>SVD rank size</i></b> and <b><i>SVD steps</i></b>.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use Dynamic quantization","localized":"","hint":"Picks a per-layer weight type automatically instead of one type everywhere. Each layer starts at the <b><i>Quantization type</i></b> (the minimum) and steps up to higher precision until its error meets the <b><i>Dynamic loss threshold</i></b>.<br>Protects error-sensitive layers at the cost of a larger model and slower load.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use Hadamard rotations","localized":"","hint":"Applies a Hadamard rotation before quantizing to spread out weight outliers, which can improve accuracy at low bit widths.<br>Group size is set by <b><i>Hadamard group size</i></b>.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use quantized MatMul","localized":"","hint":"Runs matrix multiplications in <b>int8</b> or <b>fp8</b> instead of <b>bf16</b> / <b>fp16</b>, which can speed up inference on supported hardware.<br>Compute type is set by <b><i>Quantized MatMul type</i></b>.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use quantized MatMul with conv","localized":"","hint":"Runs quantized matrix multiplication on convolutional layers, such as those in UNets like <i>SDXL</i>. This is the convolution counterpart of <b><i>Use quantized MatMul</i></b> and is controlled independently of it.<br><br>Only affects conv layers that are quantized, so it needs <b><i>Quantize convolutional layers</i></b>.<br><br>Disabled by default.","reload":"model","ui":"settings_quantization"},
|
||||
{"id":"","label":"Use line break as prompt segment marker","localized":"","hint":"","ui":"settings_text_encoder"},
|
||||
{"id":"","label":"Use zeros for prompt padding","localized":"","hint":"Force full zero tensor when prompt is empty to remove any residual noise","ui":"settings_text_encoder"},
|
||||
{"id":"","label":"Upcast sampling","localized":"","hint":"Usually produces similar results to --no-half with better performance while using less memory","ui":"settings_cuda"},
|
||||
|
||||
Reference in New Issue
Block a user