add models_not_to_quant option

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -59,6 +59,9 @@ Also, there are quite a few offloading improvements and many quality-of-life cha
   - improve offloading of models with controlnet  
   - more aggressive offloading of controlnets with lowvram flag  
 - **Quantization**
+  - option to specify models types not to quantize: *settings -> quantization*  
+    allows for having quantization enabled, but skipping specific model types that do not need it  
+    *example*: `sd, sdxl`  
   - **sdnq**
     - add quantized matmul support for all quantization types and group sizes  
     - improve the performance of low bit quants  

TODO.md

@@ -6,15 +6,12 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 
 - Remote TE  
 - [Canvas](https://konvajs.org/)  
-- [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915)  
+- Refactor: [Modular pipelines and guiders](https://github.com/huggingface/diffusers/issues/11915)  
 - Refactor: Sampler options  
 - Refactor: [GGUF](https://huggingface.co/docs/diffusers/main/en/quantization/gguf)  
-- Feature: Diffusers [group offloading](https://github.com/vladmandic/sdnext/issues/4049)  
 - Feature: LoRA add OMI format support for SD35/FLUX.1  
 - Video Core: API  
 - Video LTX: TeaCache and others, API, Conditioning preprocess Video: LTX API  
-- <https://huggingface.co/tencent/HunyuanImage-2.1>
-- <https://huggingface.co/Alpha-VLLM/Lumina-DiMOO>
 
 ### Blocked items
 
@@ -39,22 +36,20 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 - Remove: CodeFormer  
 - Remove: GFPGAN  
 - ModernUI: Lite vs Expert mode  
-
-### Future Considerations
-- [TensorRT](https://github.com/huggingface/diffusers/pull/11173)  
+- Engine: TensorRT acceleration
 
 ### New models
 
 #### Diffusers:WiP
 - [Magi](https://github.com/SandAI-org/MAGI-1)(https://github.com/huggingface/diffusers/pull/11713)  
 - [SEVA](https://github.com/huggingface/diffusers/pull/11440)  
-#### External:Unified/MultiModal
+#### External
+- [HunyuanImage](https://huggingface.co/tencent/HunyuanImage-2.1)
+- [Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO)
 - [Ming](https://github.com/inclusionAI/Ming)  
 - [Liquid](https://github.com/FoundationVision/Liquid)  
-#### External:Image2Image/Editing
 - [Step1X](https://github.com/stepfun-ai/Step1X-Edit)  
 - [SD3 UltraEdit](https://github.com/HaozheZhao/UltraEdit)  
-#### External:Video
 - [WAN2GP](https://github.com/deepbeepmeep/Wan2GP)  
 - [SelfForcing](https://github.com/guandeh17/Self-Forcing)  
 - [DiffusionForcing](https://github.com/kwsong0113/diffusion-forcing-transformer)  
@@ -80,7 +75,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 - lora: add t5 key support for sd35/f1
 - lora: maybe force imediate quantization
 - model load: force-reloading entire model as loading transformers only leads to massive memory usage
-- model load: group offload
 - model load: implement model in-memory caching
 - modernui: monkey-patch for missing tabs.select event
 - modules/lora/lora_extract.py:188:9: W0511: TODO: lora: support pre-quantized flux

installer.py

@@ -1278,7 +1278,7 @@ def install_optional():
     install('pillow-jxl-plugin==1.3.4', ignore=True, quiet=True)
     install('optimum-quanto==0.2.7', ignore=True, quiet=True)
     install('torchao==0.10.0', ignore=True, quiet=True)
-    install('bitsandbytes==0.46.1', ignore=True, quiet=True)
+    install('bitsandbytes==0.47.0', ignore=True, quiet=True)
     install('pynvml', ignore=True, quiet=True)
     install('ultralytics==8.3.40', ignore=True, quiet=True)
     install('Cython', ignore=True, quiet=True)

javascript/extraNetworks.js

@@ -178,7 +178,7 @@ async function filterExtraNetworksForTab(searchTerm) {
     found += cards.filter((elem) => elem.style.display === '').length;
   }
   const t1 = performance.now();
-  log(`filterExtraNetworks: text="${searchTerm}" items=${items} match=${found} time=${Math.round(1000 * (t1 - t0)) / 1000000}`);
+  log(`filterExtraNetworks: text="${searchTerm}" items=${items} match=${found} time=${Math.round(t1 - t0)}`);
 }
 
 function tryToRemoveExtraNetworkFromPrompt(textarea, text) {
@@ -214,6 +214,7 @@ function tryToRemoveExtraNetworkFromPrompt(textarea, text) {
 }
 
 function sortExtraNetworks(fixed = 'no') {
+  const t0 = performance.now();
   const sortDesc = ['Default', 'Name [A-Z]', 'Name [Z-A]', 'Date [Newest]', 'Date [Oldest]', 'Size [Largest]', 'Size [Smallest]'];
   const pagename = getENActivePage();
   if (!pagename) return 'sort error: unknown page';
@@ -241,7 +242,8 @@ function sortExtraNetworks(fixed = 'no') {
     for (const card of cards) pg.appendChild(card);
   }
   const desc = sortDesc[sortVal];
-  log('sortNetworks', { name: pagename, val: sortVal, order: desc, fixed: fixed === 'fixed', items: num });
+  const t1 = performance.now();
+  log('sortNetworks', { name: pagename, val: sortVal, order: desc, fixed: fixed === 'fixed', items: num, time: Math.round(t1 - t0) });
   return desc;
 }
 

javascript/loader.js

@@ -30,7 +30,7 @@ async function removeSplash() {
   const splash = document.getElementById('splash');
   if (splash) splash.remove();
   log('removeSplash');
-  const t = Math.round(performance.now() - appStartTime) / 1000;
+  const t = Math.round(performance.now() - appStartTime);
   log('startupTime', t);
   xhrPost(`${window.api}/log`, { message: `ready time=${t}` });
 }

javascript/script.js

@@ -14,7 +14,7 @@ function logFn(func) {
     const t0 = performance.now();
     const returnValue = func(...arguments);
     const t1 = performance.now();
-    log(func.name, Math.round(t1 - t0) / 1000);
+    log(func.name, `time=${Math.round(t1 - t0)}`);
     return returnValue;
   };
 }

javascript/settings.js

@@ -18,10 +18,12 @@ const AppyOpts = [
 ];
 
 async function updateOpts(json_string) {
+  const t0 = performance.now();
   const settings_data = JSON.parse(json_string);
   const new_opts = settings_data.values;
   opts_metadata = settings_data.metadata;
 
+  const t1 = performance.now();
   for (const op of monitoredOpts) {
     const key = Object.keys(op)[0];
     const callback = op[key];
@@ -37,8 +39,9 @@ async function updateOpts(json_string) {
     if (callback) callback(new_opts[key], opts[key]);
   }
 
+  const t2 = performance.now();
   window.opts = new_opts;
-  log('updateOpts', Object.keys(new_opts).length);
+  log('updateOpts', `settings=${Object.keys(new_opts).length} callbacks=${Math.round(t2 - t1)} apply=${Math.round(t1 - t0)}`);
   Object.entries(opts_metadata).forEach(([opt, meta]) => {
     if (!opts_tabs[meta.tab_name]) opts_tabs[meta.tab_name] = {};
     if (!opts_tabs[meta.tab_name].unsaved_keys) opts_tabs[meta.tab_name].unsaved_keys = new Set();

javascript/startup.js

@@ -4,6 +4,7 @@ window.subpath = '';
 
 async function initStartup() {
   const t0 = performance.now();
+  log('gradio', `time=${Math.round(t0 - appStartTime)}`);
   log('initStartup');
   if (window.setupLogger) await setupLogger();
 

modules/model_quant.py

@@ -300,7 +300,7 @@ def load_bnb(msg='', silent=False):
     if not installed('bitsandbytes'):
         if devices.backend == 'cuda':
             # forcing a version will uninstall the multi-backend-refactor branch of bnb
-            install('bitsandbytes==0.46.1', quiet=True)
+            install('bitsandbytes==0.47.0', quiet=True)
             log.warning('Quantization: bitsandbytes installed please restart')
     try:
         import bitsandbytes
@@ -757,8 +757,20 @@ def get_dit_args(load_config:dict=None, module:str=None, device_map:bool=False,
     return config, quant_args
 
 
+def dont_quant():
+    from modules import shared
+    models_list = re.split(r'[ ,]+', shared.opts.models_not_to_quant)
+    models_list = [m.lower().strip() for m in models_list]
+    if shared.sd_model_type.lower() in models_list:
+        shared.log.debug(f'Quantization: model={shared.sd_model_type} skip')
+        return True
+    return False
+
+
 def do_post_load_quant(sd_model, allow=True):
     from modules import shared
+    if dont_quant():
+        return sd_model
     if shared.opts.sdnq_quantize_weights and (shared.opts.sdnq_quantize_mode == 'post' or (allow and shared.opts.sdnq_quantize_mode == 'auto')):
         shared.log.debug('Load model: post_quant=sdnq')
         sd_model = sdnq_quantize_weights(sd_model)

modules/processing_diffusers.py

@@ -144,7 +144,8 @@ def process_base(p: processing.StableDiffusionProcessing):
         denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
         denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
         num_frames=getattr(p, 'frames', 1),
-        output_type='latent',
+        # output_type='latent',
+        output_type='np',
         clip_skip=p.clip_skip,
         desc=desc,
     )
@@ -348,7 +349,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output):
         for i in range(len(output.images)):
             image = output.images[i]
             noise_level = round(350 * p.denoising_strength)
-            output_type='latent'
+            output_type = 'latent'
             if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__ or 'Kandinsky' in shared.sd_refiner.__class__.__name__:
                 image = processing_vae.vae_decode(latents=image, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.width, height=p.height)
                 p.extra_generation_params['Noise level'] = noise_level

modules/processing_vae.py

@@ -137,6 +137,10 @@ def full_vae_decode(latents, model):
             latents = latents.to(devices.dtype_vae)
         else:
             latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype)
+        # if getattr(model.vae.post_quant_conv, "bias", None) is not None:
+            # model.vae.post_quant_conv.bias = torch.nn.Parameter(model.vae.post_quant_conv.bias.to(devices.device), requires_grad=False)
+        # if getattr(model.vae.post_quant_conv, "weight", None) is not None:
+            # model.vae.post_quant_conv.weight = torch.nn.Parameter(model.vae.post_quant_conv.weight.to(devices.device), requires_grad=False)
     else:
         latents = latents.to(model.vae.dtype)
 

modules/sd_offload.py

@@ -71,11 +71,11 @@ def apply_group_offload(sd_model, op:str='model'):
     offload_dct = {
         'onload_device': devices.device,
         'offload_device': devices.cpu,
-        'offload_type': 'block_level', # 'leaf_level',
-        'num_blocks_per_group': 1,
+        'offload_type': shared.opts.group_offload_type,
+        'num_blocks_per_group': shared.opts.group_offload_blocks,
         'non_blocking': shared.opts.diffusers_offload_nonblocking,
-        'use_stream': False,
-        'record_stream': False,
+        'use_stream': shared.opts.group_offload_stream,
+        'record_stream': shared.opts.group_offload_record,
         'low_cpu_mem_usage': False,
     }
     shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} options={offload_dct}')

modules/shared.py

@@ -167,16 +167,24 @@ options_templates.update(options_section(('model_options', "Model Options"), {
 options_templates.update(options_section(('offload', "Model Offloading"), {
     "offload_sep": OptionInfo("<h2>Model Offloading</h2>", "", gr.HTML),
     "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'group', 'model', 'sequential']}),
-    "diffusers_offload_pre": OptionInfo(True, "Offload during pre-forward"),
     "diffusers_offload_nonblocking": OptionInfo(False, "Non-blocking move operations"),
-    "diffusers_offload_min_gpu_memory": OptionInfo(startup_offload_min_gpu, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }),
-    "diffusers_offload_max_gpu_memory": OptionInfo(startup_offload_max_gpu, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }),
-    "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }),
+    "offload_balanced_sep": OptionInfo("<h2>Balanced Offload</h2>", "", gr.HTML),
+    "diffusers_offload_pre": OptionInfo(True, "Offload during pre-forward"),
+    "diffusers_offload_min_gpu_memory": OptionInfo(startup_offload_min_gpu, "Offload low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }),
+    "diffusers_offload_max_gpu_memory": OptionInfo(startup_offload_max_gpu, "Offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }),
+    "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }),
     "diffusers_offload_always": OptionInfo(startup_offload_always, "Modules to always offload"),
     "diffusers_offload_never": OptionInfo(startup_offload_never, "Modules to never offload"),
+    "offload_group_sep": OptionInfo("<h2>Group Offload</h2>", "", gr.HTML),
+    "group_offload_type": OptionInfo("block_level", "Group offload type", gr.Radio, {"choices": ['block_level', 'leaf_level']}),
+    "group_offload_stream": OptionInfo(False, "Use torch streams", gr.Checkbox),
+    'group_offload_record': OptionInfo(False, "Record torch streams", gr.Checkbox),
+    'group_offload_blocks': OptionInfo(1, "Offload blocks", gr.Number),
 }))
 
 options_templates.update(options_section(("quantization", "Model Quantization"), {
+    "models_not_to_quant": OptionInfo("", "List of model types not to quantize"),
+
     "sdnq_quantize_sep": OptionInfo("<h2>SDNQ: SD.Next Quantization</h2>", "", gr.HTML),
     "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "TE", "LLM", "Control", "VAE"]}),
     "sdnq_quantize_mode": OptionInfo("auto", "Quantization mode", gr.Dropdown, {"choices": ["auto", "pre", "post"]}),

pipelines/model_wanai.py

@@ -105,8 +105,8 @@ def load_wan(checkpoint_info, diffusers_load_config={}):
     diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanPipeline
     diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["wanai"] = diffusers.WanImageToVideoPipeline
 
-    # sd_hijack_te.init_hijack(pipe)
-    # sd_hijack_vae.init_hijack(pipe)
+    sd_hijack_te.init_hijack(pipe)
+    sd_hijack_vae.init_hijack(pipe)
 
     devices.torch_gc()
     return pipe