lora: keep parsed network data through pipeline

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -127,6 +127,7 @@ Plus inevitable bug-fixes...
   - log: ansi color handling
   - lora: cleanup tags
   - lora: support transformer ref models
+  - lora: keep parsed network data through pipeline
   - lucida: handle requirements
   - lumina-dimoo: attention-kwargs, thanks @Anai-Guo
   - metadata: fix wildcard info

modules/control/proc/hed.py

@@ -102,8 +102,8 @@ class HEDdetector:
         if scribble:
             detected_map = nms(detected_map, 127, 3.0)
             detected_map = cv2.GaussianBlur(detected_map, (0, 0), 3.0)
-            detected_map[detected_map > 4] = 255
-            detected_map[detected_map < 255] = 0
+            detected_map[detected_map > 4] = 255 # pylint: disable=unsupported-assignment-operation
+            detected_map[detected_map < 255] = 0 # pylint: disable=unsupported-assignment-operation
         if opts.control_move_processor:
             self.model.to('cpu')
         if output_type == "pil":

modules/extra_networks.py

@@ -168,17 +168,20 @@ def parse_prompt(prompt: str | None) -> tuple[str, defaultdict[str, list[ExtraNe
     return updated_prompt, res
 
 
-def parse_prompts(prompts: list[str], extra_data: defaultdict[str, list[ExtraNetworkParams]] | None = None):
-    updated_prompt_list: list[str] = []
-    extra_data = extra_data or defaultdict(list)
+def parse_prompts(
+    prompts: list[str],
+    extra_data: defaultdict[str, list[ExtraNetworkParams]] | None = None,
+):
+    updated_prompts: list[str] = []
+    if extra_data is None:
+        extra_data = defaultdict(list)
     for prompt in prompts:
         updated_prompt, parsed_extra_data = parse_prompt(prompt)
-        if not extra_data:
-            extra_data = parsed_extra_data
-        elif parsed_extra_data:
-            extra_data = parsed_extra_data
-        else:
-            pass
-        updated_prompt_list.append(updated_prompt)
+        if parsed_extra_data:
+            for key, values in parsed_extra_data.items():
+                for item in values:
+                    if item not in extra_data[key]:
+                        extra_data[key].append(item)
 
-    return updated_prompt_list, extra_data
+        updated_prompts.append(updated_prompt)
+    return updated_prompts, extra_data

modules/ltx/ltx_util.py

@@ -249,7 +249,7 @@ def get_conditions(width, height, condition_strength, condition_images, conditio
 def get_prompts(p):
     prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles)
     negative = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles)
-    prompts, networks = extra_networks.parse_prompts([prompt])
+    prompts, networks = extra_networks.parse_prompts([prompt], p.network_data)
     prompt = prompts[0] if len(prompts) > 0 else prompt
     return prompt, negative, networks
 

modules/processing.py

@@ -510,8 +510,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
             if not p.prompts:
                 break
-            p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts)
-
+            p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data)
 
             if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
                 p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)

modules/processing_diffusers.py

@@ -307,7 +307,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
             sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
             if len(p.refiner_prompt) > 0:
                 prompts = len(output.images)* [p.refiner_prompt]
-                prompts, p.network_data = extra_networks.parse_prompts(prompts)
+                prompts, p.network_data = extra_networks.parse_prompts(prompts, p.network_data)
                 reset_prompts = True
             if reset_prompts or ('base' in p.skip):
                 extra_networks.activate(p)

pipelines/minimax/minimax_nunchaku.py

@@ -1,5 +1,5 @@
 import diffusers
-from modules import shared
+from modules import shared, devices
 from modules.logger import log
 
 
@@ -9,8 +9,9 @@ def load_nunchaku(repo_id, load_config=None):
     from modules.attention import hijack_kernels
     hijack_kernels()
 
+    ## naive approach using diffusers class
     cls_name = diffusers.MiniMaxH3Transformer3DModel
-    log.debug(f'Load model: transformer="{repo_id}" subfolder="calibrated-8x20" cls={cls_name.__name__} loader="nunchaku-lite" args={load_config}')
+    log.debug(f'Load model: transformer="{repo_id}" subfolder="calibrated-8x20" cls={cls_name.__name__} loader="nunchaku-lite" patch=False args={load_config}')
     transformer = cls_name.from_pretrained(
         repo_id,
         subfolder="calibrated-8x20",
@@ -18,3 +19,43 @@ def load_nunchaku(repo_id, load_config=None):
         **load_config,
     )
     return transformer
+
+
+def load_nunchaku_patched(repo_id, load_config=None): # pylint: disable=unused-argument
+    import torch
+    from huggingface_hub import hf_hub_download
+    from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3RotaryPosEmbed
+
+    from modules.attention import hijack_kernels
+    hijack_kernels()
+
+    from installer import install
+    install('git+https://github.com/rootonchair/nunchaku-lite', 'nunchaku-lite')
+    from nunchaku_lite import core
+
+    filename = "svdq-int4_r32-minimax-h3-t2va.safetensors"
+    orig_repo = "MiniMaxAI/MiniMax-H3"
+    cls_name = diffusers.MiniMaxH3Transformer3DModel
+
+    log.debug(f'Load model: transformer="{repo_id}" fn="{filename}" cls={cls_name.__name__} loader="nunchaku-lite" patch=True args={load_config}')
+    ckpt = hf_hub_download(repo_id,
+                           filename=filename,
+                           cache_dir=shared.opts.diffusers_dir
+                          )
+    config = diffusers.MiniMaxH3Transformer3DModel.load_config(orig_repo, subfolder="transformer")
+    with torch.device("meta"):
+        transformer: diffusers.MiniMaxH3Transformer3DModel = diffusers.MiniMaxH3Transformer3DModel.from_config(config)
+
+    # rope.inv_freq is a non-persistent buffer (absent from the checkpoint), so the meta-assign load would leave it on the meta device — rebuild it off-meta first.
+    transformer.rope.inv_freq = MiniMaxH3RotaryPosEmbed(rope_freq_dim=transformer.config.rope_freq_dim, rope_theta=transformer.config.rope_theta).inv_freq # pylint: disable=no-member
+    core._patch_component(transformer, # pylint: disable=protected-access
+                          ckpt,
+                          target="manifest",
+                          precision="int4",
+                          torch_dtype=devices.dtype,
+                          device=devices.device,
+                          strict=True,
+                          adapter_options=None,
+                          assign=True,
+                         )
+    return transformer

scripts/dlss_ext.py

@@ -301,14 +301,14 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe
         t0 = time.time()
         if p:
             p.extra_generation_params["DLSSSuperSample"] = True
-        log.debug(f'DLSS: method=SuperSample quality="{ss_vsr_quality}" mode="{ss_size_mode}" scale={ss_scale_factor} width={ss_width} height={ss_height}')
+        log.info(f'DLSS: method=SuperSample quality="{ss_vsr_quality}" mode="{ss_size_mode}" scale={ss_scale_factor} width={ss_width} height={ss_height}')
         if ss_append:
             originals.extend(current_images)
         output = supersample(pkg_path, current_images, ss_vsr_quality, ss_size_mode, ss_scale_factor, ss_width, ss_height)
         if debug:
             log.trace(f'DLSS: method=SuperSample images={len(output) if output else 0} time={time.time() - t0:.3f}')
         if output:
-            images.extend(output)
+            images = output
             current_images = output
         t.ts('supersample', t0)
 
@@ -318,14 +318,14 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe
         t0 = time.time()
         if p:
             p.extra_generation_params["DLSSNeuralRender"] = True
-        log.debug(f'DLSS: method=NeuralRender style={nr_style} intensity={nr_intensity} tone={nr_local_tone} structure={nr_local_structure} skin={nr_skin_structure} scale={nr_upscaling_factor} preset={nr_preset} mask={nr_automatic_mask} model={nr_model_preset}')
+        log.info(f'DLSS: method=NeuralRender style={nr_style} intensity={nr_intensity} tone={nr_local_tone} structure={nr_local_structure} skin={nr_skin_structure} scale={nr_upscaling_factor} preset={nr_preset} mask={nr_automatic_mask} model={nr_model_preset}')
         if nr_append:
             originals.extend(current_images)
         output = neuralrender(pkg_path, current_images, nr_style, nr_intensity, nr_local_tone, nr_local_structure, nr_skin_structure, nr_upscaling_factor, nr_preset, nr_automatic_mask, nr_model_preset)
         if debug:
             log.trace(f'DLSS: method=NeuralRender images={len(output) if output else 0} time={time.time() - t0:.3f}')
         if output:
-            images.extend(output)
+            images = output
             current_images = output
         t.ts('neuralrender', t0)
 
@@ -333,12 +333,12 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe
         t0 = time.time()
         if p:
             p.extra_generation_params["DLSSFrameGen"] = True
-        log.debug(f'DLSS: method=FrameGen source={fg_source_fps} target={fg_target_fps} engine={fg_engine}')
+        log.info(f'DLSS: method=FrameGen source={fg_source_fps} target={fg_target_fps} engine={fg_engine}')
         output = framegen(pkg_path, current_images, fg_source_fps, fg_target_fps, fg_engine)
         if debug:
             log.trace(f'DLSS: method=FrameGen images={len(output) if output else 0} time={time.time() - t0:.3f}')
         if output:
-            images.extend(output)
+            images = output
             current_images = output
         t.ts('framegen', t0)
 

scripts/prompt_enhance_ext.py

@@ -6,7 +6,7 @@ import torch
 import transformers
 import gradio as gr
 from PIL import Image
-from modules import scripts_manager, shared, devices, errors, processing, sd_models, sd_modules, timer
+from modules import scripts_manager, shared, devices, errors, processing, sd_models, sd_modules, timer, extra_networks
 from modules import ui_control_helpers
 from modules.sd_offload_aux import register_aux, deregister_aux, move_aux_to_gpu, offload_aux
 from modules.logger import log
@@ -732,6 +732,8 @@ class PromptEnhanceScript(scripts_manager.Script):
         p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles)
         p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles)
         shared.prompt_styles.apply_styles_to_extra(p)
+        prompts, p.network_data = extra_networks.parse_prompts([p.prompt], p.network_data)
+        p.prompt = prompts[0]
         p.styles = []
         jobid = shared.state.begin('LLM')
         p.extra_generation_params['LLM'] = get_model_repo_from_display(llm_model)