From 489cbf804875e25c67bda7f6f95b7aa4ea614d49 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 13 Sep 2026 16:31:16 +0200 Subject: [PATCH] lora: keep parsed network data through pipeline Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/control/proc/hed.py | 4 +-- modules/extra_networks.py | 25 ++++++++------- modules/ltx/ltx_util.py | 2 +- modules/processing.py | 3 +- modules/processing_diffusers.py | 2 +- pipelines/minimax/minimax_nunchaku.py | 45 +++++++++++++++++++++++++-- scripts/dlss_ext.py | 12 +++---- scripts/prompt_enhance_ext.py | 4 ++- 9 files changed, 72 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7ed11f00..e073bd292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,6 +127,7 @@ Plus inevitable bug-fixes... - log: ansi color handling - lora: cleanup tags - lora: support transformer ref models + - lora: keep parsed network data through pipeline - lucida: handle requirements - lumina-dimoo: attention-kwargs, thanks @Anai-Guo - metadata: fix wildcard info diff --git a/modules/control/proc/hed.py b/modules/control/proc/hed.py index 06610921e..fb04e24fe 100644 --- a/modules/control/proc/hed.py +++ b/modules/control/proc/hed.py @@ -102,8 +102,8 @@ class HEDdetector: if scribble: detected_map = nms(detected_map, 127, 3.0) detected_map = cv2.GaussianBlur(detected_map, (0, 0), 3.0) - detected_map[detected_map > 4] = 255 - detected_map[detected_map < 255] = 0 + detected_map[detected_map > 4] = 255 # pylint: disable=unsupported-assignment-operation + detected_map[detected_map < 255] = 0 # pylint: disable=unsupported-assignment-operation if opts.control_move_processor: self.model.to('cpu') if output_type == "pil": diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 3d57d4d7f..678e21494 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -168,17 +168,20 @@ def parse_prompt(prompt: str | None) -> tuple[str, defaultdict[str, list[ExtraNe return updated_prompt, res -def parse_prompts(prompts: list[str], extra_data: defaultdict[str, list[ExtraNetworkParams]] | None = None): - updated_prompt_list: list[str] = [] - extra_data = extra_data or defaultdict(list) +def parse_prompts( + prompts: list[str], + extra_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, +): + updated_prompts: list[str] = [] + if extra_data is None: + extra_data = defaultdict(list) for prompt in prompts: updated_prompt, parsed_extra_data = parse_prompt(prompt) - if not extra_data: - extra_data = parsed_extra_data - elif parsed_extra_data: - extra_data = parsed_extra_data - else: - pass - updated_prompt_list.append(updated_prompt) + if parsed_extra_data: + for key, values in parsed_extra_data.items(): + for item in values: + if item not in extra_data[key]: + extra_data[key].append(item) - return updated_prompt_list, extra_data + updated_prompts.append(updated_prompt) + return updated_prompts, extra_data diff --git a/modules/ltx/ltx_util.py b/modules/ltx/ltx_util.py index bfbffa1bb..76f8ec9d7 100644 --- a/modules/ltx/ltx_util.py +++ b/modules/ltx/ltx_util.py @@ -249,7 +249,7 @@ def get_conditions(width, height, condition_strength, condition_images, conditio def get_prompts(p): prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) negative = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) - prompts, networks = extra_networks.parse_prompts([prompt]) + prompts, networks = extra_networks.parse_prompts([prompt], p.network_data) prompt = prompts[0] if len(prompts) > 0 else prompt return prompt, negative, networks diff --git a/modules/processing.py b/modules/processing.py index ad407b004..3adb1a721 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -510,8 +510,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) if not p.prompts: break - p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts) - + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner): p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index fcc33037a..3b8d5e129 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -307,7 +307,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta if len(p.refiner_prompt) > 0: prompts = len(output.images)* [p.refiner_prompt] - prompts, p.network_data = extra_networks.parse_prompts(prompts) + prompts, p.network_data = extra_networks.parse_prompts(prompts, p.network_data) reset_prompts = True if reset_prompts or ('base' in p.skip): extra_networks.activate(p) diff --git a/pipelines/minimax/minimax_nunchaku.py b/pipelines/minimax/minimax_nunchaku.py index d72de72f4..bb3baf248 100644 --- a/pipelines/minimax/minimax_nunchaku.py +++ b/pipelines/minimax/minimax_nunchaku.py @@ -1,5 +1,5 @@ import diffusers -from modules import shared +from modules import shared, devices from modules.logger import log @@ -9,8 +9,9 @@ def load_nunchaku(repo_id, load_config=None): from modules.attention import hijack_kernels hijack_kernels() + ## naive approach using diffusers class cls_name = diffusers.MiniMaxH3Transformer3DModel - log.debug(f'Load model: transformer="{repo_id}" subfolder="calibrated-8x20" cls={cls_name.__name__} loader="nunchaku-lite" args={load_config}') + log.debug(f'Load model: transformer="{repo_id}" subfolder="calibrated-8x20" cls={cls_name.__name__} loader="nunchaku-lite" patch=False args={load_config}') transformer = cls_name.from_pretrained( repo_id, subfolder="calibrated-8x20", @@ -18,3 +19,43 @@ def load_nunchaku(repo_id, load_config=None): **load_config, ) return transformer + + +def load_nunchaku_patched(repo_id, load_config=None): # pylint: disable=unused-argument + import torch + from huggingface_hub import hf_hub_download + from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3RotaryPosEmbed + + from modules.attention import hijack_kernels + hijack_kernels() + + from installer import install + install('git+https://github.com/rootonchair/nunchaku-lite', 'nunchaku-lite') + from nunchaku_lite import core + + filename = "svdq-int4_r32-minimax-h3-t2va.safetensors" + orig_repo = "MiniMaxAI/MiniMax-H3" + cls_name = diffusers.MiniMaxH3Transformer3DModel + + log.debug(f'Load model: transformer="{repo_id}" fn="{filename}" cls={cls_name.__name__} loader="nunchaku-lite" patch=True args={load_config}') + ckpt = hf_hub_download(repo_id, + filename=filename, + cache_dir=shared.opts.diffusers_dir + ) + config = diffusers.MiniMaxH3Transformer3DModel.load_config(orig_repo, subfolder="transformer") + with torch.device("meta"): + transformer: diffusers.MiniMaxH3Transformer3DModel = diffusers.MiniMaxH3Transformer3DModel.from_config(config) + + # rope.inv_freq is a non-persistent buffer (absent from the checkpoint), so the meta-assign load would leave it on the meta device — rebuild it off-meta first. + transformer.rope.inv_freq = MiniMaxH3RotaryPosEmbed(rope_freq_dim=transformer.config.rope_freq_dim, rope_theta=transformer.config.rope_theta).inv_freq # pylint: disable=no-member + core._patch_component(transformer, # pylint: disable=protected-access + ckpt, + target="manifest", + precision="int4", + torch_dtype=devices.dtype, + device=devices.device, + strict=True, + adapter_options=None, + assign=True, + ) + return transformer diff --git a/scripts/dlss_ext.py b/scripts/dlss_ext.py index 1c744a5f3..00a42670b 100644 --- a/scripts/dlss_ext.py +++ b/scripts/dlss_ext.py @@ -301,14 +301,14 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe t0 = time.time() if p: p.extra_generation_params["DLSSSuperSample"] = True - log.debug(f'DLSS: method=SuperSample quality="{ss_vsr_quality}" mode="{ss_size_mode}" scale={ss_scale_factor} width={ss_width} height={ss_height}') + log.info(f'DLSS: method=SuperSample quality="{ss_vsr_quality}" mode="{ss_size_mode}" scale={ss_scale_factor} width={ss_width} height={ss_height}') if ss_append: originals.extend(current_images) output = supersample(pkg_path, current_images, ss_vsr_quality, ss_size_mode, ss_scale_factor, ss_width, ss_height) if debug: log.trace(f'DLSS: method=SuperSample images={len(output) if output else 0} time={time.time() - t0:.3f}') if output: - images.extend(output) + images = output current_images = output t.ts('supersample', t0) @@ -318,14 +318,14 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe t0 = time.time() if p: p.extra_generation_params["DLSSNeuralRender"] = True - log.debug(f'DLSS: method=NeuralRender style={nr_style} intensity={nr_intensity} tone={nr_local_tone} structure={nr_local_structure} skin={nr_skin_structure} scale={nr_upscaling_factor} preset={nr_preset} mask={nr_automatic_mask} model={nr_model_preset}') + log.info(f'DLSS: method=NeuralRender style={nr_style} intensity={nr_intensity} tone={nr_local_tone} structure={nr_local_structure} skin={nr_skin_structure} scale={nr_upscaling_factor} preset={nr_preset} mask={nr_automatic_mask} model={nr_model_preset}') if nr_append: originals.extend(current_images) output = neuralrender(pkg_path, current_images, nr_style, nr_intensity, nr_local_tone, nr_local_structure, nr_skin_structure, nr_upscaling_factor, nr_preset, nr_automatic_mask, nr_model_preset) if debug: log.trace(f'DLSS: method=NeuralRender images={len(output) if output else 0} time={time.time() - t0:.3f}') if output: - images.extend(output) + images = output current_images = output t.ts('neuralrender', t0) @@ -333,12 +333,12 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe t0 = time.time() if p: p.extra_generation_params["DLSSFrameGen"] = True - log.debug(f'DLSS: method=FrameGen source={fg_source_fps} target={fg_target_fps} engine={fg_engine}') + log.info(f'DLSS: method=FrameGen source={fg_source_fps} target={fg_target_fps} engine={fg_engine}') output = framegen(pkg_path, current_images, fg_source_fps, fg_target_fps, fg_engine) if debug: log.trace(f'DLSS: method=FrameGen images={len(output) if output else 0} time={time.time() - t0:.3f}') if output: - images.extend(output) + images = output current_images = output t.ts('framegen', t0) diff --git a/scripts/prompt_enhance_ext.py b/scripts/prompt_enhance_ext.py index dfb4edba6..105c7fe8b 100644 --- a/scripts/prompt_enhance_ext.py +++ b/scripts/prompt_enhance_ext.py @@ -6,7 +6,7 @@ import torch import transformers import gradio as gr from PIL import Image -from modules import scripts_manager, shared, devices, errors, processing, sd_models, sd_modules, timer +from modules import scripts_manager, shared, devices, errors, processing, sd_models, sd_modules, timer, extra_networks from modules import ui_control_helpers from modules.sd_offload_aux import register_aux, deregister_aux, move_aux_to_gpu, offload_aux from modules.logger import log @@ -732,6 +732,8 @@ class PromptEnhanceScript(scripts_manager.Script): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) shared.prompt_styles.apply_styles_to_extra(p) + prompts, p.network_data = extra_networks.parse_prompts([p.prompt], p.network_data) + p.prompt = prompts[0] p.styles = [] jobid = shared.state.begin('LLM') p.extra_generation_params['LLM'] = get_model_repo_from_display(llm_model)