From 5ac87e105d986e707d9c74b166ba6342154cea19 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 12 Aug 2026 03:17:45 +0100 Subject: [PATCH 1/3] fix(video): fail the load when a modular component is missing load_components leaves a failed component as None and carries on, so the pipe reaches generation short of one and fails somewhere unrelated. Compare against what the workflow declares and refuse a pipe missing any of it. The conditioner fallback goes too, since loading it another way just defers the failure into generation. --- modules/video_models/video_modular.py | 64 ++++++++++++++++++++++----- pipelines/model_minimax.py | 11 +++-- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/modules/video_models/video_modular.py b/modules/video_models/video_modular.py index 8e057a6b1..6a70ebfb4 100644 --- a/modules/video_models/video_modular.py +++ b/modules/video_models/video_modular.py @@ -22,6 +22,49 @@ def is_modular(obj) -> bool: return 'Modular' in cls.__name__ +def component_quant_config(pipe) -> dict: + """Per-component quantization config, read off the pipeline's own component specs. + + Component names differ per architecture, so denoisers and text encoders are + recognized by the class each spec declares rather than listed here. The result + carries no default entry, so anything unrecognized loads unquantized. + """ + model_args = model_quant.create_config(module='Model') + config = {} + for name, spec in getattr(pipe, '_component_specs', {}).items(): # pylint: disable=protected-access + if getattr(spec, 'default_creation_method', None) != 'from_pretrained': + continue + cls = getattr(spec, 'type_hint', None) + origin = getattr(cls, '__module__', '') or '' + cls_name = getattr(cls, '__name__', '') or '' + if origin.startswith('transformers') and 'text_encoder' in name: + # MiniMax-H3's conditioner keeps its vision tower unquantized, which other arches on the same + # class do not do; a second modular arch would want this passed in rather than assumed here + te_args = model_quant.create_config(module='TE', modules_to_not_convert=['.model.visual']) + if 'quantization_config' in te_args: + config[name] = te_args['quantization_config'] + elif origin.startswith('diffusers') and ('Transformer' in cls_name or 'UNet' in cls_name) and 'quantization_config' in model_args: + config[name] = model_args['quantization_config'] + return config + + +def missing_components(pipe, workflow: str | None) -> list: + """Components the loaded workflow declares that did not materialize. + + A partition the workflow does not use is absent by design, so the comparison is + against the workflow's own expected components rather than every declared spec. + """ + blocks = getattr(pipe, '_blocks', None) # pylint: disable=protected-access + if blocks is None: + return [] + try: + expected = blocks.get_workflow(workflow) if workflow else blocks + names = [spec.name for spec in expected.expected_components] + except Exception: + names = list(getattr(pipe, '_component_specs', {})) # pylint: disable=protected-access + return [name for name in names if getattr(pipe, name, None) is None] + + def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision: str | None = None, offline_args: dict | None = None, base: bool = False): if repo_cls is None or isinstance(repo_cls, str): log.error(f'Load modular: repo="{repo}" cls="{repo_cls}" pipeline class not found: diffusers too old') @@ -37,19 +80,10 @@ def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision cache_dir=cache_dir, **offline_args, ) - # workflow selection stays out of from_pretrained: pruning the blocks tree to one task would disable runtime auto-dispatch between them; only the component fetch is restricted + # the workflow restricts the component fetch only: passing it to from_pretrained instead would prune the blocks tree to one task and disable runtime dispatch between them load_kwargs = {} - quant_config = {} - quant_args = model_quant.create_config(module='Model') - # TODO load_modular: need to handle component names dynamically - if 'quantization_config' in quant_args: - quant_config['transformer'] = quant_args['quantization_config'] - quant_config['transformer_ref'] = quant_args['quantization_config'] - te_args = model_quant.create_config(module='TE', modules_to_not_convert=['.model.visual']) # the conditioner's vision tower stays unquantized: quantized vision blocks have no validated precedent and only run for keyframe workflows - if 'quantization_config' in te_args: - quant_config['text_encoder'] = te_args['quantization_config'] + quant_config = component_quant_config(pipe) if quant_config: - # per-component dict without a default entry: only the listed components quantize while loading, everything else loads unquantized load_kwargs['quantization_config'] = quant_config log.debug(f'Load modular: quant={next(iter(quant_config.values())).__class__.__name__} modules={list(quant_config)}') pipe.load_components( @@ -61,10 +95,16 @@ def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision ) loaded = [name for name, component in pipe.components.items() if component is not None] empty = [name for name, component in pipe.components.items() if component is None] - pipe.sdnext_video_workflow = workflow # lets a pipe loaded outside the video registry report its own workflow + missing = missing_components(pipe, workflow) + pipe.sdnext_missing_components = missing # a caller that can recover a component clears its own entry + pipe.sdnext_video_workflow = workflow # the workflow this pipe was loaded for, which is what the reference-workflow guard reads; the executed task is chosen per request if hasattr(pipe, 'min_duration') and hasattr(pipe, 'fps'): pipe.sdnext_supported_min_frames = int(pipe.min_duration * pipe.fps) # fresh pipes report the true floor; still mode gates per instance log.info(f'Load modular: cls={pipe.__class__.__name__} workflow={workflow} components={loaded} empty={empty} time={time.time()-t0:.2f}') + if missing: + # load_components builds each component in its own try/except and reports a failure as a warning on the + # diffusers logger, so the reason is in the log above this line rather than in the exception path + log.error(f'Load modular: cls={pipe.__class__.__name__} workflow={workflow} missing={missing} components the workflow requires did not load') return pipe except Exception as e: log.error(f'Load modular: repo="{repo}" workflow={workflow} {e}') diff --git a/pipelines/model_minimax.py b/pipelines/model_minimax.py index 4ca367e69..5ff95bba9 100644 --- a/pipelines/model_minimax.py +++ b/pipelines/model_minimax.py @@ -23,12 +23,11 @@ def load_minimax(checkpoint_info, diffusers_load_config=None): # pylint: disable ) if pipe is None: return None - if pipe.text_encoder is None: - # TODO minimax missing te: we should never be here - import transformers - from pipelines import generic - text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3VLForConditionalGeneration, load_config=diffusers_load_config, allow_shared=False) - pipe.update_components(text_encoder=text_encoder) + missing = video_modular.missing_components(pipe, workflow) + if missing: + # a component that failed to build is unusable, and loading it by another route only defers the failure into generation as corrupt output + log.error(f'Load model: type=MiniMaxH3 repo="{repo_id}" workflow={workflow} missing={missing}') + return None video_modular.install_state_hook(pipe) video_load.loaded_model = None # image-path load invalidates the video tab's name cache From 113a74d91ab9617251fca3c60679f3fd91bcfea0 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 12 Aug 2026 03:17:45 +0100 Subject: [PATCH 2/3] fix(model): treat an unpopulated modular pipeline as not loaded None from a family loader means unhandled, so the chain falls through to the folder loader, which for a modular pipe builds an object holding only its from_config helpers and installs it. Refuse it when none of the from_pretrained specs materialized. --- modules/sd_models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/sd_models.py b/modules/sd_models.py index 1c5cf229d..daa4771ae 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -996,6 +996,17 @@ def load_diffuser(checkpoint_info: CheckpointInfo | None = None, op='model', rev log.error(f'Load {op}: name="{checkpoint_info.name if checkpoint_info is not None else None}" not loaded') return + # a family loader that returns None falls through to the generic folder loader, and for a modular pipeline that + # yields an object holding nothing but its from_config helpers, since a modular load registers the fetched + # components empty until load_components runs. it reports as a loaded model and then fails on the first + # component something reaches, far from the cause + specs = getattr(sd_model, '_component_specs', None) # pylint: disable=protected-access + if isinstance(specs, dict): + fetched = [name for name, spec in specs.items() if getattr(spec, 'default_creation_method', None) == 'from_pretrained'] + if len(fetched) > 0 and all(getattr(sd_model, name, None) is None for name in fetched): + log.error(f'Load {op}: name="{checkpoint_info.name if checkpoint_info is not None else None}" cls={sd_model.__class__.__name__} no components loaded') + return + set_overrides(sd_model, checkpoint_info, model_type) set_defaults(sd_model, checkpoint_info) From d62ca45eaa05b6a3800d85c1ebff3ce9249287cf Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 12 Aug 2026 03:17:45 +0100 Subject: [PATCH 3/3] refactor(video): load the modular denoiser and text encoder through the shared loaders Selected from the specs the pipe already declares, update_components first and load_components for the rest, so only what the workflow declares gets fetched. Weights land in hfcache. Covers the TODO on component names. Encoder sharing left off: shared_te_map matches a substring of the repo name and 4b hits _dynamic_4bit, redirecting Qwen3-VL to another repo. --- modules/video_models/video_modular.py | 64 ++++++++++++++++----------- pipelines/model_minimax.py | 1 + 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/modules/video_models/video_modular.py b/modules/video_models/video_modular.py index 6a70ebfb4..e272cb3e1 100644 --- a/modules/video_models/video_modular.py +++ b/modules/video_models/video_modular.py @@ -1,7 +1,7 @@ import time import logging import torch -from modules import shared, errors, devices, model_quant +from modules import shared, errors, devices from modules.logger import log @@ -22,30 +22,44 @@ def is_modular(obj) -> bool: return 'Modular' in cls.__name__ -def component_quant_config(pipe) -> dict: - """Per-component quantization config, read off the pipeline's own component specs. +def preload_components(pipe, workflow: str | None, load_config: dict | None = None) -> dict: + """Load the denoiser and text encoder through the shared loaders rather than the pipeline's own. - Component names differ per architecture, so denoisers and text encoders are - recognized by the class each spec declares rather than listed here. The result - carries no default entry, so anything unrecognized loads unquantized. + `load_components` fetches every component into the pipeline's cache directory with no + single-file override, no shared text encoder and no per-component quantization control. + The shared loaders do all three, and everything they need is already on the spec: repo, + subfolder and class. Components differ per architecture, so each is recognized by the + class its spec declares rather than by name. + + Only what the loaded workflow asks for is fetched, so an unused checkpoint partition is + never pulled. `load_components` afterwards loads whatever is still unset, which is the + tokenizer, processors, schedulers and VAEs. """ - model_args = model_quant.create_config(module='Model') - config = {} - for name, spec in getattr(pipe, '_component_specs', {}).items(): # pylint: disable=protected-access - if getattr(spec, 'default_creation_method', None) != 'from_pretrained': + from pipelines import generic + specs = getattr(pipe, '_component_specs', {}) # pylint: disable=protected-access + loaded = {} + for name in missing_components(pipe, workflow): + spec = specs.get(name) + if spec is None or getattr(spec, 'default_creation_method', None) != 'from_pretrained': continue + repo = getattr(spec, 'pretrained_model_name_or_path', None) cls = getattr(spec, 'type_hint', None) + if not repo or cls is None: + continue origin = getattr(cls, '__module__', '') or '' cls_name = getattr(cls, '__name__', '') or '' - if origin.startswith('transformers') and 'text_encoder' in name: - # MiniMax-H3's conditioner keeps its vision tower unquantized, which other arches on the same - # class do not do; a second modular arch would want this passed in rather than assumed here - te_args = model_quant.create_config(module='TE', modules_to_not_convert=['.model.visual']) - if 'quantization_config' in te_args: - config[name] = te_args['quantization_config'] - elif origin.startswith('diffusers') and ('Transformer' in cls_name or 'UNet' in cls_name) and 'quantization_config' in model_args: - config[name] = model_args['quantization_config'] - return config + subfolder = getattr(spec, 'subfolder', None) or name + component = None + if origin.startswith('diffusers') and ('Transformer' in cls_name or 'UNet' in cls_name): + component = generic.load_transformer(repo, cls_name=cls, load_config=load_config, subfolder=subfolder) + elif origin.startswith('transformers') and 'text_encoder' in name: + # sharing stays off: the shared map matches on class plus a substring of the repo name, so a + # quantized repo can be redirected to an unrelated model's encoder. enable it per arch once + # the pipeline's own encoder is known to be interchangeable with the shared one + component = generic.load_text_encoder(repo, cls_name=cls, load_config=load_config, subfolder=subfolder, allow_shared=False) + if component is not None: + loaded[name] = component + return loaded def missing_components(pipe, workflow: str | None) -> list: @@ -65,7 +79,7 @@ def missing_components(pipe, workflow: str | None) -> list: return [name for name in names if getattr(pipe, name, None) is None] -def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision: str | None = None, offline_args: dict | None = None, base: bool = False): +def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision: str | None = None, offline_args: dict | None = None, base: bool = False, load_config: dict | None = None): if repo_cls is None or isinstance(repo_cls, str): log.error(f'Load modular: repo="{repo}" cls="{repo_cls}" pipeline class not found: diffusers too old') return None @@ -81,16 +95,14 @@ def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision **offline_args, ) # the workflow restricts the component fetch only: passing it to from_pretrained instead would prune the blocks tree to one task and disable runtime dispatch between them - load_kwargs = {} - quant_config = component_quant_config(pipe) - if quant_config: - load_kwargs['quantization_config'] = quant_config - log.debug(f'Load modular: quant={next(iter(quant_config.values())).__class__.__name__} modules={list(quant_config)}') + preloaded = preload_components(pipe, workflow, load_config=load_config) + if preloaded: + pipe.update_components(**preloaded) # registered before the rest, which load_components then skips + log.debug(f'Load modular: preloaded={list(preloaded)}') pipe.load_components( workflow=workflow, dtype=devices.dtype, cache_dir=cache_dir, - **load_kwargs, **offline_args, ) loaded = [name for name, component in pipe.components.items() if component is not None] diff --git a/pipelines/model_minimax.py b/pipelines/model_minimax.py index 5ff95bba9..0d13cd065 100644 --- a/pipelines/model_minimax.py +++ b/pipelines/model_minimax.py @@ -20,6 +20,7 @@ def load_minimax(checkpoint_info, diffusers_load_config=None): # pylint: disable workflow=workflow, offline_args=offline_args, base=True, + load_config=diffusers_load_config, ) if pipe is None: return None