modular handle module with remote-code

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -119,6 +119,7 @@ Plus inevitable bug-fixes...
   - lucida: handle requirements
   - lumina-dimoo: attention-kwargs, thanks @Anai-Guo
   - minimax: crop image to video aspect ratio
+  - modular: handle module with remote-code
   - network: improve type/version lookup
   - offline: honor offline mode for more models, thanks @ryanmeador
   - openvino: optimize recompile checks and lora loading

modules/minimax/minimax_ui.py

@@ -24,6 +24,7 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_
             with gr.Accordion(open=True, label='Parameters', elem_id='minimax_param_accordion') as _param_accordion:
                 with gr.Row():
                     width, height = ui_sections.create_resolution_inputs('minimax', default_width=1024, default_height=576, step=32)
+                    btn_detect_image_size = ToolButton(value=ui_symbols.detect, elem_id="minimax_resize_detect_size")
                 with gr.Row():
                     steps = gr.Slider(minimum=2, maximum=100, step=1, label="MiniMax steps", elem_id='minimax_steps', value=30)
                     frames = gr.Slider(label='MiniMax frames', minimum=22, maximum=362, step=17, value=124, elem_id='minimax_frames')
@@ -68,8 +69,18 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_
         model_info = next((m for m in models['MiniMax'] if m.name == model_name), None)
         minimax_video.load_model(model_info.name if model_info is not None else None)
 
+    def on_image_size(init_image):
+        if init_image is not None:
+            try:
+                width, height = init_image.size
+                return gr.update(value=width), gr.update(value=height)
+            except Exception:
+                pass
+        return gr.update(), gr.update()
+
     model.change(fn=on_change, inputs=[model, init_image], outputs=[workflow, input_accordion, reference_accordion], show_progress='hidden')
     init_image.change(fn=on_change, inputs=[model, init_image], outputs=[workflow, input_accordion, reference_accordion], show_progress='hidden')
+    btn_detect_image_size.click(fn=on_image_size, inputs=[init_image], outputs=[width, height])
     btn_load.click(fn=on_load, inputs=[model], outputs=[])
 
     task_id = gr.Textbox(visible=False, value='')

modules/modular_load.py

@@ -59,18 +59,21 @@ def preload_components(pipe, workflow: str | None, load_config: dict | None = No
         if spec is None or getattr(spec, 'default_creation_method', None) != 'from_pretrained':
             continue
         repo = getattr(spec, 'pretrained_model_name_or_path', None)
-        cls = getattr(spec, 'type_hint', None)
+        cls = getattr(spec, 'type_hint', None) or {}
         if not repo or cls is None:
             continue
         origin = getattr(cls, '__module__', '') or ''
-        cls_name = getattr(cls, '__name__', '') or '' # TODO preload: components with remote code resolve to cls none
+        cls_name = getattr(cls, '__name__', '') or ''
         subfolder = getattr(spec, 'subfolder', None) or name
         component = None
         if origin.startswith('diffusers') and ('Transformer' in cls_name or 'UNet' in cls_name):
             component = generic.load_transformer(repo, cls_name=cls, load_config=load_config, subfolder=subfolder, trust_remote_code=True)
-        elif origin.startswith('transformers') and 'text_encoder' in name:
+        elif origin.startswith('transformers') and ('text_encoder' in name):
             # shared substitution is on: the map matches class plus a substring of the repo name, so its entries have to run narrow before broad
             component = generic.load_text_encoder(repo, cls_name=cls, load_config=load_config, subfolder=subfolder)
+        if 'transformer' in name:
+            # fallback for component with remote-code as it does not have resolvable cls
+            component = generic.load_transformer(repo, cls_name=None, load_config=load_config, subfolder=subfolder, trust_remote_code=True)
         if component is not None:
             loaded[name] = component
     return loaded

pipelines/generic_transformer.py

@@ -48,6 +48,9 @@ def load_transformer(
         modules_to_not_convert = []
     if modules_dtype_dict is None:
         modules_dtype_dict = {}
+    if cls_name is None:
+        from diffusers import AutoModel
+        cls_name = AutoModel
     offline_args = {'local_files_only': True} if shared.opts.offline_mode else {}
     jobid = shared.state.begin('Load DiT')
     try:
@@ -75,11 +78,14 @@ def load_transformer(
             if trust_remote_code:
                 load_args['trust_remote_code'] = True
             load_kwargs = {**load_args, **quant_args, **offline_args, **kwargs}
-            return cls_name.from_pretrained(
+            module = cls_name.from_pretrained(
                 repo_id,
                 cache_dir=shared.opts.hfcache_dir,
                 **load_kwargs,
             )
+            if cls_name.__name__ == 'AutoModel':
+                log.debug(f'Load model: transformer="{repo_id}" cls={module.__class__.__name__}')
+            return module
 
         local_file = None
         override_name = None
@@ -158,13 +164,11 @@ def load_transformer(
                 **load_kwargs,
             )
 
-        # 4. default loading from diffusers repo (also the fallback when an
-        # incompatible override is dropped above)
+        # 4. default loading from local file (also the fallback when an incompatible override is dropped above)        # 5. default loading from diffusers repo (also the fallback when an incompatible override is dropped above)
         else:
             transformer = load_from_repo()
 
-        # mark the dropdown selection as loaded so the slot's onchange callback
-        # does not force a redundant full reload for an already-consumed override
+        # mark the dropdown selection as loaded so the slot's onchange callback, does not force a redundant full reload for an already-consumed override
         if transformer is not None and override_name is not None and getattr(shared.opts, override_opt, None) == override_name:
             setattr(sd_unet, tracker_attr, override_name)
 
@@ -192,8 +196,7 @@ def load_transformer(
         log.debug(f'Load model: transformer="{repo_id}" quant="{quant_type}" size={module_size:.3f} params={param_num:.3f} memory={module_memory}')
 
     try:
-        # quantized models legitimately report the storage dtype (e.g. fp8 comfy_quant
-        # adopted via SDNQ); the compute dtype lives in the dequantizers, not the params
+        # quantized models legitimately report the storage dtype (e.g. fp8 comfy_quant adopted via SDNQ); the compute dtype lives in the dequantizers, not the params
         if getattr(transformer, 'quantization_config', None) is None:
             actual_dtype = transformer.dtype
             if isinstance(actual_dtype, torch.dtype) and isinstance(dtype, torch.dtype) and actual_dtype != dtype: