moondream2, controlnetunion, etc.

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Update for 2025-01-24
+## Update for 2025-01-26
 
 - **Contributing**:  
   - if you'd like to contribute, please see updated [contributing](https://github.com/vladmandic/automatic/blob/dev/CONTRIBUTING) guidelines
@@ -36,7 +36,7 @@
   - use the us server by default on linux  
   - use pytorch test branch on windows  
   - extend the supported python versions  
-  - removed diffusers attention hijack as it is a duplicate of dynamic attention bmm  
+  - improve sdpa dynamic attention  
 - **Torch FP8**
   - uses torch `float8_e4m3fn` or `float8_e5m2` as data storage and performs dynamic upcasting to compute `dtype` as needed  
   - compatible with most `unet` and `transformer` models: e.g. *sd15, sdxl, sd35, flux.1, hunyuan-video, ltx-video, etc.*  
@@ -77,6 +77,10 @@
   - xyz grid handle invalid values  
   - omnigen pipeline handle float seeds  
   - correct logging of docker status on logs, thanks @kmscode  
+  - fix omnigen  
+  - fix docker status reporting  
+  - vlm/vqa with moondream2  
+  - rocm do not override triton installation  
 
 ## Update for 2025-01-15
 

modules/control/run.py

@@ -148,7 +148,9 @@ def check_active(p, unit_type, units):
             active_end.append(float(u.end))
             p.guess_mode = u.guess
             if isinstance(u.mode, str):
-                p.control_mode = u.choices.index(u.mode) if u.mode in u.choices else 0
+                if not hasattr(p, 'control_mode'):
+                    p.control_mode = []
+                p.control_mode.append(u.choices.index(u.mode) if u.mode in u.choices else 0)
                 p.is_tile = p.is_tile or 'tile' in u.mode.lower()
                 p.control_tile = u.tile
                 p.extra_generation_params["Control mode"] = u.mode
@@ -427,8 +429,6 @@ def control_run(state: str = '',
     else:
         original_pipeline = None
 
-    possible = sd_models.get_call(pipe).keys()
-
     try:
         with devices.inference_context():
             if isinstance(inputs, str): # only video, the rest is a list
@@ -460,6 +460,7 @@ def control_run(state: str = '',
                 if pipe is None: # pipe may have been reset externally
                     pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits)
                     debug_log(f'Control pipeline reinit: class={pipe.__class__.__name__}')
+                possible = sd_models.get_call(pipe).keys()
                 processed_image = None
                 if frame is not None:
                     inputs = [Image.fromarray(frame)] # cv2 to pil

modules/control/units/controlnet.py

@@ -291,7 +291,7 @@ class ControlNet():
                         log.debug(f'Control {what} model NNCF Compress: id="{model_id}"')
                         from installer import install
                         install('nncf==2.7.0', quiet=True)
-                        from modules.sd_models_compile import nncf_compress_model
+                        from modules.model_quant import nncf_compress_model
                         self.model = nncf_compress_model(self.model)
                     except Exception as e:
                         log.error(f'Control {what} model NNCF Compression failed: id="{model_id}" {e}')
@@ -299,7 +299,7 @@ class ControlNet():
                     try:
                         log.debug(f'Control {what} model Optimum Quanto: id="{model_id}"')
                         model_quant.load_quanto('Load model: type=ControlNet')
-                        from modules.sd_models_compile import optimum_quanto_model
+                        from modules.model_quant import optimum_quanto_model
                         self.model = optimum_quanto_model(self.model)
                     except Exception as e:
                         log.error(f'Control {what} model Optimum Quanto: id="{model_id}" {e}')
@@ -335,9 +335,15 @@ class ControlNetPipeline():
             return
         elif detect.is_sdxl(pipeline) and len(controlnets) > 0:
             from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetUnionPipeline
-            if controlnet.__class__.__name__ == 'ControlNetUnionModel':
+            classes = [c.__class__.__name__ for c in controlnets]
+            if any(c == 'ControlNetUnionModel' for c in classes):
+                if not all(c == 'ControlNetUnionModel' for c in classes):
+                    log.warning(f'Control {what}: units={classes} mixed type')
                 cls = StableDiffusionXLControlNetUnionPipeline
-                controlnets = controlnets[0] # using only first one
+                if len(controlnets) > 1:
+                    # TODO controlnet-union multi-unit
+                    log.warning(f'Control {what}: units={classes} supports single unit only')
+                controlnets = controlnets[0]
             else:
                 cls = StableDiffusionXLControlNetPipeline
             self.pipeline = cls(

modules/processing_class.py

@@ -354,7 +354,8 @@ class StableDiffusionProcessing:
         raise NotImplementedError
 
     def close(self):
-        self.sampler = None # pylint: disable=attribute-defined-outside-init
+        self.sampler = None
+        self.scripts = None
 
 
 class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):

modules/vqa.py

@@ -117,7 +117,12 @@ def pix(question: str, image: Image.Image, repo: str = None):
 def moondream(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
-        model = transformers.AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True, cache_dir=shared.opts.hfcache_dir) # revision = "2024-03-05"
+        model = transformers.AutoModelForCausalLM.from_pretrained(
+            repo,
+            revision="2024-08-26",
+            trust_remote_code=True,
+            cache_dir=shared.opts.hfcache_dir
+        )
         processor = transformers.AutoTokenizer.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
         loaded = repo
         model.eval()