add ip_adapter to api and fix control

CHANGELOG.md

@@ -9,7 +9,7 @@
 - Control API scripts compatibility
 
 
-## Update for 2024-03-28
+## Update for 2024-03-29
 
 - **Features**:
   - **Gallery**: list, preview, search through all your images and videos!  
@@ -24,11 +24,14 @@
     both can still be installed by user if desired  
 - **Improvements**:
   - Styles apply wildcards to params
-  - Add API endpoint `/sdapi/v1/control` and util `cli/simple-control.py`  
+  - Add API endpoint `/sdapi/v1/control` and CLI util `cli/simple-control.py`  
     (in addition to previously added `/sdapi/v1/preprocessors` and `/sdapi/v1/masking`)  
     example:
-    > simple-control.py --prompt cat --input ~/generative/Samples/cutie-512.png --output /tmp/test.png --processed /tmp/proc.png --type controlnet --control 'Canny:Canny FP16:0.7, OpenPose:OpenPose FP16:0.8'
-  - Add API endpoint `/sdapi/v1/vqa` and util `cli/simple-vqa.py`
+    > simple-control.py --prompt 'woman in the city' --sampler UniPC --steps 20  
+    > --input ~/generative/Samples/cutie-512.png --output /tmp/test.png --processed /tmp/proc.png  
+    > --control 'Canny:Canny FP16:0.7, OpenPose:OpenPose FP16:0.8' --type controlnet  
+    > --ipadapter 'Plus:~/generative/Samples/cutie-512.png:0.5'  
+  - Add API endpoint `/sdapi/v1/vqa` and CLI util `cli/simple-vqa.py`
   - Make metadata in full screen viewer optional
   - Add VAE civitai scan metadata/preview
 - **Fixes**:

cli/simple-control.py

@@ -87,7 +87,24 @@ def generate(args): # pylint: disable=redefined-outer-name
                 'start': float(u[3].strip()) if len(u) > 3 else 0.0,
                 'end': float(u[4].strip()) if len(u) > 4 else 1.0,
             })
-        log.info(f"control options: {options['control']}")
+
+    if args.ipadapter is not None:
+        options['ip_adapter'] = []
+        for ipadapter in args.ipadapter.split(','):
+            u = ipadapter.split(':')
+            if len(u) < 2:
+                log.error(f'invalid ipadapter: {ipadapter}')
+                continue
+            if not os.path.exists(u[1].strip()):
+                log.error(f'invalid ipadapter image: {u[1]}')
+                continue
+            options['ip_adapter'].append({
+                'adapter': u[0].strip(),
+                'images': [encode(u[1].strip())],
+                'scale': float(u[2].strip()) if len(u) > 2 else 1.0,
+                'start': float(u[3].strip()) if len(u) > 3 else 0.1,
+                'end': float(u[4].strip()) if len(u) > 4 else 1.0,
+            })
 
     if args.mask is not None:
         options['mask'] = encode(args.mask)
@@ -108,8 +125,10 @@ def generate(args): # pylint: disable=redefined-outer-name
                 image.save(output)
                 log.info(f'image saved: size={image.size} filename={output}')
 
-    get_image(data['images'], args.output)
-    get_image(data['processed'], args.processed)
+    if 'images' in data:
+        get_image(data['images'], args.output)
+    if 'processed' in data:
+        get_image(data['processed'], args.processed)
 
 
 if __name__ == "__main__":
@@ -127,6 +146,7 @@ if __name__ == "__main__":
     parser.add_argument('--model', required=False, help='model name')
     parser.add_argument('--type', required=False, help='control type')
     parser.add_argument('--control', required=False, help='control units')
+    parser.add_argument('--ipadapter', required=False, help='ipadapter units')
     args = parser.parse_args()
     log.info(f'img2img: {args}')
     generate(args)

cli/simple-txt2img.js

@@ -20,6 +20,7 @@ const sd_options = {
   cfg_scale: 6,
   width: 512,
   height: 512,
+  /*
   // enable second pass
   enable_hr: true,
   // second pass: upscale
@@ -35,6 +36,7 @@ const sd_options = {
   refiner_start: 0.8,
   refiner_prompt: '',
   refiner_negative: '',
+  */
   // api return options
   save_images: false,
   send_images: true,

modules/api/api.py

@@ -105,34 +105,6 @@ class Api:
         shared.log.info(f'Browser session: user={user} client={req.client.host} agent={agent}')
         return {}
 
-    def prepare_img_gen_request(self, request):
-        if hasattr(request, "face") and request.face and not request.script_name and (not request.alwayson_scripts or "face" not in request.alwayson_scripts.keys()):
-            request.script_name = "face"
-            request.script_args = [
-                request.face.mode,
-                request.face.source_images,
-                request.face.ip_model,
-                request.face.ip_override_sampler,
-                request.face.ip_cache_model,
-                request.face.ip_strength,
-                request.face.ip_structure,
-                request.face.id_strength,
-                request.face.id_conditioning,
-                request.face.id_cache,
-                request.face.pm_trigger,
-                request.face.pm_strength,
-                request.face.pm_start,
-                request.face.fs_cache
-            ]
-            del request.face
-
-        if hasattr(request, "ip_adapter") and request.ip_adapter and request.script_name != "IP Adapter" and (not request.alwayson_scripts or "IP Adapter" not in request.alwayson_scripts.keys()):
-            request.alwayson_scripts = {} if request.alwayson_scripts is None else request.alwayson_scripts
-            request.alwayson_scripts["IP Adapter"] = {
-                "args": [request.ip_adapter.adapter, request.ip_adapter.scale, request.ip_adapter.image]
-            }
-            del request.ip_adapter
-
     def set_upscalers(self, req: dict):
         reqDict = vars(req)
         reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None)

modules/api/control.py

@@ -28,7 +28,7 @@ ReqControl = models.create_model_from_signature(
         {"key": "send_images", "type": bool, "default": True},
         {"key": "save_images", "type": bool, "default": False},
         {"key": "alwayson_scripts", "type": dict, "default": {}},
-        {"key": "ip_adapter", "type": Optional[models.ItemIPAdapter], "default": None, "exclude": True},
+        {"key": "ip_adapter", "type": Optional[List[models.ItemIPAdapter]], "default": None, "exclude": True},
         {"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True},
         {"key": "control", "type": Optional[List[ItemControl]], "default": [], "exclude": True},
     ]
@@ -93,9 +93,27 @@ class APIControl():
             ]
             del req.face
 
+    def prepare_ip_adapter(self, request):
+        if hasattr(request, "ip_adapter") and request.ip_adapter:
+            args = { 'ip_adapter_names': [], 'ip_adapter_scales': [], 'ip_adapter_starts': [], 'ip_adapter_ends': [], 'ip_adapter_images': [] }
+            for ipadapter in request.ip_adapter:
+                if not ipadapter.images or len(ipadapter.images) == 0:
+                    continue
+                args['ip_adapter_names'].append(ipadapter.adapter)
+                args['ip_adapter_scales'].append(ipadapter.scale)
+                args['ip_adapter_starts'].append(ipadapter.start)
+                args['ip_adapter_ends'].append(ipadapter.end)
+                args['ip_adapter_images'].append([helpers.decode_base64_to_image(x) for x in ipadapter.images])
+            del request.ip_adapter
+            return args
+        else:
+            return {}
+
     def prepare_control(self, req):
         from modules.control.unit import Unit, unit_types
         req.units = []
+        if req.unit_type is None:
+            return req.control
         if req.unit_type not in unit_types:
             shared.log.error(f'Control uknown unit type: type={req.unit_type} available={unit_types}')
             return req.control
@@ -122,7 +140,6 @@ class APIControl():
         # prepare args
         args = req.copy(update={  # Override __init__ params
             "sampler_index": processing_helpers.get_sampler_index(req.sampler_name),
-            "no_save": not req.save_images,
             "is_generator": False,
             "inputs": [helpers.decode_base64_to_image(x) for x in req.inputs] if req.inputs else None,
             "inits": [helpers.decode_base64_to_image(x) for x in req.inits] if req.inits else None,
@@ -134,10 +151,10 @@ class APIControl():
         # run
         with self.queue_lock:
             shared.state.begin('api-control', api=True)
-
             output_images = []
             output_processed = []
             output_info = ''
+            run.control_set({ 'do_not_save_grid': not req.save_images, 'do_not_save_samples': not req.save_images, **self.prepare_ip_adapter(req) })
             res = run.control_run(**args)
             for item in res:
                 if len(item) > 0 and (isinstance(item[0], list) or item[0] is None): # output_images
@@ -146,7 +163,6 @@ class APIControl():
                     output_info += item[2] if len(item) > 2 and item[2] is not None else ''
                 else:
                     output_info += item
-
             shared.state.end(api=False)
 
         # return

modules/api/generate.py

@@ -22,7 +22,6 @@ class APIGenerate():
         args.pop('alwayson_scripts', None)
         args.pop('face', None)
         args.pop('face_id', None)
-        args.pop('ip_adapter', None)
         args.pop('save_images', None)
         return args
 
@@ -61,6 +60,23 @@ class APIGenerate():
             ]
             del request.face
 
+    def prepare_ip_adapter(self, request, p):
+        if hasattr(request, "ip_adapter") and request.ip_adapter:
+            p.ip_adapter_names = []
+            p.ip_adapter_scales = []
+            p.ip_adapter_starts = []
+            p.ip_adapter_ends = []
+            p.ip_adapter_images = []
+            for ipadapter in request.ip_adapter:
+                if not ipadapter.images or len(ipadapter.images) == 0:
+                    continue
+                p.ip_adapter_names.append(ipadapter.adapter)
+                p.ip_adapter_scales.append(ipadapter.scale)
+                p.ip_adapter_starts.append(ipadapter.start)
+                p.ip_adapter_ends.append(ipadapter.end)
+                p.ip_adapter_images.append([helpers.decode_base64_to_image(x) for x in ipadapter.images])
+            del request.ip_adapter
+
     def post_text2img(self, txt2imgreq: models.ReqTxt2Img):
         self.prepare_face_module(txt2imgreq)
         script_runner = scripts.scripts_txt2img
@@ -81,6 +97,7 @@ class APIGenerate():
         send_images = args.pop('send_images', True)
         with self.queue_lock:
             p = StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)
+            self.prepare_ip_adapter(txt2imgreq, p)
             p.scripts = script_runner
             p.outpath_grids = shared.opts.outdir_grids or shared.opts.outdir_txt2img_grids
             p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_txt2img_samples
@@ -123,6 +140,7 @@ class APIGenerate():
         send_images = args.pop('send_images', True)
         with self.queue_lock:
             p = StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)
+            self.prepare_ip_adapter(img2imgreq, p)
             p.init_images = [helpers.decode_base64_to_image(x) for x in init_images]
             p.scripts = script_runner
             p.outpath_grids = shared.opts.outdir_img2img_grids

modules/api/middleware.py

@@ -43,7 +43,7 @@ def setup_middleware(app: FastAPI, cmd_opts):
             endpoint = req.scope.get('path', 'err')
             token = req.cookies.get("access-token") or req.cookies.get("access-token-unsecure")
             if (cmd_opts.api_log or cmd_opts.api_only) and endpoint.startswith('/sdapi'):
-                if '/sdapi/v1/log' or '/sdapi/v1/browser' in endpoint:
+                if '/sdapi/v1/log' in endpoint or '/sdapi/v1/browser' in endpoint:
                     return res
                 log.info('API {user} {code} {prot}/{ver} {method} {endpoint} {cli} {duration}'.format( # pylint: disable=consider-using-f-string, logging-format-interpolation
                     user = app.tokens.get(token) if hasattr(app, 'tokens') else None,

modules/api/models.py

@@ -149,9 +149,11 @@ class ItemEmbedding(BaseModel):
     vectors: int = Field(title="Vectors", description="The number of vectors in the embedding")
 
 class ItemIPAdapter(BaseModel):
-    adapter: str = Field(title="Adapter", default="Base", description="Adapter to use")
-    image: str = Field(title="Image", default="", description="Adapter image, must be a base64 string containing the image's data.")
-    scale: float = Field(title="Scale", default=0.5, gt=0, le=1, description="Scale of the adapter image, must be between 0 and 1.")
+    adapter: str = Field(title="Adapter", default="Base", description="")
+    images: List[str] = Field(title="Image", default=[], description="")
+    scale: float = Field(title="Scale", default=0.5, gt=0, le=1, description="")
+    start: float = Field(title="Start", default=0.0, gt=0, le=1, description="")
+    end: float = Field(title="End", default=1.0, gt=0, le=1, description="")
 
 class ItemFace(BaseModel):
     mode: str = Field(title="Mode", default="FaceID", description="The mode to use (available values: FaceID, FaceSwap, PhotoMaker, InstantID).")
@@ -204,7 +206,7 @@ ReqTxt2Img = PydanticModelGenerator(
         {"key": "send_images", "type": bool, "default": True},
         {"key": "save_images", "type": bool, "default": False},
         {"key": "alwayson_scripts", "type": dict, "default": {}},
-        {"key": "ip_adapter", "type": Optional[ItemIPAdapter], "default": None, "exclude": True},
+        {"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True},
         {"key": "face", "type": Optional[ItemFace], "default": None, "exclude": True},
     ]
 ).generate_model()
@@ -229,7 +231,7 @@ ReqImg2Img = PydanticModelGenerator(
         {"key": "send_images", "type": bool, "default": True},
         {"key": "save_images", "type": bool, "default": False},
         {"key": "alwayson_scripts", "type": dict, "default": {}},
-        {"key": "ip_adapter", "type": Optional[ItemIPAdapter], "default": None, "exclude": True},
+        {"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True},
         {"key": "face_id", "type": Optional[ItemFace], "default": None, "exclude": True},
     ]
 ).generate_model()

modules/control/run.py

@@ -22,6 +22,7 @@ debug('Trace: CONTROL')
 pipe = None
 instance = None
 original_pipeline = None
+p_extra_args = {}
 
 
 def restore_pipeline():
@@ -42,8 +43,19 @@ def terminate(msg):
     return msg
 
 
-def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], inits: List[Image.Image] = [], mask: Image.Image = None, unit_type: str = None, is_generator: bool = True, input_type: int = 0,
-                prompt: str = '', negative: str = '', styles: List[str] = [], steps: int = 20, sampler_index: int = None,
+def control_set(kwargs):
+    if kwargs:
+        global p_extra_args # pylint: disable=global-statement
+        p_extra_args = {}
+        debug(f'Control extra args: {kwargs}')
+    for k, v in kwargs.items():
+        p_extra_args[k] = v
+
+
+def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], inits: List[Image.Image] = [], mask: Image.Image = None, unit_type: str = None, is_generator: bool = True,
+                input_type: int = 0,
+                prompt: str = '', negative: str = '', styles: List[str] = [],
+                steps: int = 20, sampler_index: int = None,
                 seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1,
                 cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, sag_scale: float = 0.0, cfg_end: float = 1.0,
                 full_quality: bool = True, restore_faces: bool = False, tiling: bool = False,
@@ -56,7 +68,6 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
                 enable_hr: bool = False, hr_sampler_index: int = None, hr_denoising_strength: float = 0.3, hr_upscaler: str = None, hr_force: bool = False, hr_second_pass_steps: int = 20,
                 hr_scale: float = 1.0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0.0, refiner_prompt: str = '', refiner_negative: str = '',
                 video_skip_frames: int = 0, video_type: str = 'None', video_duration: float = 2.0, video_loop: bool = False, video_pad: int = 0, video_interpolate: int = 0,
-                no_save: bool = False,
                 *input_script_args
         ):
     global instance, pipe, original_pipeline # pylint: disable=global-statement
@@ -134,11 +145,14 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
     p.refiner_start = refiner_start
     p.refiner_prompt = refiner_prompt
     p.refiner_negative = refiner_negative
-    p.do_not_save_grid = no_save
-    p.do_not_save_samples = no_save
     if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0):
         p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.width * p.hr_scale / 8), 8 * int(p.height * p.hr_scale / 8)
 
+    global p_extra_args # pylint: disable=global-statement
+    for k, v in p_extra_args.items():
+        setattr(p, k, v)
+    p_extra_args = {}
+
     if shared.sd_model is None:
         shared.log.warning('Model not loaded')
         return [], '', '', 'Error: model not loaded'

modules/face/instantid.py

@@ -22,7 +22,7 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_images, stre
         return None
 
     c = shared.sd_model.__class__.__name__ if shared.sd_loaded else ''
-    if c != 'StableDiffusionXLPipeline':
+    if c != 'StableDiffusionXLPipeline' and c != 'StableDiffusionXLInstantIDPipeline':
         shared.log.warning(f'InstantID invalid base model: current={c} required=StableDiffusionXLPipeline')
         return None
 

modules/ui_control.py

@@ -460,7 +460,6 @@ def create_ui(_blocks: gr.Blocks=None):
                 input_script_args = scripts.scripts_current.setup_ui(parent='control', accordion=True)
 
             # handlers
-
             for btn in input_buttons:
                 btn.click(fn=helpers.copy_input, inputs=[input_mode, btn, input_image, input_resize, input_inpaint], outputs=[input_image, input_resize, input_inpaint], _js='controlInputMode')
                 btn.click(fn=helpers.transfer_input, inputs=[btn], outputs=[input_image, input_resize, input_inpaint] + input_buttons)
@@ -571,7 +570,6 @@ def create_ui(_blocks: gr.Blocks=None):
             generation_parameters_copypaste.register_paste_params_button(bindings)
             masking.bind_controls([input_image, input_inpaint, input_resize], preview_process, output_image)
 
-
             if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only
                 from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs, test_lite
                 gr.HTML('<br><h1>Debug</h1><br>')