From bd7ba768b1418cc1ea5e70186e35e2c4c36271f9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Mar 2024 10:26:27 -0400 Subject: [PATCH] add ip_adapter to api and fix control --- CHANGELOG.md | 11 +++++++---- cli/simple-control.py | 26 +++++++++++++++++++++++--- cli/simple-txt2img.js | 2 ++ modules/api/api.py | 28 ---------------------------- modules/api/control.py | 24 ++++++++++++++++++++---- modules/api/generate.py | 20 +++++++++++++++++++- modules/api/middleware.py | 2 +- modules/api/models.py | 12 +++++++----- modules/control/run.py | 24 +++++++++++++++++++----- modules/face/instantid.py | 2 +- modules/ui_control.py | 2 -- 11 files changed, 99 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11d9990d4..577f4163a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Control API scripts compatibility -## Update for 2024-03-28 +## Update for 2024-03-29 - **Features**: - **Gallery**: list, preview, search through all your images and videos! @@ -24,11 +24,14 @@ both can still be installed by user if desired - **Improvements**: - Styles apply wildcards to params - - Add API endpoint `/sdapi/v1/control` and util `cli/simple-control.py` + - Add API endpoint `/sdapi/v1/control` and CLI util `cli/simple-control.py` (in addition to previously added `/sdapi/v1/preprocessors` and `/sdapi/v1/masking`) example: - > simple-control.py --prompt cat --input ~/generative/Samples/cutie-512.png --output /tmp/test.png --processed /tmp/proc.png --type controlnet --control 'Canny:Canny FP16:0.7, OpenPose:OpenPose FP16:0.8' - - Add API endpoint `/sdapi/v1/vqa` and util `cli/simple-vqa.py` + > simple-control.py --prompt 'woman in the city' --sampler UniPC --steps 20 + > --input ~/generative/Samples/cutie-512.png --output /tmp/test.png --processed /tmp/proc.png + > --control 'Canny:Canny FP16:0.7, OpenPose:OpenPose FP16:0.8' --type controlnet + > --ipadapter 'Plus:~/generative/Samples/cutie-512.png:0.5' + - Add API endpoint `/sdapi/v1/vqa` and CLI util `cli/simple-vqa.py` - Make metadata in full screen viewer optional - Add VAE civitai scan metadata/preview - **Fixes**: diff --git a/cli/simple-control.py b/cli/simple-control.py index 4eddeaf6c..a0246fdcc 100755 --- a/cli/simple-control.py +++ b/cli/simple-control.py @@ -87,7 +87,24 @@ def generate(args): # pylint: disable=redefined-outer-name 'start': float(u[3].strip()) if len(u) > 3 else 0.0, 'end': float(u[4].strip()) if len(u) > 4 else 1.0, }) - log.info(f"control options: {options['control']}") + + if args.ipadapter is not None: + options['ip_adapter'] = [] + for ipadapter in args.ipadapter.split(','): + u = ipadapter.split(':') + if len(u) < 2: + log.error(f'invalid ipadapter: {ipadapter}') + continue + if not os.path.exists(u[1].strip()): + log.error(f'invalid ipadapter image: {u[1]}') + continue + options['ip_adapter'].append({ + 'adapter': u[0].strip(), + 'images': [encode(u[1].strip())], + 'scale': float(u[2].strip()) if len(u) > 2 else 1.0, + 'start': float(u[3].strip()) if len(u) > 3 else 0.1, + 'end': float(u[4].strip()) if len(u) > 4 else 1.0, + }) if args.mask is not None: options['mask'] = encode(args.mask) @@ -108,8 +125,10 @@ def generate(args): # pylint: disable=redefined-outer-name image.save(output) log.info(f'image saved: size={image.size} filename={output}') - get_image(data['images'], args.output) - get_image(data['processed'], args.processed) + if 'images' in data: + get_image(data['images'], args.output) + if 'processed' in data: + get_image(data['processed'], args.processed) if __name__ == "__main__": @@ -127,6 +146,7 @@ if __name__ == "__main__": parser.add_argument('--model', required=False, help='model name') parser.add_argument('--type', required=False, help='control type') parser.add_argument('--control', required=False, help='control units') + parser.add_argument('--ipadapter', required=False, help='ipadapter units') args = parser.parse_args() log.info(f'img2img: {args}') generate(args) diff --git a/cli/simple-txt2img.js b/cli/simple-txt2img.js index ae4d7af0c..46d09b3a2 100755 --- a/cli/simple-txt2img.js +++ b/cli/simple-txt2img.js @@ -20,6 +20,7 @@ const sd_options = { cfg_scale: 6, width: 512, height: 512, + /* // enable second pass enable_hr: true, // second pass: upscale @@ -35,6 +36,7 @@ const sd_options = { refiner_start: 0.8, refiner_prompt: '', refiner_negative: '', + */ // api return options save_images: false, send_images: true, diff --git a/modules/api/api.py b/modules/api/api.py index b4d5da9c9..e4dce4cec 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -105,34 +105,6 @@ class Api: shared.log.info(f'Browser session: user={user} client={req.client.host} agent={agent}') return {} - def prepare_img_gen_request(self, request): - if hasattr(request, "face") and request.face and not request.script_name and (not request.alwayson_scripts or "face" not in request.alwayson_scripts.keys()): - request.script_name = "face" - request.script_args = [ - request.face.mode, - request.face.source_images, - request.face.ip_model, - request.face.ip_override_sampler, - request.face.ip_cache_model, - request.face.ip_strength, - request.face.ip_structure, - request.face.id_strength, - request.face.id_conditioning, - request.face.id_cache, - request.face.pm_trigger, - request.face.pm_strength, - request.face.pm_start, - request.face.fs_cache - ] - del request.face - - if hasattr(request, "ip_adapter") and request.ip_adapter and request.script_name != "IP Adapter" and (not request.alwayson_scripts or "IP Adapter" not in request.alwayson_scripts.keys()): - request.alwayson_scripts = {} if request.alwayson_scripts is None else request.alwayson_scripts - request.alwayson_scripts["IP Adapter"] = { - "args": [request.ip_adapter.adapter, request.ip_adapter.scale, request.ip_adapter.image] - } - del request.ip_adapter - def set_upscalers(self, req: dict): reqDict = vars(req) reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None) diff --git a/modules/api/control.py b/modules/api/control.py index f34ba7cfc..99df3fb89 100644 --- a/modules/api/control.py +++ b/modules/api/control.py @@ -28,7 +28,7 @@ ReqControl = models.create_model_from_signature( {"key": "send_images", "type": bool, "default": True}, {"key": "save_images", "type": bool, "default": False}, {"key": "alwayson_scripts", "type": dict, "default": {}}, - {"key": "ip_adapter", "type": Optional[models.ItemIPAdapter], "default": None, "exclude": True}, + {"key": "ip_adapter", "type": Optional[List[models.ItemIPAdapter]], "default": None, "exclude": True}, {"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True}, {"key": "control", "type": Optional[List[ItemControl]], "default": [], "exclude": True}, ] @@ -93,9 +93,27 @@ class APIControl(): ] del req.face + def prepare_ip_adapter(self, request): + if hasattr(request, "ip_adapter") and request.ip_adapter: + args = { 'ip_adapter_names': [], 'ip_adapter_scales': [], 'ip_adapter_starts': [], 'ip_adapter_ends': [], 'ip_adapter_images': [] } + for ipadapter in request.ip_adapter: + if not ipadapter.images or len(ipadapter.images) == 0: + continue + args['ip_adapter_names'].append(ipadapter.adapter) + args['ip_adapter_scales'].append(ipadapter.scale) + args['ip_adapter_starts'].append(ipadapter.start) + args['ip_adapter_ends'].append(ipadapter.end) + args['ip_adapter_images'].append([helpers.decode_base64_to_image(x) for x in ipadapter.images]) + del request.ip_adapter + return args + else: + return {} + def prepare_control(self, req): from modules.control.unit import Unit, unit_types req.units = [] + if req.unit_type is None: + return req.control if req.unit_type not in unit_types: shared.log.error(f'Control uknown unit type: type={req.unit_type} available={unit_types}') return req.control @@ -122,7 +140,6 @@ class APIControl(): # prepare args args = req.copy(update={ # Override __init__ params "sampler_index": processing_helpers.get_sampler_index(req.sampler_name), - "no_save": not req.save_images, "is_generator": False, "inputs": [helpers.decode_base64_to_image(x) for x in req.inputs] if req.inputs else None, "inits": [helpers.decode_base64_to_image(x) for x in req.inits] if req.inits else None, @@ -134,10 +151,10 @@ class APIControl(): # run with self.queue_lock: shared.state.begin('api-control', api=True) - output_images = [] output_processed = [] output_info = '' + run.control_set({ 'do_not_save_grid': not req.save_images, 'do_not_save_samples': not req.save_images, **self.prepare_ip_adapter(req) }) res = run.control_run(**args) for item in res: if len(item) > 0 and (isinstance(item[0], list) or item[0] is None): # output_images @@ -146,7 +163,6 @@ class APIControl(): output_info += item[2] if len(item) > 2 and item[2] is not None else '' else: output_info += item - shared.state.end(api=False) # return diff --git a/modules/api/generate.py b/modules/api/generate.py index dda3fe98c..8a6e2749e 100644 --- a/modules/api/generate.py +++ b/modules/api/generate.py @@ -22,7 +22,6 @@ class APIGenerate(): args.pop('alwayson_scripts', None) args.pop('face', None) args.pop('face_id', None) - args.pop('ip_adapter', None) args.pop('save_images', None) return args @@ -61,6 +60,23 @@ class APIGenerate(): ] del request.face + def prepare_ip_adapter(self, request, p): + if hasattr(request, "ip_adapter") and request.ip_adapter: + p.ip_adapter_names = [] + p.ip_adapter_scales = [] + p.ip_adapter_starts = [] + p.ip_adapter_ends = [] + p.ip_adapter_images = [] + for ipadapter in request.ip_adapter: + if not ipadapter.images or len(ipadapter.images) == 0: + continue + p.ip_adapter_names.append(ipadapter.adapter) + p.ip_adapter_scales.append(ipadapter.scale) + p.ip_adapter_starts.append(ipadapter.start) + p.ip_adapter_ends.append(ipadapter.end) + p.ip_adapter_images.append([helpers.decode_base64_to_image(x) for x in ipadapter.images]) + del request.ip_adapter + def post_text2img(self, txt2imgreq: models.ReqTxt2Img): self.prepare_face_module(txt2imgreq) script_runner = scripts.scripts_txt2img @@ -81,6 +97,7 @@ class APIGenerate(): send_images = args.pop('send_images', True) with self.queue_lock: p = StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args) + self.prepare_ip_adapter(txt2imgreq, p) p.scripts = script_runner p.outpath_grids = shared.opts.outdir_grids or shared.opts.outdir_txt2img_grids p.outpath_samples = shared.opts.outdir_samples or shared.opts.outdir_txt2img_samples @@ -123,6 +140,7 @@ class APIGenerate(): send_images = args.pop('send_images', True) with self.queue_lock: p = StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args) + self.prepare_ip_adapter(img2imgreq, p) p.init_images = [helpers.decode_base64_to_image(x) for x in init_images] p.scripts = script_runner p.outpath_grids = shared.opts.outdir_img2img_grids diff --git a/modules/api/middleware.py b/modules/api/middleware.py index 540275bea..095c5b23d 100644 --- a/modules/api/middleware.py +++ b/modules/api/middleware.py @@ -43,7 +43,7 @@ def setup_middleware(app: FastAPI, cmd_opts): endpoint = req.scope.get('path', 'err') token = req.cookies.get("access-token") or req.cookies.get("access-token-unsecure") if (cmd_opts.api_log or cmd_opts.api_only) and endpoint.startswith('/sdapi'): - if '/sdapi/v1/log' or '/sdapi/v1/browser' in endpoint: + if '/sdapi/v1/log' in endpoint or '/sdapi/v1/browser' in endpoint: return res log.info('API {user} {code} {prot}/{ver} {method} {endpoint} {cli} {duration}'.format( # pylint: disable=consider-using-f-string, logging-format-interpolation user = app.tokens.get(token) if hasattr(app, 'tokens') else None, diff --git a/modules/api/models.py b/modules/api/models.py index ab8efd0a9..898d9bc31 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -149,9 +149,11 @@ class ItemEmbedding(BaseModel): vectors: int = Field(title="Vectors", description="The number of vectors in the embedding") class ItemIPAdapter(BaseModel): - adapter: str = Field(title="Adapter", default="Base", description="Adapter to use") - image: str = Field(title="Image", default="", description="Adapter image, must be a base64 string containing the image's data.") - scale: float = Field(title="Scale", default=0.5, gt=0, le=1, description="Scale of the adapter image, must be between 0 and 1.") + adapter: str = Field(title="Adapter", default="Base", description="") + images: List[str] = Field(title="Image", default=[], description="") + scale: float = Field(title="Scale", default=0.5, gt=0, le=1, description="") + start: float = Field(title="Start", default=0.0, gt=0, le=1, description="") + end: float = Field(title="End", default=1.0, gt=0, le=1, description="") class ItemFace(BaseModel): mode: str = Field(title="Mode", default="FaceID", description="The mode to use (available values: FaceID, FaceSwap, PhotoMaker, InstantID).") @@ -204,7 +206,7 @@ ReqTxt2Img = PydanticModelGenerator( {"key": "send_images", "type": bool, "default": True}, {"key": "save_images", "type": bool, "default": False}, {"key": "alwayson_scripts", "type": dict, "default": {}}, - {"key": "ip_adapter", "type": Optional[ItemIPAdapter], "default": None, "exclude": True}, + {"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True}, {"key": "face", "type": Optional[ItemFace], "default": None, "exclude": True}, ] ).generate_model() @@ -229,7 +231,7 @@ ReqImg2Img = PydanticModelGenerator( {"key": "send_images", "type": bool, "default": True}, {"key": "save_images", "type": bool, "default": False}, {"key": "alwayson_scripts", "type": dict, "default": {}}, - {"key": "ip_adapter", "type": Optional[ItemIPAdapter], "default": None, "exclude": True}, + {"key": "ip_adapter", "type": Optional[List[ItemIPAdapter]], "default": None, "exclude": True}, {"key": "face_id", "type": Optional[ItemFace], "default": None, "exclude": True}, ] ).generate_model() diff --git a/modules/control/run.py b/modules/control/run.py index 3236cc48e..db8e6d062 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -22,6 +22,7 @@ debug('Trace: CONTROL') pipe = None instance = None original_pipeline = None +p_extra_args = {} def restore_pipeline(): @@ -42,8 +43,19 @@ def terminate(msg): return msg -def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], inits: List[Image.Image] = [], mask: Image.Image = None, unit_type: str = None, is_generator: bool = True, input_type: int = 0, - prompt: str = '', negative: str = '', styles: List[str] = [], steps: int = 20, sampler_index: int = None, +def control_set(kwargs): + if kwargs: + global p_extra_args # pylint: disable=global-statement + p_extra_args = {} + debug(f'Control extra args: {kwargs}') + for k, v in kwargs.items(): + p_extra_args[k] = v + + +def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], inits: List[Image.Image] = [], mask: Image.Image = None, unit_type: str = None, is_generator: bool = True, + input_type: int = 0, + prompt: str = '', negative: str = '', styles: List[str] = [], + steps: int = 20, sampler_index: int = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, sag_scale: float = 0.0, cfg_end: float = 1.0, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, @@ -56,7 +68,6 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini enable_hr: bool = False, hr_sampler_index: int = None, hr_denoising_strength: float = 0.3, hr_upscaler: str = None, hr_force: bool = False, hr_second_pass_steps: int = 20, hr_scale: float = 1.0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0.0, refiner_prompt: str = '', refiner_negative: str = '', video_skip_frames: int = 0, video_type: str = 'None', video_duration: float = 2.0, video_loop: bool = False, video_pad: int = 0, video_interpolate: int = 0, - no_save: bool = False, *input_script_args ): global instance, pipe, original_pipeline # pylint: disable=global-statement @@ -134,11 +145,14 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini p.refiner_start = refiner_start p.refiner_prompt = refiner_prompt p.refiner_negative = refiner_negative - p.do_not_save_grid = no_save - p.do_not_save_samples = no_save if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0): p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.width * p.hr_scale / 8), 8 * int(p.height * p.hr_scale / 8) + global p_extra_args # pylint: disable=global-statement + for k, v in p_extra_args.items(): + setattr(p, k, v) + p_extra_args = {} + if shared.sd_model is None: shared.log.warning('Model not loaded') return [], '', '', 'Error: model not loaded' diff --git a/modules/face/instantid.py b/modules/face/instantid.py index c170e0b26..e519218df 100644 --- a/modules/face/instantid.py +++ b/modules/face/instantid.py @@ -22,7 +22,7 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_images, stre return None c = shared.sd_model.__class__.__name__ if shared.sd_loaded else '' - if c != 'StableDiffusionXLPipeline': + if c != 'StableDiffusionXLPipeline' and c != 'StableDiffusionXLInstantIDPipeline': shared.log.warning(f'InstantID invalid base model: current={c} required=StableDiffusionXLPipeline') return None diff --git a/modules/ui_control.py b/modules/ui_control.py index 9cab9d290..4abbd559d 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -460,7 +460,6 @@ def create_ui(_blocks: gr.Blocks=None): input_script_args = scripts.scripts_current.setup_ui(parent='control', accordion=True) # handlers - for btn in input_buttons: btn.click(fn=helpers.copy_input, inputs=[input_mode, btn, input_image, input_resize, input_inpaint], outputs=[input_image, input_resize, input_inpaint], _js='controlInputMode') btn.click(fn=helpers.transfer_input, inputs=[btn], outputs=[input_image, input_resize, input_inpaint] + input_buttons) @@ -571,7 +570,6 @@ def create_ui(_blocks: gr.Blocks=None): generation_parameters_copypaste.register_paste_params_button(bindings) masking.bind_controls([input_image, input_inpaint, input_resize], preview_process, output_image) - if os.environ.get('SD_CONTROL_DEBUG', None) is not None: # debug only from modules.control.test import test_processors, test_controlnets, test_adapters, test_xs, test_lite gr.HTML('

Debug


')