diff --git a/CHANGELOG.md b/CHANGELOG.md index da162f12e..4e80de52b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - EDM samplers for Playground require `diffusers==0.27.0` - StableCascade requires diffusers `kashif/diffusers.git@wuerstchen-v3` -## Update for 2024-03-01 +## Update for 2024-03-02 - [Playground v2.5](https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic) - new model version from Playground: based on SDXL, but with some cool new concepts @@ -299,7 +299,7 @@ Further details: - full implementation for *SD15* and *SD-XL*, to use simply select from *Scripts* **Base** (93MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-H-14* (2.5GB) as image encoder **Plus** (150MB) uses *InsightFace* to generate face embeds and *CLIP-ViT-H-14-laion2B* (3.8GB) as image encoder - **SXDL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder + **SDXL** (1022MB) uses *InsightFace* to generate face embeds and *OpenCLIP-ViT-bigG-14* (3.7GB) as image encoder - [FaceSwap](https://github.com/deepinsight/insightface/blob/master/examples/in_swapper/README.md) - face swap performs face swapping at the end of generation - based on InsightFace in-swapper @@ -316,7 +316,7 @@ Further details: - [IPAdapter](https://huggingface.co/h94/IP-Adapter) - additional models for *SD15* and *SD-XL*, to use simply select from *Scripts*: **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face - **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL + **SDXL**: Base SDXL, Base ViT-H SDXL, Plus ViT-H SDXL, Plus Face ViT-H SDXL - enable use via api, thanks @trojaner - [Segmind SegMoE](https://github.com/segmind/segmoe) - initial support for reference models diff --git a/cli/simple-mask.py b/cli/simple-mask.py index 83e962a23..2ea12234e 100755 --- a/cli/simple-mask.py +++ b/cli/simple-mask.py @@ -56,7 +56,7 @@ def info(args): # pylint: disable=redefined-outer-name req = { 'image': image, 'mask': mask, - 'type': 'Composite', + 'type': args.type or 'Composite', 'params': { 'auto_mask': 'Grayscale' if mask is None else None }, } data = post('/sdapi/v1/mask', req) @@ -65,6 +65,9 @@ def info(args): # pylint: disable=redefined-outer-name b64 = data['mask'].split(',',1)[0] image = Image.open(io.BytesIO(base64.b64decode(b64))) log.info(f'received image: size={image.size} time={t1-t0:.2f}') + if args.output: + image.save(args.output) + log.info(f'saved image: fn={args.output}') else: log.info(f'received: {data} time={t1-t0:.2f}') @@ -73,6 +76,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description = 'simple-info') parser.add_argument('--input', required=True, help='input image') parser.add_argument('--mask', required=False, help='input mask') + parser.add_argument('--type', required=False, help='output mask type') + parser.add_argument('--output', required=False, help='output image') args = parser.parse_args() log.info(f'info: {args}') info(args) diff --git a/cli/simple-preprocess.py b/cli/simple-preprocess.py index 81bfea77d..2b96750bf 100755 --- a/cli/simple-preprocess.py +++ b/cli/simple-preprocess.py @@ -49,7 +49,7 @@ def info(args): # pylint: disable=redefined-outer-name models = get('/sdapi/v1/preprocessors') log.info(f'models: {models}') req = { - 'model': 'Canny', + 'model': args.model or 'Canny', 'image': base64.b64encode(content).decode(), 'config': { 'low_threshold': 50 }, } @@ -59,6 +59,9 @@ def info(args): # pylint: disable=redefined-outer-name b64 = data['image'].split(',',1)[0] image = Image.open(io.BytesIO(base64.b64decode(b64))) log.info(f'received image: size={image.size} time={t1-t0:.2f}') + if args.output: + image.save(args.output) + log.info(f'saved image: fn={args.output}') else: log.info(f'received: {data} time={t1-t0:.2f}') @@ -66,6 +69,8 @@ def info(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": parser = argparse.ArgumentParser(description = 'simple-info') parser.add_argument('--input', required=True, help='input image') + parser.add_argument('--model', required=True, help='preprocessing model') + parser.add_argument('--output', required=False, help='output image') args = parser.parse_args() log.info(f'info: {args}') info(args) diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 75ac3803f..461f8c5f1 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 75ac3803feae51e6edf50f6bf76919b202629598 +Subproject commit 461f8c5f169f5b69971e846279479b050d330744 diff --git a/modules/api/api.py b/modules/api/api.py index d7647c67d..398719aa3 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -5,8 +5,7 @@ from fastapi import FastAPI, APIRouter, Depends, Request from fastapi.security import HTTPBasic, HTTPBasicCredentials from fastapi.exceptions import HTTPException from modules import errors, shared, postprocessing -from modules.api import models, endpoints, script, helpers, server, nvml, generate -from modules.control import api as control_api +from modules.api import models, endpoints, script, helpers, server, nvml, generate, process, control errors.install() @@ -29,6 +28,8 @@ class Api: self.app = app self.queue_lock = queue_lock self.generate = generate.APIGenerate(queue_lock) + self.process = process.APIProcess(queue_lock) + self.control = control.APIControl(queue_lock) # server api self.add_api_route("/sdapi/v1/motd", server.get_motd, methods=["GET"], response_model=str) @@ -49,18 +50,19 @@ class Api: # core api using locking self.add_api_route("/sdapi/v1/txt2img", self.generate.post_text2img, methods=["POST"], response_model=models.ResTxt2Img) self.add_api_route("/sdapi/v1/img2img", self.generate.post_img2img, methods=["POST"], response_model=models.ResImg2Img) + self.add_api_route("/sdapi/v1/control", self.control.post_control, methods=["POST"], response_model=control.ResControl) self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ResProcessImage) self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ResProcessBatch) - self.add_api_route("/sdapi/v1/preprocess", control_api.post_preprocess, methods=["POST"]) - self.add_api_route("/sdapi/v1/mask", control_api.post_mask, methods=["POST"]) + self.add_api_route("/sdapi/v1/preprocess", self.process.post_preprocess, methods=["POST"]) + self.add_api_route("/sdapi/v1/mask", self.process.post_mask, methods=["POST"]) # api dealing with optional scripts self.add_api_route("/sdapi/v1/scripts", script.get_scripts_list, methods=["GET"], response_model=models.ResScripts) self.add_api_route("/sdapi/v1/script-info", script.get_script_info, methods=["GET"], response_model=List[models.ItemScript]) # enumerator api - self.add_api_route("/sdapi/v1/preprocessors", control_api.get_preprocess, methods=["GET"]) - self.add_api_route("/sdapi/v1/masking", control_api.get_mask, methods=["GET"]) + self.add_api_route("/sdapi/v1/preprocessors", self.process.get_preprocess, methods=["GET"], response_model=List[process.ItemPreprocess]) + self.add_api_route("/sdapi/v1/masking", self.process.get_mask, methods=["GET"], response_model=process.ItemMask) self.add_api_route("/sdapi/v1/interrogate", endpoints.get_interrogate, methods=["GET"], response_model=List[str]) self.add_api_route("/sdapi/v1/samplers", endpoints.get_samplers, methods=["GET"], response_model=List[models.ItemSampler]) self.add_api_route("/sdapi/v1/upscalers", endpoints.get_upscalers, methods=["GET"], response_model=List[models.ItemUpscaler]) diff --git a/modules/api/control.py b/modules/api/control.py new file mode 100644 index 000000000..519f68378 --- /dev/null +++ b/modules/api/control.py @@ -0,0 +1,114 @@ +from typing import Optional, List +from threading import Lock +from pydantic import BaseModel, Field # pylint: disable=no-name-in-module +from modules import errors, shared, scripts, ui +from modules.api import script, helpers +from modules.processing import StableDiffusionProcessingControl +from modules.control import run as run_control + +# TODO control api +# should use control.run, not process_images directly + +errors.install() + + +class ReqControl(BaseModel): + pass + +class ResControl(BaseModel): + images: List[str] = Field(default=None, title="Image", description="The generated images in base64 format.") + params: dict = Field(default={}, title="Settings", description="Process settings") + info: str = Field(default="", title="Info", description="Process info") + + +class APIControl(): + def __init__(self, queue_lock: Lock): + self.queue_lock = queue_lock + self.default_script_arg = [] + + def sanitize_args(self, args: dict): + args = vars(args) + args.pop('include_init_images', None) # this is meant to be done by "exclude": True in model + args.pop('script_name', None) + args.pop('script_args', None) # will refeed them to the pipeline directly after initializing them + args.pop('alwayson_scripts', None) + args.pop('face', None) + args.pop('face_id', None) + args.pop('ip_adapter', None) + args.pop('save_images', None) + return args + + def sanitize_b64(self, request): + def sanitize_str(args: list): + for idx in range(0, len(args)): + if isinstance(args[idx], str) and len(args[idx]) >= 1000: + args[idx] = f"" + + if hasattr(request, "alwayson_scripts") and request.alwayson_scripts: + for script_name in request.alwayson_scripts.keys(): + script_obj = request.alwayson_scripts[script_name] + if script_obj and "args" in script_obj and script_obj["args"]: + sanitize_str(script_obj["args"]) + if hasattr(request, "script_args") and request.script_args: + sanitize_str(request.script_args) + + def prepare_face_module(self, request): + if hasattr(request, "face") and request.face and not request.script_name and (not request.alwayson_scripts or "face" not in request.alwayson_scripts.keys()): + request.script_name = "face" + request.script_args = [ + request.face.mode, + request.face.source_images, + request.face.ip_model, + request.face.ip_override_sampler, + request.face.ip_cache_model, + request.face.ip_strength, + request.face.ip_structure, + request.face.id_strength, + request.face.id_conditioning, + request.face.id_cache, + request.face.pm_trigger, + request.face.pm_strength, + request.face.pm_start, + request.face.fs_cache + ] + del request.face + + def post_control(self, req: ReqControl): + self.prepare_face_module(req) + + # prepare script + script_runner = scripts.scripts_control + if not script_runner.scripts: + script_runner.initialize_scripts(False) + ui.create_ui(None) + if not self.default_script_arg: + self.default_script_arg = script.init_default_script_args(script_runner) + + # prepare args + args = req.copy(update={ # Override __init__ params + "sampler_name": helpers.validate_sampler_name(req.sampler_name or req.sampler_index), + "sampler_index": None, + "do_not_save_samples": not req.save_images, + "do_not_save_grid": not req.save_images, + "init_images": [helpers.decode_base64_to_image(x) for x in req.init_images] if req.init_images else None, + "mask": helpers.decode_base64_to_image(req.mask) if req.mask else None, + }) + args = self.sanitize_args(args) + send_images = args.pop('send_images', True) + + # run + with self.queue_lock: + shared.state.begin('api-control', api=True) + + # selectable_scripts, selectable_script_idx = script.get_selectable_script(req.script_name, script_runner) + # script_args = script.init_script_args(p, req, self.default_script_arg, selectable_scripts, selectable_script_idx, script_runner) + # output_images, _processed_images, output_info = run_control(**args, **script_args) + output_images = None + output_info = None + + shared.state.end(api=False) + + # return + b64images = list(map(helpers.encode_pil_to_base64, output_images)) if send_images else [] + self.sanitize_b64(req) + return ResControl(images=b64images, params=vars(req), info=output_info) diff --git a/modules/api/generate.py b/modules/api/generate.py index 4f674d716..dda3fe98c 100644 --- a/modules/api/generate.py +++ b/modules/api/generate.py @@ -1,5 +1,5 @@ from threading import Lock -from fastapi.exceptions import HTTPException +from fastapi.responses import JSONResponse from modules import errors, shared, scripts, ui from modules.api import models, script, helpers from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images @@ -100,7 +100,7 @@ class APIGenerate(): self.prepare_face_module(img2imgreq) init_images = img2imgreq.init_images if init_images is None: - raise HTTPException(status_code=404, detail="Init image not found") + return JSONResponse(status_code=400, content={"error": "Init image is none"}) mask = img2imgreq.mask if mask: mask = helpers.decode_base64_to_image(mask) diff --git a/modules/api/models.py b/modules/api/models.py index f8195938a..b0e56d8a2 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -208,7 +208,7 @@ ReqTxt2Img = PydanticModelGenerator( StableDiffusionTxt2ImgProcessingAPI = ReqTxt2Img class ResTxt2Img(BaseModel): - images: List[str] = Field(default=None, title="Image", description="The generated image in base64 format.") + images: List[str] = Field(default=None, title="Image", description="The generated images in base64 format.") parameters: dict info: str @@ -233,7 +233,7 @@ ReqImg2Img = PydanticModelGenerator( StableDiffusionImg2ImgProcessingAPI = ReqImg2Img class ResImg2Img(BaseModel): - images: List[str] = Field(default=None, title="Image", description="The generated image in base64 format.") + images: List[str] = Field(default=None, title="Image", description="The generated images in base64 format.") parameters: dict info: str diff --git a/modules/api/process.py b/modules/api/process.py new file mode 100644 index 000000000..343b6efb4 --- /dev/null +++ b/modules/api/process.py @@ -0,0 +1,100 @@ +from typing import Optional, List +from threading import Lock +from pydantic import BaseModel, Field # pylint: disable=no-name-in-module +from fastapi.responses import JSONResponse +from modules.api.helpers import decode_base64_to_image, encode_pil_to_base64 +from modules import errors, shared + + +processor = None # cached instance of processor +errors.install() + + +class ReqPreprocess(BaseModel): + image: str = Field(title="Image", description="The base64 encoded image") + model: str = Field(title="Model", description="The model to use for preprocessing") + params: Optional[dict] = Field(default={}, title="Settings", description="Preprocessor settings") + +class ResPreprocess(BaseModel): + model: str = Field(default='', title="Model", description="The processor model used") + image: str = Field(default='', title="Image", description="The processed image in base64 format") + +class ReqMask(BaseModel): + image: str = Field(title="Image", description="The base64 encoded image") + type: str = Field(title="Mask type", description="Type of masking image to return") + mask: Optional[str] = Field(title="Mask", description="If optional maks image is not provided auto-masking will be performed") + model: Optional[str] = Field(title="Model", description="The model to use for preprocessing") + params: Optional[dict] = Field(default={}, title="Settings", description="Preprocessor settings") + +class ResMask(BaseModel): + mask: str = Field(default='', title="Image", description="The processed image in base64 format") + +class ItemPreprocess(BaseModel): + name: str = Field(title="Name") + params: dict = Field(title="Params") + +class ItemMask(BaseModel): + models: List[str] = Field(title="Models") + colormaps: List[str] = Field(title="Color maps") + params: dict = Field(title="Params") + types: List[str] = Field(title="Types") + + +class APIProcess(): + def __init__(self, queue_lock: Lock): + self.queue_lock = queue_lock + + def get_preprocess(self): + from modules.control import processors + items = [] + for k, v in processors.config.items(): + items.append(ItemPreprocess(name=k, params=v.get('params', {}))) + return items + + def post_preprocess(self, req: ReqPreprocess): + global processor # pylint: disable=global-statement + from modules.control import processors + models = list(processors.config) + if req.model not in models: + return JSONResponse(status_code=400, content={"error": f"Processor model not found: id={req.model}"}) + image = decode_base64_to_image(req.image) + if processor is None or processor.processor_id != req.model: + with self.queue_lock: + processor = processors.Processor(req.model) + for k, v in req.params.items(): + if k not in processors.config[processor.processor_id]['params']: + return JSONResponse(status_code=400, content={"error": f"Processor invalid parameter: id={req.model} {k}={v}"}) + shared.state.begin('api-preprocess', api=True) + processed = processor(image, local_config=req.params) + image = encode_pil_to_base64(processed) + shared.state.end(api=False) + return ResPreprocess(model=processor.processor_id, image=image) + + def get_mask(self): + from modules import masking + return ItemMask(models=list(masking.MODELS), colormaps=masking.COLORMAP, params=vars(masking.opts), types=masking.TYPES) + + def post_mask(self, req: ReqMask): + from modules import masking + if req.model: + if req.model not in masking.MODELS: + return JSONResponse(status_code=400, content={"error": f"Mask model not found: id={req.model}"}) + else: + masking.init_model(req.model) + if req.type not in masking.TYPES: + return JSONResponse(status_code=400, content={"error": f"Mask type not found: id={req.type}"}) + image = decode_base64_to_image(req.image) + mask = decode_base64_to_image(req.mask) if req.mask else None + for k, v in req.params.items(): + if not hasattr(masking.opts, k): + return JSONResponse(status_code=400, content={"error": f"Mask invalid parameter: {k}={v}"}) + else: + setattr(masking.opts, k, v) + shared.state.begin('api-mask', api=True) + with self.queue_lock: + processed = masking.run_mask(input_image=image, input_mask=mask, return_type=req.type) + shared.state.end(api=False) + if processed is None: + return JSONResponse(status_code=400, content={"error": "Mask is none"}) + image = encode_pil_to_base64(processed) + return ResMask(mask=image) diff --git a/modules/control/api.py b/modules/control/api.py deleted file mode 100644 index af0ca7f04..000000000 --- a/modules/control/api.py +++ /dev/null @@ -1,89 +0,0 @@ -from typing import Optional -from pydantic import BaseModel, Field # pylint: disable=no-name-in-module -from fastapi.responses import JSONResponse -from modules.api.helpers import decode_base64_to_image, encode_pil_to_base64 - - -processor = None # cached instance of processor - - -class ReqPreprocess(BaseModel): - image: str = Field(title="Image", description="The base64 encoded image") - model: str = Field(title="Model", description="The model to use for preprocessing") - params: Optional[dict] = Field(default={}, title="Settings", description="Preprocessor settings") - - -class ResPreprocess(BaseModel): - model: str = Field(default='', title="Model", description="The processor model used") - image: str = Field(default='', title="Image", description="The processed image in base64 format") - - -def get_preprocess(): - from modules.control import processors - p = {} - for k, v in processors.config.items(): - p[k] = v.get('params') - return JSONResponse(p) - - -def post_preprocess(req: ReqPreprocess): - global processor # pylint: disable=global-statement - from modules.control import processors - models = list(processors.config) - if req.model not in models: - return JSONResponse(status_code=400, content={"error": f"Processor model not found: id={req.model}"}) - image = decode_base64_to_image(req.image) - if processor is None or processor.processor_id != req.model: - processor = processors.Processor(req.model) - for k, v in req.params.items(): - if k not in processors.config[processor.processor_id]['params']: - return JSONResponse(status_code=400, content={"error": f"Processor invalid parameter: id={req.model} {k}={v}"}) - processed = processor(image, local_config=req.params) - image = encode_pil_to_base64(processed) - return ResPreprocess(model=processor.processor_id, image=image) - - -class ReqMask(BaseModel): - image: str = Field(title="Image", description="The base64 encoded image") - type: str = Field(title="Mask type", description="Type of masking image to return") - mask: Optional[str] = Field(title="Mask", description="If optional maks image is not provided auto-masking will be performed") - model: Optional[str] = Field(title="Model", description="The model to use for preprocessing") - params: Optional[dict] = Field(default={}, title="Settings", description="Preprocessor settings") - - -class ResMask(BaseModel): - mask: str = Field(default='', title="Image", description="The processed image in base64 format") - - -def get_mask(): - from modules import masking - res = { - 'models': list(masking.MODELS), - 'colormaps': masking.COLORMAP, - 'params': vars(masking.opts), - 'types': masking.TYPES, - } - return JSONResponse(res) - - -def post_mask(req: ReqMask): - from modules import masking - if req.model: - if req.model not in masking.MODELS: - return JSONResponse(status_code=400, content={"error": f"Mask model not found: id={req.model}"}) - else: - masking.init_model(req.model) - if req.type not in masking.TYPES: - return JSONResponse(status_code=400, content={"error": f"Mask type not found: id={req.type}"}) - image = decode_base64_to_image(req.image) - mask = decode_base64_to_image(req.mask) if req.mask else None - for k, v in req.params.items(): - if not hasattr(masking.opts, k): - return JSONResponse(status_code=400, content={"error": f"Mask invalid parameter: {k}={v}"}) - else: - setattr(masking.opts, k, v) - processed = masking.run_mask(input_image=image, input_mask=mask, return_type=req.type) - if processed is None: - return JSONResponse(status_code=400, content={"error": "Mask is none"}) - image = encode_pil_to_base64(processed) - return ResMask(mask=image) diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 32cfdb486..217d0698c 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -23,10 +23,10 @@ ADAPTERS = { 'Plus': 'ip-adapter-plus_sd15.safetensors', 'Plus Face': 'ip-adapter-plus-face_sd15.safetensors', 'Full Face': 'ip-adapter-full-face_sd15.safetensors', - 'Base SXDL': 'ip-adapter_sdxl.safetensors', - 'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors', - 'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors', - 'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', + 'Base SDXL': 'ip-adapter_sdxl.safetensors', + 'Base ViT-H SDXL': 'ip-adapter_sdxl_vit-h.safetensors', + 'Plus ViT-H SDXL': 'ip-adapter-plus_sdxl_vit-h.safetensors', + 'Plus Face ViT-H SDXL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors', } diff --git a/modules/processing.py b/modules/processing.py index 521d4502b..f50501f06 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -6,7 +6,7 @@ import numpy as np from PIL import Image from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, face_restoration, sd_hijack_freeu, sd_models, sd_vae, processing_helpers from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet -from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img # pylint: disable=unused-import +from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl # pylint: disable=unused-import from modules.processing_info import create_infotext diff --git a/modules/shared.py b/modules/shared.py index 6a1e4039c..e3138b26e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -68,6 +68,7 @@ restricted_opts = { "outdir_txt2img_samples", "outdir_img2img_samples", "outdir_extras_samples", + "outdir_control_samples", "outdir_grids", "outdir_txt2img_grids", "outdir_save",