diff --git a/CHANGELOG.md b/CHANGELOG.md index 90829a543..f2ea9a227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ both can still be installed by user if desired - **Improvements**: - Styles apply wildcards to params + - Add API endpoint `/sdapi/v1/vqa` and util `cli/simple-vqa.py` - Make metadata in full screen viewer optional - Add VAE civitai scan metadata/preview - **Fixes**: diff --git a/cli/image-palette.py b/cli/image-palette.py old mode 100644 new mode 100755 diff --git a/cli/lcm-convert.py b/cli/lcm-convert.py old mode 100644 new mode 100755 index c2d7c266b..9feeb98de --- a/cli/lcm-convert.py +++ b/cli/lcm-convert.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import os import argparse import torch diff --git a/cli/simple-vqa.py b/cli/simple-vqa.py new file mode 100755 index 000000000..0ac181b7c --- /dev/null +++ b/cli/simple-vqa.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +import os +import time +import base64 +import logging +import argparse +import requests +import urllib3 + + +sd_url = os.environ.get('SDAPI_URL', "http://127.0.0.1:7860") +sd_username = os.environ.get('SDAPI_USR', None) +sd_password = os.environ.get('SDAPI_PWD', None) + + +logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s') +log = logging.getLogger(__name__) +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def auth(): + if sd_username is not None and sd_password is not None: + return requests.auth.HTTPBasicAuth(sd_username, sd_password) + return None + + +def get(endpoint: str, dct: dict = None): + req = requests.get(f'{sd_url}{endpoint}', json=dct, timeout=300, verify=False, auth=auth()) + if req.status_code != 200: + return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } + else: + return req.json() + + +def post(endpoint: str, dct: dict = None): + req = requests.post(f'{sd_url}{endpoint}', json = dct, timeout=300, verify=False, auth=auth()) + if req.status_code != 200: + return { 'error': req.status_code, 'reason': req.reason, 'url': req.url } + else: + return req.json() + + +def info(args): # pylint: disable=redefined-outer-name + t0 = time.time() + with open(args.input, 'rb') as f: + content = f.read() + dct = { 'image': base64.b64encode(content).decode() } + if args.model is not None: + dct['model'] = args.model + if args.question is not None: + dct['question'] = args.question + data = post('/sdapi/v1/vqa', dct) + t1 = time.time() + log.info(f'answer: {data} time={t1-t0:.2f}') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'simple-info') + parser.add_argument('--input', required=True, help='input image') + parser.add_argument('--model', required=False, help='vqa model') + parser.add_argument('--question', required=False, help='question') + args = parser.parse_args() + log.info(f'info: {args}') + info(args) diff --git a/modules/api/api.py b/modules/api/api.py index 2a8f68c51..b4d5da9c9 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -78,6 +78,7 @@ class Api: # functional api self.add_api_route("/sdapi/v1/png-info", endpoints.post_pnginfo, methods=["POST"], response_model=models.ResImageInfo) self.add_api_route("/sdapi/v1/interrogate", endpoints.post_interrogate, methods=["POST"]) + self.add_api_route("/sdapi/v1/vqa", endpoints.post_vqa, methods=["POST"]) self.add_api_route("/sdapi/v1/refresh-checkpoints", endpoints.post_refresh_checkpoints, methods=["POST"]) self.add_api_route("/sdapi/v1/unload-checkpoint", endpoints.post_unload_checkpoint, methods=["POST"]) self.add_api_route("/sdapi/v1/reload-checkpoint", endpoints.post_reload_checkpoint, methods=["POST"]) diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 9f7efe078..1e7f4431d 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -100,6 +100,16 @@ def post_interrogate(req: models.ReqInterrogate): medium, artist, movement, trending, flavor = analyze_image(image, model=req.model) return models.ResInterrogate(caption=caption, medium=medium, artist=artist, movement=movement, trending=trending, flavor=flavor) +def post_vqa(req: models.ReqVQA): + if req.image is None or len(req.image) < 64: + raise HTTPException(status_code=404, detail="Image not found") + image = helpers.decode_base64_to_image(req.image) + image = image.convert('RGB') + from modules import vqa + print('HERE', req.question, req.model) + answer = vqa.interrogate(req.question, image, req.model) + return models.ResVQA(answer=answer) + def post_unload_checkpoint(): from modules import sd_models sd_models.unload_model_weights(op='model') diff --git a/modules/api/models.py b/modules/api/models.py index b0e56d8a2..c11af5b58 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -305,6 +305,14 @@ class ResInterrogate(BaseModel): trending: Optional[str] = Field(default=None, title="Medium", description="Image trending.") flavor: Optional[str] = Field(default=None, title="Medium", description="Image flavor.") +class ReqVQA(BaseModel): + image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.") + model: str = Field(default="Moondream 2", title="Model", description="The interrogate model used.") + question: str = Field(default="describe the image", title="Question", description="Question to ask the model.") + +class ResVQA(BaseModel): + answer: Optional[str] = Field(default=None, title="Answer", description="The generated answer for the image.") + class ResTrain(BaseModel): info: str = Field(title="Train info", description="Response string from train embedding or hypernetwork task.") diff --git a/modules/vqa.py b/modules/vqa.py index 3de7bef91..8344b15bf 100644 --- a/modules/vqa.py +++ b/modules/vqa.py @@ -124,13 +124,18 @@ def moondream(question: str, image: Image.Image, repo: str = None): return response -def interrogate(vqa_question, vqa_image, vqa_model): - vqa_model = MODELS.get(vqa_model, None) +def interrogate(vqa_question, vqa_image, vqa_model_req): + vqa_model = MODELS.get(vqa_model_req, None) shared.log.debug(f'VQA: model="{vqa_model}" question="{vqa_question}" image={vqa_image}') if vqa_image is None: answer = 'no image provided' - if vqa_model is None: + return answer + if vqa_model_req is None: answer = 'no model selected' + return answer + if vqa_model is None: + answer = f'unknown: model={vqa_model_req} available={MODELS.keys()}' + return answer if 'git' in vqa_model.lower(): answer = git(vqa_question, vqa_image, vqa_model) if 'vilt' in vqa_model.lower():