Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 01:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

add vqa api

Browse Source
This commit is contained in:
Vladimir Mandic
2024-03-26 09:24:01 -04:00
parent db11bc59d1
commit 902f02300d
8 changed files with 93 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/api/api.py
+1
View File
@@ -78,6 +78,7 @@ class Api:
# functional api
self.add_api_route("/sdapi/v1/png-info", endpoints.post_pnginfo, methods=["POST"], response_model=models.ResImageInfo)
self.add_api_route("/sdapi/v1/interrogate", endpoints.post_interrogate, methods=["POST"])
self.add_api_route("/sdapi/v1/vqa", endpoints.post_vqa, methods=["POST"])
self.add_api_route("/sdapi/v1/refresh-checkpoints", endpoints.post_refresh_checkpoints, methods=["POST"])
self.add_api_route("/sdapi/v1/unload-checkpoint", endpoints.post_unload_checkpoint, methods=["POST"])
self.add_api_route("/sdapi/v1/reload-checkpoint", endpoints.post_reload_checkpoint, methods=["POST"])
modules/api/endpoints.py
+10
View File
@@ -100,6 +100,16 @@ def post_interrogate(req: models.ReqInterrogate):
medium, artist, movement, trending, flavor = analyze_image(image, model=req.model)
return models.ResInterrogate(caption=caption, medium=medium, artist=artist, movement=movement, trending=trending, flavor=flavor)
def post_vqa(req: models.ReqVQA):
if req.image is None or len(req.image) < 64:
raise HTTPException(status_code=404, detail="Image not found")
image = helpers.decode_base64_to_image(req.image)
image = image.convert('RGB')
from modules import vqa
print('HERE', req.question, req.model)
answer = vqa.interrogate(req.question, image, req.model)
return models.ResVQA(answer=answer)
def post_unload_checkpoint():
from modules import sd_models
sd_models.unload_model_weights(op='model')
modules/api/models.py
+8
View File
@@ -305,6 +305,14 @@ class ResInterrogate(BaseModel):
trending: Optional[str] = Field(default=None, title="Medium", description="Image trending.")
flavor: Optional[str] = Field(default=None, title="Medium", description="Image flavor.")
class ReqVQA(BaseModel):
image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.")
model: str = Field(default="Moondream 2", title="Model", description="The interrogate model used.")
question: str = Field(default="describe the image", title="Question", description="Question to ask the model.")
class ResVQA(BaseModel):
answer: Optional[str] = Field(default=None, title="Answer", description="The generated answer for the image.")
class ResTrain(BaseModel):
info: str = Field(title="Train info", description="Response string from train embedding or hypernetwork task.")