Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-20 01:31:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

add joycaption

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-02-15 08:32:37 -05:00
parent 1f2fc929f7
commit abd439d97c
4 changed files with 122 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/interrogate/vqa.py
+5 -1
View File
@@ -37,7 +37,8 @@ vlm_models = {
"Microsoft GIT VQA Large": "microsoft/git-large-vqav2", # 1.6GB
"ToriiGate 0.4 2B": "Minthy/ToriiGate-v0.4-2B",
"ViLT Base": "dandelin/vilt-b32-finetuned-vqa", # 0.5GB
"JoyTag": "fancyfeast/joytag",
"JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB
"JoyTag": "fancyfeast/joytag", # 17.4GB
}
vlm_prompts = [
'<CAPTION>',
@@ -348,6 +349,9 @@ def interrogate(question, image, model_name):
elif 'joytag' in vqa_model.lower():
from modules.interrogate import joytag
answer = joytag.predict(image)
elif 'joycaption' in vqa_model.lower():
from modules.interrogate import joycaption
answer = joycaption.predict(question, image)
else:
answer = 'unknown model'
except Exception as e: