diff --git a/CHANGELOG.md b/CHANGELOG.md index 8567165be..40c437b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,13 @@ TODO: - massive updates to [Wiki](https://github.com/vladmandic/automatic/wiki) with over 20 new pages and articles, now includes guides for nearly all major features thanks @GenesisArtemis! -- support for **HunyuanDiT 1.2** -- add support for DoRA networks, thanks @AI-Casanova! -- add support for [uv](https://pypi.org/project/uv/), extremely fast installer, thanks @Yoinky3000! +- support for [HunyuanDiT 1.2](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers) +- support for DoRA networks, thanks @AI-Casanova! +- support for [CogFlorence 2 Large](https://huggingface.co/thwri/CogFlorence-2-Large-Freeze) VLM model +- support for [uv](https://pypi.org/project/uv/), extremely fast installer, thanks @Yoinky3000! to use, simply add `--uv` to your command line params - enable `florence` VLM for all platforms, thanks @lshqqytiger! +- add SD3 with FP16 T5 to list of detected models - fix executing extensions with zero params - add support for embeddings bundled in LoRA, thanks @AI-Casanova! - fix executing extensions with zero params @@ -22,7 +24,7 @@ TODO: - fix diffusers version detection for SD3 - fix current step for higher order samplers - fix control input type video -- add SD3 with FP16 T5 to list of detected models +- fix reset pipeline at the end of each iteration - multiple ModernUI fixes ## Update for 2024-06-23 diff --git a/modules/vqa.py b/modules/vqa.py index fd994ccd2..87ea8431c 100644 --- a/modules/vqa.py +++ b/modules/vqa.py @@ -1,3 +1,4 @@ +import json import torch import transformers import transformers.dynamic_module_utils @@ -11,6 +12,7 @@ loaded: str = None MODELS = { "MS Florence 2 Base": "microsoft/Florence-2-base", # 0.5GB "MS Florence 2 Large": "microsoft/Florence-2-large", # 1.5GB + "CogFlorence 2 Large": "thwri/CogFlorence-2-Large-Freeze", # 1.6GB "Moondream 2": "vikhyatk/moondream2", # 3.7GB "GIT TextCaps Base": "microsoft/git-base-textcaps", # 0.7GB "GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB @@ -166,6 +168,11 @@ def florence(question: str, image: Image.Image, repo: str = None): if 'task' in response: response = response['task'] + if 'answer' in response: + response = response['answer'] + if isinstance(response, dict): + response = json.dumps(response) + response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip() shared.log.debug(f'VQA: task={task} response="{response}"') return response