diff --git a/CHANGELOG.md b/CHANGELOG.md index d48bce68d..db8625033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,8 @@ allows for using many different guidance methods: *CFG, CFGZero, PAG, APG, SLG, SEG, TCFG, FDG* - **Fixes** + - **Microsoft Florence 2** both base and large variants + *note* this will trigger download of the new variant of the model, feel free to delete older variant in `huggingface` folder - ui: fix image metadata display when switching selected image in control tab - framepack: add explicit hf-login before framepack load - framepack: patch solver for unsupported gpus diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index d034f611d..e56b4f695 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit d034f611df9931d87059a8256a2c77cc82f8e11c +Subproject commit e56b4f69549e6654c953346082ff3947256c9636 diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 578c96093..5518ff803 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -27,8 +27,10 @@ vlm_models = { "Apple FastVLM 0.5B": "apple/FastVLM-0.5B", "Apple FastVLM 1.5B": "apple/FastVLM-1.5B", "Apple FastVLM 7B": "apple/FastVLM-7B", - "Microsoft Florence 2 Base": "microsoft/Florence-2-base-ft", # 0.5GB - "Microsoft Florence 2 Large": "microsoft/Florence-2-large-ft", # 1.5GB + # "Microsoft Florence 2 Base": "microsoft/Florence-2-base-ft", # 0.5GB + # "Microsoft Florence 2 Large": "microsoft/Florence-2-large-ft", # 1.5GB + "Microsoft Florence 2 Base": "florence-community/Florence-2-base-ft", # 0.5GB + "Microsoft Florence 2 Large": "florence-community/Florence-2-large-ft", # 1.5GB "MiaoshouAI PromptGen 1.5 Base": "MiaoshouAI/Florence-2-base-PromptGen-v1.5@c06a5f02cc6071a5d65ee5d294cf3732d3097540", # 1.1GB "MiaoshouAI PromptGen 1.5 Large": "MiaoshouAI/Florence-2-large-PromptGen-v1.5@28a42440e39c9c32b83f7ae74ec2b3d1540404f0", # 3.3GB "MiaoshouAI PromptGen 2.0 Base": "MiaoshouAI/Florence-2-base-PromptGen-v2.0", # 1.1GB @@ -354,7 +356,6 @@ def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, - _attn_implementation="eager", **quant_args, ) processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) @@ -536,6 +537,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"') transformers.dynamic_module_utils.get_imports = get_imports model = None + """ model = transformers.AutoModelForCausalLM.from_pretrained( repo, trust_remote_code=True, @@ -544,6 +546,15 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str cache_dir=shared.opts.hfcache_dir, **quant_args, ) + """ + model = transformers.Florence2ForConditionalGeneration.from_pretrained( + repo, + dtype=torch.bfloat16, + revision=revision, + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args, + ) processor = transformers.AutoProcessor.from_pretrained(repo, trust_remote_code=True, revision=revision, cache_dir=shared.opts.hfcache_dir) transformers.dynamic_module_utils.get_imports = _get_imports loaded = repo diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index aea0e5730..221d815dc 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -159,7 +159,7 @@ class Script(scripts_manager.Script): trust_remote_code=True, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, - _attn_implementation="eager", + # _attn_implementation="eager", **gguf_args, **quant_args, )