add ms florence

CHANGELOG.md

@@ -1,8 +1,8 @@
 # Change Log for SD.Next
 
-## Update for 2024-06-21
+## Update for 2024-06-23
 
-### Highlights for 2024-06-21
+### Highlights for 2024-06-23
 
 Following zero-day **SD3** release, a week later here's a refresh with 10+ improvements  
 including full prompt attention, support for compressed weights, additional text-encoder quantization modes.  
@@ -12,6 +12,7 @@ But there's more than SD3:
 - support for **PixArt-Sigma** in small/medium/large variants  
 - support for **HunyuanDiT 1.1**  
 - additional **NNCF weights compression** support: SD3, PixArt, ControlNet, Lora
+- integration of **MS Florence** VLM/VQA *Base* and *Large* models  
 - (finally) new release of **Torch-DirectML**  
 - additional efficiencies for users with low vram gpus  
 - over 20 overall fixes  
@@ -39,6 +40,8 @@ But there's more than SD3:
   *note* by default pixart-Σ uses full fp16 t5 encoder with large memory footprint  
   simply select in *settings -> model -> text encoder* before or after model load  
 - **HunyuanDiT**: support for model version 1.1  
+- **MS Florence**: integration of Microsoft Florence VLM/VQA Base and Large models  
+  simply select in *process -> visual query*!
 
 ### Improvements: General
 

installer.py

@@ -894,6 +894,7 @@ def install_submodules(force=True):
                 branch(name)
         except Exception:
             log.error(f'Error updating submodule: {submodule}')
+    setup_logging()
     if args.profile:
         print_profile(pr, 'Submodule')
     return '\n'.join(res)

modules/api/models.py

@@ -313,7 +313,7 @@ class ResInterrogate(BaseModel):
 
 class ReqVQA(BaseModel):
     image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.")
-    model: str = Field(default="Moondream 2", title="Model", description="The interrogate model used.")
+    model: str = Field(default="MS Florence 2 Base", title="Model", description="The interrogate model used.")
     question: str = Field(default="describe the image", title="Question", description="Question to ask the model.")
 
 class ResVQA(BaseModel):

modules/ui_postprocessing.py

@@ -80,7 +80,7 @@ def create_ui():
                     with gr.Row():
                         vqa_answer = gr.Textbox(label="Answer", lines=3)
                     with gr.Row(elem_id='interrogate_buttons_query'):
-                        vqa_model = gr.Dropdown(list(vqa.MODELS), value='Moondream 2', label='VQA Model')
+                        vqa_model = gr.Dropdown(list(vqa.MODELS), value='MS Florence 2 Base', label='VQA Model')
                         vqa_submit = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary')
                     vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_image, vqa_model], outputs=[vqa_answer])
 

modules/vqa.py

@@ -8,6 +8,8 @@ processor = None
 model = None
 loaded: str = None
 MODELS = {
+    "MS Florence 2 Base": "microsoft/Florence-2-base", # 0.5GB
+    "MS Florence 2 Large": "microsoft/Florence-2-large", # 1.5GB
     "Moondream 2": "vikhyatk/moondream2", # 3.7GB
     "GIT TextCaps Base": "microsoft/git-base-textcaps", # 0.7GB
     "GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB
@@ -124,7 +126,44 @@ def moondream(question: str, image: Image.Image, repo: str = None):
     return response
 
 
+def florence(question: str, image: Image.Image, repo: str = None):
+    global processor, model, loaded # pylint: disable=global-statement
+    if model is None or loaded != repo:
+        model = transformers.AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True)
+        processor = transformers.AutoProcessor.from_pretrained(repo, trust_remote_code=True)
+        loaded = repo
+        model.eval()
+    model.to(devices.device, devices.dtype)
+    shared.log.debug(f'VQA: class={model.__class__.__name__} processor={processor.__class__} model={repo}')
+
+    if question.startswith('<'):
+        task = question.split('>', 1)[0] + '>'
+    else:
+        task = '<MORE_DETAILED_CAPTION>'
+        question = task + question
+    inputs = processor(text=question, images=image, return_tensors="pt")
+    input_ids = inputs['input_ids'].to(devices.device)
+    pixel_values = inputs['pixel_values'].to(devices.device, devices.dtype)
+    with devices.inference_context():
+        generated_ids = model.generate(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            max_new_tokens=1024,
+            num_beams=3,
+            do_sample=False
+        )
+        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+        response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height))
+
+    if 'task' in response:
+        response = response['task']
+    shared.log.debug(f'VQA: task={task} response="{response}"')
+    return response
+
+
 def interrogate(vqa_question, vqa_image, vqa_model_req):
+    from installer import install
+    install('flash_attn', quiet=True)
     vqa_model = MODELS.get(vqa_model_req, None)
     shared.log.debug(f'VQA: model="{vqa_model}" question="{vqa_question}" image={vqa_image}')
     if vqa_image is None:
@@ -146,6 +185,8 @@ def interrogate(vqa_question, vqa_image, vqa_model_req):
         answer = pix(vqa_question, vqa_image, vqa_model)
     if 'moondream2' in vqa_model.lower():
         answer = moondream(vqa_question, vqa_image, vqa_model)
+    if 'florence' in vqa_model.lower():
+        answer = florence(vqa_question, vqa_image, vqa_model)
     else:
         answer = 'unknown model'
     if model is not None: