From 72f6cf6a1f639512effa3b7749a169cde91e73df Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 23 Jun 2024 08:50:06 -0400 Subject: [PATCH] add ms florence --- CHANGELOG.md | 7 ++++-- installer.py | 1 + modules/api/models.py | 2 +- modules/ui_postprocessing.py | 2 +- modules/vqa.py | 41 ++++++++++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d44a2f09..19c962a9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2024-06-21 +## Update for 2024-06-23 -### Highlights for 2024-06-21 +### Highlights for 2024-06-23 Following zero-day **SD3** release, a week later here's a refresh with 10+ improvements including full prompt attention, support for compressed weights, additional text-encoder quantization modes. @@ -12,6 +12,7 @@ But there's more than SD3: - support for **PixArt-Sigma** in small/medium/large variants - support for **HunyuanDiT 1.1** - additional **NNCF weights compression** support: SD3, PixArt, ControlNet, Lora +- integration of **MS Florence** VLM/VQA *Base* and *Large* models - (finally) new release of **Torch-DirectML** - additional efficiencies for users with low vram gpus - over 20 overall fixes @@ -39,6 +40,8 @@ But there's more than SD3: *note* by default pixart-Σ uses full fp16 t5 encoder with large memory footprint simply select in *settings -> model -> text encoder* before or after model load - **HunyuanDiT**: support for model version 1.1 +- **MS Florence**: integration of Microsoft Florence VLM/VQA Base and Large models + simply select in *process -> visual query*! ### Improvements: General diff --git a/installer.py b/installer.py index aaaad060b..2bffe8276 100644 --- a/installer.py +++ b/installer.py @@ -894,6 +894,7 @@ def install_submodules(force=True): branch(name) except Exception: log.error(f'Error updating submodule: {submodule}') + setup_logging() if args.profile: print_profile(pr, 'Submodule') return '\n'.join(res) diff --git a/modules/api/models.py b/modules/api/models.py index 8437e91ba..5813fdcc6 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -313,7 +313,7 @@ class ResInterrogate(BaseModel): class ReqVQA(BaseModel): image: str = Field(default="", title="Image", description="Image to work on, must be a Base64 string containing the image's data.") - model: str = Field(default="Moondream 2", title="Model", description="The interrogate model used.") + model: str = Field(default="MS Florence 2 Base", title="Model", description="The interrogate model used.") question: str = Field(default="describe the image", title="Question", description="Question to ask the model.") class ResVQA(BaseModel): diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index 42d1f9847..b1948a27b 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -80,7 +80,7 @@ def create_ui(): with gr.Row(): vqa_answer = gr.Textbox(label="Answer", lines=3) with gr.Row(elem_id='interrogate_buttons_query'): - vqa_model = gr.Dropdown(list(vqa.MODELS), value='Moondream 2', label='VQA Model') + vqa_model = gr.Dropdown(list(vqa.MODELS), value='MS Florence 2 Base', label='VQA Model') vqa_submit = gr.Button("Interrogate", elem_id="interrogate_btn_interrogate", variant='primary') vqa_submit.click(vqa.interrogate, inputs=[vqa_question, vqa_image, vqa_model], outputs=[vqa_answer]) diff --git a/modules/vqa.py b/modules/vqa.py index 8344b15bf..357a604d9 100644 --- a/modules/vqa.py +++ b/modules/vqa.py @@ -8,6 +8,8 @@ processor = None model = None loaded: str = None MODELS = { + "MS Florence 2 Base": "microsoft/Florence-2-base", # 0.5GB + "MS Florence 2 Large": "microsoft/Florence-2-large", # 1.5GB "Moondream 2": "vikhyatk/moondream2", # 3.7GB "GIT TextCaps Base": "microsoft/git-base-textcaps", # 0.7GB "GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB @@ -124,7 +126,44 @@ def moondream(question: str, image: Image.Image, repo: str = None): return response +def florence(question: str, image: Image.Image, repo: str = None): + global processor, model, loaded # pylint: disable=global-statement + if model is None or loaded != repo: + model = transformers.AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True) + processor = transformers.AutoProcessor.from_pretrained(repo, trust_remote_code=True) + loaded = repo + model.eval() + model.to(devices.device, devices.dtype) + shared.log.debug(f'VQA: class={model.__class__.__name__} processor={processor.__class__} model={repo}') + + if question.startswith('<'): + task = question.split('>', 1)[0] + '>' + else: + task = '' + question = task + question + inputs = processor(text=question, images=image, return_tensors="pt") + input_ids = inputs['input_ids'].to(devices.device) + pixel_values = inputs['pixel_values'].to(devices.device, devices.dtype) + with devices.inference_context(): + generated_ids = model.generate( + input_ids=input_ids, + pixel_values=pixel_values, + max_new_tokens=1024, + num_beams=3, + do_sample=False + ) + generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] + response = processor.post_process_generation(generated_text, task="task", image_size=(image.width, image.height)) + + if 'task' in response: + response = response['task'] + shared.log.debug(f'VQA: task={task} response="{response}"') + return response + + def interrogate(vqa_question, vqa_image, vqa_model_req): + from installer import install + install('flash_attn', quiet=True) vqa_model = MODELS.get(vqa_model_req, None) shared.log.debug(f'VQA: model="{vqa_model}" question="{vqa_question}" image={vqa_image}') if vqa_image is None: @@ -146,6 +185,8 @@ def interrogate(vqa_question, vqa_image, vqa_model_req): answer = pix(vqa_question, vqa_image, vqa_model) if 'moondream2' in vqa_model.lower(): answer = moondream(vqa_question, vqa_image, vqa_model) + if 'florence' in vqa_model.lower(): + answer = florence(vqa_question, vqa_image, vqa_model) else: answer = 'unknown model' if model is not None: