diff --git a/CHANGELOG.md b/CHANGELOG.md index 072af4c91..6314aa6af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,10 +72,10 @@ Plus support for **CogView-4**, **SANA 1.5**, new CLiP models, improvements to r - new [Caption](https://github.com/vladmandic/sdnext/wiki/Caption) guide - new [VAE](https://github.com/vladmandic/sdnext/wiki/VAE) guide - updated [SD3](https://github.com/vladmandic/sdnext/wiki/SD3) guide - - updated [ZLUDA](https://github.com/vladmandic/sdnext/wiki/ZLUDA) guide - - updated [OpenVINO](https://github.com/vladmandic/sdnext/wiki/OpenVINO) guide - - updated [AMD-ROCm](https://github.com/vladmandic/sdnext/wiki/AMD-ROCm) guide - - upte [Intel-ARC](https://github.com/vladmandic/sdnext/wiki/Intel-ARC) guide + - updated [ZLUDA](https://github.com/vladmandic/sdnext/wiki/ZLUDA) guide + - updated [OpenVINO](https://github.com/vladmandic/sdnext/wiki/OpenVINO) guide + - updated [AMD-ROCm](https://github.com/vladmandic/sdnext/wiki/AMD-ROCm) guide + - upte [Intel-ARC](https://github.com/vladmandic/sdnext/wiki/Intel-ARC) guide - **Remote VAE** - add support for remote vae encode in addition to remote vae decode - used by *img2img, inpaint, hires, detailer* @@ -83,9 +83,11 @@ Plus support for **CogView-4**, **SANA 1.5**, new CLiP models, improvements to r - add remote vae info to metadata, thanks @iDeNoh - remote vae use `scaling_factor` and `shift_factor` - **Caption/VLM** - - [Google Gemma 3 4B](https://huggingface.co/google/gemma-3-4b-it) + - [Google Gemma 3](https://huggingface.co/google/gemma-3-4b-it) 4B simply select from list of available models in caption tab - - add option to set system prompt for vlm models that support it: *Gemma, Smol, Qwen* + - [ByteDance/Sa2VA](https://huggingface.co/ByteDance/Sa2VA-1B) 1B, 4B + simply select from list of available models in caption tab + - add option to set system prompt for vlm models that support it: *Gemma, Smol, Qwen* - [NudeNet](https://github.com/vladmandic/sd-extension-nudenet/) extension updates - add detection of prompt language and alphabet and filter based on those values - add image policy checks using `LlavaGuard` VLM to detect policy violations (and reasons) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 2bdec3543..3db5251e8 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -41,6 +41,8 @@ vlm_models = { "AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B", "AIDC Ovis2 2B": "AIDC-AI/Ovis2-2B", "AIDC Ovis2 4B": "AIDC-AI/Ovis2-4B", + "ByteDance Sa2VA 1B": "ByteDance/Sa2VA-1B", + "ByteDance Sa2VA 4B": "ByteDance/Sa2VA-4B", # "OpenGVLab InternVL 2.5 1B": "OpenGVLab/InternVL2_5-1B" # "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken # "nVidia Eagle 2 1B": "nvidia/Eagle2-1B", # not compatible with latest transformers @@ -72,7 +74,7 @@ def b64(image): def clean(response, question): - strip = ['---', '\r', '\t', '**', '"', '“', '”', 'Assistant:', 'Caption:'] + strip = ['---', '\r', '\t', '**', '"', '“', '”', 'Assistant:', 'Caption:', '<|im_end|>'] if isinstance(response, dict): if 'task' in response: response = response['task'] @@ -451,6 +453,39 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str return response +def sa2(question: str, image: Image.Image, repo: str = None): + global processor, model, loaded # pylint: disable=global-statement + if model is None or loaded != repo: + model = transformers.AutoModel.from_pretrained( + repo, + torch_dtype=devices.dtype, + low_cpu_mem_usage=True, + use_flash_attn=False, + trust_remote_code=True) + model = model.eval() + processor = transformers.AutoTokenizer.from_pretrained( + repo, + trust_remote_code=True, + use_fast=False, + ) + loaded = repo + model = model.to(devices.device, devices.dtype) + if question.startswith('<'): + task = question.split('>', 1)[0] + '>' + else: + task = '' + input_dict = { + 'image': image, + 'text': f'{task}', + 'past_text': '', + 'mask_prompts': None, + 'tokenizer': processor, + } + return_dict = model.predict_forward(**input_dict) + response = return_dict["prediction"] # the text format answer + return response + + def interrogate(question, system_prompt, prompt, image, model_name, quiet:bool=False): if not quiet: shared.state.begin('Interrogate') @@ -516,6 +551,8 @@ def interrogate(question, system_prompt, prompt, image, model_name, quiet:bool=F answer = gemma(question, image, vqa_model, system_prompt) elif 'ovis' in vqa_model.lower(): answer = ovis(question, image, vqa_model) + elif 'sa2' in vqa_model.lower(): + answer = sa2(question, image, vqa_model) else: answer = 'unknown model' except Exception as e: