diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b8b9c52..ba80790a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,21 +2,30 @@ ## Update for 2025-03-15 +### TODO + - Gemma3 requires `git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3` + - **Models** - [THUDM CogView 4 6B](https://huggingface.co/THUDM/CogView4-6B) - new foundation model for image generation based o T5-XXL text encoder and a flow-based diffusion transformer + new foundation model for image generation based o GLM-4 text encoder and a flow-based diffusion transformer fully supports offloading and on-the-fly quantization simply select from *networks -> models -> reference* + *note* cogview4 is compatible with flowmatching samplers - New [zer0int CLiP-L](https://huggingface.co/zer0int/CLIP-Registers-Gated_MLP-ViT-L-14) models: download text encoders into folder set in settings -> system paths -> text encoders (default is `models/Text-encoder`) load using *settings -> text encoder* *tip*: add *sd_text_encoder* to your *settings -> user interface -> quicksettings* list to have it appear at the top of the ui +- **Caption/VLM** + - [Google Gemma 3 4B](https://huggingface.co/google/gemma-3-4b-it) + simply select from list of available models in caption tab + - add option to set system prompt for vlm models that support it: *Gemma, Smol, Qwen* - **Wiki/Docs** - updated [Models](https://github.com/vladmandic/sdnext/wiki/Models) info - Updated SD3 - **Other** - add remote vae info to metadata, thanks @iDeNoh - add quantization support to **CogView-3Plus** + - update `diffusers` - **Fixes** - fix installer not starting when older version of `rich` is installed - fix circular imports when debug flags are enabled diff --git a/installer.py b/installer.py index 153dae193..ca93e49df 100644 --- a/installer.py +++ b/installer.py @@ -538,7 +538,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all or args.skip_git: return - sha = 'b75b204a584e29ebf4e80a61be11458e9ed56e3e' # diffusers commit hash + sha = '82188cef0487837b8c70fc3f36ea63c05c85f341' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 7bdc31c6a..ee30fac44 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -113,7 +113,7 @@ def post_vqa(req: models.ReqVQA): image = helpers.decode_base64_to_image(req.image) image = image.convert('RGB') from modules.interrogate import vqa - answer = vqa.interrogate(req.question, '', image, req.model) + answer = vqa.interrogate(req.question, req.system, '', image, req.model) return models.ResVQA(answer=answer) def post_unload_checkpoint(): diff --git a/modules/interrogate/interrogate.py b/modules/interrogate/interrogate.py index f68212d91..ce3f75193 100644 --- a/modules/interrogate/interrogate.py +++ b/modules/interrogate/interrogate.py @@ -28,7 +28,7 @@ def interrogate(image): elif shared.opts.interrogate_default_type == 'VLM': shared.log.info(f'Interrogate: type={shared.opts.interrogate_default_type} vlm="{shared.opts.interrogate_vlm_model}" prompt="{shared.opts.interrogate_vlm_prompt}"') from modules.interrogate import vqa - prompt = vqa.interrogate(image=image, model_name=shared.opts.interrogate_vlm_model, question=shared.opts.interrogate_vlm_prompt, prompt=None) + prompt = vqa.interrogate(image=image, model_name=shared.opts.interrogate_vlm_model, question=shared.opts.interrogate_vlm_prompt, prompt=None, system_prompt=shared.opts.interrogate_vlm_system) shared.log.debug(f'Interrogate: time={time.time()-t0:.2f} answer="{prompt}"') return prompt else: diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index afe5aac09..5dc88f459 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -22,19 +22,20 @@ vlm_models = { "CogFlorence 2.0 Large": "thwri/CogFlorence-2-Large-Freeze", # 1.6GB "CogFlorence 2.2 Large": "thwri/CogFlorence-2.2-Large", # 1.6GB "Moondream 2": "vikhyatk/moondream2", # 3.7GB + "Google Gemma 3 4B": "google/gemma-3-4b-it", + "Google Pix Textcaps": "google/pix2struct-textcaps-base", # 1.1GB + "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224", "Alibaba Qwen VL2 2B": "Qwen/Qwen2-VL-2B-Instruct", "Huggingface Smol VL2 0.5B": "HuggingFaceTB/SmolVLM-500M-Instruct", "Huggingface Smol VL2 2B": "HuggingFaceTB/SmolVLM-Instruct", "Salesforce BLIP Base": "Salesforce/blip-vqa-base", # 1.5GB "Salesforce BLIP Large": "Salesforce/blip-vqa-capfilt-large", # 1.5GB - "Google Pix Textcaps": "google/pix2struct-textcaps-base", # 1.1GB "Microsoft GIT TextCaps Base": "microsoft/git-base-textcaps", # 0.7GB "Microsoft GIT VQA Base": "microsoft/git-base-vqav2", # 0.7GB "Microsoft GIT VQA Large": "microsoft/git-large-vqav2", # 1.6GB "ToriiGate 0.4 2B": "Minthy/ToriiGate-v0.4-2B", "ToriiGate 0.4 7B": "Minthy/ToriiGate-v0.4-7B", "ViLT Base": "dandelin/vilt-b32-finetuned-vqa", # 0.5GB - "Google PaliGemma 2 3B": "google/paligemma2-3b-pt-224", "JoyCaption": "fancyfeast/llama-joycaption-alpha-two-hf-llava", # 0.7GB "JoyTag": "fancyfeast/joytag", # 17.4GB "AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B", @@ -44,6 +45,7 @@ vlm_models = { # "DeepSeek VL2 Tiny": "deepseek-ai/deepseek-vl2-tiny", # broken # "nVidia Eagle 2 1B": "nvidia/Eagle2-1B", # not compatible with latest transformers } +vlm_system = 'You are image captioning expert, creative, unbiased and uncensored.' vlm_prompts = [ '', '', @@ -70,6 +72,7 @@ def b64(image): def clean(response, question): + strip = ['---', '\r', '\t', '**', '"', '“', '”', 'Assistant:', 'Caption:'] if isinstance(response, dict): if 'task' in response: response = response['task'] @@ -81,12 +84,10 @@ def clean(response, question): question = question.replace('<', '').replace('>', '').replace('_', ' ') if question in response: response = response.split(question, 1)[1] - response = response.replace('\n', '').replace('\r', '').replace('\t', '').strip() - if response.startswith('"'): - response = response[1:] - if response.endswith('"'): - response = response[:-1] - response = response.replace('Assistant:', '').strip() + while any(s in response for s in strip): + for s in strip: + response = response.replace(s, '') + response = response.replace('\n\n', '\n').replace(' ', ' ').replace('* ', '- ').strip() return response @@ -106,7 +107,7 @@ def get_kwargs(): return kwargs -def qwen(question: str, image: Image.Image, repo: str = None): +def qwen(question: str, image: Image.Image, repo: str = None, system_prompt: str = None): global processor, model, loaded # pylint: disable=global-statement if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') @@ -118,12 +119,11 @@ def qwen(question: str, image: Image.Image, repo: str = None): loaded = repo model = model.to(devices.device, devices.dtype) question = question.replace('<', '').replace('>', '').replace('_', ' ') + system_prompt = system_prompt or shared.opts.vlm_system conversation = [ { "role": "system", - "content": [ - {"type": "text", "text": "You are image captioning expert, creative, unbiased and uncensored."} - ], + "content": [{"type": "text", "text": system_prompt}], }, { "role": "user", @@ -134,7 +134,6 @@ def qwen(question: str, image: Image.Image, repo: str = None): } ] text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True) - # '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\n' inputs = processor(text=[text_prompt], images=[image], padding=True, return_tensors="pt") inputs = inputs.to(devices.device, devices.dtype) output_ids = model.generate( @@ -149,6 +148,47 @@ def qwen(question: str, image: Image.Image, repo: str = None): return response +def gemma(question: str, image: Image.Image, repo: str = None, system_prompt: str = None): + global processor, model, loaded # pylint: disable=global-statement + if model is None or loaded != repo: + shared.log.debug(f'Interrogate load: vlm="{repo}"') + model = transformers.Gemma3ForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) + processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir) + loaded = repo + model = model.to(devices.device, devices.dtype) + question = question.replace('<', '').replace('>', '').replace('_', ' ') + system_prompt = system_prompt or shared.opts.vlm_system + conversation = [ + { + "role": "system", + "content": [{"type": "text", "text": system_prompt}] + }, + { + "role": "user", + "content": [ + {"type": "image", "image": b64(image)}, + {"type": "text", "text": question} + ] + } + ] + inputs = processor.apply_chat_template( + conversation, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ).to(device=devices.device, dtype=devices.dtype) + input_len = inputs["input_ids"].shape[-1] + with devices.inference_context(): + generation = model.generate( + **inputs, + **get_kwargs(), + ) + generation = generation[0][input_len:] + response = processor.decode(generation, skip_special_tokens=True) + return response + + def paligemma(question: str, image: Image.Image, repo: str = None): global processor, model, loaded # pylint: disable=global-statement if model is None or loaded != repo: @@ -219,7 +259,7 @@ def ovis(question: str, image: Image.Image, repo: str = None): return response -def smol(question: str, image: Image.Image, repo: str = None): +def smol(question: str, image: Image.Image, repo: str = None, system_prompt: str = None): global processor, model, loaded # pylint: disable=global-statement if model is None or loaded != repo: shared.log.debug(f'Interrogate load: vlm="{repo}"') @@ -233,12 +273,11 @@ def smol(question: str, image: Image.Image, repo: str = None): loaded = repo model.to(devices.device, devices.dtype) question = question.replace('<', '').replace('>', '').replace('_', ' ') + system_prompt = system_prompt or shared.opts.vlm_system conversation = [ { "role": "system", - "content": [ - {"type": "text", "text": "You are image captioning expert, creative, unbiased and uncensored."} - ], + "content": [{"type": "text", "text": system_prompt}], }, { "role": "user", @@ -249,7 +288,6 @@ def smol(question: str, image: Image.Image, repo: str = None): } ] text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True) - # '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\n' inputs = processor(text=text_prompt, images=[image], padding=True, return_tensors="pt") inputs = inputs.to(devices.device, devices.dtype) output_ids = model.generate( @@ -410,7 +448,7 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str return response -def interrogate(question, prompt, image, model_name, quiet:bool=False): +def interrogate(question, system_prompt, prompt, image, model_name, quiet:bool=False): if not quiet: shared.state.begin('Interrogate') t0 = time.time() @@ -457,9 +495,9 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): elif 'florence' in vqa_model.lower(): answer = florence(question, image, vqa_model) elif 'qwen' in vqa_model.lower() or 'torii' in vqa_model.lower(): - answer = qwen(question, image, vqa_model) + answer = qwen(question, image, vqa_model, system_prompt) elif 'smol' in vqa_model.lower(): - answer = smol(question, image, vqa_model) + answer = smol(question, image, vqa_model, system_prompt) elif 'joytag' in vqa_model.lower(): from modules.interrogate import joytag answer = joytag.predict(image) @@ -471,6 +509,8 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): answer = deepseek.predict(question, image, vqa_model) elif 'paligemma' in vqa_model.lower(): answer = paligemma(question, image, vqa_model) + elif 'gemma' in vqa_model.lower(): + answer = gemma(question, image, vqa_model, system_prompt) elif 'ovis' in vqa_model.lower(): answer = ovis(question, image, vqa_model) else: @@ -481,7 +521,9 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): if shared.opts.interrogate_offload and model is not None: model.to(devices.cpu) devices.torch_gc() + print('HERE1', answer) answer = clean(answer, question) + print('HERE2', answer) t1 = time.time() if not quiet: shared.log.debug(f'Interrogate: type=vlm model="{model_name}" repo="{vqa_model}" args={get_kwargs()} time={t1-t0:.2f}') @@ -489,7 +531,7 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): return answer -def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, write, append, recursive): +def batch(model_name, system_prompt, batch_files, batch_folder, batch_str, question, prompt, write, append, recursive): class BatchWriter: def __init__(self, folder, mode='w'): self.folder = folder @@ -536,7 +578,7 @@ def batch(model_name, batch_files, batch_folder, batch_str, question, prompt, wr if shared.state.interrupted: break image = Image.open(file) - prompt = interrogate(question, prompt, image, model_name, quiet=True) + prompt = interrogate(question, system_prompt, prompt, image, model_name, quiet=True) prompts.append(prompt) if write: writer.add(file, prompt) diff --git a/modules/model_cogview.py b/modules/model_cogview.py index f76a95ed3..8ced40ce2 100644 --- a/modules/model_cogview.py +++ b/modules/model_cogview.py @@ -76,7 +76,7 @@ def load_cogview4(checkpoint_info, diffusers_load_config={}): ) diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Text Encoder') - text_encoder = transformers.T5EncoderModel.from_pretrained( + text_encoder = transformers.AutoModelForCausalLM.from_pretrained( repo_id, subfolder="text_encoder", cache_dir=shared.opts.diffusers_dir, @@ -91,5 +91,9 @@ def load_cogview4(checkpoint_info, diffusers_load_config={}): cache_dir=shared.opts.diffusers_dir, **diffusers_load_config, ) + if shared.opts.diffusers_eval: + pipe.text_encoder.eval() + pipe.transformer.eval() + pipe.enable_model_cpu_offload() # TODO cogview4: balanced offload does not work for GlmModel devices.torch_gc() return pipe diff --git a/modules/pixelsmith/pixelsmith_pipeline.py b/modules/pixelsmith/pixelsmith_pipeline.py index 4e04b2d92..702ee67f6 100644 --- a/modules/pixelsmith/pixelsmith_pipeline.py +++ b/modules/pixelsmith/pixelsmith_pipeline.py @@ -133,7 +133,6 @@ class PAGIdentitySelfAttnProcessor: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -248,7 +247,6 @@ class PAGCFGIdentitySelfAttnProcessor: value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 54dafb940..be819d658 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -139,7 +139,7 @@ def full_vae_decode(latents, model): if latents_mean and latents_std: latents_mean = (torch.tensor(latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)) latents_std = (torch.tensor(latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)) - latents = latents * latents_std / scaling_factor + latents_mean + latents = ((latents * latents_std) / scaling_factor) + latents_mean else: latents = latents / scaling_factor if shift_factor: diff --git a/modules/schedulers/scheduler_dpm_flowmatch.py b/modules/schedulers/scheduler_dpm_flowmatch.py index 69452aca9..ab9aa47a9 100644 --- a/modules/schedulers/scheduler_dpm_flowmatch.py +++ b/modules/schedulers/scheduler_dpm_flowmatch.py @@ -230,6 +230,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): device: Union[str, torch.device] = None, sigmas: Optional[List[float]] = None, mu: Optional[float] = None, + timesteps: Optional[torch.Tensor] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -355,12 +356,12 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): sigma_min = sigmas[-1] sigmas = np.linspace(1.0, sigma_min, num_inference_steps) sigmas = torch.from_numpy(sigmas).to(dtype=torch.float64, device=device) - + if self.config.use_dynamic_shifting: sigmas = self.time_shift(mu, 1.0, sigmas) else: sigmas = self.config.shift * sigmas / (1 + (self.config.shift - 1) * sigmas) - + timesteps = sigmas * self.config.num_train_timesteps self.timesteps = timesteps.to(device=device) self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) @@ -517,7 +518,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): model_output = sample - sigma * model_output d = (sample - model_output) / sigma dt = sigma_next - sigma - sample = sample + d * dt + sample = sample + d * dt else: # DPM-Solver2 sigma_mid = sigma.log().lerp(sigma_next.log(), 0.5).exp() @@ -596,7 +597,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): elif self.config.algorithm_type == "dpmsolver++2M": if self.config.solver_order == 2: t, t_next = t_fn(sigma), t_fn(sigma_next) - h = t_next - t + h = t_next - t if self.model_outputs[-2] is None or sigma_next == 0: sample = (sigma_fn(t_next) / sigma_fn(t)) * sample - (-h).expm1() * model_output else: @@ -703,7 +704,7 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): if self.config.use_noise_sampler: sample = sample + self.noise_sampler(sigma_fn(t), sigma_fn(t_next)) * self.config.s_noise * su else: - sample = sample + noise * self.config.s_noise * su + sample = sample + noise * self.config.s_noise * su del x_2 del denoised_2 del d @@ -745,13 +746,13 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): t, s = -sigma.log(), -sigma_next.log() h = s - t h_eta = h * 2 - + # 3. Delta timestep dt = sigma_next - sigma sample = sample + model_output * dt sample = torch.exp(-h_eta) * sample + (-h_eta).expm1().neg() * model_output - + if self.h_2 is not None: r0 = self.h_1 / h r1 = self.h_2 / h @@ -780,13 +781,13 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): sample = sample + self.noise_sampler(sigma, sigma_next) * sigma_next * (-2 * h).expm1().neg().sqrt() * self.config.s_noise else: sample = sample + noise * sigma_next * (-2 * h).expm1().neg().sqrt() * self.config.s_noise - + self.h_2 = self.h_1 self.h_1 = h if not self.config.use_noise_sampler and noise is not None: del noise prev_sample = sample - + # Cast sample back to expected dtype prev_sample = prev_sample.to(model_output.dtype) diff --git a/modules/sd_offload.py b/modules/sd_offload.py index 33ce1b2c0..2c4126209 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -10,8 +10,9 @@ from modules.timer import process as process_timer debug_move = shared.log.trace if os.environ.get('SD_MOVE_DEBUG', None) is not None else lambda *args, **kwargs: None -should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen', 'hunyuanvideo', 'cogvideox', 'mochi'] +should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen', 'hunyuanvideo', 'cogvideox', 'mochi', 'cogview4'] offload_hook_instance = None +balanced_offload_exclude = ['OmniGenPipeline', 'CogView4Pipeline'] def get_signature(cls): @@ -193,8 +194,7 @@ def apply_balanced_offload(sd_model, exclude=[]): if sd_model is None: return sd_model t0 = time.time() - excluded = ['OmniGenPipeline'] - if sd_model.__class__.__name__ in excluded: + if sd_model.__class__.__name__ in balanced_offload_exclude: return sd_model cached = True checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else None diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index accd6b0ed..67ed8e8ee 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -79,7 +79,7 @@ def create_sampler(name, model): shared.log.debug(f'Sampler: "{name}" config={config.options}') return sampler elif shared.native: - FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana', 'HunyuanVideoPipeline'] + FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana', 'HunyuanVideoPipeline', 'CogView4Pipeline'] if 'KDiffusion' in model.__class__.__name__: return None if not any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' in name: diff --git a/modules/shared.py b/modules/shared.py index 962bf05d0..10cb1007e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -19,7 +19,7 @@ from modules.dml import memory_providers, default_memory_provider, directml_do_h from modules.onnx_impl import initialize_onnx, execution_providers from modules.memstats import memory_stats, ram_stats # pylint: disable=unused-import from modules.interrogate.openclip import caption_models, caption_types, get_clip_models, refresh_clip_models, category_types -from modules.interrogate.vqa import vlm_models, vlm_prompts +from modules.interrogate.vqa import vlm_models, vlm_prompts, vlm_system from modules.ui_components import DropdownEditable from modules.options import OptionInfo import modules.memmon @@ -855,6 +855,7 @@ options_templates.update(options_section(('interrogate', "Interrogate"), { "interrogate_vlm_sep": OptionInfo("

VLM

", "", gr.HTML), "interrogate_vlm_model": OptionInfo(list(vlm_models)[0], "VLM: default model", gr.Dropdown, {"choices": list(vlm_models)}), "interrogate_vlm_prompt": OptionInfo(vlm_prompts[2], "VLM: default prompt", DropdownEditable, {"choices": vlm_prompts }), + "interrogate_vlm_system": OptionInfo(vlm_system, "VLM: default prompt"), "interrogate_vlm_num_beams": OptionInfo(3, "VLM: num beams", gr.Slider, {"minimum": 1, "maximum": 16, "step": 1, "visible": False}), "interrogate_vlm_max_length": OptionInfo(512, "VLM: max length", gr.Slider, {"minimum": 1, "maximum": 4096, "step": 1, "visible": False}), "interrogate_vlm_do_sample": OptionInfo(False, "VLM: use sample method"), diff --git a/modules/ui_caption.py b/modules/ui_caption.py index 04a16702c..474427e9d 100644 --- a/modules/ui_caption.py +++ b/modules/ui_caption.py @@ -35,6 +35,8 @@ def create_ui(): with gr.Tabs(elem_id="mode_caption"): with gr.Tab("VLM Caption", elem_id="tab_vlm_caption"): from modules.interrogate import vqa + with gr.Row(): + vlm_system = gr.Textbox(label="System prompt", value=vqa.vlm_system, lines=1, elem_id='vlm_system') with gr.Row(): vlm_question = gr.Dropdown(label="Predefined question", allow_custom_value=False, choices=vqa.vlm_prompts, value=vqa.vlm_prompts[2], elem_id='vlm_question') with gr.Row(): @@ -114,7 +116,7 @@ def create_ui(): btn_clip_analyze_img = gr.Button("Analyze", variant='primary', elem_id="btn_clip_analyze_img") with gr.Column(variant='compact', elem_id='interrogate_output'): with gr.Row(elem_id='interrogate_output_prompt'): - prompt = gr.Textbox(label="Answer", lines=8, placeholder="ai generated image description") + prompt = gr.Textbox(label="Answer", lines=12, placeholder="ai generated image description") with gr.Row(elem_id='interrogate_output_classes'): medium = gr.Label(elem_id="interrogate_label_medium", label="Medium", num_top_classes=5, visible=False) artist = gr.Label(elem_id="interrogate_label_artist", label="Artist", num_top_classes=5, visible=False) @@ -127,8 +129,8 @@ def create_ui(): btn_clip_interrogate_img.click(openclip.interrogate_image, inputs=[image, clip_model, blip_model, clip_mode], outputs=[prompt]) btn_clip_analyze_img.click(openclip.analyze_image, inputs=[image, clip_model, blip_model], outputs=[medium, artist, movement, trending, flavor]) btn_clip_interrogate_batch.click(fn=openclip.interrogate_batch, inputs=[clip_batch_files, clip_batch_folder, clip_batch_str, clip_model, blip_model, clip_mode, clip_save_output, clip_save_append, clip_folder_recursive], outputs=[prompt]) - btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_prompt, image, vlm_model], outputs=[prompt]) - btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append, vlm_folder_recursive], outputs=[prompt]) + btn_vlm_caption.click(fn=vqa.interrogate, inputs=[vlm_question, vlm_system, vlm_prompt, image, vlm_model], outputs=[prompt]) + btn_vlm_caption_batch.click(fn=vqa.batch, inputs=[vlm_model, vlm_system, vlm_batch_files, vlm_batch_folder, vlm_batch_str, vlm_question, vlm_prompt, vlm_save_output, vlm_save_append, vlm_folder_recursive], outputs=[prompt]) for tabname, button in copy_interrogate_buttons.items(): generation_parameters_copypaste.register_paste_params_button(generation_parameters_copypaste.ParamBinding(paste_button=button, tabname=tabname, source_text_component=prompt, source_image_component=image,)) diff --git a/requirements.txt b/requirements.txt index f61890cbe..e64ada91b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,18 +41,18 @@ torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.3 tqdm==4.67.1 -accelerate==1.3.0 +accelerate==1.5.2 opencv-contrib-python-headless==4.9.0.80 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.28.1 +huggingface_hub==0.29.3 numexpr==2.8.8 numpy==1.26.4 numba==0.59.1 protobuf==4.25.3 pytorch_lightning==1.9.4 tokenizers==0.21.0 -transformers==4.48.3 +transformers==4.49.0 urllib3==1.26.19 Pillow==10.4.0 timm==0.9.16