From 29de324ff769eb68af0c0a91f8284fa97f5a2c4a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Sep 2026 21:53:29 +0200 Subject: [PATCH] update all google stuff Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 11 +++ constraints.txt | 2 +- modules/caption/gemini.py | 2 +- modules/caption/models_def.py | 12 +-- modules/modular.py | 2 +- modules/processing.py | 2 +- modules/video_models/google_omni.py | 137 ++++++++++++++++++++++++++++ modules/video_models/google_veo.py | 36 +++----- modules/video_models/models_def.py | 49 +++++++++- modules/video_models/video_load.py | 4 + pipelines/model_google.py | 2 +- scripts/prompt_enhance/options.py | 20 ++-- 12 files changed, 234 insertions(+), 45 deletions(-) create mode 100644 modules/video_models/google_omni.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ae5858a..b53f17b52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,17 @@ All-about-optimizations: - implement progress and preview - intercept and profiling hooks - on-demand convert standard model on-demand +- **Google** + - updated support for google models in text, image and video workflows + *note*: requires google api key + - [Google Veo](https://ai.google.dev/gemini-api/docs/veo) in *preview*, *fast* and *lite* variants + workflows: *t2v, i2v* + - [Google Omni](https://ai.google.dev/gemini-api/docs/omni) in *flash* variant + workflows: *t2v, i2v* + - [Google Nano Banana](https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-image) in *2* and *2 lite* and *pro* variants + workflows: *caption* + - [Google Gemini](https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash) in *flash* and *pro* variants + workflows: *caption, prompt-enhance* - **Compute** - cuda: update `torch==2.14.0` with `cuda==13.2` - openvino: update `openvino==2026.3.1` with `torch==2.13.0` diff --git a/constraints.txt b/constraints.txt index 1eb39948b..2ba2bddf7 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,3 +1,3 @@ fastapi==0.124.4 -numpy==2.5.2 Pillow==12.2.0 +# numpy==2.5.2 diff --git a/modules/caption/gemini.py b/modules/caption/gemini.py index c5a6f8e70..b36021205 100644 --- a/modules/caption/gemini.py +++ b/modules/caption/gemini.py @@ -11,7 +11,7 @@ class GoogleGeminiPipeline(): def __init__(self, model_name: str): self.model = model_name.split(' (')[0] from installer import install - install('google-genai==1.52.0') + install('google-genai==2.22.0') from google import genai # pylint: disable=no-name-in-module args = self.get_args() self.client = genai.Client(**args) diff --git a/modules/caption/models_def.py b/modules/caption/models_def.py index 82d529616..63519cd1d 100644 --- a/modules/caption/models_def.py +++ b/modules/caption/models_def.py @@ -74,13 +74,13 @@ vlm_models = { "AIDC Ovis2 2B": "AIDC-AI/Ovis2-2B", "AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B", # cloud - f"Google Gemini 3.5 Flash {ui_symbols.cloud}": "google/gemini-3.5-flash", - f"Google Gemini 3.1 Pro {ui_symbols.cloud}": "gemini-3.1-pro-preview", + f"Google Gemini 3.8 Flash {ui_symbols.cloud}": "gemini-3.8-flash", + f"Google Gemini 3.7 Flash {ui_symbols.cloud}": "gemini-3.7-flash", + f"Google Gemini 3.6 Flash {ui_symbols.cloud}": "gemini-3.6-flash", + f"Google Gemini 3.5 Flash {ui_symbols.cloud}": "gemini-3.5-flash", + f"Google Gemini 3.5 Flash Lite {ui_symbols.cloud}": "gemini-3.5-flash-lite", f"Google Gemini 3.1 Flash Lite {ui_symbols.cloud}": "gemini-3.1-flash-lite", - f"Google Gemini 3.1 Flash Lite Preview {ui_symbols.cloud}": "gemini-3.1-flash-lite-preview", - f"Google Gemini 2.5 Pro {ui_symbols.cloud}": "gemini-2.5-pro", - f"Google Gemini 2.5 Flash {ui_symbols.cloud}": "gemini-2.5-flash", - f"Google Gemini 2.5 Flash Lite {ui_symbols.cloud}": "gemini-2.5-flash-lite", + f"Google Gemini 3.1 Pro {ui_symbols.cloud}": "gemini-3.1-pro-preview", } # Default model diff --git a/modules/modular.py b/modules/modular.py index 34fd7d75b..5e8eeb9c6 100644 --- a/modules/modular.py +++ b/modules/modular.py @@ -27,7 +27,7 @@ def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool: return False compatible = get_modular_class(diffusion_pipeline) is not None if not compatible: - log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline') + log.warning(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline') return compatible diff --git a/modules/processing.py b/modules/processing.py index 7f5eb0557..010b28d72 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -541,7 +541,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner): p.scripts.postprocess_batch(p, samples, batch_number=n) - if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner): + if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner) and isinstance(samples, list): p.prompts = p.all_prompts[(n * p.batch_size):((n+1) * p.batch_size)] p.negative_prompts = p.all_negative_prompts[(n * p.batch_size):((n+1) * p.batch_size)] batch_params = scripts_manager.PostprocessBatchListArgs(list(samples)) diff --git a/modules/video_models/google_omni.py b/modules/video_models/google_omni.py new file mode 100644 index 000000000..33989eec6 --- /dev/null +++ b/modules/video_models/google_omni.py @@ -0,0 +1,137 @@ +import io +import os +import base64 +import time + +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) + +from PIL import Image +from modules.logger import log + + +image_size_buckets = { + '360p': 640*360, + '720p': 1280*720, + '1080p': 1920*1080, + '4k': 3840*2160, +} +aspect_ratios_buckets = { + '16:9': 16/9, + '9:16': 9/16, +} + + +def google_requirements(): + from installer import install + install('google-genai==2.22.0') + # install('pydantic==2.11.7', ignore=True, quiet=True) + # reload('pydantic', '2.11.7') + + +def get_size_buckets(width: int, height: int) -> tuple[str, str]: + aspect_ratio = width / height + pixel_count = width * height + closest_size = min(image_size_buckets.items(), key=lambda x: abs(x[1] - pixel_count))[0] + closest_aspect_ratio = min(aspect_ratios_buckets.items(), key=lambda x: abs(x[1] - aspect_ratio))[0] + return closest_size, closest_aspect_ratio + + +class GoogleOmniVideoPipeline: + def __init__(self, model_name: str): + self.model = model_name + self.client = None + google_requirements() + log.debug(f'Load model: type=GoogleOmni model="{model_name}"') + + def get_args(self): + from modules.shared import opts + # Use UI settings only - env vars are intentionally ignored + api_key = opts.google_api_key + project_id = opts.google_project_id + location_id = opts.google_location_id + use_vertexai = opts.google_use_vertexai + + has_api_key = api_key and len(api_key) > 0 + has_project = project_id and len(project_id) > 0 + has_location = location_id and len(location_id) > 0 + + if use_vertexai: + if has_api_key and (has_project or has_location): + # Invalid: can't have both api_key AND project/location + log.error(f'Cloud: model="{self.model}" API key and project/location are mutually exclusive') + return None + elif has_api_key: + # Vertex AI Express Mode: api_key + vertexai, no project/location + args = {'api_key': api_key, 'vertexai': True} + elif has_project and has_location: + # Standard Vertex AI: project/location, no api_key + args = {'vertexai': True, 'project': project_id, 'location': location_id} + else: + log.error(f'Cloud: model="{self.model}" Vertex AI requires either API key (Express Mode) or project ID + location ID') + return None + else: + # Gemini Developer API: api_key only + if not has_api_key: + log.error(f'Cloud: model="{self.model}" API key not provided') + return None + args = {'api_key': api_key} + + # Debug logging + args_log = args.copy() + if args_log.get('api_key'): + args_log['api_key'] = '...' + args_log['api_key'][-4:] + log.debug(f'Cloud: model="{self.model}" args={args_log}') + return args + + def __call__(self, prompt: list[str], width: int, height: int, image: Image.Image = None): + if isinstance(prompt, list) and len(prompt) > 0: + prompt = prompt[0] + if self.client is None: + args = self.get_args() + if args is None: + return None + from google import genai # pylint: disable=no-name-in-module + self.client = genai.Client(**args) + + resolution, aspect_ratio = get_size_buckets(width, height) + response_format = { + 'type': 'video', + 'aspect_ratio': aspect_ratio, + 'resolution': resolution, + } + if image is not None: + image_bytes = io.BytesIO() + image.save(image_bytes, format='JPEG') + input_content = [ + {'type': 'image', 'data': base64.b64encode(image_bytes.getvalue()).decode('utf-8'), 'mime_type': 'image/jpeg'}, + {'type': 'text', 'text': prompt}, + ] + else: + input_content = prompt + log.debug(f'Cloud: prompt="{prompt}" size={resolution} ar={aspect_ratio} image={image} model="{self.model}" genai={genai.__version__}') + + t0 = time.time() + try: + interaction = self.client.interactions.create( + model=self.model, + input=input_content, + response_format=response_format, + ) + except Exception as e: + log.error(f'Cloud video: model="{self.model}" {e}') + return None + t1 = time.time() + log.debug(f'Cloud processing: model="{self.model}" elapsed={t1-t0:.2f}') + + try: + video_bytes = base64.b64decode(interaction.output_video.data) + return { 'bytes': video_bytes, 'images': [] } + except Exception as e: + log.error(f'Cloud download: model="{self.model}" {e}') + return None + + +def load_omni(model_name): # pylint: disable=unused-argument + pipe = GoogleOmniVideoPipeline(model_name = model_name) + return pipe diff --git a/modules/video_models/google_veo.py b/modules/video_models/google_veo.py index d0c5b3666..7c7941a9a 100644 --- a/modules/video_models/google_veo.py +++ b/modules/video_models/google_veo.py @@ -6,13 +6,13 @@ import sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from PIL import Image -from installer import install from modules.logger import log image_size_buckets = { '720p': 1280*720, '1080p': 1920*1080, + '4k': 3840*2160, } aspect_ratios_buckets = { '1:1': 1/1, @@ -30,7 +30,9 @@ aspect_ratios_buckets = { def google_requirements(): - install('google-genai==1.52.0') + from installer import install + install('google-genai==2.22.0') + # install('google-genai==1.75.0') # install('pydantic==2.11.7', ignore=True, quiet=True) # reload('pydantic', '2.11.7') @@ -110,7 +112,7 @@ class GoogleVeoVideoPipeline: log.debug(f'Cloud: model="{self.model}" args={args_log}') return args - def __call__(self, prompt: list[str], width: int, height: int, image: Image.Image = None, num_frames: int = 4*24): + def __call__(self, prompt: list[str], width: int = 1280, height: int = 720, image: Image.Image = None, num_frames: int = 4*24): from google import genai # pylint: disable=no-name-in-module if isinstance(prompt, list) and len(prompt) > 0: @@ -127,28 +129,30 @@ class GoogleVeoVideoPipeline: duration = 4 if duration > 8: duration = 8 - self.config=genai.types.GenerateVideosConfig( - # seed=42, - # fps=24, + self.config = genai.types.GenerateVideosConfig( duration_seconds=duration, aspect_ratio=aspect_ratio, resolution=resolution, - # person_generation='ALLOW_ALL', + person_generation='ALLOW_ALL', + # negative_prompt=negative_prompt, + # seed=42, + # fps=24, # safety_filter_level='BLOCK_NONE', - # negative_prompt=None, # enhance_prompt=True, # generate_audio=True, ) - log.debug(f'Cloud: prompt="{prompt}" size={resolution} ar={aspect_ratio} image={image} model="{self.model}" frames={num_frames} duration={duration}') + log.debug(f'Cloud: prompt="{prompt}" size={resolution} ar={aspect_ratio} image={image} model="{self.model}" duration={duration} genai={genai.__version__}') operation = None try: + t0 = time.time() if image is not None: operation = self.img2vid(prompt, image) else: operation = self.txt2vid(prompt) while not operation.done: - log.debug(f"Cloud processing: {operation}") + t1 = time.time() + log.debug(f"Cloud processing: {operation} elapsed={t1-t0:.2f}") time.sleep(10) operation = self.client.operations.get(operation) except Exception as e: @@ -172,15 +176,3 @@ class GoogleVeoVideoPipeline: def load_veo(model_name): # pylint: disable=unused-argument pipe = GoogleVeoVideoPipeline(model_name = model_name) return pipe - - -if __name__ == "__main__": - from installer import setup_logging # pylint: disable=ungrouped-imports - setup_logging() - log.info('test') - model = GoogleVeoVideoPipeline('veo-3.1-generate-preview') - img = Image.open('C:\\Users\\mandi\\OneDrive\\Generative\\Samples\\cartoon.png') - vid = model(['A beautiful young woman walking through the fantasy city'], 1280, 720, image=img) - if vid is not None: - with open("veo.mp4", "wb") as f: - f.write(vid['video']) diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 9bd08ff88..370f1af82 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -750,19 +750,63 @@ try: ], 'Google Veo': [ Model(name='Google Veo 3.1 T2V', - url='https://gemini.google/overview/video-generation/', + url='https://ai.google.dev/gemini-api/docs/veo', repo='veo-3.1-generate-preview', custom='GoogleVeoVideoPipeline', repo_cls=None, te_cls=None, dit_cls=None), Model(name='Google Veo 3.1 I2V', - url='https://gemini.google/overview/video-generation/', + url='https://ai.google.dev/gemini-api/docs/veo', repo='veo-3.1-generate-preview', custom='GoogleVeoVideoPipeline', repo_cls=None, te_cls=None, dit_cls=None), + Model(name='Google Veo 3.1 Fast T2V', + url='https://ai.google.dev/gemini-api/docs/veo', + repo='veo-3.1-fast-generate-preview', + custom='GoogleVeoVideoPipeline', + repo_cls=None, + te_cls=None, + dit_cls=None), + Model(name='Google Veo 3.1 Fast I2V', + url='https://ai.google.dev/gemini-api/docs/veo', + repo='veo-3.1-fast-generate-preview', + custom='GoogleVeoVideoPipeline', + repo_cls=None, + te_cls=None, + dit_cls=None), + Model(name='Google Veo 3.1 Lite T2V', + url='https://ai.google.dev/gemini-api/docs/veo', + repo='veo-3.1-lite-generate-preview', + custom='GoogleVeoVideoPipeline', + repo_cls=None, + te_cls=None, + dit_cls=None), + Model(name='Google Veo 3.1 Lite I2V', + url='https://ai.google.dev/gemini-api/docs/veo', + repo='veo-3.1-lite-generate-preview', + custom='GoogleVeoVideoPipeline', + repo_cls=None, + te_cls=None, + dit_cls=None), + ], + 'Google Omni': [ + Model(name='Google Omni 1.1 Flash T2V', + url='https://ai.google.dev/gemini-api/docs/omni', + repo='gemini-omni-1.1-flash', + custom='GoogleOmniVideoPipeline', + repo_cls=None, + te_cls=None, + dit_cls=None), + Model(name='Google Omni 1.1 Flash I2V', + url='https://ai.google.dev/gemini-api/docs/omni', + repo='gemini-omni-1.1-flash', + custom='GoogleOmniVideoPipeline', + repo_cls=None, + te_cls=None, + dit_cls=None), ], } t1 = time.time() @@ -866,6 +910,7 @@ CLASS_MODES = { # the mode a pipeline class implies, for rows whose name declare 'Kandinsky5I2VPipeline': 'i2v', 'MiniMaxH3ModularPipeline': 'workflow', 'GoogleVeoVideoPipeline': 't2v', + 'GoogleOmniVideoPipeline': 't2v', } diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 1e4343846..ada9533dd 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -50,6 +50,10 @@ def load_custom(model_name: str): from modules.video_models.google_veo import load_veo pipe = load_veo(model_name) return pipe + if 'gemini-omni' in model_name: + from modules.video_models.google_omni import load_omni + pipe = load_omni(model_name) + return pipe return None diff --git a/pipelines/model_google.py b/pipelines/model_google.py index 7048cc365..b19163962 100644 --- a/pipelines/model_google.py +++ b/pipelines/model_google.py @@ -27,7 +27,7 @@ aspect_ratios_buckets = { def google_requirements(): from installer import install # , reload - install('google-genai==1.52.0') + install('google-genai==2.22.0') def get_size_buckets(width: int, height: int) -> tuple[str, str]: diff --git a/scripts/prompt_enhance/options.py b/scripts/prompt_enhance/options.py index 1913de1bb..dc943406f 100644 --- a/scripts/prompt_enhance/options.py +++ b/scripts/prompt_enhance/options.py @@ -36,13 +36,13 @@ class Options: 'trohrbaugh/Qwen3.5-9B-heretic-v2', ] cloud = [ + 'google/gemini-3.8-flash', + 'google/gemini-3.7-flash', + 'google/gemini-3.6-flash', 'google/gemini-3.5-flash', - 'google/gemini-3.1-pro-preview', + 'google/gemini-3.5-flash-lite', 'google/gemini-3.1-flash-lite', - 'google/gemini-3.1-flash-lite-preview', - 'google/gemini-2.5-flash', - 'google/gemini-2.5-flash-lite', - 'google/gemini-2.5-pro', + 'google/gemini-3.1-pro-preview', ] models = { # Gemma @@ -83,13 +83,13 @@ class Options: 'cognitivecomputations/Dolphin3.0-Llama3.2-1B': {}, 'cognitivecomputations/Dolphin3.0-Llama3.2-3B': {}, # Gemini + 'google/gemini-3.8-flash': {}, + 'google/gemini-3.7-flash': {}, + 'google/gemini-3.6-flash': {}, 'google/gemini-3.5-flash': {}, - 'google/gemini-3.1-pro-preview': {}, + 'google/gemini-3.5-flash-lite': {}, 'google/gemini-3.1-flash-lite': {}, - 'google/gemini-3.1-flash-lite-preview': {}, - 'google/gemini-2.5-flash': {}, - 'google/gemini-2.5-flash-lite': {}, - 'google/gemini-2.5-pro': {}, + 'google/gemini-3.1-pro-preview': {}, # SmolLM 'HuggingFaceTB/SmolLM2-135M-Instruct': {}, 'HuggingFaceTB/SmolLM2-360M-Instruct': {},