Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

update all google stuff

Browse Source 
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2026-09-05 21:53:29 +02:00
parent cd88d2ae34
commit 29de324ff7
12 changed files with 234 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+11
View File
@@ -52,6 +52,17 @@ All-about-optimizations:
- implement progress and preview
- intercept and profiling hooks
- on-demand convert standard model on-demand
- **Google**
- updated support for google models in text, image and video workflows
*note*: requires google api key
- [Google Veo](https://ai.google.dev/gemini-api/docs/veo) in *preview*, *fast* and *lite* variants
workflows: *t2v, i2v*
- [Google Omni](https://ai.google.dev/gemini-api/docs/omni) in *flash* variant
workflows: *t2v, i2v*
- [Google Nano Banana](https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-image) in *2* and *2 lite* and *pro* variants
workflows: *caption*
- [Google Gemini](https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash) in *flash* and *pro* variants
workflows: *caption, prompt-enhance*
- **Compute**
- cuda: update `torch==2.14.0` with `cuda==13.2`
- openvino: update `openvino==2026.3.1` with `torch==2.13.0`
constraints.txt
+1 -1
View File
@@ -1,3 +1,3 @@
fastapi==0.124.4
numpy==2.5.2
Pillow==12.2.0
# numpy==2.5.2
modules/caption/gemini.py
+1 -1
View File
@@ -11,7 +11,7 @@ class GoogleGeminiPipeline():
def __init__(self, model_name: str):
self.model = model_name.split(' (')[0]
from installer import install
install('google-genai==1.52.0')
install('google-genai==2.22.0')
from google import genai # pylint: disable=no-name-in-module
args = self.get_args()
self.client = genai.Client(**args)
modules/caption/models_def.py
+6 -6
View File
@@ -74,13 +74,13 @@ vlm_models = {
"AIDC Ovis2 2B": "AIDC-AI/Ovis2-2B",
"AIDC Ovis2 1B": "AIDC-AI/Ovis2-1B",
# cloud
f"Google Gemini 3.5 Flash {ui_symbols.cloud}": "google/gemini-3.5-flash",
f"Google Gemini 3.1 Pro {ui_symbols.cloud}": "gemini-3.1-pro-preview",
f"Google Gemini 3.8 Flash {ui_symbols.cloud}": "gemini-3.8-flash",
f"Google Gemini 3.7 Flash {ui_symbols.cloud}": "gemini-3.7-flash",
f"Google Gemini 3.6 Flash {ui_symbols.cloud}": "gemini-3.6-flash",
f"Google Gemini 3.5 Flash {ui_symbols.cloud}": "gemini-3.5-flash",
f"Google Gemini 3.5 Flash Lite {ui_symbols.cloud}": "gemini-3.5-flash-lite",
f"Google Gemini 3.1 Flash Lite {ui_symbols.cloud}": "gemini-3.1-flash-lite",
f"Google Gemini 3.1 Flash Lite Preview {ui_symbols.cloud}": "gemini-3.1-flash-lite-preview",
f"Google Gemini 2.5 Pro {ui_symbols.cloud}": "gemini-2.5-pro",
f"Google Gemini 2.5 Flash {ui_symbols.cloud}": "gemini-2.5-flash",
f"Google Gemini 2.5 Flash Lite {ui_symbols.cloud}": "gemini-2.5-flash-lite",
f"Google Gemini 3.1 Pro {ui_symbols.cloud}": "gemini-3.1-pro-preview",
}
# Default model
modules/modular.py
+1 -1
View File
@@ -27,7 +27,7 @@ def is_compatible(diffusion_pipeline: diffusers.DiffusionPipeline) -> bool:
return False
compatible = get_modular_class(diffusion_pipeline) is not None
if not compatible:
log.debug(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline')
log.warning(f'Modular: source={diffusion_pipeline.__class__.__name__} incompatible pipeline')
return compatible
modules/processing.py
+1 -1
View File
@@ -541,7 +541,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
p.scripts.postprocess_batch(p, samples, batch_number=n)
if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner) and isinstance(samples, list):
p.prompts = p.all_prompts[(n * p.batch_size):((n+1) * p.batch_size)]
p.negative_prompts = p.all_negative_prompts[(n * p.batch_size):((n+1) * p.batch_size)]
batch_params = scripts_manager.PostprocessBatchListArgs(list(samples))
modules/video_models/google_omni.py
+137
View File
@@ -0,0 +1,137 @@
import io
import os
import base64
import time
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
from PIL import Image
from modules.logger import log
image_size_buckets = {
'360p': 640*360,
'720p': 1280*720,
'1080p': 1920*1080,
'4k': 3840*2160,
}
aspect_ratios_buckets = {
'16:9': 16/9,
'9:16': 9/16,
}
def google_requirements():
from installer import install
install('google-genai==2.22.0')
# install('pydantic==2.11.7', ignore=True, quiet=True)
# reload('pydantic', '2.11.7')
def get_size_buckets(width: int, height: int) -> tuple[str, str]:
aspect_ratio = width / height
pixel_count = width * height
closest_size = min(image_size_buckets.items(), key=lambda x: abs(x[1] - pixel_count))[0]
closest_aspect_ratio = min(aspect_ratios_buckets.items(), key=lambda x: abs(x[1] - aspect_ratio))[0]
return closest_size, closest_aspect_ratio
class GoogleOmniVideoPipeline:
def __init__(self, model_name: str):
self.model = model_name
self.client = None
google_requirements()
log.debug(f'Load model: type=GoogleOmni model="{model_name}"')
def get_args(self):
from modules.shared import opts
# Use UI settings only - env vars are intentionally ignored
api_key = opts.google_api_key
project_id = opts.google_project_id
location_id = opts.google_location_id
use_vertexai = opts.google_use_vertexai
has_api_key = api_key and len(api_key) > 0
has_project = project_id and len(project_id) > 0
has_location = location_id and len(location_id) > 0
if use_vertexai:
if has_api_key and (has_project or has_location):
# Invalid: can't have both api_key AND project/location
log.error(f'Cloud: model="{self.model}" API key and project/location are mutually exclusive')
return None
elif has_api_key:
# Vertex AI Express Mode: api_key + vertexai, no project/location
args = {'api_key': api_key, 'vertexai': True}
elif has_project and has_location:
# Standard Vertex AI: project/location, no api_key
args = {'vertexai': True, 'project': project_id, 'location': location_id}
else:
log.error(f'Cloud: model="{self.model}" Vertex AI requires either API key (Express Mode) or project ID + location ID')
return None
else:
# Gemini Developer API: api_key only
if not has_api_key:
log.error(f'Cloud: model="{self.model}" API key not provided')
return None
args = {'api_key': api_key}
# Debug logging
args_log = args.copy()
if args_log.get('api_key'):
args_log['api_key'] = '...' + args_log['api_key'][-4:]
log.debug(f'Cloud: model="{self.model}" args={args_log}')
return args
def __call__(self, prompt: list[str], width: int, height: int, image: Image.Image = None):
if isinstance(prompt, list) and len(prompt) > 0:
prompt = prompt[0]
if self.client is None:
args = self.get_args()
if args is None:
return None
from google import genai # pylint: disable=no-name-in-module
self.client = genai.Client(**args)
resolution, aspect_ratio = get_size_buckets(width, height)
response_format = {
'type': 'video',
'aspect_ratio': aspect_ratio,
'resolution': resolution,
}
if image is not None:
image_bytes = io.BytesIO()
image.save(image_bytes, format='JPEG')
input_content = [
{'type': 'image', 'data': base64.b64encode(image_bytes.getvalue()).decode('utf-8'), 'mime_type': 'image/jpeg'},
{'type': 'text', 'text': prompt},
]
else:
input_content = prompt
log.debug(f'Cloud: prompt="{prompt}" size={resolution} ar={aspect_ratio} image={image} model="{self.model}" genai={genai.__version__}')
t0 = time.time()
try:
interaction = self.client.interactions.create(
model=self.model,
input=input_content,
response_format=response_format,
)
except Exception as e:
log.error(f'Cloud video: model="{self.model}" {e}')
return None
t1 = time.time()
log.debug(f'Cloud processing: model="{self.model}" elapsed={t1-t0:.2f}')
try:
video_bytes = base64.b64decode(interaction.output_video.data)
return { 'bytes': video_bytes, 'images': [] }
except Exception as e:
log.error(f'Cloud download: model="{self.model}" {e}')
return None
def load_omni(model_name): # pylint: disable=unused-argument
pipe = GoogleOmniVideoPipeline(model_name = model_name)
return pipe
modules/video_models/google_veo.py
+14 -22
View File
@@ -6,13 +6,13 @@ import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
from PIL import Image
from installer import install
from modules.logger import log
image_size_buckets = {
'720p': 1280*720,
'1080p': 1920*1080,
'4k': 3840*2160,
}
aspect_ratios_buckets = {
'1:1': 1/1,
@@ -30,7 +30,9 @@ aspect_ratios_buckets = {
def google_requirements():
install('google-genai==1.52.0')
from installer import install
install('google-genai==2.22.0')
# install('google-genai==1.75.0')
# install('pydantic==2.11.7', ignore=True, quiet=True)
# reload('pydantic', '2.11.7')
@@ -110,7 +112,7 @@ class GoogleVeoVideoPipeline:
log.debug(f'Cloud: model="{self.model}" args={args_log}')
return args
def __call__(self, prompt: list[str], width: int, height: int, image: Image.Image = None, num_frames: int = 4*24):
def __call__(self, prompt: list[str], width: int = 1280, height: int = 720, image: Image.Image = None, num_frames: int = 4*24):
from google import genai # pylint: disable=no-name-in-module
if isinstance(prompt, list) and len(prompt) > 0:
@@ -127,28 +129,30 @@ class GoogleVeoVideoPipeline:
duration = 4
if duration > 8:
duration = 8
self.config=genai.types.GenerateVideosConfig(
# seed=42,
# fps=24,
self.config = genai.types.GenerateVideosConfig(
duration_seconds=duration,
aspect_ratio=aspect_ratio,
resolution=resolution,
# person_generation='ALLOW_ALL',
person_generation='ALLOW_ALL',
# negative_prompt=negative_prompt,
# seed=42,
# fps=24,
# safety_filter_level='BLOCK_NONE',
# negative_prompt=None,
# enhance_prompt=True,
# generate_audio=True,
)
log.debug(f'Cloud: prompt="{prompt}" size={resolution} ar={aspect_ratio} image={image} model="{self.model}" frames={num_frames} duration={duration}')
log.debug(f'Cloud: prompt="{prompt}" size={resolution} ar={aspect_ratio} image={image} model="{self.model}" duration={duration} genai={genai.__version__}')
operation = None
try:
t0 = time.time()
if image is not None:
operation = self.img2vid(prompt, image)
else:
operation = self.txt2vid(prompt)
while not operation.done:
log.debug(f"Cloud processing: {operation}")
t1 = time.time()
log.debug(f"Cloud processing: {operation} elapsed={t1-t0:.2f}")
time.sleep(10)
operation = self.client.operations.get(operation)
except Exception as e:
@@ -172,15 +176,3 @@ class GoogleVeoVideoPipeline:
def load_veo(model_name): # pylint: disable=unused-argument
pipe = GoogleVeoVideoPipeline(model_name = model_name)
return pipe
if __name__ == "__main__":
from installer import setup_logging # pylint: disable=ungrouped-imports
setup_logging()
log.info('test')
model = GoogleVeoVideoPipeline('veo-3.1-generate-preview')
img = Image.open('C:\\Users\\mandi\\OneDrive\\Generative\\Samples\\cartoon.png')
vid = model(['A beautiful young woman walking through the fantasy city'], 1280, 720, image=img)
if vid is not None:
with open("veo.mp4", "wb") as f:
f.write(vid['video'])
modules/video_models/models_def.py
+47 -2
View File
@@ -750,19 +750,63 @@ try:
],
'Google Veo': [
Model(name='Google Veo 3.1 T2V',
url='https://gemini.google/overview/video-generation/',
url='https://ai.google.dev/gemini-api/docs/veo',
repo='veo-3.1-generate-preview',
custom='GoogleVeoVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
Model(name='Google Veo 3.1 I2V',
url='https://gemini.google/overview/video-generation/',
url='https://ai.google.dev/gemini-api/docs/veo',
repo='veo-3.1-generate-preview',
custom='GoogleVeoVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
Model(name='Google Veo 3.1 Fast T2V',
url='https://ai.google.dev/gemini-api/docs/veo',
repo='veo-3.1-fast-generate-preview',
custom='GoogleVeoVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
Model(name='Google Veo 3.1 Fast I2V',
url='https://ai.google.dev/gemini-api/docs/veo',
repo='veo-3.1-fast-generate-preview',
custom='GoogleVeoVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
Model(name='Google Veo 3.1 Lite T2V',
url='https://ai.google.dev/gemini-api/docs/veo',
repo='veo-3.1-lite-generate-preview',
custom='GoogleVeoVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
Model(name='Google Veo 3.1 Lite I2V',
url='https://ai.google.dev/gemini-api/docs/veo',
repo='veo-3.1-lite-generate-preview',
custom='GoogleVeoVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
],
'Google Omni': [
Model(name='Google Omni 1.1 Flash T2V',
url='https://ai.google.dev/gemini-api/docs/omni',
repo='gemini-omni-1.1-flash',
custom='GoogleOmniVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
Model(name='Google Omni 1.1 Flash I2V',
url='https://ai.google.dev/gemini-api/docs/omni',
repo='gemini-omni-1.1-flash',
custom='GoogleOmniVideoPipeline',
repo_cls=None,
te_cls=None,
dit_cls=None),
],
}
t1 = time.time()
@@ -866,6 +910,7 @@ CLASS_MODES = { # the mode a pipeline class implies, for rows whose name declare
'Kandinsky5I2VPipeline': 'i2v',
'MiniMaxH3ModularPipeline': 'workflow',
'GoogleVeoVideoPipeline': 't2v',
'GoogleOmniVideoPipeline': 't2v',
}
modules/video_models/video_load.py
+4
View File
@@ -50,6 +50,10 @@ def load_custom(model_name: str):
from modules.video_models.google_veo import load_veo
pipe = load_veo(model_name)
return pipe
if 'gemini-omni' in model_name:
from modules.video_models.google_omni import load_omni
pipe = load_omni(model_name)
return pipe
return None
pipelines/model_google.py
+1 -1
View File
@@ -27,7 +27,7 @@ aspect_ratios_buckets = {
def google_requirements():
from installer import install # , reload
install('google-genai==1.52.0')
install('google-genai==2.22.0')
def get_size_buckets(width: int, height: int) -> tuple[str, str]:
scripts/prompt_enhance/options.py
+10 -10
View File
@@ -36,13 +36,13 @@ class Options:
'trohrbaugh/Qwen3.5-9B-heretic-v2',
]
cloud = [
'google/gemini-3.8-flash',
'google/gemini-3.7-flash',
'google/gemini-3.6-flash',
'google/gemini-3.5-flash',
'google/gemini-3.1-pro-preview',
'google/gemini-3.5-flash-lite',
'google/gemini-3.1-flash-lite',
'google/gemini-3.1-flash-lite-preview',
'google/gemini-2.5-flash',
'google/gemini-2.5-flash-lite',
'google/gemini-2.5-pro',
'google/gemini-3.1-pro-preview',
]
models = {
# Gemma
@@ -83,13 +83,13 @@ class Options:
'cognitivecomputations/Dolphin3.0-Llama3.2-1B': {},
'cognitivecomputations/Dolphin3.0-Llama3.2-3B': {},
# Gemini
'google/gemini-3.8-flash': {},
'google/gemini-3.7-flash': {},
'google/gemini-3.6-flash': {},
'google/gemini-3.5-flash': {},
'google/gemini-3.1-pro-preview': {},
'google/gemini-3.5-flash-lite': {},
'google/gemini-3.1-flash-lite': {},
'google/gemini-3.1-flash-lite-preview': {},
'google/gemini-2.5-flash': {},
'google/gemini-2.5-flash-lite': {},
'google/gemini-2.5-pro': {},
'google/gemini-3.1-pro-preview': {},
# SmolLM
'HuggingFaceTB/SmolLM2-135M-Instruct': {},
'HuggingFaceTB/SmolLM2-360M-Instruct': {},