diff --git a/CHANGELOG.md b/CHANGELOG.md index 83738aebe..b55b180d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,19 @@ ### Highlights for 2025-11-13 -TBD +New native [kanvas](https://vladmandic.github.io/sdnext-docs/Kanvas/) module for image manipulation that fully replaces img2img, inpaint and outpaint controls +And a first cloud model with **Google's Nano Banana** + +![Screenshot](https://github.com/user-attachments/assets/54b25586-b611-4d70-a28f-ee3360944034) + +[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) ### Details for 2025-11-13 +- **Models** + - [Google Gemini 2.5 Flash Nano Banana](https://blog.google/products/gemini/gemini-nano-banana-examples/) + first cloud-based model directly supported in SD.Next UI + *note*: need to set `GOOGLE_API_KEY` environment variable with your key to use this model - **Features** - **kanvas**: new module for native canvas-based image manipulation kanvas is a full replacement for *img2img, inpaint and outpaint* controls diff --git a/html/reference.json b/html/reference.json index c11d627fc..eaf9428e5 100644 --- a/html/reference.json +++ b/html/reference.json @@ -1190,6 +1190,14 @@ "preview": "shuttleai--shuttle-jaguar.jpg", "tags": "community", "skip": true + }, + + "Google Gemini 2.5 Flash Nano Banana": { + "path": "gemini-2.5-flash-image", + "desc": "Gemini can generate and process images conversationally. You can prompt Gemini with text, images, or a combination of both allowing you to create, edit, and iterate on visuals with unprecedented control.", + "preview": "gemini-2.5-flash-image.jpg", + "tags": "cloud", + "skip": true } } diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 5e884dba5..0143210ea 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -153,6 +153,10 @@ async function filterExtraNetworksForTab(searchTerm) { cards.forEach((elem) => elem.style.display = elem.dataset.tags .toLowerCase() .includes('community') ? '' : 'none'); + } else if (searchTerm === 'cloud/') { + cards.forEach((elem) => elem.style.display = elem.dataset.tags + .toLowerCase() + .includes('cloud') ? '' : 'none'); } else if (searchTerm === 'quantized/') { cards.forEach((elem) => elem.style.display = elem.dataset.tags .toLowerCase() diff --git a/javascript/settings.js b/javascript/settings.js index 96bdd24a9..1ffe449e9 100644 --- a/javascript/settings.js +++ b/javascript/settings.js @@ -178,7 +178,7 @@ async function initModels() { if (en.classList.contains('hide')) gradioApp().getElementById('txt2img_extra_networks_btn').click(); const repeat = setInterval(() => { const buttons = Array.from(gradioApp().querySelectorAll('#txt2img_model_subdirs > button')) || []; - const reference = buttons.find((b) => (b.innerText === 'Reference') || (b.innerText === 'Distilled') || (b.innerText === 'Community') || (b.innerText === 'Quantized')); + const reference = buttons.find((b) => (b.innerText === 'Reference') || (b.innerText === 'Distilled') || (b.innerText === 'Community') || (b.innerText === 'Quantized') || (b.innerText === 'Cloud')); if (reference) { clearInterval(repeat); reference.click(); diff --git a/models/Reference/gemini-2.5-flash-image.jpg b/models/Reference/gemini-2.5-flash-image.jpg new file mode 100644 index 000000000..9ed6ec95c Binary files /dev/null and b/models/Reference/gemini-2.5-flash-image.jpg differ diff --git a/modules/modeldata.py b/modules/modeldata.py index d06a09fbd..4279784aa 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -88,6 +88,9 @@ def get_model_type(pipe): model_type = 'hunyuanimage3' elif 'HunyuanImage' in name: model_type = 'hunyuanimage' + # cloud models + elif 'NanoBanana' in name: + model_type = 'nanobanana' else: model_type = name return model_type diff --git a/modules/processing_args.py b/modules/processing_args.py index bb81fda94..74bfce1fd 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -145,6 +145,8 @@ def task_specific_kwargs(p, model): task_args['image'] = Image.new('RGB', (p.width, p.height), (0, 0, 0)) # monkey-patch so wan-i2i pipeline does not error-out on t2i if ('WanVACEPipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): task_args['reference_images'] = p.init_images + if ('GoogleNanoBananaPipeline' in model_cls) and (p.init_images is not None) and (len(p.init_images) > 0): + task_args['image'] = p.init_images[0] if 'BlipDiffusionPipeline' in model_cls: if len(p.init_images) == 0: shared.log.error('BLiP diffusion requires init image') diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 2e2bfb40a..30c7d52e9 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -129,6 +129,8 @@ def guess_by_name(fn, current_guess): new_guess = 'Stable Diffusion XL' elif 'stable-video-diffusion' in fn.lower(): new_guess = 'StableVideoDiffusion' + elif 'gemini-2.5-flash-image' in fn.lower(): + new_guess = 'NanoBanana' if debug_load: shared.log.trace(f'Autodetect: method=name file="{fn}" previous="{current_guess}" current="{new_guess}"') return new_guess or current_guess diff --git a/modules/sd_models.py b/modules/sd_models.py index e6fc1fb71..bd01482f5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -49,6 +49,7 @@ pipe_switch_task_exclude = [ 'HunyuanImagePipeline', 'AuraFlowPipeline', 'ChronoEditPipeline', + 'GoogleNanoBananaPipeline', ] i2i_pipes = [ 'LEditsPPPipelineStableDiffusion', 'LEditsPPPipelineStableDiffusionXL', @@ -421,6 +422,10 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' from pipelines.model_xomni import load_xomni sd_model = load_xomni(checkpoint_info, diffusers_load_config) # pylint: disable=assignment-from-none allow_post_quant = False + elif model_type in ['NanoBanana']: + from pipelines.model_google import load_nanobanana + sd_model = load_nanobanana(checkpoint_info, diffusers_load_config) + allow_post_quant = False except Exception as e: shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') if debug_load: @@ -863,6 +868,7 @@ def load_diffuser(checkpoint_info=None, op='model', revision=None): # pylint: di modelstats.analyze() shared.log.info(f"Load {op}: family={shared.sd_model_type} time={timer.load.dct()} native={get_native(sd_model)} memory={memory_stats()}") + shared.opts.save(silent=True) class DiffusersTaskType(Enum): diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index cf7035847..7ba05c907 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -294,6 +294,7 @@ class ExtraNetworksPage: subdirs['Distilled'] = 1 subdirs['Quantized'] = 1 subdirs['Community'] = 1 + subdirs['Cloud'] = 1 subdirs[diffusers_base] = 1 if self.name == 'style' and shared.opts.extra_networks_styles: subdirs['Local'] = 1 @@ -313,11 +314,13 @@ class ExtraNetworksPage: subdirs.move_to_end('Quantized', last=True) if 'Community' in subdirs: subdirs.move_to_end('Community', last=True) + if 'Cloud' in subdirs: + subdirs.move_to_end('Cloud', last=True) subdirs_html = '' for subdir in subdirs: if len(subdir) == 0: continue - if subdir in ['All', 'Local', 'Diffusers', 'Reference', 'Distilled', 'Quantized', 'Community']: + if subdir in ['All', 'Local', 'Diffusers', 'Reference', 'Distilled', 'Quantized', 'Community', 'Cloud']: style = 'network-reference' else: style = 'network-folder' diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 2fe4db376..26d81121a 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -68,13 +68,6 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): else: path = f'{v.get("path", "")}' - ready = reference_downloaded(url) - if not ready and shared.opts.offline_mode: - count['hidden'] += 1 - continue - if ready: - count['ready'] += 1 - tag = v.get('tags', '') if tag in count: count[tag] += 1 @@ -83,6 +76,16 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): else: count['base'] += 1 + ready = reference_downloaded(url) + version = "ready" if ready else "download" + if tag == 'cloud': + version = 'cloud' + if not ready and shared.opts.offline_mode: + count['hidden'] += 1 + continue + if ready: + count['ready'] += 1 + yield { "type": 'Model', "name": name, @@ -97,7 +100,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "info": {}, "metadata": {}, "description": v.get('desc', ''), - "version": "ready" if ready else "download", + "version": version, "tags": tag, } shared.log.debug(f'Networks: type="reference" items={count}') diff --git a/pipelines/model_google.py b/pipelines/model_google.py new file mode 100644 index 000000000..b08a28222 --- /dev/null +++ b/pipelines/model_google.py @@ -0,0 +1,105 @@ +import io +import os +from PIL import Image +from installer import install, reload, log + + +image_size_buckets = { + '1M': 1024*1024, + '2M': 2048*1024, + '4M': 4096*1024, +} +aspect_ratios_buckets = { + '1:1': 1/1, + '2:3': 2/3, + '3:2': 3/2, + '4:3': 4/3, + '3:4': 3/4, + '4:5': 4/5, + '5:4': 5/4, + '16:9': 16/9, + '9:16': 9/16, + '21:9': 21/9, + '9:21': 9/21, +} + + +def get_size_buckets(width: int, height: int) -> str: + aspect_ratio = width / height + closest_aspect_ratio = min(aspect_ratios_buckets.items(), key=lambda x: abs(x[1] - aspect_ratio))[0] + pixel_count = width * height + closest_size = min(image_size_buckets.items(), key=lambda x: abs(x[1] - pixel_count))[0] + closest_aspect_ratio = min(aspect_ratios_buckets.items(), key=lambda x: abs(x[1] - aspect_ratio))[0] + return closest_size, closest_aspect_ratio + + +class GoogleNanoBananaPipeline(): + def __init__(self, model_name: str): + self.model = model_name + self.client = None + self.config = None + install('google-genai') + install('pydantic==2.11.7', ignore=True, quiet=True) + reload('pydantic', '2.11.7') + log.debug(f'Load model: type=NanoBanana model="{model_name}"') + + def txt2img(self, prompt): + return self.client.models.generate_content( + model=self.model, + config=self.config, + contents=prompt, + ) + + def img2img(self, prompt, image): + from google import genai + image_bytes = io.BytesIO() + image.save(image_bytes, format='JPEG') + return self.client.models.generate_content( + model=self.model, + config=self.config, + contents=[ + genai.types.Part.from_bytes(data=image_bytes.getvalue(), mime_type='image/jpeg'), + prompt, + ], + ) + + def __call__(self, prompt: list[str], width: int, height: int, image: Image.Image = None): + from google import genai + if self.client is None: + api_key = os.getenv("GOOGLE_API_KEY", None) + if api_key is None: + log.error(f'Cloud: model="{self.model}" GOOGLE_API_KEY environment variable not set') + return None + self.client = genai.Client(api_key=api_key, vertexai=False) + + image_size, aspect_ratio = get_size_buckets(width, height) + log.debug(f'Cloud: prompt={prompt} size={image_size} ar={aspect_ratio} image={image} model="{self.model}"') + self.config=genai.types.GenerateContentConfig( + response_modalities=["IMAGE"], + image_config=genai.types.ImageConfig(aspect_ratio=aspect_ratio, image_size=image_size) + ) + + try: + if image is not None: + response = self.img2img(prompt, image) + else: + response = self.txt2img(prompt) + except Exception as e: + log.error(f'Cloud: model="{self.model}" {e}') + return None + + image = None + if getattr(response, 'prompt_feedback', None) is not None: + log.error(f'Cloud: model="{self.model}" {response.prompt_feedback}') + if not hasattr(response, 'candidates') or (response.candidates is None) or (len(response.candidates) == 0): + log.error(f'Cloud: model="{self.model}" no images received') + return None + for part in response.candidates[0].content.parts: + if part.inline_data is not None: + image = Image.open(io.BytesIO(part.inline_data.data)) + return image + + +def load_nanobanana(checkpoint_info, diffusers_load_config): # pylint: disable=unused-argument + pipe = GoogleNanoBananaPipeline(model_name = checkpoint_info.filename) + return pipe