diff --git a/CHANGELOG.md b/CHANGELOG.md index 6163f5236..aa87a33bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,10 @@ All-about-optimizations: - improved LoRA performance and quality, especially with quantized models - newly structured attention mechanisms -- modular pipelines with new guidance +- modular pipelines with new guidance methods - support for different caching stacks - compute updates across the board +- enhanced cloud model support ### Details for 2026-09-07 @@ -51,7 +52,7 @@ All-about-optimizations: - implement progress and preview - intercept and profiling hooks - on-demand convert standard model on-demand -- **Google** +- **Cloud** - updated support for google models in text, image and video workflows *note*: requires google api key - [Google Veo](https://ai.google.dev/gemini-api/docs/veo) in *preview*, *fast* and *lite* variants @@ -62,6 +63,10 @@ All-about-optimizations: workflows: *caption* - [Google Gemini](https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash) in *flash* and *pro* variants workflows: *caption, prompt-enhance* + - added support for xai grok models + - [X.AI Grok](https://x.ai/grok) in *3*, *3 fast*, *3 mini* and *3 mini fast* variants + workflows: *caption, prompt-enhance* + *note*: requires grok api key - **Compute** - cuda: update `torch==2.14.0` with `cuda==13.2` - openvino: update `openvino==2026.3.1` with `torch==2.13.0` @@ -75,7 +80,9 @@ All-about-optimizations: - update `numpy` and `scipy` frozen requirements as required by new compute drivers *note*: this may break compatibility with some legacy packages, so report any finidings - **Other** - - Video Preview: TAESD support for **MiniMax** + - video preview: TAESD support for **MiniMax** + - support `xai grok` for prompt enhance workflows + *note*: requires grok api key - remove `/redocs` as `/docs` are primary api docs - rebuild docs site index - **Wiki/Docs**: @@ -99,6 +106,7 @@ All-about-optimizations: - vae: fetch scale factor from the model - todo: remove dead code, thanks @Anai-Guo - offline: honor offline mode for more models, thanks @ryanmeador + - prompt enhance: cloud models use correct system prompt ## Update for 2026-08-26 diff --git a/modules/caption/gemini.py b/modules/caption/gemini.py index b36021205..3297bec12 100644 --- a/modules/caption/gemini.py +++ b/modules/caption/gemini.py @@ -3,6 +3,7 @@ import os from modules import shared from modules.logger import log + debug_enabled = os.environ.get('SD_CAPTION_DEBUG', None) is not None debug_log = log.trace if debug_enabled else lambda *args, **kwargs: None @@ -63,11 +64,13 @@ class GoogleGeminiPipeline(): config['temperature'] = kwargs['temperature'] if 'max_output_tokens' in kwargs: config['max_output_tokens'] = kwargs['max_output_tokens'] - debug_log(f'Gemini config: {config}') + debug_log(f'LLM config: {config}') + debug_log(f'LLM instructions: "{instructions}"') + debug_log(f'LLM image: {image}') question = question.replace('<', '').replace('>', '').replace('_', ' ') if prefill: question += prefill - debug_log(f'Gemini question: "{question}"') + debug_log(f'LLM question: "{question}"') if image: data = io.BytesIO() @@ -80,9 +83,9 @@ class GoogleGeminiPipeline(): answer = '' try: response = self.client.models.generate_content( - model=model, - contents=contents, - config=config, + model = model, + contents = contents, + config = config, ) debug_log(f'Gemini response: {response}') answer = response.text @@ -94,8 +97,8 @@ class GoogleGeminiPipeline(): ai = None -def predict(question, image, vqa_model, system_prompt, model_name, prefill, thinking, gen_kwargs): +def predict(question, image, model_name, system_prompt, prefill, thinking, gen_kwargs): global ai # pylint: disable=global-statement if ai is None: ai = GoogleGeminiPipeline(model_name) - return ai(question, image, vqa_model, system_prompt, prefill, thinking, gen_kwargs) + return ai(question, image, model_name, system_prompt, prefill, thinking, gen_kwargs) diff --git a/modules/caption/grok.py b/modules/caption/grok.py new file mode 100644 index 000000000..2f7fb9965 --- /dev/null +++ b/modules/caption/grok.py @@ -0,0 +1,99 @@ +import io +import os +import base64 +from modules import shared +from modules.logger import log + + +debug_enabled = os.environ.get('SD_CAPTION_DEBUG', None) is not None +debug_log = log.trace if debug_enabled else lambda *args, **kwargs: None + + +class XAIGrokPipeline(): + def __init__(self, model_name: str): + self.url = 'https://api.x.ai/v1' + self.model = model_name.split(' (')[0].replace('xai/', '') + from installer import install + install('openai') + from openai import OpenAI # pylint: disable=no-name-in-module + args = self.get_args() + if not args: + return + self.client = OpenAI(**args) + log.debug(f'Load model: type=XAIGrok model="{self.model}"') + + def get_args(self): + from modules.shared import opts + # Use UI settings only - env vars are intentionally ignored + api_key = opts.xai_api_key + has_api_key = api_key and len(api_key) > 0 + if not has_api_key: # Gemini Developer API: api_key only + log.error(f'Cloud: model="{self.model}" API key not provided') + return None + args = { + 'api_key': api_key, + 'base_url': self.url, + } + # Debug logging + args_log = args.copy() + if args_log.get('api_key'): + args_log['api_key'] = '...' + args_log['api_key'][-4:] + log.debug(f'Cloud: model="{self.model}" args={args_log}') + return args + + def __call__(self, question, image, model, instructions, prefill, thinking, kwargs): + question = question.replace('<', '').replace('>', '').replace('_', ' ') + if prefill: + question += prefill + debug_log(f'LLM instructions: "{instructions}"') + debug_log(f'LLM question: "{question}"') + debug_log(f'LLM image: {image}') + answer = '' + temperature = kwargs.get('temperature', 0.0) + try: + if image is not None: + image_data = image.convert('RGB') + image_bytes = io.BytesIO() + image_data.save(image_bytes, format='JPEG') + image_bytes.seek(0) + content = [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64.b64encode(image_bytes.getvalue()).decode('utf-8')}", + "detail": "high", + }, + }, + { + "type": "text", + "text": question, + }, + ] + else: + content = question + response = self.client.chat.completions.create( + model = self.model, + messages = [ + {"role": "system", "content": instructions or shared.opts.caption_vlm_system}, + {"role": "user", "content": content}, + ], + stream = False, + temperature = temperature, + reasoning_effort = "high" if thinking else "low" + ) + text = (response.choices[0].message.content or "").strip() + debug_log(f'Grok response: {response}') + answer = text + except Exception as e: + log.error(f'Grok: {e}') + answer = f'Error: {e}' + return answer + + +ai = None + +def predict(question, image, model_name, system_prompt, prefill, thinking, gen_kwargs): + global ai # pylint: disable=global-statement + if ai is None: + ai = XAIGrokPipeline(model_name) + return ai(question, image, model_name, system_prompt, prefill, thinking, gen_kwargs) diff --git a/modules/caption/models_def.py b/modules/caption/models_def.py index 63519cd1d..f325a187b 100644 --- a/modules/caption/models_def.py +++ b/modules/caption/models_def.py @@ -81,6 +81,10 @@ vlm_models = { f"Google Gemini 3.5 Flash Lite {ui_symbols.cloud}": "gemini-3.5-flash-lite", f"Google Gemini 3.1 Flash Lite {ui_symbols.cloud}": "gemini-3.1-flash-lite", f"Google Gemini 3.1 Pro {ui_symbols.cloud}": "gemini-3.1-pro-preview", + f"X.AI Grok 3 {ui_symbols.cloud}": "grok-3-latest", + f"X.AI Grok 3 Fast {ui_symbols.cloud}": "grok-3-fast-latest", + f"X.AI Grok 3 Mini {ui_symbols.cloud}": "grok-3-mini-latest", + f"X.AI Grok 3 Mini Fast {ui_symbols.cloud}": "grok-3-mini-fast-latest", } # Default model @@ -224,5 +228,10 @@ Summary: def get_vlm_repo(display_name: str) -> str: """Look up repo ID from display name, stripping any trailing symbols.""" + from modules.logger import log name = display_name.strip() - return vlm_models.get(name, name) + model = vlm_models.get(name, None) + if model is None: + log.warning(f"Model '{name}' not found") + return name + return model diff --git a/modules/caption/vqa.py b/modules/caption/vqa.py index 784f6d941..6a00e2fa3 100644 --- a/modules/caption/vqa.py +++ b/modules/caption/vqa.py @@ -1597,7 +1597,12 @@ class VQA: handler = 'gemini' gen_kwargs = get_kwargs(self.model) from modules.caption import gemini - answer = gemini.predict(question, image, vqa_model, system_prompt, model_name, prefill, thinking_mode, gen_kwargs) + answer = gemini.predict(question, image, vqa_model, system_prompt, prefill, thinking_mode, gen_kwargs) + elif 'grok' in vqa_model.lower(): + handler = 'grok' + gen_kwargs = get_kwargs(self.model) + from modules.caption import grok + answer = grok.predict(question, image, vqa_model, system_prompt, prefill, thinking_mode, gen_kwargs) else: answer = 'unknown model' except Exception as e: diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index 66088e8a8..825722aa3 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -113,6 +113,8 @@ def create_settings(cmd_opts): "google_api_key": OptionInfo("", "Google cloud API key", gr.Textbox, secret=True, env_var='GOOGLE_API_KEY'), "google_project_id": OptionInfo("", "Google Cloud project ID", gr.Textbox, secret=True, env_var='GOOGLE_PROJECT_ID'), "google_location_id": OptionInfo("", "Google Cloud location ID", gr.Textbox), + "model_xai_sep": OptionInfo("