diff --git a/CHANGELOG.md b/CHANGELOG.md index aacd64ce1..29ab781b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,10 @@ ### Highlights for 2026-07-02 -Service-pack update with several fixes and quality-of-life improvements -Plus few new models: **Krea 2**, **Photoroom PRXPixel**, **FLUX.2 Klein 9B KV** +Service-pack update with number of fixes and quality-of-life improvements +Plus few new models: **Krea 2**, **Photoroom PRXPixel**, **FLUX.2 Klein 9B KV** and some new community models And **SDNQ** improvements: now with *NPU* support and its own native *attention* kernels! +Also couple of *experimental* features: see below for details... [Home](https://vladmandic.github.io/sdnext/) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) @@ -34,6 +35,7 @@ And **SDNQ** improvements: now with *NPU* support and its own native *attention* - add option: *model loading -> attempt to load incomplete model* disabled by default, attempts to load model by mapping it to known model even if some components are missing for example: if you place bare unet/dit finetune into stable-diffusion folder + - prompt encode caching for pass-through text-encoders - **UI** - dynamic visibility of image controls - improve main panel positioning: *portrait/landscape* @@ -44,6 +46,9 @@ And **SDNQ** improvements: now with *NPU* support and its own native *attention* - **Internal** - delay init of video models - **Experimental** + - support for **openai interface** for llm + in *prompt enhance* enable openai interface and when llm is loaded, + sdnext will start a local openai-compatible server on usual endpoints (e.g. `/v1/completions`, `/v1/chat/completions`, etc.) - support for [pruna](https://docs.pruna.ai/en/stable/compression.html) swiss-army-knife of model compression, caching and optimization see *settings -> model compile* for options *note*: pruna options compatibility varies greatly depending on platform, gpu, torch and model used @@ -65,6 +70,7 @@ And **SDNQ** improvements: now with *NPU* support and its own native *attention* - measure: handle current kanvas stage - model metadata: handle invalid metadata and strip workflows - mps: install `torchsde` as requirement + - nunchaku: gate for `cuda` only - onnxruntime: handle invalid version - onnxruntime: mark all import paths as non-critical - options: handle compatibility options diff --git a/TODO.md b/TODO.md index efb27fe06..00a6b31bc 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,7 @@ - Integrate natural language image search - [ImageDB](https://github.com/vladmandic/imagedb) - Unify *huggingface* and *diffusers* model folders +- [QuantFunc](https://huggingface.co/QuantFunc/Klein-9B-Series) ### OnHold diff --git a/data/reference-community.json b/data/reference-community.json index bbc1de240..a2ac6b8a2 100644 --- a/data/reference-community.json +++ b/data/reference-community.json @@ -169,5 +169,19 @@ "desc": "Anima 1.0 Base pre-merged with several LoRAs and quantized to uint4 using SDNQ with Hadamard. Flexible as it can be used with and without guidance.", "date": "2026 July", "size": 2.22 + }, + "FLUX.2 Klein 9B KV Merge sdnq-hadamard-svd": { + "path": "vladmandic/Flux.2-Klein-9B-KV-Merge-sdnq-hadamard-uint4", + "preview": "black-forest-labs--FLUX.2-klein-9b-kv.jpg", + "desc": "FLUX.2 Klein 9B KV pre-merged with several LoRAs and quantized to uint4 using SDNQ with Hadamard.", + "size": 11.67, + "date": "2026 July" + }, + "Krea 2 Turbo Merge sdnq-hadamard-uint4": { + "path": "vladmandic/Krea-2-Turbo-Merge-sdnq-hadamard-uint4", + "preview": "CalamitousFelicitousness--Krea-2-Turbo-Diffusers.jpg", + "desc": "Krea 2 Turbo pre-merged with several LoRAs and quantized to uint4 using SDNQ with Hadamard.", + "size": 10.55, + "date": "2026 July" } } diff --git a/modules/api/models.py b/modules/api/models.py index 5ed7743fd..cae3a44d5 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -375,6 +375,7 @@ class ReqPromptEnhance(BaseModel): process_words: Optional[str] = Field(title="Banned words", default=None, description="List of words to process") semantic_threshold: Optional[float] = Field(title="Semantic threshold", default=None, description="Semantic similarity threshold for processed words") embedding_similarity: Optional[float] = Field(title="Embedding similarity", default=None, description="Embedding similarity threshold for processed words") + use_openai: Optional[bool] = Field(title="Use OpenAI", default=False, description="Use OpenAI API for model access") class ResPromptEnhance(BaseModel): prompt: str = Field(title="Prompt", description="Enhanced prompt") diff --git a/modules/api/process.py b/modules/api/process.py index 1abf3950b..ecb1659c7 100644 --- a/modules/api/process.py +++ b/modules/api/process.py @@ -247,6 +247,7 @@ class APIProcess: process_words=req.process_words, semantic_threshold=req.semantic_threshold, embedding_similarity=req.embedding_similarity, + use_openai=req.use_openai ) elif req.type == 'video': from modules.ui_video_vlm import enhance_prompt diff --git a/modules/openai/serve.py b/modules/openai/serve.py index 5c3fa3b2f..f46f015b8 100644 --- a/modules/openai/serve.py +++ b/modules/openai/serve.py @@ -17,6 +17,7 @@ class OpenAIServer: processor=None, host: str = "127.0.0.1", port: int = 8888, + server: Optional[uvicorn.Server] = None, max_context_tokens: Optional[int] = None, max_new_tokens: Optional[int] = None, stream: Optional[bool] = None, @@ -24,7 +25,7 @@ class OpenAIServer: top_p: Optional[float] = None, top_k: Optional[int] = None, repetition_penalty: Optional[float] = None, - api_key: Optional[str] = None + api_key: Optional[str] = None, ): self.model = model self.tokenizer = tokenizer @@ -50,46 +51,55 @@ class OpenAIServer: log.info(f"OpenAI: {self.model_info}") attention = get_attention_config(self) log.debug(f'OpenAI: {attention}') - self.api_key = api_key - self._security = HTTPBearer(auto_error=False) - self._is_running = False - self._lock = threading.Lock() + + if server: + self.use_server = True + self.app = server + self.server = server + self._is_running = True + else: + self.use_server = False + self.app = FastAPI(title="SD.Next OpenAI-compatible LLM Server", version="1.0") + self.server: Optional[uvicorn.Server] = None + self._is_running = False self._startup_event = threading.Event() - self.server: Optional[uvicorn.Server] = None + self.api_key = api_key + self._lock = threading.Lock() + self._security = HTTPBearer(auto_error=False) self.thread: Optional[threading.Thread] = None - self.app = FastAPI(title="SD.Next OpenAI-compatible LLM Server", version="1.0") setup_routes(self) def start(self, timeout_seconds: float = 10.0): """Spawns the serving interface safely using an active background thread worker.""" + if self._is_running: + # log.warning("OpenAI: Server('already running')") + return with self._lock: - if self._is_running: - log.warning("OpenAI: Server('already running')") - return - self._startup_event.clear() - config = uvicorn.Config( - app=self.app, host=self.host, port=self.port, log_level="info", loop="asyncio", workers=1 - ) - self.server = uvicorn.Server(config) - self.server.install_signal_handlers = lambda *args, **kwargs: None - original_startup = self.server.startup + if self.server is None: + self._startup_event.clear() + config = uvicorn.Config(app=self.app, host=self.host, port=self.port, log_level="info", loop="asyncio", workers=1) + self.server = uvicorn.Server(config) + self.server.install_signal_handlers = lambda *args, **kwargs: None + original_startup = self.server.startup - async def patched_startup(*args, **kwargs): - await original_startup(*args, **kwargs) - self._startup_event.set() + async def patched_startup(*args, **kwargs): + await original_startup(*args, **kwargs) + self._startup_event.set() - self.server.startup = patched_startup - self.thread = threading.Thread(target=self.server.run, name="TransformersServeWorkerThread", daemon=True) - self.thread.start() - if not self._startup_event.wait(timeout=timeout_seconds): - self.stop() - raise TimeoutError("OpenAI: init timeout") + self.server.startup = patched_startup + self.thread = threading.Thread(target=self.server.run, name="TransformersServeWorkerThread", daemon=True) + self.thread.start() + if not self._startup_event.wait(timeout=timeout_seconds): + self.stop() + raise TimeoutError("OpenAI: init timeout") self._is_running = True url = f"http://{self.host}:{self.port}/v1" log.info(f"OpenAI: Server(url={url})") def stop(self, timeout_seconds: float = 5.0): """Safely winds down network sockets and detached worker threads.""" + if self.use_server: + return with self._lock: if not self._is_running or not self.server: return @@ -98,5 +108,7 @@ class OpenAIServer: self.thread.join(timeout=timeout_seconds) self.server = None self.thread = None + self.server = None + self.thread = None self._is_running = False log.info("OpenAI: Server(None)") diff --git a/modules/sd_hijack_te.py b/modules/sd_hijack_te.py index a9f7f9d29..30d86c087 100644 --- a/modules/sd_hijack_te.py +++ b/modules/sd_hijack_te.py @@ -4,6 +4,40 @@ from modules import shared, errors, timer, sd_models from modules.logger import log +class PromptCache: + def __init__(self): + self.cache = {} + self.id = None + self.max = 16 + + def get(self, prompt): + if self.id != id(shared.sd_model): + self.cache.clear() + self.id = id(shared.sd_model) + log.debug(f'Encode: prompt cache activate id={self.id} depth={len(self.cache)}') + if (isinstance(prompt, list) and len(prompt) == 1 and isinstance(prompt[0], str)): + cached = self.cache.get(prompt[0], None) + elif isinstance(prompt, str): + cached = self.cache.get(prompt, None) + else: + cached = None + if cached: + log.debug(f'Encode: prompt="{prompt}" cache={len(self.cache)} hit') + return cached + + def set(self, prompt, encoded): + if len(self.cache) >= self.max: + oldest_key = next(iter(self.cache)) + del self.cache[oldest_key] + if (isinstance(prompt, list) and len(prompt) == 1 and isinstance(prompt[0], str)): + self.cache[prompt[0]] = encoded + elif isinstance(prompt, str): + self.cache[prompt] = encoded + + +prompt_cache = PromptCache() + + def hijack_encode_prompt(*args, **kwargs): jobid = shared.state.begin('TE Encode') t0 = time.time() @@ -17,8 +51,8 @@ def hijack_encode_prompt(*args, **kwargs): if prompt is None and len(args_copy) > 0: prompt = args_copy[0] patch_prompt = True + prompt = [p.strip(", \n") if isinstance(p, str) else p for p in prompt] if isinstance(prompt, list) else prompt res = prompt - log.debug(f'Encode: prompt="{prompt}" hijack=True') if hasattr(shared.sd_model, 'before_prompt_encode'): log.debug(f'Encode: prompt="{prompt}" op=before') @@ -26,8 +60,16 @@ def hijack_encode_prompt(*args, **kwargs): if patch_prompt: args_copy[0] = res - if hasattr(shared.sd_model, 'orig_encode_prompt'): - res = shared.sd_model.orig_encode_prompt(*args_copy, **kwargs) + cached = prompt_cache.get(prompt) + if cached is not None: + res = cached + else: + log.debug(f'Encode: prompt="{prompt}" hijack=True') + if hasattr(shared.sd_model, 'orig_encode_prompt'): + res = shared.sd_model.orig_encode_prompt(*args_copy, **kwargs) + else: + res = shared.sd_model.encode_prompt(*args_copy, **kwargs) + prompt_cache.set(prompt, res) if hasattr(shared.sd_model, 'after_prompt_encode'): log.debug(f'Encode: prompt="{prompt}" op=after') diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 690d004c0..d65e549fd 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -15,7 +15,7 @@ from collections import OrderedDict import gradio as gr from PIL import Image from starlette.responses import FileResponse, JSONResponse -from modules import paths, shared, files_cache, errors, infotext, ui_symbols, ui_components, modelstats +from modules import paths, shared, devices, files_cache, errors, infotext, ui_symbols, ui_components, modelstats from modules.logger import log from modules.json_helpers import writefile @@ -321,7 +321,8 @@ class ExtraNetworksPage: subdirs['Base'] = 1 subdirs['Distilled'] = 1 subdirs['Quantized'] = 1 - subdirs['Nunchaku'] = 1 + if devices.backend == 'cuda': + subdirs['Nunchaku'] = 1 subdirs['Community'] = 1 subdirs['Cloud'] = 1 subdirs[diffusers_base] = 1 @@ -355,7 +356,9 @@ class ExtraNetworksPage: continue if subdir in ['All', 'Local', 'Diffusers']: style = 'network-local' - elif subdir in ['Base', 'Reference', 'Distilled', 'Quantized', 'Nunchaku', 'Community', 'Cloud']: + elif subdir in ['Base', 'Reference', 'Distilled', 'Quantized', 'Community', 'Cloud']: + style = 'network-reference' + elif subdir in ['Nunchaku'] and devices.backend == 'cuda': style = 'network-reference' else: style = 'network-folder' diff --git a/pipelines/model_flux2_klein.py b/pipelines/model_flux2_klein.py index b1599f18d..a6eb649bd 100644 --- a/pipelines/model_flux2_klein.py +++ b/pipelines/model_flux2_klein.py @@ -20,7 +20,7 @@ def load_flux2_klein(checkpoint_info, diffusers_load_config=None): if repo_id is None or repo_id.lower() == 'none': return None - if '-kv' in repo_id: + if '-kv' in repo_id.lower(): cls = diffusers.Flux2KleinKVPipeline else: cls = diffusers.Flux2KleinPipeline diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 63cfe3dd0..c00357b71 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -302,6 +302,7 @@ class PromptEnhanceScript(scripts_manager.Script): processor: transformers.AutoProcessor = None tokenizer: transformers.AutoTokenizer = None busy: bool = False + server = None options = Options() def title(self): @@ -316,7 +317,7 @@ class PromptEnhanceScript(scripts_manager.Script): from modules.sd_models_compile import compile_torch self.llm = compile_torch(self.llm, apply_to_components=False, op="LLM") - def load(self, name:str | None=None, model_repo:str | None=None, model_gguf:str | None=None, model_type:str | None=None, model_file:str | None=None): + def load(self, name:str | None=None, use_openai:bool=False, model_repo:str | None=None, model_gguf:str | None=None, model_type:str | None=None, model_file:str | None=None): # Strip symbols from display name if present name = get_model_repo_from_display(name) if name else self.options.default if self.busy: @@ -419,7 +420,9 @@ class PromptEnhanceScript(scripts_manager.Script): log.error(f'Prompt enhance: load {e}') if debug_enabled: errors.display(e, 'Prompt enhance') + devices.torch_gc() + self.set_openai(enable=use_openai) self.busy = False return model_repo @@ -430,6 +433,7 @@ class PromptEnhanceScript(scripts_manager.Script): def unload(self): if self.llm is not None: model_name = self.model + self.set_openai(enable=False) log.debug(f'Prompt enhance: unloading model="{model_name}"') deregister_aux('prompt_enhance') sd_models.move_model(self.llm, devices.cpu, force=True) @@ -442,6 +446,21 @@ class PromptEnhanceScript(scripts_manager.Script): else: log.debug('Prompt enhance: no model loaded') + def set_openai(self, enable: bool): + from modules.openai.serve import OpenAIServer + if enable and self.llm is not None and self.tokenizer is not None: + self.server = OpenAIServer( + model=self.llm, + tokenizer=self.tokenizer, + host="127.0.0.1", + port=8000, + server=shared.api.app, + ) + self.server.start() + elif self.server is not None: + self.server.stop() + self.server = None + def clean(self, response, keep_thinking=False, prefill_text='', keep_prefill=False): # Handle thinking tags FIRST (before generic tag removal) if '' in response or '' in response: @@ -573,6 +592,7 @@ class PromptEnhanceScript(scripts_manager.Script): process_words:str='', semantic_threshold:float=0.0, embedding_similarity:float=0.0, + use_openai:bool=False, ): # Strip symbols from model name if present model = get_model_repo_from_display(model) if model else self.options.default @@ -599,7 +619,7 @@ class PromptEnhanceScript(scripts_manager.Script): time.sleep(0.1) if not is_cloud_model(model): - self.load(model) + self.load(model, use_openai=use_openai) if seed is None or seed == -1: random.seed() @@ -859,7 +879,7 @@ class PromptEnhanceScript(scripts_manager.Script): return prompt # Return original full prompt on censorship return response - def apply(self, prompt, image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity): + def apply(self, prompt, image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity, use_openai): response = self.enhance( prompt=prompt, image=image, @@ -884,6 +904,7 @@ class PromptEnhanceScript(scripts_manager.Script): process_words=process_words, semantic_threshold=semantic_threshold, embedding_similarity=embedding_similarity, + use_openai=use_openai, ) if apply_prompt: return [response, response] @@ -918,13 +939,14 @@ class PromptEnhanceScript(scripts_manager.Script): # Set initial state based on whether default model supports vision default_is_vl = is_vision_model(Options.default) use_vision = gr.Checkbox(label='Use vision', value=False, interactive=default_is_vl, elem_id='prompt_enhance_use_vision') + use_openai = gr.Checkbox(label='OpenAI interface', value=False, elem_id='prompt_enhance_openai') gr.HTML('
') with gr.Group(): with gr.Row(): llm_model = gr.Dropdown(label='LLM model', choices=Options.get_model_choices(), value=Options.get_default_display(), interactive=True, allow_custom_value=True, elem_id='prompt_enhance_model') with gr.Row(): load_btn = gr.Button(value='Load model', elem_id='prompt_enhance_load', variant='secondary') - load_btn.click(fn=self.load, inputs=[llm_model], outputs=[]) + load_btn.click(fn=self.load, inputs=[llm_model, use_openai], outputs=[]) unload_btn = gr.Button(value='Unload model', elem_id='prompt_enhance_unload', variant='secondary') unload_btn.click(fn=self.unload, inputs=[], outputs=[]) with gr.Accordion('Custom model', open=False, elem_id='prompt_enhance_custom'): @@ -938,7 +960,7 @@ class PromptEnhanceScript(scripts_manager.Script): model_file = gr.Textbox(label='Model file', value=None, interactive=True, elem_id='prompt_enhance_model_file', placeholder='Optional GGUF model file inside GGUF model repo') with gr.Row(): custom_btn = gr.Button(value='Load custom model', elem_id='prompt_enhance_custom_load', variant='secondary') - custom_btn.click(fn=self.load, inputs=[model_repo, model_repo, model_gguf, model_type, model_file], outputs=[llm_model]) + custom_btn.click(fn=self.load, inputs=[model_repo, use_openai, model_repo, model_gguf, model_type, model_file], outputs=[llm_model]) llm_model.change(fn=self.get_custom, inputs=[llm_model], outputs=[model_repo, model_gguf, model_type, model_file]) gr.HTML('
') with gr.Accordion('Options', open=False, elem_id='prompt_enhance_options'): @@ -990,8 +1012,8 @@ class PromptEnhanceScript(scripts_manager.Script): # Update vision toggle interactivity when model changes llm_model.change(fn=self.update_vision_toggle, inputs=[llm_model], outputs=[use_vision], show_progress=False) if self.prompt: - apply_btn.click(fn=self.apply, inputs=[self.prompt, self.image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity], outputs=[prompt_output, self.prompt]) - return [self.prompt, self.image, apply_auto, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity] + apply_btn.click(fn=self.apply, inputs=[self.prompt, self.image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity, use_openai], outputs=[prompt_output, self.prompt]) + return [self.prompt, self.image, apply_auto, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity, use_openai] def after_component(self, component, **_kwargs): # searching for actual ui prompt components if getattr(component, 'elem_id', '') in ['txt2img_prompt', 'img2img_prompt', 'control_prompt', 'video_prompt']: @@ -1002,7 +1024,7 @@ class PromptEnhanceScript(scripts_manager.Script): self.image.use_original = True def before_process(self, p: processing.StableDiffusionProcessing, *args, **kwargs): # pylint: disable=unused-argument - _self_prompt, self_image, apply_auto, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity = args + _self_prompt, self_image, apply_auto, llm_model, prompt_system, prompt_prefix, prompt_suffix, min_tokens, max_tokens, do_sample, temperature, repetition_penalty, top_k, top_p, thinking_mode, nsfw_mode, use_vision, prefill_text, keep_prefill, keep_thinking, custom_args, process_words, semantic_threshold, embedding_similarity, use_openai = args if not apply_auto and not p.enhance_prompt: return if shared.state.skipped or shared.state.interrupted: @@ -1039,6 +1061,7 @@ class PromptEnhanceScript(scripts_manager.Script): process_words=process_words, semantic_threshold=semantic_threshold, embedding_similarity=embedding_similarity, + use_openai=use_openai, ) timer.process.record('prompt') shared.state.end(jobid)