rebuild ui

Signed-off-by: Vladimir Mandic <mandic00@live.com>

modules/openai/classes.py

@@ -0,0 +1,65 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional
+from transformers import StoppingCriteria
+
+
+@dataclass
+class Model:
+    name: Optional[str] = None
+    cls: str = "UnknownModelClass"
+    tokenizer: str = "UnknownTokenizerClass"
+    processor: Optional[str] = None
+    type: Optional[str] = None
+
+
+@dataclass
+class Config:
+    max_context_tokens: int = 4096
+    max_new_tokens: int = 512
+    stream: bool = False
+    temperature: float = 0.2
+    top_p: float = 0.9
+    top_k: int = 50
+    repetition_penalty: float = 1.0
+    use_cache: bool = True
+
+
+@dataclass
+class Attention:
+    implementation: str = "eager"
+    flash: bool = False
+    arch: str = ""
+
+
+@dataclass
+class Req:
+    id: int
+    client: Optional[str] = None
+    url: Optional[str] = None
+    config: dict[str, Any] = None
+
+
+class CustomTokenStopCriteria(StoppingCriteria):
+    def __init__(self, stop_ids: List[int]):
+        super().__init__()
+        self.stop_ids = stop_ids
+
+    def __call__(self, input_ids: Any, scores: Any, **kwargs) -> bool:
+        return input_ids[0][-1].item() in self.stop_ids
+
+
+class Stats:
+    id: int = 0
+    messages: int = 0
+    prompt: int = 0
+    ttft: float = 0.0
+    latency: float = 0.0
+    tps: float = 0.0
+    tools: int = 0
+    tokens: dict[str, int] = {}
+    chunks: dict[str, int] = {}
+    streaming: bool = None
+    thinking: bool = None
+
+    def __str__(self):
+        return f"Res(id={self.id}, messages={self.messages}, prompt={self.prompt}, ttft={self.ttft:.3f}, latency={self.latency:.3f}, tps={self.tps:.3f}, tools={self.tools}, tokens={self.tokens}, chunks={self.chunks}, streaming={self.streaming}, thinking={self.thinking})"

modules/openai/generate.py

@@ -0,0 +1,272 @@
+import os
+import json
+import re
+import threading
+import time
+from typing import Any, AsyncGenerator, Dict, List, Optional
+from fastapi import Request
+from fastapi.responses import StreamingResponse
+from transformers import StoppingCriteriaList, TextIteratorStreamer
+from modules.logger import log
+from .classes import CustomTokenStopCriteria, Stats
+
+
+debug_log = log.debug if os.environ.get('SD_LLM_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def enforce_rolling_context(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Ensures context bounds are respected while anchoring the base system prompt."""
+    system_message = None
+    chat_history = []
+    for msg in messages:
+        if msg.get("role") == "system":
+            system_message = msg
+        else:
+            chat_history.append(msg)
+    while len(chat_history) > 0:
+        current_payload = ([system_message] if system_message else []) + chat_history
+        try:
+            if hasattr(self.tokenizer, "apply_chat_template") and self.tokenizer.chat_template:
+                test_str = self.tokenizer.apply_chat_template(current_payload, tokenize=False, add_generation_prompt=True)
+            else:
+                test_str = "".join([m["content"] for m in current_payload])
+            total_len = len(self.tokenizer.encode(test_str))
+            if (total_len + self.config.max_new_tokens) <= self.config.max_context_tokens:
+                return current_payload
+        except Exception:
+            pass
+        if len(chat_history) >= 2:
+            chat_history = chat_history[2:]
+            log.debug("OpenAI: Context('dropped oldest conversation')")
+        elif len(chat_history) == 1:
+            chat_history.pop(0)
+            log.debug("OpenAI: Context('dropped history')")
+        else:
+            break
+    return [system_message] if system_message else []
+
+
+def parse_inline_tool_calls(text: str) -> List[Dict[str, Any]]:
+    """Parses model-specific text tags and builds OpenAI tool payload models."""
+    tool_calls = []
+    qwen_pattern = r"<\|tool_call_start\|>(.*?)(?:<\|tool_call_end\|>|$)"
+    gemma_pattern = r"<tool_call>(.*?)(?:</tool_call>|$)"
+    matches = re.findall(qwen_pattern, text, re.DOTALL) + re.findall(gemma_pattern, text, re.DOTALL)
+    for idx, match_str in enumerate(matches):
+        try:
+            parsed = json.loads(match_str.strip())
+            call_id = f"call_{int(time.time())}_{idx}"
+            if "name" in parsed:
+                args = parsed.get("arguments", {})
+                if not isinstance(args, str):
+                    args = json.dumps(args)
+                tool_calls.append({
+                    "id": call_id,
+                    "type": "function",
+                    "function": {"name": parsed["name"], "arguments": args}
+                })
+        except Exception:
+            continue
+    return tool_calls
+
+
+async def execute_generation(
+    self,
+    stats: Stats,
+    request: Request,
+    prompt: str,
+    config: Dict[str, Any],
+    stream: bool,
+    stream_options: Optional[Dict[str, Any]] = None,
+    images: Optional[List[Any]] = None,
+):
+    """Executes tensor processing, tracking streaming reasoning buffers and finish conditions."""
+    start_time = time.time()
+    if self.processor and images:
+        inputs = self.processor(text=prompt, images=images, return_tensors="pt").to(self.model.device)
+    else:
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+    model_name = getattr(self.model.config, "_name_or_path", "local-transformer")
+    req_id = f"gen-{int(time.time())}"
+    stop_ids = [self.tokenizer.eos_token_id]
+    stop_targets = [
+        "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "</tool_call>",
+        "<|tool_call_end|>", "<|end|>", "<|start_header_id|>", "<|end_header_id|>",
+        "<end_of_turn>"
+    ]
+    for t_str in stop_targets:
+        t_id = self.tokenizer.convert_tokens_to_ids(t_str)
+        if isinstance(t_id, int) and t_id > 0:
+            stop_ids.append(t_id)
+    stopping_criteria = StoppingCriteriaList([CustomTokenStopCriteria(list(set(stop_ids)))])
+    chunks = { 'reasoning': [], 'content': [] }
+
+    def execute_streaming_generation():
+        streamer = TextIteratorStreamer(self.tokenizer, skip_prompt=True, skip_special_tokens=False)
+        generation_kwargs = dict(
+            **inputs,
+            streamer=streamer,
+            pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,
+            stopping_criteria=stopping_criteria,
+            use_cache=self.config.use_cache,
+            **config
+        )
+        threading.Thread(target=self.model.generate, kwargs=generation_kwargs, daemon=True).start()
+
+        async def production_stream_decorator() -> AsyncGenerator[str, None]:
+            first_token_sent = False
+            is_thinking = False
+            token_count = 0
+            generation_start = time.time()
+            try:
+                for chunk in streamer:
+                    if await request.is_disconnected():
+                        break
+                    if not chunk:
+                        continue
+                    if any(t in chunk for t in ["<think>", "## Thought", "thought\n"]):
+                        is_thinking = True
+                        for t in ["<think>", "## Thought", "thought\n"]:
+                            chunk = chunk.replace(t, "")
+                    if "</think>" in chunk:
+                        is_thinking = False
+                        chunk = chunk.replace("</think>", "")
+                    for tag in [
+                        "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "<|tool_call_start|>",
+                        "<tool_call>", "<|end|>", "<|start_header_id|>", "<|end_header_id|>",
+                        "<end_of_turn>"
+                    ]:
+                        chunk = chunk.replace(tag, "")
+                    if not chunk:
+                        continue
+                    if not first_token_sent:
+                        stats.ttft = time.time() - start_time
+                        first_token_sent = True
+                    token_count += 1
+                    choice_delta = {}
+                    if is_thinking:
+                        chunks['reasoning'].append(chunk)
+                        choice_delta["reasoning_content"] = chunk
+                    else:
+                        chunks['content'].append(chunk)
+                        choice_delta["content"] = chunk
+
+                    finish_reason = "length" if token_count >= config.get("max_new_tokens", self.config.max_new_tokens) else None
+                    payload = {
+                        "id": req_id, "object": "chat.completion.chunk", "created": int(time.time()),
+                        "model": model_name, "choices": [{"index": 0, "delta": choice_delta, "finish_reason": finish_reason}]
+                    }
+                    yield f"data: {json.dumps(payload)}\n\n"
+                final_payload = {
+                    "id": req_id, "object": "chat.completion.chunk", "created": int(time.time()),
+                    "model": model_name, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+                }
+                yield f"data: {json.dumps(final_payload)}\n\n"
+                if stream_options and stream_options.get("include_usage"):
+                    prompt_tokens = len(inputs.input_ids[0])
+                    usage_payload = {
+                        "id": req_id,
+                        "object": "chat.completion.chunk",
+                        "created": int(time.time()),
+                        "model": model_name,
+                        "choices": [],
+                        "usage": {
+                            "prompt_tokens": prompt_tokens,
+                            "completion_tokens": token_count,
+                            "total_tokens": prompt_tokens + token_count
+                        }
+                    }
+                    yield f"data: {json.dumps(usage_payload)}\n\n"
+                yield "data: [DONE]\n\n"
+            finally:
+                duration = time.time() - generation_start
+                if duration > 0 and token_count > 0:
+                    stats.tps = token_count / duration
+                    stats.chunks['reasoning'] = len(chunks['reasoning'])
+                    stats.chunks['content'] = len(chunks['content'])
+                    stats.latency = time.time() - start_time
+                    stats.tools = 0
+                    stats.tokens['prompt'] = len(inputs.input_ids[0])
+                    stats.tokens['output'] = token_count
+                    stats.streaming = True
+                    stats.thinking = len(chunks['reasoning']) > 0
+                    text_reasoning = " ".join(c.strip() for c in chunks["reasoning"] if len(c.strip()) > 0)
+                    text_content = " ".join(c.strip() for c in chunks["content"] if len(c.strip()) > 0)
+                    debug_log(f'OpenAI chunks: reasoning="{text_reasoning}"')
+                    debug_log(f'OpenAI chunks: content="{text_content}"')
+                    log.debug(f"OpenAI: {stats}")
+        return StreamingResponse(production_stream_decorator(), media_type="text/event-stream")
+
+    def execute_direct_generation():
+        outputs = self.model.generate(
+            **inputs,
+            pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,
+            stopping_criteria=stopping_criteria,
+            use_cache=self.config.use_cache,
+            **config
+        )
+
+        generated_ids = outputs[0][inputs.input_ids.shape[-1]:]
+        raw_text = self.tokenizer.decode(generated_ids, skip_special_tokens=False)
+        reasoning_text = ""
+        for start_tag, end_tag in [("<think>", "</think>"), ("## Thought", "##"), ("thought\n", "\n\n")]:
+            if start_tag in raw_text and end_tag in raw_text:
+                parts = raw_text.split(end_tag)
+                reasoning_text = parts[0].replace(start_tag, "").strip()
+                raw_text = parts[1]
+                break
+        tool_calls = parse_inline_tool_calls(raw_text)
+        clean_text = raw_text
+        for tag in [
+            "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "</tool_call>",
+            "<|tool_call_end|>", "<|end|>", "<|start_header_id|>", "<|end_header_id|>",
+            "<end_of_turn>"
+        ]:
+            clean_text = clean_text.replace(tag, "")
+        if tool_calls:
+            clean_text = re.sub(r"<\|tool_call_start\|>.*?<\|tool_call_end\|>", "", clean_text, flags=re.DOTALL)
+            clean_text = re.sub(r"<tool_call>.*?</tool_call>", "", clean_text, flags=re.DOTALL).strip()
+
+        stats.ttft = time.time() - start_time
+        stats.latency = time.time() - start_time
+        stats.tools = len(tool_calls)
+        stats.tokens['prompt'] = len(inputs.input_ids[0])
+        stats.tokens['reasoning'] = len(reasoning_text)
+        stats.tokens['completion'] = len(generated_ids)
+        stats.tokens['total'] = len(inputs.input_ids[0]) + len(generated_ids)
+        stats.tps = stats.tokens['total'] / stats.latency if stats.latency > 0 else 0
+        stats.streaming = False
+        stats.thinking = len(reasoning_text) > 0
+        log.debug(f"OpenAI: {stats}")
+
+        debug_log(f'OpenAI response: "{clean_text}"')
+        message_payload = {"role": "assistant", "content": clean_text if clean_text else None}
+        if reasoning_text:
+            debug_log(f'OpenAI reasoning: "{reasoning_text}"')
+            message_payload["reasoning_content"] = reasoning_text
+        if tool_calls:
+            debug_log(f'OpenAI tools: {tool_calls}')
+            message_payload["tool_calls"] = tool_calls
+        f_reason = "tool_calls" if tool_calls else ("length" if len(generated_ids) >= config["max_new_tokens"] else "stop")
+
+        return {
+            "id": req_id,
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "model": model_name,
+            "choices": [{
+                "message": message_payload,
+                "index": 0,
+                "finish_reason": f_reason
+            }],
+            "usage": {
+                "prompt_tokens": len(inputs.input_ids[0]),
+                "completion_tokens": len(generated_ids),
+                "total_tokens": len(inputs.input_ids[0]) + len(generated_ids)
+            }
+        }
+
+    if stream:
+        return execute_streaming_generation()
+    else:
+        return execute_direct_generation()

modules/openai/helpers.py

@@ -0,0 +1,52 @@
+from .classes import Attention
+
+
+def get_attention_config(self):
+    attention = Attention(
+        implementation=getattr(self.model.config, "_attn_implementation", "eager"),
+        flash=getattr(self.model.config, "use_flash_attention", False)
+    )
+    if hasattr(self.model.config, "num_key_value_heads") and hasattr(self.model.config, "num_attention_heads"):
+        kv_heads = self.model.config.num_key_value_heads
+        attn_heads = self.model.config.num_attention_heads
+        if kv_heads < attn_heads:
+            attention.arch = f"arch=GQA kv_heads={kv_heads} attn_heads={attn_heads}"
+        elif kv_heads == 1:
+            attention.arch = "arch=MQA"
+        else:
+            attention.arch = "arch=MHA"
+    return attention
+
+
+def get_prompt_template(self, messages: list[str], tools) -> str:
+    model_name = getattr(self.model.config, "_name_or_path", "").lower()
+    if hasattr(self.tokenizer, "apply_chat_template") and self.tokenizer.chat_template:
+        template_kwargs = {
+            "conversation": messages,
+            "tokenize": False,
+            "add_generation_prompt": True
+        }
+        if tools:
+            template_kwargs["tools"] = tools
+        prompt_str = self.tokenizer.apply_chat_template(**template_kwargs)
+    else:
+        prompt_str = ""
+        if "llama" in model_name:
+            prompt_str += "<|begin_of_text|>"
+            for msg in messages:
+                prompt_str += f"<|start_header_id|>{msg['role']}<|end_header_id|>\n\n{msg['content']}<|eot_id|>"
+            prompt_str += "<|start_header_id|>assistant<|end_header_id|>\n\n"
+        elif "gemma" in model_name:
+            for msg in messages:
+                role_tag = msg["role"] if msg["role"] != "system" else "user"
+                prompt_str += f"<start_of_turn>{role_tag}\n{msg['content']}<end_of_turn>\n"
+            prompt_str += "<start_of_turn>assistant\n"
+        elif "qwen" in model_name:
+            for msg in messages:
+                prompt_str += f"<|im_start|>{msg['role']}\n{msg['content']}<|im_end|>\n"
+            prompt_str += "<|im_start|>assistant\n"
+        else:
+            for msg in messages:
+                prompt_str += f"<|{msg['role']}|>\n{msg['content']}\n"
+            prompt_str += "<|assistant|>\n"
+    return prompt_str

modules/openai/routes.py

@@ -0,0 +1,129 @@
+import time
+from typing import Optional
+from fastapi import Request, HTTPException, Depends
+from fastapi.security import HTTPAuthorizationCredentials
+from modules.logger import log
+from .generate import execute_generation, enforce_rolling_context
+from .helpers import get_prompt_template
+from .classes import Stats, Req
+
+
+def setup_routes(self):
+    """Exposes standard endpoints and safely routes traffic requests."""
+
+    async def verify_api_key(credentials: Optional[HTTPAuthorizationCredentials] = Depends(self._security)):
+        if not self.api_key:
+            return
+        if credentials is None or credentials.credentials != self.api_key:
+            raise HTTPException(
+                status_code=401,
+                detail="Invalid or missing API key in Authorization header."
+            )
+
+    @self.app.get("/health")
+    async def health_check():
+        return {"status": "healthy", "timestamp": time.time()}
+
+    @self.app.get("/v1/models", dependencies=[Depends(verify_api_key)])
+    async def list_models():
+        model_name = getattr(self.model.config, "_name_or_path", "local-transformer")
+        return {
+            "object": "list",
+            "data": [{
+                "id": model_name,
+                "object": "model",
+                "created": int(time.time()),
+                "owned_by": "transformers"
+            }]
+        }
+
+    @self.app.get("/v1/models/{model_id}", dependencies=[Depends(verify_api_key)])
+    async def retrieve_model(model_id: str):
+        model_name = getattr(self.model.config, "_name_or_path", "local-transformer")
+        if model_id != model_name:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Model '{model_id}' not found. Active model is '{model_name}'."
+            )
+        return {
+            "id": model_name,
+            "object": "model",
+            "created": int(time.time()),
+            "owned_by": "transformers"
+        }
+
+    @self.app.post("/v1/completions", dependencies=[Depends(verify_api_key)], tags=["production_hardened"])
+    async def text_completions(request: Request):
+        json_body = await request.json()
+        prompt_str = json_body.get("prompt", "")
+        if isinstance(prompt_str, list):
+            prompt_str = prompt_str[0] if prompt_str else ""
+        temperature = float(json_body.get("temperature", self.config.temperature))
+        config = {
+            "max_new_tokens": json_body.get("max_tokens", self.config.max_new_tokens),
+            "temperature": temperature,
+            "top_p": float(json_body.get("top_p", self.config.top_p)),
+            "top_k": int(json_body.get("top_k", self.config.top_k)),
+            "repetition_penalty": float(json_body.get("frequency_penalty", self.config.repetition_penalty)),
+            "do_sample": True if temperature > 0.0 else False
+        }
+        stats = Stats()
+        stats.id = id(request)
+        stats.prompt = len(prompt_str)
+
+        req = Req(id=id(request), config=config, client=request.scope.get('client', ('0:0.0.0', 0))[0], url=request.scope.get('path', 'err'))
+        log.debug(f"OpenAI: {req}")
+
+        return await execute_generation(
+            self,
+            stats=stats,
+            request=request,
+            prompt=prompt_str,
+            config=config,
+            stream=json_body.get("stream", self.config.stream),
+            stream_options=json_body.get("stream_options", None),
+            images=None
+        )
+
+    @self.app.post("/v1/chat/completions", dependencies=[Depends(verify_api_key)], tags=["production_hardened"])
+    async def chat_completions(request: Request):
+        json_body = await request.json()
+        messages = json_body.get("messages", [])
+        tools = json_body.get("tools", None)
+        stream_options = json_body.get("stream_options", None)
+        images = json_body.get("images", None)
+
+        temperature = float(json_body.get("temperature", self.config.temperature))
+        config = {
+            "max_new_tokens": json_body.get("max_tokens", self.config.max_new_tokens),
+            "temperature": temperature,
+            "top_p": float(json_body.get("top_p", self.config.top_p)),
+            "top_k": int(json_body.get("top_k", self.config.top_k)),
+            "repetition_penalty": float(json_body.get("frequency_penalty", self.config.repetition_penalty)),
+            "do_sample": True if temperature > 0.0 else False
+        }
+
+        sanitized_messages = enforce_rolling_context(self, messages)
+        try:
+            prompt_str = get_prompt_template(self, sanitized_messages, tools)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"LLM: template execution failure: {str(e)}") from e
+
+        stats = Stats()
+        stats.id = id(request)
+        stats.messages = len(messages)
+        stats.prompt = len(prompt_str)
+
+        req = Req(id=id(request), config=config, client=request.scope.get('client', ('0:0.0.0', 0))[0], url=request.scope.get('path', 'err'))
+        log.debug(f"OpenAI: {req}")
+
+        return await execute_generation(
+            self,
+            stats=stats,
+            request=request,
+            prompt=prompt_str,
+            config=config,
+            stream=json_body.get("stream", self.config.stream),
+            stream_options=stream_options,
+            images=images
+        )

modules/openai/serve.py

@@ -0,0 +1,102 @@
+import threading
+from typing import Optional
+import uvicorn
+from fastapi import FastAPI
+from fastapi.security import HTTPBearer
+from modules.logger import log
+from .classes import Model, Config
+from .helpers import get_attention_config
+from .routes import setup_routes
+
+
+class OpenAIServer:
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        processor=None,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        max_context_tokens: Optional[int] = None,
+        max_new_tokens: Optional[int] = None,
+        stream: Optional[bool] = None,
+        temperature: Optional[float] = None,
+        top_p: Optional[float] = None,
+        top_k: Optional[int] = None,
+        repetition_penalty: Optional[float] = None,
+        api_key: Optional[str] = None
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.processor = processor
+        self.host = host
+        self.port = port
+        self.model_info = Model(
+            name=getattr(model.config, "_name_or_path", "local-transformer"),
+            cls=model.__class__.__name__,
+            tokenizer=tokenizer.__class__.__name__,
+            processor=processor.__class__.__name__ if processor else None,
+            type=getattr(model.config, "model_type", None)
+        )
+        self.config = Config(
+            max_context_tokens=max_context_tokens if max_context_tokens is not None else 4096,
+            max_new_tokens=max_new_tokens if max_new_tokens is not None else 512,
+            stream=stream if stream is not None else False,
+            temperature=temperature if temperature is not None else 0.2,
+            top_p=top_p if top_p is not None else 0.9,
+            top_k=top_k if top_k is not None else 50,
+            repetition_penalty=repetition_penalty if repetition_penalty is not None else 1.
+        )
+        log.info(f"OpenAI: {self.model_info}")
+        attention = get_attention_config(self)
+        log.debug(f'OpenAI: {attention}')
+        self.api_key = api_key
+        self._security = HTTPBearer(auto_error=False)
+        self._is_running = False
+        self._lock = threading.Lock()
+        self._startup_event = threading.Event()
+        self.server: Optional[uvicorn.Server] = None
+        self.thread: Optional[threading.Thread] = None
+        self.app = FastAPI(title="SD.Next OpenAI-compatible LLM Server", version="1.0")
+        setup_routes(self)
+
+    def start(self, timeout_seconds: float = 10.0):
+        """Spawns the serving interface safely using an active background thread worker."""
+        with self._lock:
+            if self._is_running:
+                log.warning("OpenAI: Server('already running')")
+                return
+            self._startup_event.clear()
+            config = uvicorn.Config(
+                app=self.app, host=self.host, port=self.port, log_level="info", loop="asyncio", workers=1
+            )
+            self.server = uvicorn.Server(config)
+            self.server.install_signal_handlers = lambda *args, **kwargs: None
+            original_startup = self.server.startup
+
+            async def patched_startup(*args, **kwargs):
+                await original_startup(*args, **kwargs)
+                self._startup_event.set()
+
+            self.server.startup = patched_startup
+            self.thread = threading.Thread(target=self.server.run, name="TransformersServeWorkerThread", daemon=True)
+            self.thread.start()
+        if not self._startup_event.wait(timeout=timeout_seconds):
+            self.stop()
+            raise TimeoutError("OpenAI: init timeout")
+        self._is_running = True
+        url = f"http://{self.host}:{self.port}/v1"
+        log.info(f"OpenAI: Server(url={url})")
+
+    def stop(self, timeout_seconds: float = 5.0):
+        """Safely winds down network sockets and detached worker threads."""
+        with self._lock:
+            if not self._is_running or not self.server:
+                return
+            self.server.should_exit = True
+            if self.thread and self.thread.is_alive():
+                self.thread.join(timeout=timeout_seconds)
+            self.server = None
+            self.thread = None
+            self._is_running = False
+            log.info("OpenAI: Server(None)")

modules/openai/test.py

@@ -0,0 +1,44 @@
+import time
+import logging
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from .serve import OpenAIServer
+from modules import logger
+
+
+logger.setup_logging(debug=True)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
+logging.getLogger("asyncio").setLevel(logging.WARNING)
+
+
+logger.log.info("OpenAI: load model...")
+model = AutoModelForCausalLM.from_pretrained(
+    "Qwen/Qwen3-0.6B",
+    device_map="auto",
+    dtype=torch.bfloat16,
+    trust_remote_code=True,
+    attn_implementation="sdpa",
+)
+tokenizer = AutoTokenizer.from_pretrained(
+    "Qwen/Qwen3-0.6B",
+    trust_remote_code=True
+)
+
+logger.log.info("OpenAI: create server...")
+server = OpenAIServer(
+    model=model,
+    tokenizer=tokenizer,
+    host="127.0.0.1",
+    port=8000
+)
+
+logger.log.info("OpenAI: start server...")
+server.start()
+while True:
+    try:
+        time.sleep(1)
+    except KeyboardInterrupt:
+        server.stop()
+        break

ui/aspectRatioOverlay.ts

@@ -3,7 +3,7 @@ import { get_tab_index } from './ui';
 
 let currentWidth: number | null = null;
 let currentHeight: number | null = null;
-let arFrameTimeout: ReturnType<typeof setTimeout> | null = null;
+let arFrameTimeout: ReturnType<typeof setTimeout> | undefined;
 
 function dimensionChange(e: Event, isWidth: boolean, isHeight: boolean): void {
   const { target } = e;

ui/autocomplete.ts

@@ -586,8 +586,8 @@ const dropdown = {
 
 // -- Event handlers --
 
-let debounceTimer = null;
-let focusoutHideTimer = null;
+let debounceInput: ReturnType<typeof setTimeout> | undefined;
+let debounceFocus: ReturnType<typeof setTimeout> | undefined;
 
 function onInput(textarea) {
   if (!active) return;
@@ -607,8 +607,8 @@ function onInput(textarea) {
     dropdown.hide();
     return;
   }
-  clearTimeout(debounceTimer);
-  debounceTimer = setTimeout(() => {
+  clearTimeout(debounceInput);
+  debounceInput = setTimeout(() => {
     let results;
     if (info.mode === 'lora') {
       results = xnEngine.searchLoras(info.word);
@@ -679,15 +679,15 @@ function attachAutocomplete(textarea) {
   textarea.addEventListener('focusin', () => {
     if (dropdown.visible && dropdown.textarea && dropdown.textarea !== textarea) dropdown.hide();
     // Cancel any pending hide from a recent blur so refocusing within 200ms doesn't close the dropdown.
-    clearTimeout(focusoutHideTimer);
-    focusoutHideTimer = null;
+    clearTimeout(debounceFocus);
+    debounceFocus = undefined;
     // Re-fire input handling so a partial tag at the cursor reopens the dropdown.
     onInput(textarea);
   });
   textarea.addEventListener('focusout', () => {
     // Cancel any in-flight debounced dropdown.show; otherwise it fires against a stale textarea.
-    clearTimeout(debounceTimer);
-    focusoutHideTimer = setTimeout(() => dropdown.hide(), 200);
+    clearTimeout(debounceInput);
+    debounceFocus = setTimeout(() => dropdown.hide(), 200);
   });
 }
 

ui/dist/sdnext.mjs

@@ -9959,7 +9959,7 @@ var uiReadyCallbacks = [];
 var uiTabChangeCallbacks = [];
 var optionsChangedCallbacks = [];
 var uiCurrentTab = null;
-var uiAfterUpdateTimeout = null;
+var uiAfterUpdateTimeout;
 function registerCallback(queue, callback) {
   if (queue.includes(callback)) return;
   queue.push(callback);
@@ -10035,7 +10035,7 @@ var executedOnLoaded = false;
 var ignoreElements = ["logMonitorData", "logWarnings", "logErrors", "tooltip-container", "logger"];
 var ignoreElementsSet = new Set(ignoreElements);
 var ignoreClasses = ["wrap"];
-var mutationTimer = null;
+var mutationTimer;
 var validMutations = [];
 async function mutationCallback(mutations) {
   if (mutations.length <= 0) return;
@@ -10063,7 +10063,7 @@ async function mutationCallback(mutations) {
       executeCallbacks(uiTabChangeCallbacks);
     }
     validMutations = [];
-    mutationTimer = null;
+    mutationTimer = void 0;
   }, 100);
 }
 document.addEventListener("DOMContentLoaded", () => {
@@ -10697,30 +10697,30 @@ function setupExtraNetworksForTab(tabName) {
   txtDescription.classList.add("description");
   div.appendChild(txtSearch);
   div.appendChild(txtDescription);
-  let searchTimer = null;
+  let debouceSearch;
   txtSearchValue.addEventListener("input", (evt) => {
-    if (searchTimer) clearTimeout(searchTimer);
-    searchTimer = setTimeout(async () => {
+    if (debouceSearch) clearTimeout(debouceSearch);
+    debouceSearch = setTimeout(async () => {
       await filterExtraNetworksForTab(txtSearchValue.value.toLowerCase());
-      searchTimer = null;
+      debouceSearch = void 0;
     }, 100);
   });
-  let hoverTimer = null;
+  let debounceHover;
   let previousCard = null;
   if (window.opts.extra_networks_fetch) {
     gradioApp().getElementById(`${tabName}_extra_tabs`).onmouseover = async (e) => {
       const el2 = e.target.closest(".card");
       if (!el2 || el2.title === previousCard) return;
-      if (!hoverTimer) {
-        hoverTimer = setTimeout(() => {
+      if (!debounceHover) {
+        debounceHover = setTimeout(() => {
           readCardDescription(el2.dataset.page, el2.dataset.name);
           readCardTags(el2, el2.dataset.tags);
           previousCard = el2.title;
         }, 300);
       }
       el2.onmouseout = () => {
-        clearTimeout(hoverTimer);
-        hoverTimer = null;
+        clearTimeout(debounceHover);
+        debounceHover = void 0;
       };
     };
   }
@@ -11045,7 +11045,7 @@ function requestProgress(id_task = "undefined", progressEl = null, galleryEl = n
     livePreview.appendChild(img);
     img.onload = () => {
       img.style.width = `min(100%, max(${img.naturalWidth}px, 512px))`;
-      parentGallery.style.minHeight = `min(82vh, ${img.naturalWidth}px)`;
+      parentGallery.style.minHeight = `min(82vh, ${img.naturalHeight}px)`;
       parentGallery.style.maxHeight = `min(82vh, ${img.naturalHeight}px)`;
       parentGallery.style.overflow = "hidden";
     };
@@ -11134,11 +11134,16 @@ var fontSizeApplyRaf = 0;
 var pendingFontSize = null;
 var appliedFontSize = null;
 var cachedGradioRoot = null;
+var resizeDebounce;
 var wait_time = 800;
 var token_timeouts = {};
 var uiLoaded = false;
 var promptsInitialized = false;
 window.args_to_array = Array.from;
+function set_theme(theme) {
+  const gradioURL = window.location.href;
+  if (!gradioURL.includes("?__theme=")) window.location.replace(`${gradioURL}?__theme=${theme}`);
+}
 function update_token_counter(button_id) {
   if (token_timeouts[button_id]) clearTimeout(token_timeouts[button_id]);
   token_timeouts[button_id] = setTimeout(() => gradioApp().getElementById(button_id)?.click(), wait_time);
@@ -11689,7 +11694,10 @@ function resolutionChange(ar, width, height) {
     else if (h > w) width = Math.round(height * w / h);
   } catch {
   }
-  if (window.resizeStage) window.resizeStage(width, height);
+  if (window.resizeStage) {
+    clearTimeout(resizeDebounce);
+    resizeDebounce = setTimeout(() => window.resizeStage(width, height), 250);
+  }
   return [ar, width, height];
 }
 async function reconnectUI() {
@@ -11771,6 +11779,7 @@ window.currentImageResolutionimg2img = currentImageResolutionimg2img;
 window.currentImageResolutioncontrol = currentImageResolutioncontrol;
 window.updateImg2imgResizeToTextAfterChangingImage = updateImg2imgResizeToTextAfterChangingImage;
 window.create_submit_args = create_submit_args;
+window.set_theme = set_theme;
 
 // ui/inputAccordion.ts
 function inputAccordionChecked(id, checked) {
@@ -12200,7 +12209,7 @@ function markIfModified(setting_name, value) {
   if (changed_items.size > 0) tab_nav_indicator.title += `click to reset ${changed_items.size} unapplied changes in this tab
 `;
   if (saved.size > 0) tab_nav_indicator.title += `${saved.size} custom values
-${unsaved.size} default values}`;
+${unsaved.size} default values`;
 }
 window.markIfModified = markIfModified;
 function updateAllOpts() {
@@ -13110,8 +13119,8 @@ var SimpleProgressBar = class {
   #textDiv = document.createElement("div");
   #text = document.createElement("span");
   #visible = false;
-  #hideTimeout = null;
-  #interval = null;
+  #hideTimeout;
+  #interval;
   #max = 0;
   /** @type {Set} */
   #monitoredSet;
@@ -13141,7 +13150,7 @@ var SimpleProgressBar = class {
   clear() {
     this.#stop();
     clearTimeout(this.#hideTimeout);
-    this.#hideTimeout = null;
+    this.#hideTimeout = void 0;
     this.#container.style.display = "none";
     this.#visible = false;
     this.#progress.style.width = "0";
@@ -13149,7 +13158,7 @@ var SimpleProgressBar = class {
   }
   #update(loaded, max) {
     if (this.#hideTimeout) {
-      this.#hideTimeout = null;
+      this.#hideTimeout = void 0;
     }
     this.#progress.style.width = `${Math.floor(loaded / max * 100)}%`;
     this.#text.textContent = `${loaded}/${max}`;
@@ -13159,9 +13168,7 @@ var SimpleProgressBar = class {
     }
     if (loaded >= max) {
       this.#stop();
-      this.#hideTimeout = setTimeout(() => {
-        this.clear();
-      }, 1e3);
+      this.#hideTimeout = setTimeout(() => this.clear(), 1e3);
     }
   }
   #stop() {
@@ -14986,8 +14993,8 @@ var dropdown = {
     this.hide();
   }
 };
-var debounceTimer = null;
-var focusoutHideTimer = null;
+var debounceInput;
+var debounceFocus;
 function onInput(textarea) {
   if (!active) return;
   if (textarea.dataset.imeActive === "1") return;
@@ -15004,8 +15011,8 @@ function onInput(textarea) {
     dropdown.hide();
     return;
   }
-  clearTimeout(debounceTimer);
-  debounceTimer = setTimeout(() => {
+  clearTimeout(debounceInput);
+  debounceInput = setTimeout(() => {
     let results;
     if (info.mode === "lora") {
       results = xnEngine.searchLoras(info.word);
@@ -15072,13 +15079,13 @@ function attachAutocomplete(textarea) {
   });
   textarea.addEventListener("focusin", () => {
     if (dropdown.visible && dropdown.textarea && dropdown.textarea !== textarea) dropdown.hide();
-    clearTimeout(focusoutHideTimer);
-    focusoutHideTimer = null;
+    clearTimeout(debounceFocus);
+    debounceFocus = void 0;
     onInput(textarea);
   });
   textarea.addEventListener("focusout", () => {
-    clearTimeout(debounceTimer);
-    focusoutHideTimer = setTimeout(() => dropdown.hide(), 200);
+    clearTimeout(debounceInput);
+    debounceFocus = setTimeout(() => dropdown.hide(), 200);
   });
 }
 var PROMPT_IDS = [
@@ -15210,12 +15217,12 @@ var localeData = {
   type: 2,
   hint: null,
   btn: null,
-  expandTimeout: null,
+  expandTimeout: void 0,
   // Property for expansion timeout
   currentElement: null
   // Track current element for expansion
 };
-var localeTimeout = null;
+var localeTimeout;
 var isTouchDevice = "ontouchstart" in window;
 async function cycleLocale() {
   clearTimeout(localeTimeout);
@@ -15292,7 +15299,7 @@ async function tooltipHideDelegated(e) {
 async function tooltipShow(e) {
   if (localeData.expandTimeout) {
     clearTimeout(localeData.expandTimeout);
-    localeData.expandTimeout = null;
+    localeData.expandTimeout = void 0;
   }
   localeData.hint.classList.remove("tooltip-expanded");
   localeData.currentElement = e.target;
@@ -15348,7 +15355,7 @@ async function tooltipShow(e) {
 async function tooltipHide(e) {
   if (localeData.expandTimeout) {
     clearTimeout(localeData.expandTimeout);
-    localeData.expandTimeout = null;
+    localeData.expandTimeout = void 0;
   }
   localeData.hint.classList.remove("tooltip-show", "tooltip-expanded");
   localeData.currentElement = null;
@@ -16358,7 +16365,7 @@ window.refreshHistory = refreshHistory;
 // ui/aspectRatioOverlay.ts
 var currentWidth = null;
 var currentHeight = null;
-var arFrameTimeout = null;
+var arFrameTimeout;
 function dimensionChange(e, isWidth, isHeight) {
   const { target } = e;
   if (!(target instanceof HTMLInputElement)) return;

ui/extraNetworks.ts

@@ -494,32 +494,32 @@ function setupExtraNetworksForTab(tabName) {
   txtDescription.classList.add('description');
   div.appendChild(txtSearch);
   div.appendChild(txtDescription);
-  let searchTimer = null;
+  let debouceSearch: ReturnType<typeof setTimeout> | undefined;
   txtSearchValue.addEventListener('input', (evt) => {
-    if (searchTimer) clearTimeout(searchTimer);
-    searchTimer = setTimeout(async () => {
+    if (debouceSearch) clearTimeout(debouceSearch);
+    debouceSearch = setTimeout(async () => {
       await filterExtraNetworksForTab(txtSearchValue.value.toLowerCase());
-      searchTimer = null;
+      debouceSearch = undefined;
     }, 100);
   });
 
   // card hover
-  let hoverTimer = null;
-  let previousCard = null;
+  let debounceHover: ReturnType<typeof setTimeout> | undefined;
+  let previousCard: string | null = null;
   if (window.opts.extra_networks_fetch) {
     gradioApp().getElementById(`${tabName}_extra_tabs`).onmouseover = async (e) => {
       const el = e.target.closest('.card'); // bubble-up to card
       if (!el || (el.title === previousCard)) return;
-      if (!hoverTimer) {
-        hoverTimer = setTimeout(() => {
+      if (!debounceHover) {
+        debounceHover = setTimeout(() => {
           readCardDescription(el.dataset.page, el.dataset.name);
           readCardTags(el, el.dataset.tags);
           previousCard = el.title;
         }, 300);
       }
       el.onmouseout = () => {
-        clearTimeout(hoverTimer);
-        hoverTimer = null;
+        clearTimeout(debounceHover);
+        debounceHover = undefined;
       };
     };
   }

ui/gallery.ts

@@ -266,8 +266,8 @@ class SimpleProgressBar {
   #textDiv = document.createElement('div');
   #text = document.createElement('span');
   #visible = false;
-  #hideTimeout = null;
-  #interval = null;
+  #hideTimeout: ReturnType<typeof setTimeout> | undefined;
+  #interval: ReturnType<typeof setTimeout> | undefined;
   #max = 0;
   /** @type {Set} */
   #monitoredSet;
@@ -302,7 +302,7 @@ class SimpleProgressBar {
   clear() {
     this.#stop();
     clearTimeout(this.#hideTimeout);
-    this.#hideTimeout = null;
+    this.#hideTimeout = undefined;
     this.#container.style.display = 'none';
     this.#visible = false;
     this.#progress.style.width = '0';
@@ -311,7 +311,7 @@ class SimpleProgressBar {
 
   #update(loaded, max) {
     if (this.#hideTimeout) {
-      this.#hideTimeout = null;
+      this.#hideTimeout = undefined;
     }
 
     this.#progress.style.width = `${Math.floor((loaded / max) * 100)}%`;
@@ -323,9 +323,7 @@ class SimpleProgressBar {
     }
     if (loaded >= max) {
       this.#stop();
-      this.#hideTimeout = setTimeout(() => {
-        this.clear();
-      }, 1000);
+      this.#hideTimeout = setTimeout(() => this.clear(), 1000);
     }
   }
 

ui/script.ts

@@ -48,7 +48,7 @@ export const uiTabChangeCallbacks = [];
 export const optionsChangedCallbacks = [];
 
 let uiCurrentTab = null;
-let uiAfterUpdateTimeout = null;
+let uiAfterUpdateTimeout: ReturnType<typeof setTimeout> | undefined;
 
 function registerCallback(queue, callback) {
   if (queue.includes(callback)) return;
@@ -137,7 +137,7 @@ const ignoreElements = ['logMonitorData', 'logWarnings', 'logErrors', 'tooltip-c
 const ignoreElementsSet = new Set(ignoreElements);
 const ignoreClasses = ['wrap'];
 
-let mutationTimer = null;
+let mutationTimer: ReturnType<typeof setTimeout> | undefined;
 let validMutations = [];
 
 async function mutationCallback(mutations) {
@@ -167,7 +167,7 @@ async function mutationCallback(mutations) {
       executeCallbacks(uiTabChangeCallbacks);
     }
     validMutations = [];
-    mutationTimer = null;
+    mutationTimer = undefined;
   }, 100);
 }
 

ui/setHints.ts

@@ -15,10 +15,10 @@ const localeData = {
   type: 2,
   hint: null,
   btn: null,
-  expandTimeout: null, // Property for expansion timeout
+  expandTimeout: undefined, // Property for expansion timeout
   currentElement: null, // Track current element for expansion
 };
-let localeTimeout = null;
+let localeTimeout: ReturnType<typeof setTimeout> | undefined;
 const isTouchDevice = 'ontouchstart' in window;
 
 async function cycleLocale() {
@@ -104,7 +104,7 @@ async function tooltipHideDelegated(e) {
 async function tooltipShow(e) {
   if (localeData.expandTimeout) { // clear any existing expansion timeout
     clearTimeout(localeData.expandTimeout);
-    localeData.expandTimeout = null;
+    localeData.expandTimeout = undefined;
   }
 
   localeData.hint.classList.remove('tooltip-expanded'); // remove expanded class and reset current element
@@ -167,7 +167,7 @@ async function tooltipShow(e) {
 async function tooltipHide(e) {
   if (localeData.expandTimeout) {
     clearTimeout(localeData.expandTimeout);
-    localeData.expandTimeout = null;
+    localeData.expandTimeout = undefined;
   }
   localeData.hint.classList.remove('tooltip-show', 'tooltip-expanded');
   localeData.currentElement = null;

ui/settings.ts

@@ -147,7 +147,7 @@ async function onAfterUiUpdateCallback() {
   });
 
   const settingsSearch = gradioApp().querySelectorAll('#settings_search > label > textarea')[0];
-  let settingsTimer;
+  let settingsTimer: ReturnType<typeof setTimeout> | undefined;
   let settingSearchValue = '';
 
   function doSettingsSearch() {

ui/ui.ts

@@ -15,6 +15,7 @@ let fontSizeApplyRaf = 0;
 let pendingFontSize: number | null = null;
 let appliedFontSize: number | null = null;
 let cachedGradioRoot: any = null;
+let resizeDebounce: ReturnType<typeof setTimeout> | undefined;
 const wait_time = 800;
 const token_timeouts = {};
 let uiLoaded = false;
@@ -773,7 +774,11 @@ export function resolutionChange(ar: string, width: number, height: number) {
     if (w > h) height = Math.round(width * h / w);
     else if (h > w) width = Math.round(height * w / h);
   } catch { /**/ }
-  if (window.resizeStage) window.resizeStage(width, height); // notify kanvas
+  // min/max size handled in gradio debounce
+  if (window.resizeStage) {
+    clearTimeout(resizeDebounce);
+    resizeDebounce = setTimeout(() => window.resizeStage(width, height), 250); // notify kanvas
+  }
   return [ar, width, height];
 }