rebuild ui

Signed-off-by: Vladimir Mandic <mandic00@live.com>

modules/openai/classes.py

@@ -0,0 +1,65 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional
+from transformers import StoppingCriteria
+
+
+@dataclass
+class Model:
+    name: Optional[str] = None
+    cls: str = "UnknownModelClass"
+    tokenizer: str = "UnknownTokenizerClass"
+    processor: Optional[str] = None
+    type: Optional[str] = None
+
+
+@dataclass
+class Config:
+    max_context_tokens: int = 4096
+    max_new_tokens: int = 512
+    stream: bool = False
+    temperature: float = 0.2
+    top_p: float = 0.9
+    top_k: int = 50
+    repetition_penalty: float = 1.0
+    use_cache: bool = True
+
+
+@dataclass
+class Attention:
+    implementation: str = "eager"
+    flash: bool = False
+    arch: str = ""
+
+
+@dataclass
+class Req:
+    id: int
+    client: Optional[str] = None
+    url: Optional[str] = None
+    config: dict[str, Any] = None
+
+
+class CustomTokenStopCriteria(StoppingCriteria):
+    def __init__(self, stop_ids: List[int]):
+        super().__init__()
+        self.stop_ids = stop_ids
+
+    def __call__(self, input_ids: Any, scores: Any, **kwargs) -> bool:
+        return input_ids[0][-1].item() in self.stop_ids
+
+
+class Stats:
+    id: int = 0
+    messages: int = 0
+    prompt: int = 0
+    ttft: float = 0.0
+    latency: float = 0.0
+    tps: float = 0.0
+    tools: int = 0
+    tokens: dict[str, int] = {}
+    chunks: dict[str, int] = {}
+    streaming: bool = None
+    thinking: bool = None
+
+    def __str__(self):
+        return f"Res(id={self.id}, messages={self.messages}, prompt={self.prompt}, ttft={self.ttft:.3f}, latency={self.latency:.3f}, tps={self.tps:.3f}, tools={self.tools}, tokens={self.tokens}, chunks={self.chunks}, streaming={self.streaming}, thinking={self.thinking})"

modules/openai/generate.py

@@ -0,0 +1,272 @@
+import os
+import json
+import re
+import threading
+import time
+from typing import Any, AsyncGenerator, Dict, List, Optional
+from fastapi import Request
+from fastapi.responses import StreamingResponse
+from transformers import StoppingCriteriaList, TextIteratorStreamer
+from modules.logger import log
+from .classes import CustomTokenStopCriteria, Stats
+
+
+debug_log = log.debug if os.environ.get('SD_LLM_DEBUG', None) is not None else lambda *args, **kwargs: None
+
+
+def enforce_rolling_context(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Ensures context bounds are respected while anchoring the base system prompt."""
+    system_message = None
+    chat_history = []
+    for msg in messages:
+        if msg.get("role") == "system":
+            system_message = msg
+        else:
+            chat_history.append(msg)
+    while len(chat_history) > 0:
+        current_payload = ([system_message] if system_message else []) + chat_history
+        try:
+            if hasattr(self.tokenizer, "apply_chat_template") and self.tokenizer.chat_template:
+                test_str = self.tokenizer.apply_chat_template(current_payload, tokenize=False, add_generation_prompt=True)
+            else:
+                test_str = "".join([m["content"] for m in current_payload])
+            total_len = len(self.tokenizer.encode(test_str))
+            if (total_len + self.config.max_new_tokens) <= self.config.max_context_tokens:
+                return current_payload
+        except Exception:
+            pass
+        if len(chat_history) >= 2:
+            chat_history = chat_history[2:]
+            log.debug("OpenAI: Context('dropped oldest conversation')")
+        elif len(chat_history) == 1:
+            chat_history.pop(0)
+            log.debug("OpenAI: Context('dropped history')")
+        else:
+            break
+    return [system_message] if system_message else []
+
+
+def parse_inline_tool_calls(text: str) -> List[Dict[str, Any]]:
+    """Parses model-specific text tags and builds OpenAI tool payload models."""
+    tool_calls = []
+    qwen_pattern = r"<\|tool_call_start\|>(.*?)(?:<\|tool_call_end\|>|$)"
+    gemma_pattern = r"<tool_call>(.*?)(?:</tool_call>|$)"
+    matches = re.findall(qwen_pattern, text, re.DOTALL) + re.findall(gemma_pattern, text, re.DOTALL)
+    for idx, match_str in enumerate(matches):
+        try:
+            parsed = json.loads(match_str.strip())
+            call_id = f"call_{int(time.time())}_{idx}"
+            if "name" in parsed:
+                args = parsed.get("arguments", {})
+                if not isinstance(args, str):
+                    args = json.dumps(args)
+                tool_calls.append({
+                    "id": call_id,
+                    "type": "function",
+                    "function": {"name": parsed["name"], "arguments": args}
+                })
+        except Exception:
+            continue
+    return tool_calls
+
+
+async def execute_generation(
+    self,
+    stats: Stats,
+    request: Request,
+    prompt: str,
+    config: Dict[str, Any],
+    stream: bool,
+    stream_options: Optional[Dict[str, Any]] = None,
+    images: Optional[List[Any]] = None,
+):
+    """Executes tensor processing, tracking streaming reasoning buffers and finish conditions."""
+    start_time = time.time()
+    if self.processor and images:
+        inputs = self.processor(text=prompt, images=images, return_tensors="pt").to(self.model.device)
+    else:
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+    model_name = getattr(self.model.config, "_name_or_path", "local-transformer")
+    req_id = f"gen-{int(time.time())}"
+    stop_ids = [self.tokenizer.eos_token_id]
+    stop_targets = [
+        "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "</tool_call>",
+        "<|tool_call_end|>", "<|end|>", "<|start_header_id|>", "<|end_header_id|>",
+        "<end_of_turn>"
+    ]
+    for t_str in stop_targets:
+        t_id = self.tokenizer.convert_tokens_to_ids(t_str)
+        if isinstance(t_id, int) and t_id > 0:
+            stop_ids.append(t_id)
+    stopping_criteria = StoppingCriteriaList([CustomTokenStopCriteria(list(set(stop_ids)))])
+    chunks = { 'reasoning': [], 'content': [] }
+
+    def execute_streaming_generation():
+        streamer = TextIteratorStreamer(self.tokenizer, skip_prompt=True, skip_special_tokens=False)
+        generation_kwargs = dict(
+            **inputs,
+            streamer=streamer,
+            pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,
+            stopping_criteria=stopping_criteria,
+            use_cache=self.config.use_cache,
+            **config
+        )
+        threading.Thread(target=self.model.generate, kwargs=generation_kwargs, daemon=True).start()
+
+        async def production_stream_decorator() -> AsyncGenerator[str, None]:
+            first_token_sent = False
+            is_thinking = False
+            token_count = 0
+            generation_start = time.time()
+            try:
+                for chunk in streamer:
+                    if await request.is_disconnected():
+                        break
+                    if not chunk:
+                        continue
+                    if any(t in chunk for t in ["<think>", "## Thought", "thought\n"]):
+                        is_thinking = True
+                        for t in ["<think>", "## Thought", "thought\n"]:
+                            chunk = chunk.replace(t, "")
+                    if "</think>" in chunk:
+                        is_thinking = False
+                        chunk = chunk.replace("</think>", "")
+                    for tag in [
+                        "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "<|tool_call_start|>",
+                        "<tool_call>", "<|end|>", "<|start_header_id|>", "<|end_header_id|>",
+                        "<end_of_turn>"
+                    ]:
+                        chunk = chunk.replace(tag, "")
+                    if not chunk:
+                        continue
+                    if not first_token_sent:
+                        stats.ttft = time.time() - start_time
+                        first_token_sent = True
+                    token_count += 1
+                    choice_delta = {}
+                    if is_thinking:
+                        chunks['reasoning'].append(chunk)
+                        choice_delta["reasoning_content"] = chunk
+                    else:
+                        chunks['content'].append(chunk)
+                        choice_delta["content"] = chunk
+
+                    finish_reason = "length" if token_count >= config.get("max_new_tokens", self.config.max_new_tokens) else None
+                    payload = {
+                        "id": req_id, "object": "chat.completion.chunk", "created": int(time.time()),
+                        "model": model_name, "choices": [{"index": 0, "delta": choice_delta, "finish_reason": finish_reason}]
+                    }
+                    yield f"data: {json.dumps(payload)}\n\n"
+                final_payload = {
+                    "id": req_id, "object": "chat.completion.chunk", "created": int(time.time()),
+                    "model": model_name, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+                }
+                yield f"data: {json.dumps(final_payload)}\n\n"
+                if stream_options and stream_options.get("include_usage"):
+                    prompt_tokens = len(inputs.input_ids[0])
+                    usage_payload = {
+                        "id": req_id,
+                        "object": "chat.completion.chunk",
+                        "created": int(time.time()),
+                        "model": model_name,
+                        "choices": [],
+                        "usage": {
+                            "prompt_tokens": prompt_tokens,
+                            "completion_tokens": token_count,
+                            "total_tokens": prompt_tokens + token_count
+                        }
+                    }
+                    yield f"data: {json.dumps(usage_payload)}\n\n"
+                yield "data: [DONE]\n\n"
+            finally:
+                duration = time.time() - generation_start
+                if duration > 0 and token_count > 0:
+                    stats.tps = token_count / duration
+                    stats.chunks['reasoning'] = len(chunks['reasoning'])
+                    stats.chunks['content'] = len(chunks['content'])
+                    stats.latency = time.time() - start_time
+                    stats.tools = 0
+                    stats.tokens['prompt'] = len(inputs.input_ids[0])
+                    stats.tokens['output'] = token_count
+                    stats.streaming = True
+                    stats.thinking = len(chunks['reasoning']) > 0
+                    text_reasoning = " ".join(c.strip() for c in chunks["reasoning"] if len(c.strip()) > 0)
+                    text_content = " ".join(c.strip() for c in chunks["content"] if len(c.strip()) > 0)
+                    debug_log(f'OpenAI chunks: reasoning="{text_reasoning}"')
+                    debug_log(f'OpenAI chunks: content="{text_content}"')
+                    log.debug(f"OpenAI: {stats}")
+        return StreamingResponse(production_stream_decorator(), media_type="text/event-stream")
+
+    def execute_direct_generation():
+        outputs = self.model.generate(
+            **inputs,
+            pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,
+            stopping_criteria=stopping_criteria,
+            use_cache=self.config.use_cache,
+            **config
+        )
+
+        generated_ids = outputs[0][inputs.input_ids.shape[-1]:]
+        raw_text = self.tokenizer.decode(generated_ids, skip_special_tokens=False)
+        reasoning_text = ""
+        for start_tag, end_tag in [("<think>", "</think>"), ("## Thought", "##"), ("thought\n", "\n\n")]:
+            if start_tag in raw_text and end_tag in raw_text:
+                parts = raw_text.split(end_tag)
+                reasoning_text = parts[0].replace(start_tag, "").strip()
+                raw_text = parts[1]
+                break
+        tool_calls = parse_inline_tool_calls(raw_text)
+        clean_text = raw_text
+        for tag in [
+            "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "</tool_call>",
+            "<|tool_call_end|>", "<|end|>", "<|start_header_id|>", "<|end_header_id|>",
+            "<end_of_turn>"
+        ]:
+            clean_text = clean_text.replace(tag, "")
+        if tool_calls:
+            clean_text = re.sub(r"<\|tool_call_start\|>.*?<\|tool_call_end\|>", "", clean_text, flags=re.DOTALL)
+            clean_text = re.sub(r"<tool_call>.*?</tool_call>", "", clean_text, flags=re.DOTALL).strip()
+
+        stats.ttft = time.time() - start_time
+        stats.latency = time.time() - start_time
+        stats.tools = len(tool_calls)
+        stats.tokens['prompt'] = len(inputs.input_ids[0])
+        stats.tokens['reasoning'] = len(reasoning_text)
+        stats.tokens['completion'] = len(generated_ids)
+        stats.tokens['total'] = len(inputs.input_ids[0]) + len(generated_ids)
+        stats.tps = stats.tokens['total'] / stats.latency if stats.latency > 0 else 0
+        stats.streaming = False
+        stats.thinking = len(reasoning_text) > 0
+        log.debug(f"OpenAI: {stats}")
+
+        debug_log(f'OpenAI response: "{clean_text}"')
+        message_payload = {"role": "assistant", "content": clean_text if clean_text else None}
+        if reasoning_text:
+            debug_log(f'OpenAI reasoning: "{reasoning_text}"')
+            message_payload["reasoning_content"] = reasoning_text
+        if tool_calls:
+            debug_log(f'OpenAI tools: {tool_calls}')
+            message_payload["tool_calls"] = tool_calls
+        f_reason = "tool_calls" if tool_calls else ("length" if len(generated_ids) >= config["max_new_tokens"] else "stop")
+
+        return {
+            "id": req_id,
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "model": model_name,
+            "choices": [{
+                "message": message_payload,
+                "index": 0,
+                "finish_reason": f_reason
+            }],
+            "usage": {
+                "prompt_tokens": len(inputs.input_ids[0]),
+                "completion_tokens": len(generated_ids),
+                "total_tokens": len(inputs.input_ids[0]) + len(generated_ids)
+            }
+        }
+
+    if stream:
+        return execute_streaming_generation()
+    else:
+        return execute_direct_generation()

modules/openai/helpers.py

@@ -0,0 +1,52 @@
+from .classes import Attention
+
+
+def get_attention_config(self):
+    attention = Attention(
+        implementation=getattr(self.model.config, "_attn_implementation", "eager"),
+        flash=getattr(self.model.config, "use_flash_attention", False)
+    )
+    if hasattr(self.model.config, "num_key_value_heads") and hasattr(self.model.config, "num_attention_heads"):
+        kv_heads = self.model.config.num_key_value_heads
+        attn_heads = self.model.config.num_attention_heads
+        if kv_heads < attn_heads:
+            attention.arch = f"arch=GQA kv_heads={kv_heads} attn_heads={attn_heads}"
+        elif kv_heads == 1:
+            attention.arch = "arch=MQA"
+        else:
+            attention.arch = "arch=MHA"
+    return attention
+
+
+def get_prompt_template(self, messages: list[str], tools) -> str:
+    model_name = getattr(self.model.config, "_name_or_path", "").lower()
+    if hasattr(self.tokenizer, "apply_chat_template") and self.tokenizer.chat_template:
+        template_kwargs = {
+            "conversation": messages,
+            "tokenize": False,
+            "add_generation_prompt": True
+        }
+        if tools:
+            template_kwargs["tools"] = tools
+        prompt_str = self.tokenizer.apply_chat_template(**template_kwargs)
+    else:
+        prompt_str = ""
+        if "llama" in model_name:
+            prompt_str += "<|begin_of_text|>"
+            for msg in messages:
+                prompt_str += f"<|start_header_id|>{msg['role']}<|end_header_id|>\n\n{msg['content']}<|eot_id|>"
+            prompt_str += "<|start_header_id|>assistant<|end_header_id|>\n\n"
+        elif "gemma" in model_name:
+            for msg in messages:
+                role_tag = msg["role"] if msg["role"] != "system" else "user"
+                prompt_str += f"<start_of_turn>{role_tag}\n{msg['content']}<end_of_turn>\n"
+            prompt_str += "<start_of_turn>assistant\n"
+        elif "qwen" in model_name:
+            for msg in messages:
+                prompt_str += f"<|im_start|>{msg['role']}\n{msg['content']}<|im_end|>\n"
+            prompt_str += "<|im_start|>assistant\n"
+        else:
+            for msg in messages:
+                prompt_str += f"<|{msg['role']}|>\n{msg['content']}\n"
+            prompt_str += "<|assistant|>\n"
+    return prompt_str

modules/openai/routes.py

@@ -0,0 +1,129 @@
+import time
+from typing import Optional
+from fastapi import Request, HTTPException, Depends
+from fastapi.security import HTTPAuthorizationCredentials
+from modules.logger import log
+from .generate import execute_generation, enforce_rolling_context
+from .helpers import get_prompt_template
+from .classes import Stats, Req
+
+
+def setup_routes(self):
+    """Exposes standard endpoints and safely routes traffic requests."""
+
+    async def verify_api_key(credentials: Optional[HTTPAuthorizationCredentials] = Depends(self._security)):
+        if not self.api_key:
+            return
+        if credentials is None or credentials.credentials != self.api_key:
+            raise HTTPException(
+                status_code=401,
+                detail="Invalid or missing API key in Authorization header."
+            )
+
+    @self.app.get("/health")
+    async def health_check():
+        return {"status": "healthy", "timestamp": time.time()}
+
+    @self.app.get("/v1/models", dependencies=[Depends(verify_api_key)])
+    async def list_models():
+        model_name = getattr(self.model.config, "_name_or_path", "local-transformer")
+        return {
+            "object": "list",
+            "data": [{
+                "id": model_name,
+                "object": "model",
+                "created": int(time.time()),
+                "owned_by": "transformers"
+            }]
+        }
+
+    @self.app.get("/v1/models/{model_id}", dependencies=[Depends(verify_api_key)])
+    async def retrieve_model(model_id: str):
+        model_name = getattr(self.model.config, "_name_or_path", "local-transformer")
+        if model_id != model_name:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Model '{model_id}' not found. Active model is '{model_name}'."
+            )
+        return {
+            "id": model_name,
+            "object": "model",
+            "created": int(time.time()),
+            "owned_by": "transformers"
+        }
+
+    @self.app.post("/v1/completions", dependencies=[Depends(verify_api_key)], tags=["production_hardened"])
+    async def text_completions(request: Request):
+        json_body = await request.json()
+        prompt_str = json_body.get("prompt", "")
+        if isinstance(prompt_str, list):
+            prompt_str = prompt_str[0] if prompt_str else ""
+        temperature = float(json_body.get("temperature", self.config.temperature))
+        config = {
+            "max_new_tokens": json_body.get("max_tokens", self.config.max_new_tokens),
+            "temperature": temperature,
+            "top_p": float(json_body.get("top_p", self.config.top_p)),
+            "top_k": int(json_body.get("top_k", self.config.top_k)),
+            "repetition_penalty": float(json_body.get("frequency_penalty", self.config.repetition_penalty)),
+            "do_sample": True if temperature > 0.0 else False
+        }
+        stats = Stats()
+        stats.id = id(request)
+        stats.prompt = len(prompt_str)
+
+        req = Req(id=id(request), config=config, client=request.scope.get('client', ('0:0.0.0', 0))[0], url=request.scope.get('path', 'err'))
+        log.debug(f"OpenAI: {req}")
+
+        return await execute_generation(
+            self,
+            stats=stats,
+            request=request,
+            prompt=prompt_str,
+            config=config,
+            stream=json_body.get("stream", self.config.stream),
+            stream_options=json_body.get("stream_options", None),
+            images=None
+        )
+
+    @self.app.post("/v1/chat/completions", dependencies=[Depends(verify_api_key)], tags=["production_hardened"])
+    async def chat_completions(request: Request):
+        json_body = await request.json()
+        messages = json_body.get("messages", [])
+        tools = json_body.get("tools", None)
+        stream_options = json_body.get("stream_options", None)
+        images = json_body.get("images", None)
+
+        temperature = float(json_body.get("temperature", self.config.temperature))
+        config = {
+            "max_new_tokens": json_body.get("max_tokens", self.config.max_new_tokens),
+            "temperature": temperature,
+            "top_p": float(json_body.get("top_p", self.config.top_p)),
+            "top_k": int(json_body.get("top_k", self.config.top_k)),
+            "repetition_penalty": float(json_body.get("frequency_penalty", self.config.repetition_penalty)),
+            "do_sample": True if temperature > 0.0 else False
+        }
+
+        sanitized_messages = enforce_rolling_context(self, messages)
+        try:
+            prompt_str = get_prompt_template(self, sanitized_messages, tools)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"LLM: template execution failure: {str(e)}") from e
+
+        stats = Stats()
+        stats.id = id(request)
+        stats.messages = len(messages)
+        stats.prompt = len(prompt_str)
+
+        req = Req(id=id(request), config=config, client=request.scope.get('client', ('0:0.0.0', 0))[0], url=request.scope.get('path', 'err'))
+        log.debug(f"OpenAI: {req}")
+
+        return await execute_generation(
+            self,
+            stats=stats,
+            request=request,
+            prompt=prompt_str,
+            config=config,
+            stream=json_body.get("stream", self.config.stream),
+            stream_options=stream_options,
+            images=images
+        )

modules/openai/serve.py

@@ -0,0 +1,102 @@
+import threading
+from typing import Optional
+import uvicorn
+from fastapi import FastAPI
+from fastapi.security import HTTPBearer
+from modules.logger import log
+from .classes import Model, Config
+from .helpers import get_attention_config
+from .routes import setup_routes
+
+
+class OpenAIServer:
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        processor=None,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        max_context_tokens: Optional[int] = None,
+        max_new_tokens: Optional[int] = None,
+        stream: Optional[bool] = None,
+        temperature: Optional[float] = None,
+        top_p: Optional[float] = None,
+        top_k: Optional[int] = None,
+        repetition_penalty: Optional[float] = None,
+        api_key: Optional[str] = None
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.processor = processor
+        self.host = host
+        self.port = port
+        self.model_info = Model(
+            name=getattr(model.config, "_name_or_path", "local-transformer"),
+            cls=model.__class__.__name__,
+            tokenizer=tokenizer.__class__.__name__,
+            processor=processor.__class__.__name__ if processor else None,
+            type=getattr(model.config, "model_type", None)
+        )
+        self.config = Config(
+            max_context_tokens=max_context_tokens if max_context_tokens is not None else 4096,
+            max_new_tokens=max_new_tokens if max_new_tokens is not None else 512,
+            stream=stream if stream is not None else False,
+            temperature=temperature if temperature is not None else 0.2,
+            top_p=top_p if top_p is not None else 0.9,
+            top_k=top_k if top_k is not None else 50,
+            repetition_penalty=repetition_penalty if repetition_penalty is not None else 1.
+        )
+        log.info(f"OpenAI: {self.model_info}")
+        attention = get_attention_config(self)
+        log.debug(f'OpenAI: {attention}')
+        self.api_key = api_key
+        self._security = HTTPBearer(auto_error=False)
+        self._is_running = False
+        self._lock = threading.Lock()
+        self._startup_event = threading.Event()
+        self.server: Optional[uvicorn.Server] = None
+        self.thread: Optional[threading.Thread] = None
+        self.app = FastAPI(title="SD.Next OpenAI-compatible LLM Server", version="1.0")
+        setup_routes(self)
+
+    def start(self, timeout_seconds: float = 10.0):
+        """Spawns the serving interface safely using an active background thread worker."""
+        with self._lock:
+            if self._is_running:
+                log.warning("OpenAI: Server('already running')")
+                return
+            self._startup_event.clear()
+            config = uvicorn.Config(
+                app=self.app, host=self.host, port=self.port, log_level="info", loop="asyncio", workers=1
+            )
+            self.server = uvicorn.Server(config)
+            self.server.install_signal_handlers = lambda *args, **kwargs: None
+            original_startup = self.server.startup
+
+            async def patched_startup(*args, **kwargs):
+                await original_startup(*args, **kwargs)
+                self._startup_event.set()
+
+            self.server.startup = patched_startup
+            self.thread = threading.Thread(target=self.server.run, name="TransformersServeWorkerThread", daemon=True)
+            self.thread.start()
+        if not self._startup_event.wait(timeout=timeout_seconds):
+            self.stop()
+            raise TimeoutError("OpenAI: init timeout")
+        self._is_running = True
+        url = f"http://{self.host}:{self.port}/v1"
+        log.info(f"OpenAI: Server(url={url})")
+
+    def stop(self, timeout_seconds: float = 5.0):
+        """Safely winds down network sockets and detached worker threads."""
+        with self._lock:
+            if not self._is_running or not self.server:
+                return
+            self.server.should_exit = True
+            if self.thread and self.thread.is_alive():
+                self.thread.join(timeout=timeout_seconds)
+            self.server = None
+            self.thread = None
+            self._is_running = False
+            log.info("OpenAI: Server(None)")

modules/openai/test.py

@@ -0,0 +1,44 @@
+import time
+import logging
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from .serve import OpenAIServer
+from modules import logger
+
+
+logger.setup_logging(debug=True)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
+logging.getLogger("asyncio").setLevel(logging.WARNING)
+
+
+logger.log.info("OpenAI: load model...")
+model = AutoModelForCausalLM.from_pretrained(
+    "Qwen/Qwen3-0.6B",
+    device_map="auto",
+    dtype=torch.bfloat16,
+    trust_remote_code=True,
+    attn_implementation="sdpa",
+)
+tokenizer = AutoTokenizer.from_pretrained(
+    "Qwen/Qwen3-0.6B",
+    trust_remote_code=True
+)
+
+logger.log.info("OpenAI: create server...")
+server = OpenAIServer(
+    model=model,
+    tokenizer=tokenizer,
+    host="127.0.0.1",
+    port=8000
+)
+
+logger.log.info("OpenAI: start server...")
+server.start()
+while True:
+    try:
+        time.sleep(1)
+    except KeyboardInterrupt:
+        server.stop()
+        break