server state history

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -48,6 +48,13 @@
   - **Sana** model loader optimizations
   - add explicit offload after encode prompt  
     configure in *settings -> text encoder -> offload*  
+- **API**  
+  - new [API Wiki](https://github.com/vladmandic/sdnext/wiki/API)  
+  - server will now maintain job history which can be queried via API  
+    so you can check previous jobs as well as request any previously generated images/videos  
+  - history endpoint: `/sdapi/v1/history?id={id}`  
+  - download endpoint: `/file={filename}`  
+  - progress api `/sdapi/v1/progress` now also include task id in the response  
 - **Other**
   - text/image/control/video pipeline vs task compatibility check  
   - **HiDream-I1, FLUX.1, SD3.x** add HF gated access auth check  
@@ -65,7 +72,7 @@
   - do not force gc at end of processing  
   - add `SD_LORA_DUMP` env variable for dev/diag to dump lora/model keys  
 - **Wiki**  
-  - new *Nunchaku* page  
+  - new *Nunchaku*, *API* pages  
   - updated *HiDream, Quantization, NNCF, Video, Docker, WSL* pages  
 - **Fixes**
   - HunyuanVideo-I2V with latest transformers  

cli/api-history.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+"""
+get list of all history jobs or a specific job
+"""
+
+import sys
+import logging
+import urllib3
+import requests
+
+
+url = "http://127.0.0.1:7860"
+user = ""
+password = ""
+
+log_format = '%(asctime)s %(levelname)s: %(message)s'
+logging.basicConfig(level = logging.INFO, format = log_format)
+log = logging.getLogger("sd")
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+log.info('state history')
+sys.argv.pop(0)
+task_id = sys.argv[0] if len(sys.argv) == 1 else ''
+auth = requests.auth.HTTPBasicAuth(user, password) if len(user) > 0 and len(password) > 0 else None
+req = requests.get(f'{url}/sdapi/v1/history?id={task_id}', verify=False, auth=auth, timeout=60)
+if req.status_code != 200:
+    log.error({ 'url': req.url, 'request': req.status_code, 'reason': req.reason })
+    exit(1)
+res = req.json()
+for item in res:
+    log.info(item)

cli/api-progress.py

@@ -48,6 +48,7 @@ while True:
         status = progress()
         # {'progress': 0.0, 'eta_relative': 0.0, 'state': {'skipped': False, 'interrupted': False, 'job': '', 'job_count': 0, 'job_timestamp': '20250316110822', 'job_no': 0, 'sampling_step': 20, 'sampling_steps': 20}, 'current_image': None, 'textinfo': None}
         state = status.get('state', {})
+        task_id = status.get('id', None)
         job_timestamp = state.get('job_timestamp', None)
         job_progress = status.get('progress', 0)
         eta_relative = status.get('eta_relative', 0)
@@ -61,7 +62,7 @@ while True:
             job_timestamp = datetime.datetime.strptime(job_timestamp, "%Y%m%d%H%M%S") if job_timestamp != '0' else datetime.datetime.now()
             elapsed = datetime.datetime.now() - job_timestamp
             timeout = round(opts.timeout - elapsed.total_seconds())
-            log.info(f'sdnext: last="{job_timestamp}" elapsed={elapsed} timeout={timeout} progress={job_progress} eta={eta_relative} step={sampling_step}/{sampling_steps} job="{job}"')
+            log.info(f'sdnext: id={task_id} last="{job_timestamp}" elapsed={elapsed} timeout={timeout} progress={job_progress} eta={eta_relative} step={sampling_step}/{sampling_steps} job="{job}"')
             if timeout < 0:
                 log.warning(f'sdnext reached: timeout={opts.timeout} action={opts.action}')
                 os.system(opts.action)

cli/image-search.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 from typing import Union
 import os
 import re

cli/load-unet.py

@@ -1,3 +1,5 @@
+# test for manually loading unet state_dict
+
 import torch
 import diffusers
 

modules/api/api.py

@@ -46,6 +46,7 @@ class Api:
         self.add_api_route("/sdapi/v1/status", server.get_status, methods=["GET"], response_model=models.ResStatus)
         self.add_api_route("/sdapi/v1/platform", server.get_platform, methods=["GET"])
         self.add_api_route("/sdapi/v1/progress", server.get_progress, methods=["GET"], response_model=models.ResProgress)
+        self.add_api_route("/sdapi/v1/history", server.get_history, methods=["GET"], response_model=list[models.ResHistory])
         self.add_api_route("/sdapi/v1/interrupt", server.post_interrupt, methods=["POST"])
         self.add_api_route("/sdapi/v1/skip", server.post_skip, methods=["POST"])
         self.add_api_route("/sdapi/v1/shutdown", server.post_shutdown, methods=["POST"])
@@ -92,8 +93,8 @@ class Api:
         self.add_api_route("/sdapi/v1/unload-checkpoint", endpoints.post_unload_checkpoint, methods=["POST"])
         self.add_api_route("/sdapi/v1/reload-checkpoint", endpoints.post_reload_checkpoint, methods=["POST"])
         self.add_api_route("/sdapi/v1/refresh-vae", endpoints.post_refresh_vae, methods=["POST"])
-        self.add_api_route("/sdapi/v1/history", endpoints.get_history, methods=["GET"], response_model=List[str])
-        self.add_api_route("/sdapi/v1/history", endpoints.post_history, methods=["POST"], response_model=int)
+        self.add_api_route("/sdapi/v1/latents", endpoints.get_latent_history, methods=["GET"], response_model=List[str])
+        self.add_api_route("/sdapi/v1/latents", endpoints.post_latent_history, methods=["POST"], response_model=int)
 
         # lora api
         if shared.native:

modules/api/endpoints.py

@@ -170,9 +170,9 @@ def post_pnginfo(req: models.ReqImageInfo):
     script_callbacks.infotext_pasted_callback(geninfo, params)
     return models.ResImageInfo(info=geninfo, items=items, parameters=params)
 
-def get_history():
+def get_latent_history():
     return shared.history.list
 
-def post_history(req: models.ReqHistory):
+def post_latent_history(req: models.ReqLatentHistory):
     shared.history.index = shared.history.find(req.name)
     return shared.history.index

modules/api/models.py

@@ -296,20 +296,33 @@ class ReqPostLog(BaseModel):
     debug: Optional[str] = Field(title="Debug message", description="The debug message to log")
     error: Optional[str] = Field(title="Error message", description="The error message to log")
 
+class ReqHistory(BaseModel):
+    id: str = Field(default=None, title="Task ID", description="Task ID")
+
 class ReqProgress(BaseModel):
     skip_current_image: bool = Field(default=False, title="Skip current image", description="Skip current image serialization")
 
 class ResProgress(BaseModel):
+    id: str = Field(title="TaskID", description="Task ID")
     progress: float = Field(title="Progress", description="The progress with a range of 0 to 1")
     eta_relative: float = Field(title="ETA in secs")
     state: dict = Field(title="State", description="The current state snapshot")
     current_image: str = Field(default=None, title="Current image", description="The current image in base64 format. opts.show_progress_every_n_steps is required for this to work.")
     textinfo: str = Field(default=None, title="Info text", description="Info text used by WebUI.")
 
+class ResHistory(BaseModel):
+    id: str = Field(title="ID", description="Task ID")
+    job: str = Field(title="Job", description="Job name")
+    op: str = Field(title="Operation", description="Operation name")
+    start: Union[float, None] = Field(title="Start", description="Start time")
+    end: Union[float, None] = Field(title="End", description="End time")
+    outputs: List[str] = Field(title="Outputs", description="List of filenames")
+
 class ResStatus(BaseModel):
     status: str = Field(title="Status", description="Current status")
-    task: str = Field(title="Task", description="Current task")
+    task: str = Field(title="Task", description="Current job")
     timestamp: Optional[str] = Field(title="Timestamp", description="Timestamp of the current job")
+    current: str = Field(title="Task", description="Current job")
     id: str = Field(title="ID", description="ID of the current task")
     job: int = Field(title="Job", description="Current job")
     jobs: int = Field(title="Jobs", description="Total jobs")
@@ -343,7 +356,7 @@ class ReqVQA(BaseModel):
     model: str = Field(default="Microsoft Florence 2 Base", title="Model", description="The interrogate model used.")
     question: str = Field(default="describe the image", title="Question", description="Question to ask the model.")
 
-class ReqHistory(BaseModel):
+class ReqLatentHistory(BaseModel):
     name: str = Field(title="Name", description="Name of the history item to select")
 
 class ResVQA(BaseModel):

modules/api/server.py

@@ -78,9 +78,17 @@ def set_config(req: Dict[str, Any]):
 def get_cmd_flags():
     return vars(shared.cmd_opts)
 
+def get_history(req: models.ReqHistory = Depends()):
+    if req.id is not None and len(req.id) > 0:
+        res = [item for item in shared.state.state_history if item['id'] == req.id]
+    else:
+        res = shared.state.state_history
+    res = [models.ResHistory(**item) for item in res]
+    return res
+
 def get_progress(req: models.ReqProgress = Depends()):
     if shared.state.job_count == 0:
-        return models.ResProgress(progress=0, eta_relative=0, state=shared.state.dict(), textinfo=shared.state.textinfo)
+        return models.ResProgress(id=shared.state.id, progress=0, eta_relative=0, state=shared.state.dict(), textinfo=shared.state.textinfo)
     shared.state.do_set_current_image()
     current_image = None
     if shared.state.current_image and not req.skip_current_image:
@@ -94,7 +102,7 @@ def get_progress(req: models.ReqProgress = Depends()):
     progress = min((current / total) if current > 0 and total > 0 else 0, 1)
     time_since_start = time.time() - shared.state.time_start
     eta_relative = (time_since_start / progress) - time_since_start if progress > 0 else 0
-    res = models.ResProgress(progress=round(progress, 2), eta_relative=round(eta_relative, 2), current_image=current_image, textinfo=shared.state.textinfo, state=shared.state.dict(), )
+    res = models.ResProgress(id=shared.state.id, progress=round(progress, 2), eta_relative=round(eta_relative, 2), current_image=current_image, textinfo=shared.state.textinfo, state=shared.state.dict(), )
     return res
 
 def get_status():

modules/call_queue.py

@@ -46,7 +46,11 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False, name=None):
     def f(*args, extra_outputs_array=extra_outputs, **kwargs):
         t = time.perf_counter()
         shared.mem_mon.reset()
-        shared.state.begin(job_name)
+        if len(args) > 0 and type(args[0]) == str and args[0][0:5] == "task(" and args[0][-1] == ")":
+            task_id = args[0]
+        else:
+            task_id = 0
+        shared.state.begin(job_name, task_id=task_id)
         try:
             if shared.cmd_opts.profile:
                 pr = cProfile.Profile()

modules/images.py

@@ -198,6 +198,7 @@ def save_image(image,
     exifinfo += params.pnginfo.get(pnginfo_section_name, '')
     filename, extension = os.path.splitext(params.filename)
     filename_txt = f"{filename}.txt" if shared.opts.save_txt and len(exifinfo) > 0 else None
+    shared.state.outputs(params.filename)
     save_queue.put((params.image, filename, extension, params, exifinfo, filename_txt)) # actual save is executed in a thread that polls data from queue
     save_queue.join()
     if not hasattr(params.image, 'already_saved_as'):

modules/shared_state.py

@@ -1,18 +1,24 @@
 import os
+import re
 import sys
+import uuid
 import time
 import datetime
 from modules.errors import log, display
 
 
 debug_output = os.environ.get('SD_STATE_DEBUG', None)
+debug_history = debug_output or os.environ.get('SD_STATE_HISTORY', None)
 
 
 class State:
     job_history = []
     task_history = []
+    state_history = []
     image_history = 0
     latent_history = 0
+    id = 0
+    results = []
     skipped = False
     interrupted = False
     paused = False
@@ -39,6 +45,7 @@ class State:
     disable_preview = False
     preview_job = -1
     time_start = None
+    time_end = None
     need_restart = False
     server_start = time.time()
     oom = False
@@ -104,7 +111,8 @@ class State:
         from modules.api import models
         res = models.ResStatus(
             task=self.job,
-            id=progress.current_task or '',
+            current=progress.current_task or '',
+            id=self.id,
             job=max(self.job_no, 0),
             jobs=max(self.frame_count, self.job_count, self.job_no),
             total=self.total_jobs,
@@ -131,7 +139,30 @@ class State:
             res.status = 'running' if self.job != '' else 'idle'
         return res
 
-    def begin(self, title="", api=None):
+    def history(self, op:str):
+        job = { 'id': self.id, 'job': self.job.lower(), 'op': op.lower(), 'start': self.time_start, 'end': self.time_end, 'outputs': self.results }
+        self.state_history.append(job)
+        l = len(self.state_history)
+        if l > 10000:
+            del self.state_history[0]
+        if debug_history:
+            log.trace(f'State history: jobs={l} {job}')
+
+    def outputs(self, results):
+        if isinstance(results, list):
+            self.results += results
+        else:
+            self.results.append(results)
+
+    def get_id(self, task_id):
+        if task_id is None or task_id == 0:
+            task_id = uuid.uuid4().hex[:15]
+        if not isinstance(task_id, str):
+            task_id = str(task_id)
+        match = re.search(r'\((.*?)\)', task_id)
+        return match.group(1) if match else task_id
+
+    def begin(self, title="", task_id=0, api=None):
         import modules.devices
         self.job_history.append(title)
         self.total_jobs += 1
@@ -144,6 +175,8 @@ class State:
         self.id_live_preview = 0
         self.interrupted = False
         self.preview_job = -1
+        self.results = []
+        self.id = self.get_id(task_id)
         self.job = title
         self.job_count = 0
         self.frame_count = 0
@@ -159,6 +192,7 @@ class State:
         self.prediction_type = "epsilon"
         self.api = api or self.api
         self.time_start = time.time()
+        self.history('begin')
         if debug_output:
             log.trace(f'State begin: {self}')
         modules.devices.torch_gc()
@@ -171,6 +205,8 @@ class State:
             self.time_start = time.time()
         if debug_output:
             log.trace(f'State end: {self}')
+        self.time_end = time.time()
+        self.history('end')
         self.job = ""
         self.job_count = 0
         self.job_no = 0
@@ -197,6 +233,7 @@ class State:
             self.sampling_steps += steps * jobs
             self.job_count += jobs
         self.job = job
+        self.history('update')
         if debug_output:
             log.trace(f'State update: {self} steps={steps} jobs={jobs}')
 

modules/video.py

@@ -79,6 +79,7 @@ def save_video(p, images, filename = None, video_type: str = 'none', duration: f
     if not filename.lower().endswith(ext):
         filename += f'.{ext}'
     filename = namegen.sanitize(filename)
+    shared.state.outputs(filename)
     if not sync:
         threading.Thread(target=save_video_atomic, args=(images, filename, video_type, duration, loop, interpolate, scale, pad, change)).start()
     else:

scripts/pulid_ext.py

@@ -242,7 +242,7 @@ class Script(scripts.Script):
                     )[0]
             info = processing.create_infotext(p)
             processed = processing.Processed(p, [output], info=info)
-            shared.state.end('PuLID')
+            shared.state.end()
         else: # let processing run the pipeline
             p.task_args['id_embedding'] = id_embedding
             p.task_args['uncond_id_embedding'] = uncond_id_embedding