fix progress eta reporting

Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
2025-03-16 11:22:29 -04:00
parent f9a692daf6
commit a3e10fd24c
3 changed files with 16 additions and 8 deletions
+1
View File
@@ -43,6 +43,7 @@
- fix memory stats not displaying the ram usage
- fix **RunPod** memory limit reporting
- fix flux ipadapter with start/stop values
- fix progress api `eta_relative`
- **IPEX**
- add `--upgrade` to torch_command when using `--use-nightly` for *ipex* and *rocm*
- add xpu to profiler
+13 -6
View File
@@ -18,7 +18,7 @@ class Dot(dict):
opts = Dot({
"timeout": 3600,
"frequency": 60,
"frequency": 1,
"action": "sudo shutdown now",
"url": "http://127.0.0.1:7860",
"user": "",
@@ -46,15 +46,22 @@ log.info(f'sdnext monitor started: {opts}')
while True:
try:
status = progress()
# {'progress': 0.0, 'eta_relative': 0.0, 'state': {'skipped': False, 'interrupted': False, 'job': '', 'job_count': 0, 'job_timestamp': '20250316110822', 'job_no': 0, 'sampling_step': 20, 'sampling_steps': 20}, 'current_image': None, 'textinfo': None}
state = status.get('state', {})
last_job = state.get('job_timestamp', None)
if last_job is None:
job_timestamp = state.get('job_timestamp', None)
job_progress = status.get('progress', 0)
eta_relative = status.get('eta_relative', 0)
job = state.get('job', '')
job_timestamp = state.get('job_timestamp', None)
sampling_step = state.get('sampling_step', 0)
sampling_steps = state.get('sampling_steps', 0)
if job_timestamp is None:
log.warning(f'sdnext montoring cannot get last job info: {status}')
else:
last_job = datetime.datetime.strptime(last_job, "%Y%m%d%H%M%S")
elapsed = datetime.datetime.now() - last_job
job_timestamp = datetime.datetime.strptime(job_timestamp, "%Y%m%d%H%M%S") if job_timestamp != '0' else datetime.datetime.now()
elapsed = datetime.datetime.now() - job_timestamp
timeout = round(opts.timeout - elapsed.total_seconds())
log.info(f'sdnext: last_job={last_job} elapsed={elapsed} timeout={timeout}')
log.info(f'sdnext: last="{job_timestamp}" elapsed={elapsed} timeout={timeout} progress={job_progress} eta={eta_relative} step={sampling_step}/{sampling_steps} job="{job}"')
if timeout < 0:
log.warning(f'sdnext reached: timeout={opts.timeout} action={opts.action}')
os.system(opts.action)
+2 -2
View File
@@ -91,10 +91,10 @@ def get_progress(req: models.ReqProgress = Depends()):
step_y = max(shared.state.sampling_steps, 1)
current = step_y * batch_x + step_x
total = step_y * batch_y
progress = current / total if current > 0 and total > 0 else 0
progress = min((current / total) if current > 0 and total > 0 else 0, 1)
time_since_start = time.time() - shared.state.time_start
eta_relative = (time_since_start / progress) - time_since_start if progress > 0 else 0
res = models.ResProgress(progress=progress, eta_relative=eta_relative, state=shared.state.dict(), current_image=current_image, textinfo=shared.state.textinfo)
res = models.ResProgress(progress=round(progress, 2), eta_relative=round(eta_relative, 2), current_image=current_image, textinfo=shared.state.textinfo, state=shared.state.dict(), )
return res
def get_status():