mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
fix progress eta reporting
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
@@ -43,6 +43,7 @@
|
||||
- fix memory stats not displaying the ram usage
|
||||
- fix **RunPod** memory limit reporting
|
||||
- fix flux ipadapter with start/stop values
|
||||
- fix progress api `eta_relative`
|
||||
- **IPEX**
|
||||
- add `--upgrade` to torch_command when using `--use-nightly` for *ipex* and *rocm*
|
||||
- add xpu to profiler
|
||||
|
||||
+13
-6
@@ -18,7 +18,7 @@ class Dot(dict):
|
||||
|
||||
opts = Dot({
|
||||
"timeout": 3600,
|
||||
"frequency": 60,
|
||||
"frequency": 1,
|
||||
"action": "sudo shutdown now",
|
||||
"url": "http://127.0.0.1:7860",
|
||||
"user": "",
|
||||
@@ -46,15 +46,22 @@ log.info(f'sdnext monitor started: {opts}')
|
||||
while True:
|
||||
try:
|
||||
status = progress()
|
||||
# {'progress': 0.0, 'eta_relative': 0.0, 'state': {'skipped': False, 'interrupted': False, 'job': '', 'job_count': 0, 'job_timestamp': '20250316110822', 'job_no': 0, 'sampling_step': 20, 'sampling_steps': 20}, 'current_image': None, 'textinfo': None}
|
||||
state = status.get('state', {})
|
||||
last_job = state.get('job_timestamp', None)
|
||||
if last_job is None:
|
||||
job_timestamp = state.get('job_timestamp', None)
|
||||
job_progress = status.get('progress', 0)
|
||||
eta_relative = status.get('eta_relative', 0)
|
||||
job = state.get('job', '')
|
||||
job_timestamp = state.get('job_timestamp', None)
|
||||
sampling_step = state.get('sampling_step', 0)
|
||||
sampling_steps = state.get('sampling_steps', 0)
|
||||
if job_timestamp is None:
|
||||
log.warning(f'sdnext montoring cannot get last job info: {status}')
|
||||
else:
|
||||
last_job = datetime.datetime.strptime(last_job, "%Y%m%d%H%M%S")
|
||||
elapsed = datetime.datetime.now() - last_job
|
||||
job_timestamp = datetime.datetime.strptime(job_timestamp, "%Y%m%d%H%M%S") if job_timestamp != '0' else datetime.datetime.now()
|
||||
elapsed = datetime.datetime.now() - job_timestamp
|
||||
timeout = round(opts.timeout - elapsed.total_seconds())
|
||||
log.info(f'sdnext: last_job={last_job} elapsed={elapsed} timeout={timeout}')
|
||||
log.info(f'sdnext: last="{job_timestamp}" elapsed={elapsed} timeout={timeout} progress={job_progress} eta={eta_relative} step={sampling_step}/{sampling_steps} job="{job}"')
|
||||
if timeout < 0:
|
||||
log.warning(f'sdnext reached: timeout={opts.timeout} action={opts.action}')
|
||||
os.system(opts.action)
|
||||
|
||||
@@ -91,10 +91,10 @@ def get_progress(req: models.ReqProgress = Depends()):
|
||||
step_y = max(shared.state.sampling_steps, 1)
|
||||
current = step_y * batch_x + step_x
|
||||
total = step_y * batch_y
|
||||
progress = current / total if current > 0 and total > 0 else 0
|
||||
progress = min((current / total) if current > 0 and total > 0 else 0, 1)
|
||||
time_since_start = time.time() - shared.state.time_start
|
||||
eta_relative = (time_since_start / progress) - time_since_start if progress > 0 else 0
|
||||
res = models.ResProgress(progress=progress, eta_relative=eta_relative, state=shared.state.dict(), current_image=current_image, textinfo=shared.state.textinfo)
|
||||
res = models.ResProgress(progress=round(progress, 2), eta_relative=round(eta_relative, 2), current_image=current_image, textinfo=shared.state.textinfo, state=shared.state.dict(), )
|
||||
return res
|
||||
|
||||
def get_status():
|
||||
|
||||
Reference in New Issue
Block a user