diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c18adf53..adbe101e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,8 +48,8 @@ Plus inevitable bug-fixes... - **DLSS** - add DLSS support for: *NeuralRender, SuperSample and FrameGen* dlls 5 caused quite a stir, but combined with generative ai it becomes a nice tool - - available via *extras -> dlss* as part of generate workflow or as a standalone *processing* workflow - *todo*: video support will be added in the future + - available as part of image/video generate workflows via *extras -> dlss* + or as a standalone *processing* workflow - *note*: requires nvidia rtx gpu, windows platform and compatible gpu drivers but...it can be used from wsl2: unpack required package on windows host and you can access it from the wsl2 environment - *install*: requires [DLSS 5 Visual Enhancer](https://github.com/Merserk/dlss5-visual-enhancer/releases/tag/v7.0) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index b3e6c8f75..dbef993e1 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit b3e6c8f75cf65b19a3c3309c2fb6a69336471625 +Subproject commit dbef993e1a95cfd39a99c4ccbef8d1e0322500be diff --git a/scripts/dlss/controller_cli.py b/scripts/dlss/controller_cli.py index f50a97696..368c664a7 100644 --- a/scripts/dlss/controller_cli.py +++ b/scripts/dlss/controller_cli.py @@ -164,7 +164,7 @@ class DLSSController: log.trace(f'DLSS controller start: result={response.get("result")}') return True - def _send(self, request: dict, timeout: float = 60.0): + def _send(self, request: dict, timeout: float = 300.0): process = self.process if process is None or process.stdin is None or process.stdout is None: return None @@ -208,7 +208,7 @@ class DLSSController: log.trace(f'DLSS controller stray output: {raw!r}') continue # skip any non-JSON noise emitted before the JSON response line - def call(self, pkg_path: str, command: str, kwargs: dict, timeout: float = 60.0) -> dict: + def call(self, pkg_path: str, command: str, kwargs: dict, timeout: float = 600.0) -> dict: with self.lock: if not self.ensure_installed(pkg_path): return { 'status': 'error', 'result': None, 'error': { 'code': 'not_installed', 'message': 'controller is not installed' } } @@ -217,7 +217,7 @@ class DLSSController: encoded_kwargs = { key: _encode_value(value) for key, value in kwargs.items() } request = { 'request_id': str(uuid.uuid4()), 'command': command, 'args': [], 'kwargs': encoded_kwargs } if debug: - log.trace(f'DLSS controller request: command={command}') + log.trace(f'DLSS controller request: command={command} timeout={timeout}') response = self._send(request, timeout=timeout) if response is None: return { 'status': 'error', 'result': None, 'error': { 'code': 'not_ready', 'message': 'controller is not responding' } } diff --git a/scripts/dlss/controller_srv.py b/scripts/dlss/controller_srv.py index 44117c248..fc920030c 100644 --- a/scripts/dlss/controller_srv.py +++ b/scripts/dlss/controller_srv.py @@ -113,7 +113,7 @@ def _dispatch_command(command: str, request_id: str, args: tuple[Any, ...], kwar if command == "render": images = kwargs.get("images") if images is None: - raise StandaloneError("invalid_arguments", "Missing required 'images' argument for render command.") + raise StandaloneError("invalid_arguments", "NeuralRender: missing required images") options = _coerce_options(kwargs.get("options"), default=RenderOptions(), option_type=RenderOptions) result = DLSSNeuralRenderer()(np.asarray(images), options) return _response(request_id, status="ok", result=result, diagnostics={"shape": list(result.shape)}) @@ -121,7 +121,7 @@ def _dispatch_command(command: str, request_id: str, args: tuple[Any, ...], kwar if command == "upscale": images = kwargs.get("images") if images is None: - raise StandaloneError("invalid_arguments", "Missing required 'images' argument for upscale command.") + raise StandaloneError("invalid_arguments", "SuperSample: missing required images") options = _coerce_options(kwargs.get("options"), default=UpscaleOptions(), option_type=UpscaleOptions) result = DLSSSuperSample()(np.asarray(images), options) return _response(request_id, status="ok", result=result, diagnostics={"shape": list(result.shape)}) @@ -129,16 +129,16 @@ def _dispatch_command(command: str, request_id: str, args: tuple[Any, ...], kwar if command == "framegen": frames = kwargs.get("frames") if frames is None: - raise StandaloneError("invalid_arguments", "Missing required 'frames' argument for framegen command.") + raise StandaloneError("invalid_arguments", "FrameGen: missing required frames") source_fps = kwargs.get("source_fps") target_fps = kwargs.get("target_fps") if source_fps is None or target_fps is None: - raise StandaloneError("invalid_arguments", "framegen requires both 'source_fps' and 'target_fps'.") + raise StandaloneError("invalid_arguments", "FrameGen: missing source/target FPS") options = _coerce_options(kwargs.get("options"), default=InterpolationOptions(), option_type=InterpolationOptions) result = DLSSFrameGen()(np.asarray(frames), source_fps, target_fps, options) return _response(request_id, status="ok", result=result, diagnostics={"shape": list(result.shape)}) - raise StandaloneError("invalid_arguments", f"Unsupported controller command: {command!r}") + raise StandaloneError("invalid_arguments", f"Controller: unsupported command: {command!r}") def _controller_worker(request_queue: mp.Queue, response_queue: mp.Queue, busy: Any, current_request_id: Any, current_command: Any) -> None: @@ -279,7 +279,7 @@ class ControllerClient: return {"request_id": "shutdown", "status": "ok", "result": {"shutdown": True}, "error": None, "diagnostics": {}} response = self._send_and_wait("shutdown") if self.process.is_alive(): - self.process.join(timeout=5.0) + self.process.join(timeout=10.0) return response def close(self) -> None: @@ -289,7 +289,7 @@ class ControllerClient: pass if self.process is not None and self.process.is_alive(): self.process.terminate() - self.process.join(timeout=5.0) + self.process.join(timeout=10.0) def __enter__(self) -> "ControllerClient": return self.start() diff --git a/scripts/dlss/framegen.py b/scripts/dlss/framegen.py index 4dda189b0..dfb07ac4d 100644 --- a/scripts/dlss/framegen.py +++ b/scripts/dlss/framegen.py @@ -83,7 +83,7 @@ class DLSSFrameGen: options.validate() batch, _, height, width = validate_nchw(frames, name="frames") if width < 64 or height < 64: - raise StandaloneError("invalid_dimensions", "Frame interpolation requires frames at least 64x64 pixels.") + raise StandaloneError("invalid_dimensions", "FrameGen: invalid resolution") source_rate = resolve_target_rate(source_fps) target_rate = resolve_target_rate(target_fps) own_controller = controller or JobController() @@ -93,9 +93,7 @@ class DLSSFrameGen: capabilities = probe_frame_interpolation_capabilities(options.ai_gpu_uuid) log.debug(f'DLSSFrameGen: capabilities={capabilities}') if not capabilities.available: - raise StandaloneError( - "feature_unavailable", - "DLSS Frame Generation is unavailable. " + capabilities.detail, + raise StandaloneError("feature_unavailable", "FrameGen: unavailable. " + capabilities.detail, ) plan = choose_interpolation_plan( source_rate, @@ -115,10 +113,7 @@ class DLSSFrameGen: expected = output_frame_count(Fraction(batch, 1) / source_rate, target_rate) if len(result) != expected: log.error(f'DLSSFrameGen: result length={len(result)} expected={expected}') - raise StandaloneError( - "invalid_native_output", - f"Interpolation produced {len(result)} frames; expected {expected}.", - ) + raise StandaloneError("invalid_native_output", f"FrameGen: interpolation produced {len(result)} frames; expected {expected}.") output = np.stack([rgba_to_rgb_nchw(item.rgba)[0] for item in result], axis=0) log.debug(f'DLSSFrameGen: output={output.shape}') self.diagnostics = { @@ -134,7 +129,7 @@ class DLSSFrameGen: raise except Exception as exc: log.error(f'DLSSFrameGen: unexpected exception {exc}') - raise StandaloneError("processing_failed", f"DLSS frame interpolation failed: {exc}") from exc + raise StandaloneError("processing_failed", f"FrameGen: failed: {exc}") from exc self.last_report = { "input_shape": tuple(frames.shape), "output_shape": tuple(output.shape), @@ -182,7 +177,7 @@ class DLSSFrameGen: candidates: list[_TimedFrame] = [] for source in source_frames: if controller.cancel.is_set(): - raise StandaloneError("cancelled", "Frame interpolation was cancelled.") + raise StandaloneError("cancelled", "FrameGen: cancelled") items = [source] for stage in stages: next_items: list[_TimedFrame] = [] diff --git a/scripts/dlss/render.py b/scripts/dlss/render.py index 4e34b6e65..faa548a02 100644 --- a/scripts/dlss/render.py +++ b/scripts/dlss/render.py @@ -77,9 +77,10 @@ class DLSSNeuralRenderer: log.info('DLSSNeuralRenderer: call') options = options or RenderOptions() options.validate() - batch, _, height, width = validate_nchw(images, name="images") + batch, _channels, height, width = validate_nchw(images, name="images") + log.debug(f'DLSSNeuralRenderer: input={images.shape}') if width < 64 or height < 64: - raise StandaloneError("invalid_dimensions", "DLSS Neural Rendering requires images at least 64x64 pixels.") + raise StandaloneError("invalid_dimensions", "NeuralRender: invalid resolution") output_width, output_height = resolve_output_size(width, height, options.upscaling_factor) own_controller = controller or JobController() log.debug(f'DLSSNeuralRenderer: controller={own_controller}') @@ -93,37 +94,38 @@ class DLSSNeuralRenderer: factor, mode = resolve_upscaling_mode(options.upscaling_factor) native_settings = resolve_native_settings(options.source_options()) session_diagnostics: list[dict[str, Any]] = [] + session = DLSSFrameSession( + input_width=width, + input_height=height, + output_width=output_width, + output_height=output_height, + frame_count=batch, + warmup_frames=options.warmup_frames, + factor=factor, + mode=mode, + native_settings=native_settings, + gpu=gpu, + runtime_bundle=prepared.runtime_bundle, + controller=active_controller, + ) + log.debug(f'DLSSNeuralRenderer: session={session}') for index in range(batch): if active_controller.cancel.is_set(): - raise StandaloneError("cancelled", "Neural rendering was cancelled.") + raise StandaloneError("cancelled", "NeuralRender: cancelled.") try: - session = DLSSFrameSession( - input_width=width, - input_height=height, - output_width=output_width, - output_height=output_height, - frame_count=1, - warmup_frames=options.warmup_frames, - factor=factor, - mode=mode, - native_settings=native_settings, - gpu=gpu, - runtime_bundle=prepared.runtime_bundle, - controller=active_controller, - ) - log.debug(f'DLSSNeuralRenderer: session={session}') rgb = nchw_image_to_hwc(images, index, name="images") rgba = rgb_to_rgba(rgb) render_input = resize_fit(rgba, session.render_width, session.render_height) motion = np.zeros((session.render_height, session.render_width, 2), dtype=np.float16) + log.debug(f'DLSSNeuralRenderer: index={index} processes={render_input.shape}') processed, _ = session.process( - index=0, + index=index, rgba=render_input, motion=motion, reset=True, pts=0, ) - log.debug(f'DLSSNeuralRenderer: processed={processed.shape}') + log.debug(f'DLSSNeuralRenderer: index={index} processed={processed.shape}') outputs.append(rgba_to_rgb_nchw(processed)[0]) session_diagnostics.append({ "render_width": session.render_width, @@ -134,12 +136,12 @@ class DLSSNeuralRenderer: }) for l in session.worker_logs or []: log.debug(f'DLSSNeuralRenderer worker: {l}') - session.close() except Exception as e: log.error(f'DLSSNeuralRenderer: exception {e}') if session is not None and not session.closed: session.abort() raise + session.close() self.diagnostics = { "gpu": dict(gpu), "runtime_bundle": prepared.runtime_bundle, @@ -149,7 +151,7 @@ class DLSSNeuralRenderer: raise except Exception as exc: log.error(f'DLSSNeuralRenderer: unexpected exception {exc}') - raise StandaloneError("processing_failed", f"DLSS Neural Rendering failed: {exc}") from exc + raise StandaloneError("processing_failed", f"NeuralRender failed: {exc}") from exc result = np.ascontiguousarray(np.stack(outputs, axis=0)) self.last_report = { "input_shape": tuple(images.shape), diff --git a/scripts/dlss/supersample.py b/scripts/dlss/supersample.py index cf874ed29..6feac58a6 100644 --- a/scripts/dlss/supersample.py +++ b/scripts/dlss/supersample.py @@ -86,7 +86,7 @@ class DLSSSuperSample: log.debug(f'DLSSSuperSample: session={session}') for index in range(batch): if active_controller.cancel.is_set(): - raise StandaloneError("cancelled", "Upscale was cancelled.") + raise StandaloneError("cancelled", "SuperSample: cancelled.") frame = nchw_image_to_hwc(images, index, name="images") worker_input = srgb_to_worker(frame) worker_output = session.process_frame(worker_input) @@ -104,7 +104,7 @@ class DLSSSuperSample: raise except Exception as exc: log.error(f'DLSSSuperSample: unexpected exception {exc}') - raise StandaloneError("processing_failed", f"RTX Video upscaling failed: {exc}") from exc + raise StandaloneError("processing_failed", f"SuperSample: failed: {exc}") from exc result = np.ascontiguousarray(np.stack(outputs, axis=0)) self.last_report = { "input_shape": tuple(images.shape), diff --git a/scripts/dlss_ext.py b/scripts/dlss_ext.py index 3df979471..b34707396 100644 --- a/scripts/dlss_ext.py +++ b/scripts/dlss_ext.py @@ -8,7 +8,7 @@ from scripts.dlss import controller_cli as c debug = os.environ.get('SD_DLSS_DEBUG', None) is not None -FPS_CHOICES = ['23.976', '24', '25', '29.97', '30', '50', '59.94', '60', '90', '119.88', '120', '144', '165', '180', '240', '360', '480'] +FPS_CHOICES = ['23.976', '25', '29.97', '30', '50', '59.94', '60', '90', '119.88', '120', '144', '165', '180', '240', '360', '480'] def create_ui(parent): @@ -158,7 +158,15 @@ def supersample(pkg_path, images, ss_vsr_quality, ss_size_mode, ss_scale_factor, 'height': int(ss_height), 'aspect_lock': False, } - response = c.controller.call(pkg_path, 'upscale', { 'images': c.images_to_nchw(images), 'options': options }) + frames = c.images_to_nchw(images) + if debug: + log.trace(f'DLSS: method=SuperSample input={frames.shape} options={options}') + response = c.controller.call( + pkg_path, + 'upscale', + { 'images': frames, 'options': options }, + timeout=300.0, + ) if response.get('status') != 'ok': error = response.get('error') or {} log.error(f'DLSS: {error.get("message")}') @@ -184,7 +192,15 @@ def neuralrender(pkg_path, images, nr_style, nr_intensity, nr_local_tone, nr_loc 'automatic_mask': bool(nr_automatic_mask), 'dlss_model_preset': nr_model_preset, } - response = c.controller.call(pkg_path, 'render', { 'images': c.images_to_nchw(images), 'options': options }) + frames = c.images_to_nchw(images) + if debug: + log.trace(f'DLSS: method=NeuralRender input={frames.shape} options={options}') + response = c.controller.call( + pkg_path, + 'render', + { 'images': frames, 'options': options }, + timeout=600.0, + ) if response.get('status') != 'ok': error = response.get('error') or {} log.error(f'DLSS: {error.get("message")}') @@ -202,10 +218,13 @@ def framegen(pkg_path, images, fg_source_fps, fg_target_fps, fg_engine): log.warning('DLSS: FrameGen requires at least two frames, skipping') return None options = { 'ai_gpu_uuid': 'auto', 'engine': fg_engine } + frames = c.images_to_nchw(images) + if debug: + log.trace(f'DLSS: method=FrameGen input={frames.shape} options={options}') response = c.controller.call( pkg_path, 'framegen', - { 'frames': c.images_to_nchw(images), 'source_fps': fg_source_fps, 'target_fps': fg_target_fps, 'options': options }, - timeout=120.0, + { 'frames': frames, 'source_fps': fg_source_fps, 'target_fps': fg_target_fps, 'options': options }, + timeout=300.0, ) if response.get('status') != 'ok': error = response.get('error') or {} @@ -249,8 +268,8 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe ss_width = int(ss_width) ss_height = int(ss_height) ss_scale_factor = float(ss_scale_factor) - fg_source_fps = float(fg_source_fps) - fg_target_fps = float(fg_target_fps) + fg_source_fps = str(fg_source_fps) + fg_target_fps = str(fg_target_fps) images = [] originals = [] @@ -300,13 +319,15 @@ def dlss(p: processing.StableDiffusionProcessing | None, pp: processing.Processe current_images = output t.ts('framegen', t0) - log.debug(f'DLSS: images={len(images)} {t.summary(min_time=0)}') + log.debug(f'DLSS: frames={len(images)} {t.summary(min_time=0)}') pp.images = images pp.originals = originals return pp class DLSSScript(scripts_manager.Script): + video_capable = scripts_manager.AlwaysVisible + def title(self): return 'nVidia DLSS' diff --git a/ui/js/login.js b/ui/js/login.js index 02fb71fc3..0ff90ce39 100644 --- a/ui/js/login.js +++ b/ui/js/login.js @@ -34,6 +34,9 @@ function forceLogin() { document.body.appendChild(form); const status = form.querySelector('#loginStatus'); + if (!status) { + console.error('forceLogin', 'loginStatus element not found'); + } form.addEventListener('submit', (event) => { event.preventDefault();