Merge branch 'dev' of https://github.com/vladmandic/sdnext into dev

Signed-off-by: Vladimir Mandic <57876960+vladmandic@users.noreply.github.com>

cli/nvidia-smi.py

@@ -13,9 +13,8 @@ def get_nvidia_smi(output='dict'):
     if smi is None:
         log.error("nvidia-smi not found")
         return None
-    result = subprocess.run(f'"{smi}" -q -x', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    xml = result.stdout.decode(encoding="utf8", errors="ignore")
-    d = xmltodict.parse(xml)
+    result = subprocess.run(f'"{smi}" -q -x', shell=True, check=False, env=os.environ, capture_output=True, text=True)
+    d = xmltodict.parse(result.stdout)
     if 'nvidia_smi_log' in d:
         d = d['nvidia_smi_log']
     if 'gpu' in d and 'supported_clocks' in d['gpu']:

installer.py

@@ -200,14 +200,32 @@ def uninstall(package, quiet = False):
     return res
 
 
-def run(cmd: str, arg: str):
-    result = subprocess.run(f'"{cmd}" {arg}', shell=True, check=False, env=os.environ, capture_output=True)
-    txt = result.stdout.decode(encoding="utf8", errors="ignore")
-    if len(result.stderr) > 0:
-        txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
-    txt = txt.strip()
-    debug(f'Exec {cmd}: {txt}')
-    return txt
+def run(cmd: str, *args: str, **kwargs):
+    """Run command and arguments with `subprocess.run`.
+
+    Default run options are `shell=True, check=False, env=os.environ`.
+
+    Args:
+        cmd (str): Main command to run.
+        *args (str): Additional command arguments.
+        **kwargs: `subprocess.run` option overrides.
+
+    Returns:
+        tuple[CompletedProcess[str], str]: Tuple with the results and the combined `stdout` and `stderr` values.
+    """
+    options = {
+        "check": False,
+        "env": os.environ,
+    }
+    options |= kwargs  # Override defaults with passed kwargs
+    result = subprocess.run(f'"{cmd}" {" ".join(args)}', **options, shell=True, capture_output=True, text=True)
+    result.stdout = result.stdout.strip()
+    result.stderr = result.stderr.strip()
+    txt = result.stdout
+    if result.stderr:
+        # Put newline between outputs only if stdout isn't empty
+        txt += "\n" + result.stderr if txt else result.stderr
+    return result, txt
 
 
 def cleanup_broken_packages():
@@ -246,18 +264,13 @@ def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True):
     all_args = f'{pip_log}{arg} {env_args}'.strip()
     if not quiet:
         log.debug(f'Running: {pipCmd}="{all_args}"')
-    result = subprocess.run(f'"{sys.executable}" -m {pipCmd} {all_args}', shell=True, check=False, env=os.environ, capture_output=True)
-    txt = result.stdout.decode(encoding="utf8", errors="ignore")
+    result, txt = run(sys.executable, "-m", pipCmd, all_args)
     if len(result.stderr) > 0:
         if uv and result.returncode != 0:
-            err = result.stderr.decode(encoding="utf8", errors="ignore")
             log.warning(f'Install: cmd="{pipCmd}" args="{all_args}" cannot use uv, fallback to pip')
-            debug(f'Install: uv pip error: {err}')
+            debug(f'Install: uv pip error: {result.stderr}')
             cleanup_broken_packages()
             return pip(originalArg, ignore, quiet, uv=False)
-        else:
-            txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
-    txt = txt.strip()
     debug(f'Install {pipCmd}: {txt}')
     if result.returncode != 0 and not ignore:
         errors.append(f'pip: {package}')
@@ -294,25 +307,21 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal
     git_cmd = os.environ.get('GIT', "git")
     if git_cmd != "git":
         git_cmd = os.path.abspath(git_cmd)
-    result = subprocess.run(f'"{git_cmd}" {arg}', check=False, shell=True, env=os.environ, capture_output=True, cwd=folder or '.')
-    stdout = result.stdout.decode(encoding="utf8", errors="ignore")
-    if len(result.stderr) > 0:
-        stdout += ('\n' if len(stdout) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
-    stdout = stdout.strip()
+    result, txt = run(git_cmd, arg, cwd=folder or ".")
     if result.returncode != 0 and not ignore:
         if folder is None:
             folder = 'root'
-        if "couldn't find remote ref" in stdout: # not a git repo
+        if "couldn't find remote ref" in txt: # not a git repo
             log.error(f'Git: folder="{folder}" could not identify repository')
-        elif "no submodule mapping found" in stdout:
+        elif "no submodule mapping found" in txt:
             log.warning(f'Git: folder="{folder}" submodules changed')
-        elif 'or stash them' in stdout:
+        elif 'or stash them' in txt:
             log.error(f'Git: folder="{folder}" local changes detected')
         else:
-            log.error(f'Git: folder="{folder}" arg="{arg}" output={stdout}')
+            log.error(f'Git: folder="{folder}" arg="{arg}" output={txt}')
         errors.append(f'git: {folder}')
     ts('git', t_start)
-    return stdout
+    return txt
 
 
 # reattach as needed as head can get detached
@@ -471,7 +480,7 @@ def check_diffusers():
     t_start = time.time()
     if args.skip_all:
         return
-    target_commit = '4a2833c1c226362677e165a363fecb4da331e122' # diffusers commit hash
+    target_commit = '8ec0a5ccad96957c10388d2d2acc7fdd8e0fab84' # diffusers commit hash
     # if args.use_rocm or args.use_zluda or args.use_directml:
     #     sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
     pkg = package_spec('diffusers')
@@ -927,13 +936,10 @@ def run_extension_installer(folder):
             if os.environ.get('PYTHONPATH', None) is not None:
                 seperator = ';' if sys.platform == 'win32' else ':'
                 env['PYTHONPATH'] += seperator + os.environ.get('PYTHONPATH', None)
-            result = subprocess.run(f'"{sys.executable}" "{path_installer}"', shell=True, env=env, check=False, capture_output=True, cwd=folder)
-            txt = result.stdout.decode(encoding="utf8", errors="ignore")
-            debug(f'Extension installer: file="{path_installer}" {txt}')
+            result, txt = run(sys.executable, path_installer, env=env, cwd=folder)
+            debug(f'Extension installer: file="{path_installer}" {result.stdout}')
             if result.returncode != 0:
                 errors.append(f'ext: {os.path.basename(folder)}')
-                if len(result.stderr) > 0:
-                    txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore")
                 log.error(f'Extension installer error: {path_installer}')
                 log.debug(txt)
     except Exception as e:
@@ -1264,29 +1270,24 @@ def get_version(force=False):
         except Exception:
             pass
         try:
-            res = subprocess.run('git log --pretty=format:"%h %ad" -1 --date=short', capture_output=True, shell=True, check=True)
-            ver = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '  '
+            ver = run('git', 'log --pretty=format:"%h %ad" -1 --date=short', check=True)[0].stdout or '  '
             commit, updated = ver.split(' ')
             version['commit'], version['updated'] = commit, updated
         except Exception as e:
             log.warning(f'Version: where=commit {e}')
         try:
-            res = subprocess.run('git remote get-url origin', capture_output=True, shell=True, check=True)
-            origin = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else ''
-            res = subprocess.run('git rev-parse --abbrev-ref HEAD', capture_output=True, shell=True, check=True)
-            branch_name = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else ''
-            version['url'] = origin.replace('\n', '').removesuffix('.git') + '/tree/' + branch_name.replace('\n', '')
-            version['branch'] = branch_name.replace('\n', '')
+            origin = run('git', 'remote get-url origin', check=True)[0].stdout
+            branch_name = run('git', 'rev-parse --abbrev-ref HEAD', check=True)[0].stdout
+            version['url'] = origin.removesuffix('.git') + '/tree/' + branch_name
+            version['branch'] = branch_name
             if version['branch'] == 'HEAD':
                 log.warning('Version: detached state detected')
         except Exception as e:
             log.warning(f'Version: where=branch {e}')
         try:
             if os.path.exists('extensions-builtin/sdnext-modernui'):
-                res = subprocess.run('git rev-parse --abbrev-ref HEAD', capture_output=True, shell=True, check=True, cwd='extensions-builtin/sdnext-modernui')
-                branch_ui = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else ''
-                branch_ui = 'dev' if 'dev' in branch_ui else 'main'
-                version['ui'] = branch_ui
+                branch_ui = run('git', 'rev-parse --abbrev-ref HEAD', check=True, cwd='extensions-builtin/sdnext-modernui')[0].stdout
+                version['ui'] = 'dev' if 'dev' in branch_ui else 'main'
             else:
                 version['ui'] = 'unavailable'
         except Exception as e:
@@ -1296,10 +1297,8 @@ def get_version(force=False):
             if os.environ.get('SD_KANVAS_DISABLE', None) is not None:
                 version['kanvas'] = 'disabled'
             elif os.path.exists('extensions-builtin/sdnext-kanvas'):
-                res = subprocess.run('git rev-parse --abbrev-ref HEAD', capture_output=True, shell=True, check=True, cwd='extensions-builtin/sdnext-kanvas')
-                branch_kanvas = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else ''
-                branch_kanvas = 'dev' if 'dev' in branch_kanvas else 'main'
-                version['kanvas'] = branch_kanvas
+                branch_kanvas = run('git', 'rev-parse --abbrev-ref HEAD', check=True, cwd='extensions-builtin/sdnext-kanvas')[0].stdout
+                version['kanvas'] = 'dev' if 'dev' in branch_kanvas else 'main'
             else:
                 version['kanvas'] = 'unavailable'
         except Exception as e:
@@ -1487,8 +1486,7 @@ def get_state():
         def _get_commit(item):
             ext, ext_dir = item
             try:
-                res = subprocess.run('git rev-parse HEAD', capture_output=True, shell=True, check=False, cwd=ext_dir)
-                return ext, res.stdout.decode(encoding='utf8', errors='ignore').strip()
+                return ext, run('git', 'rev-parse HEAD', cwd=ext_dir)[0].stdout
             except Exception:
                 return ext, ''
 

launch.py

@@ -97,13 +97,13 @@ def run(command, desc=None, errdesc=None, custom_env=None, live=False): # compat
         if result.returncode != 0:
             raise RuntimeError(f"""{errdesc or 'Error running command'} Command: {command} Error code: {result.returncode}""")
         return ''
-    result = subprocess.run(command, capture_output=True, check=False, shell=True, env=os.environ if custom_env is None else custom_env)
+    result = subprocess.run(command, capture_output=True, check=False, shell=True, env=os.environ if custom_env is None else custom_env, text=True)
     if result.returncode != 0:
         raise RuntimeError(f"""{errdesc or 'Error running command'}: {command} code: {result.returncode}
-{result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stdout)>0 else ''}
-{result.stderr.decode(encoding="utf8", errors="ignore") if len(result.stderr)>0 else ''}
+{result.stdout}
+{result.stderr}
 """)
-    return result.stdout.decode(encoding="utf8", errors="ignore")
+    return result.stdout
 
 
 def check_run(command): # compatbility function

modules/caption/deepbooru.py

@@ -205,6 +205,7 @@ def batch(
     Returns:
         Combined tag results
     """
+    import os
     import time
     from pathlib import Path
     import rich.progress as rp
@@ -214,55 +215,15 @@ def batch(
 
     # Collect image files
     image_files = []
-    image_extensions = {'.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif'}
-
-    # From file picker
-    if batch_files:
-        for f in batch_files:
-            if isinstance(f, dict):
-                image_files.append(Path(f['name']))
-            elif hasattr(f, 'name'):
-                image_files.append(Path(f.name))
-            else:
-                image_files.append(Path(f))
-
-    # From folder picker
-    if batch_folder:
-        folder_path = None
-        if isinstance(batch_folder, list) and len(batch_folder) > 0:
-            f = batch_folder[0]
-            if isinstance(f, dict):
-                folder_path = Path(f['name']).parent
-            elif hasattr(f, 'name'):
-                folder_path = Path(f.name).parent
-        if folder_path and folder_path.is_dir():
-            if recursive:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.rglob(f'*{ext}'))
-            else:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.glob(f'*{ext}'))
-
-    # From string path
-    if batch_str and batch_str.strip():
+    if batch_files is not None:
+        image_files += [f.name for f in batch_files]
+    if batch_folder is not None:
+        image_files += [f.name for f in batch_folder]
+    if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
+        image_extensions = {'.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif'}
         folder_path = Path(batch_str.strip())
-        if folder_path.is_dir():
-            if recursive:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.rglob(f'*{ext}'))
-            else:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.glob(f'*{ext}'))
-
-    # Remove duplicates while preserving order
-    seen = set()
-    unique_files = []
-    for f in image_files:
-        f_resolved = f.resolve()
-        if f_resolved not in seen:
-            seen.add(f_resolved)
-            unique_files.append(f)
-    image_files = unique_files
+        for ext in image_extensions:
+            image_files.extend(str(p) for p in (folder_path.rglob(f'*{ext}') if recursive else folder_path.glob(f'*{ext}')))
 
     if not image_files:
         log.warning('DeepBooru batch: no images found')
@@ -280,25 +241,26 @@ def batch(
 
     with pbar:
         task = pbar.add_task(total=len(image_files), description='starting...')
-        for img_path in image_files:
-            pbar.update(task, advance=1, description=str(img_path.name))
+        for file in image_files:
+            file_name = os.path.basename(file)
+            pbar.update(task, advance=1, description=file_name)
             try:
                 if shared.state.interrupted:
                     log.info('DeepBooru batch: interrupted')
                     break
 
-                image = Image.open(img_path)
+                image = Image.open(file)
                 tags_str = model.tag_multi(image, **kwargs)
 
                 if save_output:
                     from modules.caption import tagger
-                    tagger.save_tags_to_file(img_path, tags_str, save_append)
+                    tagger.save_tags_to_file(Path(file), tags_str, save_append)
 
-                results.append(f'{img_path.name}: {tags_str[:100]}...' if len(tags_str) > 100 else f'{img_path.name}: {tags_str}')
+                results.append(f'{file_name}: {tags_str[:100]}...' if len(tags_str) > 100 else f'{file_name}: {tags_str}')
 
             except Exception as e:
-                log.error(f'DeepBooru batch: file="{img_path}" error={e}')
-                results.append(f'{img_path.name}: ERROR - {e}')
+                log.error(f'DeepBooru batch: file="{file}" error={e}')
+                results.append(f'{file_name}: ERROR - {e}')
 
     model.stop()
     elapsed = time.time() - t0

modules/caption/waifudiffusion.py

@@ -419,6 +419,7 @@ def batch(
     Returns:
         Combined tag results
     """
+    import os
     from pathlib import Path
 
     # Load model
@@ -429,55 +430,15 @@ def batch(
 
     # Collect image files
     image_files = []
-    image_extensions = {'.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif'}
-
-    # From file picker
-    if batch_files:
-        for f in batch_files:
-            if isinstance(f, dict):
-                image_files.append(Path(f['name']))
-            elif hasattr(f, 'name'):
-                image_files.append(Path(f.name))
-            else:
-                image_files.append(Path(f))
-
-    # From folder picker
-    if batch_folder:
-        folder_path = None
-        if isinstance(batch_folder, list) and len(batch_folder) > 0:
-            f = batch_folder[0]
-            if isinstance(f, dict):
-                folder_path = Path(f['name']).parent
-            elif hasattr(f, 'name'):
-                folder_path = Path(f.name).parent
-        if folder_path and folder_path.is_dir():
-            if recursive:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.rglob(f'*{ext}'))
-            else:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.glob(f'*{ext}'))
-
-    # From string path
-    if batch_str and batch_str.strip():
+    if batch_files is not None:
+        image_files += [f.name for f in batch_files]
+    if batch_folder is not None:
+        image_files += [f.name for f in batch_folder]
+    if batch_str is not None and len(batch_str) > 0 and os.path.exists(batch_str) and os.path.isdir(batch_str):
+        image_extensions = {'.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif'}
         folder_path = Path(batch_str.strip())
-        if folder_path.is_dir():
-            if recursive:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.rglob(f'*{ext}'))
-            else:
-                for ext in image_extensions:
-                    image_files.extend(folder_path.glob(f'*{ext}'))
-
-    # Remove duplicates while preserving order
-    seen = set()
-    unique_files = []
-    for f in image_files:
-        f_resolved = f.resolve()
-        if f_resolved not in seen:
-            seen.add(f_resolved)
-            unique_files.append(f)
-    image_files = unique_files
+        for ext in image_extensions:
+            image_files.extend(str(p) for p in (folder_path.rglob(f'*{ext}') if recursive else folder_path.glob(f'*{ext}')))
 
     if not image_files:
         log.warning('WaifuDiffusion batch: no images found')
@@ -496,25 +457,26 @@ def batch(
 
     with pbar:
         task = pbar.add_task(total=len(image_files), description='starting...')
-        for img_path in image_files:
-            pbar.update(task, advance=1, description=str(img_path.name))
+        for file in image_files:
+            file_name = os.path.basename(file)
+            pbar.update(task, advance=1, description=file_name)
             try:
                 if shared.state.interrupted:
                     log.info('WaifuDiffusion batch: interrupted')
                     break
 
-                image = Image.open(img_path)
+                image = Image.open(file)
                 tags_str = tagger.predict(image, **kwargs)
 
                 if save_output:
                     from modules.caption import tagger as tagger_module
-                    tagger_module.save_tags_to_file(img_path, tags_str, save_append)
+                    tagger_module.save_tags_to_file(Path(file), tags_str, save_append)
 
-                results.append(f'{img_path.name}: {tags_str[:100]}...' if len(tags_str) > 100 else f'{img_path.name}: {tags_str}')
+                results.append(f'{file_name}: {tags_str[:100]}...' if len(tags_str) > 100 else f'{file_name}: {tags_str}')
 
             except Exception as e:
-                log.error(f'WaifuDiffusion batch: file="{img_path}" error={e}')
-                results.append(f'{img_path.name}: ERROR - {e}')
+                log.error(f'WaifuDiffusion batch: file="{file}" error={e}')
+                results.append(f'{file_name}: ERROR - {e}')
 
     elapsed = time.time() - t0
     log.info(f'WaifuDiffusion batch: complete images={len(results)} time={elapsed:.1f}s')

modules/devices.py

@@ -104,8 +104,8 @@ def get_gpu_info():
         elif torch.cuda.is_available() and torch.version.cuda:
             try:
                 import subprocess
-                result = subprocess.run('nvidia-smi --query-gpu=driver_version --format=csv,noheader', shell=True, check=False, env=os.environ, capture_output=True)
-                version = result.stdout.decode(encoding="utf8", errors="ignore").strip()
+                result = subprocess.run('nvidia-smi --query-gpu=driver_version --format=csv,noheader', shell=True, check=False, env=os.environ, capture_output=True, text=True)
+                version = result.stdout.strip()
                 return version
             except Exception:
                 return ''

modules/rocm.py

@@ -31,8 +31,8 @@ def dirname(path_: str, r: int = 1) -> str:
 
 
 def spawn(command: str | list[str], cwd: os.PathLike = '.') -> str:
-    process = subprocess.run(command, cwd=cwd, shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
-    return process.stdout.decode(encoding="utf8", errors="ignore")
+    process = subprocess.run(command, cwd=cwd, shell=True, check=False, capture_output=True, text=True)
+    return process.stdout
 
 
 def load_library_global(path_: str):

modules/ui_caption.py

@@ -270,6 +270,7 @@ def create_ui():
                         with gr.Row():
                             wd_show_scores = gr.Checkbox(label='Show confidence scores', value=shared.opts.tagger_show_scores, elem_id='wd_show_scores')
                     gr.HTML('<style>#wd_character_threshold:has(input:disabled), #wd_include_rating:has(input:disabled) { opacity: 0.5; }</style>')
+                    gr.HTML('<style>#vlm_batch_files, #vlm_batch_folder, #clip_batch_files, #clip_batch_folder, #wd_batch_files, #wd_batch_folder { max-height: 10em; overflow-y: auto !important; }</style>')
                     with gr.Accordion(label='Caption: Batch', open=False, visible=True):
                         with gr.Row():
                             wd_batch_files = gr.File(label="Files", show_label=True, file_count='multiple', file_types=['image'], interactive=True, height=100, elem_id='wd_batch_files')

modules/ui_common.py

@@ -249,12 +249,14 @@ def open_folder(result_gallery, gallery_index = 0):
         path = os.path.normpath(folder)
         if platform.system() == "Windows":
             os.startfile(path) # pylint: disable=no-member
-        elif platform.system() == "Darwin":
-            subprocess.Popen(["open", path]) # pylint: disable=consider-using-with
+            return
+        if platform.system() == "Darwin":
+            opener = "open"
         elif "microsoft-standard-WSL2" in platform.uname().release:
-            subprocess.Popen(["wsl-open", path]) # pylint: disable=consider-using-with
+            opener = "wslview" if shutil.which("wslview") is not None else "wsl-open"
         else:
-            subprocess.Popen(["xdg-open", path]) # pylint: disable=consider-using-with
+            opener = "xdg-open"
+        subprocess.Popen([opener, path])  # pylint: disable=consider-using-with
 
 
 def create_output_panel(tabname, preview=True, prompt=None, height=None, transfer=True, scale=1, result_info=None):