From 186afd4bac8ede295cbe4afd941404182a153cd1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 2 Sep 2026 10:58:59 +0200 Subject: [PATCH] update openvino, refactor compile overrides, fix lora load Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 24 ++++++++------ installer.py | 4 +-- modules/api/api.py | 2 +- modules/api/docs.py | 5 +-- modules/attention/router.py | 2 +- modules/lora/lora_convert.py | 5 +-- modules/lora/lora_diffusers.py | 15 +++++---- modules/lora/lora_load.py | 4 +-- modules/sd_models.py | 3 ++ modules/sd_models_compile.py | 60 +++++++++++++++++++++++++--------- modules/sd_offload.py | 2 +- modules/ui_settings.py | 10 ------ webui.py | 2 +- 13 files changed, 84 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a61d5d52..d917c9976 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2026-09-01 +## Update for 2026-09-02 - **LoRA** - *TODO*: see [LoRA docs](https://vladmandic.github.io/sdnext-docs/LoRA) for all of the improvements and usage instructions @@ -34,23 +34,27 @@ - implement progress and preview - intercept and profiling hooks - on-demand convert standard model on-demand +- **Compute** + - openvino: update `openvino==2026.3.1` with `torch==2.13.0` - **Other** - Video Preview: TAESD support for **MiniMax** - new optional transformer hooks: *settings -> compute add-ons* *PAG: Perturbed attention guidance, PAB: Pyramid attention broadcast, FBC: First Block Cache, FC: Faster Cache, LS: Layer Skip, MC: Mag Cache, TS: TaylorSeer* *note*: compatibility of different methods varies across different models + - remove `/redocs` as `/docs` are primary api docs - **Wiki/Docs**: - [MiniMax](https://vladmandic.github.io/sdnext-docs/MiniMax) updated with LoRA, Turbo and other improvements - **Fixes** - - unnecessary secondary prompt if same - - js fetch exception handling - - detailer handling of stop/skip/pause - - cleanup dead rife code, thanks @Anai-Guo - - lumina-dimoo attention-kwargs, thanks @Anai-Guo - - improve network type/version lookup - - cleanup lora tags - - xyz grid apply bool values - - vdm scheduler fix steps, thanks @zjn20030811 + - prompt: unnecessary secondary prompt if same + - ui: js fetch exception handling + - detailer: handling of stop/skip/pause + - rife: cleanup dead code, thanks @Anai-Guo + - lumina-dimoo: attention-kwargs, thanks @Anai-Guo + - network: improve type/version lookup + - lora: cleanup tags + - xyz grid: apply bool values + - vdm scheduler: fix steps, thanks @zjn20030811 + - openvino: optimize recompile checks and lora loading ## Update for 2026-08-26 diff --git a/installer.py b/installer.py index e21e5dcbb..42f8ae462 100644 --- a/installer.py +++ b/installer.py @@ -808,10 +808,10 @@ def install_openvino(): if sys.platform == 'darwin': torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.11.0 torchvision==0.26.0') else: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.11.0+cpu torchvision==0.26.0 --index-url https://download.pytorch.org/whl/cpu') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.13.0+cpu torchvision==0.28.0 --index-url https://download.pytorch.org/whl/cpu') if not (args.skip_all or args.skip_requirements): - install(os.environ.get('OPENVINO_COMMAND', 'openvino==2026.2.1'), 'openvino') + install(os.environ.get('OPENVINO_COMMAND', 'openvino==2026.3.1'), 'openvino') ts('openvino', t_start) return torch_command diff --git a/modules/api/api.py b/modules/api/api.py index ee32648d3..5b018d1e3 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -29,7 +29,7 @@ class Api: self.router = APIRouter() if shared.cmd_opts.docs: docs.create_docs(app) - docs.create_redocs(app) + # docs.create_redocs(app) self.app = app self.queue_lock = queue_lock self.generate = generate.APIGenerate(queue_lock) diff --git a/modules/api/docs.py b/modules/api/docs.py index 208bf3a56..584b91b11 100644 --- a/modules/api/docs.py +++ b/modules/api/docs.py @@ -1,8 +1,6 @@ import json from starlette.responses import HTMLResponse from fastapi import FastAPI -from fastapi.openapi.docs import get_redoc_html, swagger_ui_default_parameters -from fastapi.encoders import jsonable_encoder def get_swagger_ui_html(*, @@ -16,6 +14,8 @@ def get_swagger_ui_html(*, init_oauth: dict | None = None, swagger_ui_parameters: dict | None = None, ) -> HTMLResponse: + from fastapi.encoders import jsonable_encoder + from fastapi.openapi.docs import swagger_ui_default_parameters current_swagger_ui_parameters = swagger_ui_default_parameters.copy() if swagger_ui_parameters: current_swagger_ui_parameters.update(swagger_ui_parameters) @@ -79,6 +79,7 @@ def create_docs(app: FastAPI): def create_redocs(app: FastAPI): + from fastapi.openapi.docs import get_redoc_html @app.get("/redocs", include_in_schema=False) # override for the default fastapi redocs route async def custom_redoc_html(): res = get_redoc_html( diff --git a/modules/attention/router.py b/modules/attention/router.py index e1e7d80a1..10d3aab0b 100644 --- a/modules/attention/router.py +++ b/modules/attention/router.py @@ -142,7 +142,7 @@ def reapply() -> None: compiled = getattr(shared, 'compiled_model_state', None) if compiled is not None and getattr(compiled, 'is_compiled', False): torch._dynamo.reset() # pylint: disable=protected-access - log.debug('Attention: dynamo reset, compiled model resident') + log.debug('Attention: dynamo=reset compiled model resident') def report() -> dict: diff --git a/modules/lora/lora_convert.py b/modules/lora/lora_convert.py index 48c1b09b3..2c5455c00 100644 --- a/modules/lora/lora_convert.py +++ b/modules/lora/lora_convert.py @@ -136,8 +136,9 @@ class KeyConvert: sd_module = shared.sd_model.network_layer_mapping.get(flat_key, None) if sd_module is not None: key = flat_key - if debug and sd_module is None: - raise RuntimeError(f"LoRA key not found in network_layer_mapping: key={key} mapping={shared.sd_model.network_layer_mapping.keys()}") + if sd_module is None: + if debug: + raise RuntimeError(f"LoRA key not found in network_layer_mapping: key={key} mapping={shared.sd_model.network_layer_mapping.keys()}") return key, sd_module diff --git a/modules/lora/lora_diffusers.py b/modules/lora/lora_diffusers.py index e8321f8a0..96c6d110f 100644 --- a/modules/lora/lora_diffusers.py +++ b/modules/lora/lora_diffusers.py @@ -54,27 +54,30 @@ def load_per_module(sd_model: diffusers.DiffusionPipeline, filename: str, adapte def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale:float=shared.opts.extra_networks_default_multiplier, lora_module=None, reason: str = '') -> network.Network | None: t0 = time.time() name = name.replace(".", "_") + reason = 'unknown' if reason is None or len(reason) == 0 else reason sd_model: diffusers.DiffusionPipeline = getattr(shared.sd_model, "pipe", shared.sd_model) - log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers reason={reason or "unknown"} scale={lora_scale} fuse={lora_overrides.fuse_native()}:{shared.opts.lora_fuse_diffusers}') + log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers reason="{reason}" scale={lora_scale} fuse={lora_overrides.fuse_native()}:{shared.opts.lora_fuse_diffusers}') if not hasattr(sd_model, 'load_lora_weights'): - log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') + log.error(f'Network load: type=LoRA class={sd_model.__class__} method=diffusersdoes not implement load lora') return None try: if lora_module is not None and isinstance(lora_module, list) and len(lora_module) > 0: name = load_per_module(sd_model, network_on_disk.filename, adapter_name=name, lora_modules=lora_module) sd_model._lora_partial = True # pylint: disable=protected-access else: + if shared.sd_model_type in ['sd', 'sdxl']: # skip te to avoid errors when lora does not have te to start with + diffusers.loaders.lora_pipeline._load_lora_into_text_encoder = lambda *args, **kwargs: None # pylint: disable=protected-access sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name) except Exception as e: if 'already in use' in str(e): pass else: if 'following keys have not been correctly renamed' in str(e): - log.error(f'Network load: type=LoRA name="{name}" diffusers unsupported format') + log.error(f'Network load: type=LoRA name="{name}" method=diffusers unsupported format') elif 'object has no attribute' in str(e): - log.error(f'Network load: type=LoRA name="{name}" diffusers empty module') + log.error(f'Network load: type=LoRA name="{name}" method=diffusers empty module') else: - log.error(f'Network load: type=LoRA name="{name}" {e}') + log.error(f'Network load: type=LoRA name="{name}" method=diffusers {e}') if l.debug: errors.display(e, "LoRA") return None @@ -84,7 +87,7 @@ def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale list_adapters = sd_model.get_list_adapters() list_adapters = [adapter for adapters in list_adapters.values() for adapter in adapters] if name not in list_adapters: - log.error(f'Network load: type=LoRA name="{name}" adapters={list_adapters} not loaded') + log.error(f'Network load: type=LoRA name="{name}" method=diffusers adapters={list_adapters} not loaded') else: diffuser_loaded.append(name) diffuser_scales.append(lora_scale) diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index d8c150fec..eb867b326 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -183,7 +183,7 @@ def maybe_recompile_model(names, te_multipliers): backup_cuda_compile = shared.opts.cuda_compile backup_scheduler = getattr(sd_model, "scheduler", None) sd_models.unload_model_weights(op='model') - shared.opts.cuda_compile = [] + shared.opts.cuda_compile = ['LoRA'] # if its empty, it will be overridden by set_openvino_overrides() to ['Model'] which is not what we want sd_models.reload_model_weights(op='model') shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, current_task) shared.opts.cuda_compile = backup_cuda_compile @@ -354,7 +354,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)} fuse={lora_overrides.fuse_native()}:{shared.opts.lora_fuse_diffusers}') if recompile_model: - log.info("Network load: type=LoRA recompiling model") + log.info("Network load: type=LoRA model recompile required") if shared.compiled_model_state is not None: backup_lora_model = shared.compiled_model_state.lora_model else: diff --git a/modules/sd_models.py b/modules/sd_models.py index 53cc42347..37ad35067 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1070,6 +1070,8 @@ def load_diffuser(checkpoint_info: CheckpointInfo | None = None, op='model', rev try: if shared.opts.ipex_optimize: sd_model = sd_models_compile.ipex_optimize(sd_model) + if shared.cmd_opts.use_openvino or devices.backend == 'openvino': + sd_models_compile.set_openvino_overrides() if (shared.opts.cuda_compile_backend != 'none') and len(shared.opts.cuda_compile) > 0: if 'components' in shared.opts.cuda_compile_options: @@ -1596,6 +1598,7 @@ def unload_model_weights(op='model'): shared.compiled_model_state.compiled_cache.clear() shared.compiled_model_state.req_cache.clear() shared.compiled_model_state.partitioned_modules.clear() + # shared.compiled_model_state = None if (op == 'model' or op == 'dict') and model_data.sd_model: log.debug(f'Current {op}: {memory_stats()}') if not ('Model' in shared.opts.cuda_compile and (shared.opts.cuda_compile_backend == "openvino_fx" or shared.opts.cuda_compile_backend == "openvino")): diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index a6c4acf54..287006a61 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -8,10 +8,11 @@ from installer import setup_logging debug = os.environ.get('SD_COMPILE_DEBUG', None) is not None debug_log = log.trace if debug else lambda *args, **kwargs: None +deepcache_worker = None +log_once = False -#Used by OpenVINO, can be used with TensorRT or Olive -class CompiledModelState: +class CompiledModelState: # Used by OpenVINO, can be used with TensorRT or Olive def __init__(self): self.is_compiled = False self.model_hash_str = "" @@ -29,9 +30,6 @@ class CompiledModelState: self.partitioned_modules = {} -deepcache_worker = None - - def ipex_optimize(sd_model, apply_to_components=True, op="Model"): try: t0 = time.time() @@ -205,19 +203,25 @@ def compile_stablefast(sd_model): def compile_torch(sd_model, apply_to_components=True, op="Model"): try: - t0 = time.time() + global log_once # pylint: disable=global-statement + t0 = time.perf_counter() import torch._dynamo # pylint: disable=unused-import,redefined-outer-name torch._dynamo.reset() # pylint: disable=protected-access - log.debug(f"{op} compile: task=torch available={torch._dynamo.list_backends()}") # pylint: disable=protected-access is_repeated = hasattr(sd_model, 'compile_repeated_blocks') and 'repeated' in shared.opts.cuda_compile_options and not sd_model.__class__.__name__.startswith("Autoencoder") - log.debug(f"{op} compile: options={shared.opts.cuda_compile_options} mode={shared.opts.cuda_compile_mode} backend={shared.opts.cuda_compile_backend} repeated={is_repeated} components={apply_to_components} targets={shared.opts.cuda_compile}") + if not log_once: + log_once = True + log.debug(f"{op} compile: task=torch available={torch._dynamo.list_backends()}") # pylint: disable=protected-access + log.debug(f"{op} compile: options={shared.opts.cuda_compile_options} mode={shared.opts.cuda_compile_mode} backend={shared.opts.cuda_compile_backend} repeated={is_repeated} components={apply_to_components} targets={shared.opts.cuda_compile}") compiled_components = [] def torch_compile_model(model, op=None, sd_model=None): # pylint: disable=unused-argument - setup_logging() # compile messes with logging so reset is needed - log.debug(f"Compile: cls={sd_model.__class__.__name__} apply") name = model.__class__.__name__ if callable(model) else model.__name__ + if 'OptimizedModule' in name: + log.warning('Model compile: task=torch model is already compiled') + return model + log.debug(f"Model compile: cls={name} apply") + setup_logging() # compile messes with logging so reset is needed compiled_components.append(name) if is_repeated: model.compile_repeated_blocks( @@ -228,14 +232,17 @@ def compile_torch(sd_model, apply_to_components=True, op="Model"): ) elif hasattr(model, 'device') and model.device.type != "meta": return_device = model.device - model = torch.compile(model.to(devices.device), + model = model.to(devices.device) + model = torch.compile( + model, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph='fullgraph' in shared.opts.cuda_compile_options, dynamic='dynamic' in shared.opts.cuda_compile_options, ).to(return_device) else: - model = torch.compile(model, + model = torch.compile( + model, mode=shared.opts.cuda_compile_mode, backend=shared.opts.cuda_compile_backend, fullgraph='fullgraph' in shared.opts.cuda_compile_options, @@ -287,7 +294,8 @@ def compile_torch(sd_model, apply_to_components=True, op="Model"): sd_model("dummy prompt") except Exception: pass - t1 = time.time() + + t1 = time.perf_counter() log.info(f"{op} compile: task=torch components={compiled_components} time={t1-t0:.2f}") except Exception as e: log.warning(f"{op} compile: task=torch {e}") @@ -329,23 +337,43 @@ def compile_diffusers(sd_model, apply_to_components=True, op="Model"): log.warning(f'{op} compile enabled but no backend specified') return sd_model t0 = time.time() - log.info(f"{op} compile: pipeline={sd_model.__class__.__name__} backend={shared.opts.cuda_compile_backend} options={shared.opts.cuda_compile_options}") + log.info(f"{op} compile: component={sd_model.__class__.__name__} backend={shared.opts.cuda_compile_backend} options={shared.opts.cuda_compile_options}") if shared.opts.cuda_compile_backend == 'onediff': sd_model = compile_onediff(sd_model) + log.debug(f"{op} compile: task=onediff time={time.time()-t0:.2f}") elif shared.opts.cuda_compile_backend == 'stable-fast': sd_model = compile_stablefast(sd_model) + log.debug(f"{op} compile: task=stablefast time={time.time()-t0:.2f}") elif shared.opts.cuda_compile_backend == 'deep-cache': sd_model = compile_deepcache(sd_model) + log.debug(f"{op} compile: task=deepcache time={time.time()-t0:.2f}") elif shared.opts.cuda_compile_backend == 'pruna': sd_model = compile_pruna(sd_model) + log.debug(f"{op} compile: task=pruna time={time.time()-t0:.2f}") else: check_deepcache(False) sd_model = compile_torch(sd_model, apply_to_components=apply_to_components, op=op) - t1 = time.time() - log.debug(f"{op} compile: time={t1-t0:.2f}") return sd_model +def set_openvino_overrides(): + if "Model" not in shared.opts.cuda_compile: + if 'LoRA' in shared.opts.cuda_compile: + shared.opts.cuda_compile = [] + else: + shared.opts.cuda_compile.append("Model") + log.warning("OpenVINO: compile=Model setting override") + if shared.opts.cuda_compile_backend != shared.opts.openvino_compile_backend: + shared.opts.cuda_compile_backend = shared.opts.openvino_compile_backend + log.warning(f"OpenVINO: backend={shared.opts.openvino_compile_backend} setting override") + if shared.opts.diffusers_offload_mode != "none": + shared.opts.diffusers_offload_mode = "none" + log.warning("OpenVINO: offload=None setting override") + if not shared.opts.lora_force_diffusers: + shared.opts.lora_force_diffusers = True + log.warning("OpenVINO: lora=diffusers setting override") + + def openvino_recompile_model(p, hires=False, refiner=False): # recompile if a parameter changes # pylint: disable=unused-argument if (shared.opts.cuda_compile_backend == "openvino_fx" or shared.opts.cuda_compile_backend == "openvino") and ('Model' in shared.opts.cuda_compile): compile_height = p.height if not hires and hasattr(p, 'height') else p.hr_upscale_to_y diff --git a/modules/sd_offload.py b/modules/sd_offload.py index 95e4fc261..a6f1bcafb 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -106,7 +106,7 @@ def set_diffuser_offload(sd_model, op:str='model', quiet:bool=False, force:bool= process_timer.add('offload', time.time() - t0) return - if shared.opts.diffusers_offload_mode == "none": + if shared.opts.diffusers_offload_mode == "none" and devices.backend != "openvino": log.warning('Offload: type=none "use balanced offload with model type set not to offload"') apply_none_offload(sd_model, quiet=quiet) diff --git a/modules/ui_settings.py b/modules/ui_settings.py index 216895788..af3874d93 100644 --- a/modules/ui_settings.py +++ b/modules/ui_settings.py @@ -135,16 +135,6 @@ def run_settings(*args): from modules.onnx_impl import install_olive, initialize_onnx_pipelines install_olive() initialize_onnx_pipelines() - if shared.cmd_opts.use_openvino: - if "Model" not in shared.opts.cuda_compile: - log.warning("OpenVINO: Overriding Torch Compile Model") - shared.opts.cuda_compile.append("Model") - if shared.opts.cuda_compile_backend != shared.opts.openvino_compile_backend: - log.warning(f"OpenVINO: Overriding Torch Compile backend={shared.opts.openvino_compile_backend}") - shared.opts.cuda_compile_backend = shared.opts.openvino_compile_backend - if shared.opts.diffusers_offload_mode != "none": - log.warning("OpenVINO: Overriding diffusers_offload_mode=none") - shared.opts.diffusers_offload_mode = "none" if shared.opts.sd_backend != "diffusers": log.error('Legacy option: backend=original is no longer supported') shared.opts.sd_backend = "diffusers" diff --git a/webui.py b/webui.py index 547142c00..70b1595d7 100644 --- a/webui.py +++ b/webui.py @@ -448,7 +448,7 @@ def start_ui(): log.info(f'Public URL: {proto}://{public_ip}:{shared.cmd_opts.port}') if shared.cmd_opts.docs: log.info(f'API docs: {local_url[:-1]}/docs') # pylint: disable=unsubscriptable-object - log.info(f'API redocs: {local_url[:-1]}/redocs') # pylint: disable=unsubscriptable-object + # log.info(f'API redocs: {local_url[:-1]}/redocs') # pylint: disable=unsubscriptable-object if share_url is not None: log.info(f'Share URL: {share_url}') if getattr(shared.cmd_opts, 'enso', False):