mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
OpenVINO add --device-id and --use-cpu openvino
This commit is contained in:
Disty0
@@ -29,7 +29,7 @@ All Individual features are not listed here, instead check [ChangeLog](CHANGELOG
|
||||
- Support for multiple diffusion models!
|
||||
Stable Diffusion, SD-XL, Kandinsky, DeepFloyd IF, etc.
|
||||
- Fully multiplatform with platform specific autodetection and tuning performed on install
|
||||
Windows / Linux / MacOS with CPU / nVidia / AMD / IntelArc / DirectML
|
||||
Windows / Linux / MacOS with CPU / nVidia / AMD / Intel / DirectML
|
||||
- Improved prompt parser
|
||||
- Enhanced *Lora*/*Locon*/*Lyco* code supporting latest trends in training
|
||||
- Built-in queue management
|
||||
|
||||
@@ -8,86 +8,97 @@ from torch._dynamo.backends.registry import register_backend
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
from torch._inductor.compile_fx import compile_fx
|
||||
from hashlib import sha256
|
||||
from modules import shared
|
||||
|
||||
@register_backend
|
||||
@fake_tensor_unsupported
|
||||
def openvino_fx(subgraph, example_inputs):
|
||||
try:
|
||||
executor_parameters = None
|
||||
core = Core()
|
||||
if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0":
|
||||
os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1")
|
||||
model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest()
|
||||
executor_parameters = {"model_hash_str": model_hash_str}
|
||||
executor_parameters = None
|
||||
core = Core()
|
||||
if os.getenv("OPENVINO_TORCH_MODEL_CACHING") != "0":
|
||||
os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1")
|
||||
model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest()
|
||||
executor_parameters = {"model_hash_str": model_hash_str}
|
||||
|
||||
example_inputs.reverse()
|
||||
cache_root = "./cache/"
|
||||
if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None:
|
||||
cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR")
|
||||
example_inputs.reverse()
|
||||
cache_root = "./cache/"
|
||||
if os.getenv("OPENVINO_TORCH_CACHE_DIR") is not None:
|
||||
cache_root = os.getenv("OPENVINO_TORCH_CACHE_DIR")
|
||||
|
||||
if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None:
|
||||
device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE")
|
||||
elif any(openvino_cpu in cpu_module.lower() for cpu_module in shared.cmd_opts.use_cpu for openvino_cpu in ["openvino", "all"]):
|
||||
device = "CPU"
|
||||
elif shared.cmd_opts.device_id is not None:
|
||||
device = f"GPU.{shared.cmd_opts.device_id}"
|
||||
elif "GPU" in core.available_devices:
|
||||
device = "GPU"
|
||||
elif "GPU.1" in core.available_devices:
|
||||
device = "GPU.1"
|
||||
elif "GPU.0" in core.available_devices:
|
||||
device = "GPU.0"
|
||||
else:
|
||||
device = "CPU"
|
||||
shared.log.warning("OpenVINO: No compatible GPU detected!")
|
||||
os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device)
|
||||
shared.log.debug(f"OpenVINO Device: {device}")
|
||||
if shared.opts.cuda_compile_errors and device not in core.available_devices:
|
||||
shared.log.warning(f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices")
|
||||
assert device in core.available_devices, f"OpenVINO: Specified device {device} is not in the list of OpenVINO Available Devices"
|
||||
|
||||
if os.getenv("OPENVINO_TORCH_BACKEND_DEVICE") is not None:
|
||||
device = os.getenv("OPENVINO_TORCH_BACKEND_DEVICE")
|
||||
assert device in core.available_devices, "Specified device " + device + " is not in the list of OpenVINO Available Devices"
|
||||
else:
|
||||
os.environ.setdefault('OPENVINO_TORCH_BACKEND_DEVICE', device)
|
||||
#Cache saving keeps increasing the partition id
|
||||
#This loop check if non 0 partition id caches exist
|
||||
#Takes 0.002 seconds when nothing is found
|
||||
use_cached_file = False
|
||||
for i in range(100):
|
||||
file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root)
|
||||
if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"):
|
||||
use_cached_file = True
|
||||
break
|
||||
|
||||
#Cache saving keeps increasing the partition id
|
||||
#This loop check if non 0 partition id caches exist
|
||||
#Takes 0.002 seconds when nothing is found
|
||||
use_cached_file = False
|
||||
for i in range(100):
|
||||
file_name = get_cached_file_name(*example_inputs, model_hash_str=str(model_hash_str + str(i)), device=device, cache_root=cache_root)
|
||||
if file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin"):
|
||||
use_cached_file = True
|
||||
break
|
||||
if use_cached_file:
|
||||
om = core.read_model(file_name + ".xml")
|
||||
|
||||
if use_cached_file:
|
||||
om = core.read_model(file_name + ".xml")
|
||||
dtype_mapping = {
|
||||
torch.float32: Type.f32,
|
||||
torch.float64: Type.f64,
|
||||
torch.float16: Type.f16,
|
||||
torch.int64: Type.i64,
|
||||
torch.int32: Type.i32,
|
||||
torch.uint8: Type.u8,
|
||||
torch.int8: Type.i8,
|
||||
torch.bool: Type.boolean
|
||||
}
|
||||
|
||||
dtype_mapping = {
|
||||
torch.float32: Type.f32,
|
||||
torch.float64: Type.f64,
|
||||
torch.float16: Type.f16,
|
||||
torch.int64: Type.i64,
|
||||
torch.int32: Type.i32,
|
||||
torch.uint8: Type.u8,
|
||||
torch.int8: Type.i8,
|
||||
torch.bool: Type.boolean
|
||||
}
|
||||
for idx, input_data in enumerate(example_inputs):
|
||||
om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
|
||||
om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
|
||||
om.validate_nodes_and_infer_types()
|
||||
|
||||
for idx, input_data in enumerate(example_inputs):
|
||||
om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype])
|
||||
om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape)))
|
||||
om.validate_nodes_and_infer_types()
|
||||
if model_hash_str is not None:
|
||||
core.set_property({'CACHE_DIR': cache_root + '/blob'})
|
||||
|
||||
if model_hash_str is not None:
|
||||
core.set_property({'CACHE_DIR': cache_root + '/blob'})
|
||||
compiled_model = core.compile_model(om, device)
|
||||
def _call(*args):
|
||||
ov_inputs = [a.detach().cpu().numpy() for a in args]
|
||||
ov_inputs.reverse()
|
||||
res = compiled_model(ov_inputs)
|
||||
result = [torch.from_numpy(res[out]) for out in compiled_model.outputs]
|
||||
return result
|
||||
return _call
|
||||
else:
|
||||
example_inputs.reverse()
|
||||
model = make_fx(subgraph)(*example_inputs)
|
||||
with torch.no_grad():
|
||||
model.eval()
|
||||
partitioner = Partitioner()
|
||||
compiled_model = partitioner.make_partitions(model)
|
||||
|
||||
compiled_model = core.compile_model(om, device)
|
||||
def _call(*args):
|
||||
ov_inputs = [a.detach().cpu().numpy() for a in args]
|
||||
ov_inputs.reverse()
|
||||
res = compiled_model(ov_inputs)
|
||||
result = [torch.from_numpy(res[out]) for out in compiled_model.outputs]
|
||||
return result
|
||||
return _call
|
||||
else:
|
||||
example_inputs.reverse()
|
||||
model = make_fx(subgraph)(*example_inputs)
|
||||
with torch.no_grad():
|
||||
model.eval()
|
||||
partitioner = Partitioner()
|
||||
compiled_model = partitioner.make_partitions(model)
|
||||
|
||||
def _call(*args):
|
||||
res = execute(compiled_model, *args, executor="openvino",
|
||||
executor_parameters=executor_parameters)
|
||||
return res
|
||||
return _call
|
||||
except Exception:
|
||||
return compile_fx(subgraph, example_inputs)
|
||||
def _call(*args):
|
||||
res = execute(compiled_model, *args, executor="openvino",
|
||||
executor_parameters=executor_parameters)
|
||||
return res
|
||||
return _call
|
||||
|
||||
|
||||
def get_cached_file_name(*args, model_hash_str, device, cache_root):
|
||||
|
||||
Reference in New Issue
Block a user