diff --git a/configs/olive/sd/text_encoder.json b/configs/olive/sd/text_encoder.json index 3e4aeb32b..91a3d57d3 100644 --- a/configs/olive/sd/text_encoder.json +++ b/configs/olive/sd/text_encoder.json @@ -36,6 +36,17 @@ } }, "passes": { + "optimize_CPUExecutionProvider": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "clip", + "opt_level": 0, + "float16": false, + "use_gpu": false, + "keep_io_types": false + } + }, "optimize_DmlExecutionProvider": { "type": "OrtTransformersOptimization", "disable_search": true, @@ -91,6 +102,17 @@ "use_gpu": true, "keep_io_types": false } + }, + "quantization": { + "type": "OnnxDynamicQuantization", + "disable_search": true, + "config": { + "save_as_external_data": false, + "all_tensors_to_one_file": true, + "per_channel": false, + "reduce_range": false, + "MatMulConstBOnly": true + } } }, "pass_flows": [["optimize"]], diff --git a/configs/olive/sd/unet.json b/configs/olive/sd/unet.json index 1b02ce0c3..907504b06 100644 --- a/configs/olive/sd/unet.json +++ b/configs/olive/sd/unet.json @@ -53,6 +53,17 @@ } }, "passes": { + "optimize_CPUExecutionProvider": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "clip", + "opt_level": 0, + "float16": false, + "use_gpu": false, + "keep_io_types": false + } + }, "optimize_DmlExecutionProvider": { "type": "OrtTransformersOptimization", "disable_search": true, @@ -108,6 +119,17 @@ "use_gpu": true, "keep_io_types": false } + }, + "quantization": { + "type": "OnnxDynamicQuantization", + "disable_search": true, + "config": { + "save_as_external_data": false, + "all_tensors_to_one_file": true, + "per_channel": false, + "reduce_range": false, + "MatMulConstBOnly": true + } } }, "pass_flows": [["optimize"]], diff --git a/configs/olive/sd/vae_decoder.json b/configs/olive/sd/vae_decoder.json index 3653ad422..2e21c0f79 100644 --- a/configs/olive/sd/vae_decoder.json +++ b/configs/olive/sd/vae_decoder.json @@ -43,6 +43,17 @@ } }, "passes": { + "optimize_CPUExecutionProvider": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "clip", + "opt_level": 0, + "float16": false, + "use_gpu": false, + "keep_io_types": false + } + }, "optimize_DmlExecutionProvider": { "type": "OrtTransformersOptimization", "disable_search": true, @@ -98,6 +109,17 @@ "use_gpu": true, "keep_io_types": false } + }, + "quantization": { + "type": "OnnxDynamicQuantization", + "disable_search": true, + "config": { + "save_as_external_data": false, + "all_tensors_to_one_file": true, + "per_channel": false, + "reduce_range": false, + "MatMulConstBOnly": true + } } }, "pass_flows": [["optimize"]], diff --git a/configs/olive/sd/vae_encoder.json b/configs/olive/sd/vae_encoder.json index 52dc369f9..2f012b187 100644 --- a/configs/olive/sd/vae_encoder.json +++ b/configs/olive/sd/vae_encoder.json @@ -43,6 +43,17 @@ } }, "passes": { + "optimize_CPUExecutionProvider": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "clip", + "opt_level": 0, + "float16": false, + "use_gpu": false, + "keep_io_types": false + } + }, "optimize_DmlExecutionProvider": { "type": "OrtTransformersOptimization", "disable_search": true, @@ -98,6 +109,17 @@ "use_gpu": true, "keep_io_types": false } + }, + "quantization": { + "type": "OnnxDynamicQuantization", + "disable_search": true, + "config": { + "save_as_external_data": false, + "all_tensors_to_one_file": true, + "per_channel": false, + "reduce_range": false, + "MatMulConstBOnly": true + } } }, "pass_flows": [["optimize"]], diff --git a/modules/onnx_ep.py b/modules/onnx_ep.py index 148d89840..6204a375c 100644 --- a/modules/onnx_ep.py +++ b/modules/onnx_ep.py @@ -15,7 +15,7 @@ class ExecutionProvider(str, Enum): available_execution_providers: List[ExecutionProvider] = ort.get_available_providers() EP_TO_NAME = { - ExecutionProvider.CPU: "cpu", # is this a valid option? + ExecutionProvider.CPU: "gpu-cpu", # ??? ExecutionProvider.DirectML: "gpu-dml", ExecutionProvider.CUDA: "gpu-cuda", # test required ExecutionProvider.ROCm: "gpu-rocm", # test required diff --git a/modules/onnx_pipelines.py b/modules/onnx_pipelines.py index ce48fea9d..4f2af1e1b 100644 --- a/modules/onnx_pipelines.py +++ b/modules/onnx_pipelines.py @@ -244,30 +244,31 @@ class OnnxRawPipeline(OnnxPipelineBase): if flow[i] == "optimize": flow[i] = pass_key olive_config["input_model"]["config"]["model_path"] = os.path.abspath(os.path.join(in_dir, submodel, "model.onnx")) - olive_config["passes"][pass_key]["config"]["float16"] = shared.opts.olive_float16 olive_config["engine"]["execution_providers"] = [shared.opts.onnx_execution_provider] - if shared.opts.onnx_execution_provider == ExecutionProvider.CUDA or shared.opts.onnx_execution_provider == ExecutionProvider.ROCm: - if version.parse(ort.__version__) < version.parse("1.17.0"): - olive_config["passes"][pass_key]["config"]["optimization_options"] = {"enable_skip_group_norm": False} - if shared.opts.olive_float16: - olive_config["passes"][pass_key]["config"]["keep_io_types"] = False + if pass_key in olive_config["passes"]: + olive_config["passes"][pass_key]["config"]["float16"] = shared.opts.olive_float16 + if shared.opts.onnx_execution_provider == ExecutionProvider.CUDA or shared.opts.onnx_execution_provider == ExecutionProvider.ROCm: + if version.parse(ort.__version__) < version.parse("1.17.0"): + olive_config["passes"][pass_key]["config"]["optimization_options"] = {"enable_skip_group_norm": False} + if shared.opts.olive_float16: + olive_config["passes"][pass_key]["config"]["keep_io_types"] = False run(olive_config) with open(os.path.join("footprints", f"{submodel}_{EP_TO_NAME[shared.opts.onnx_execution_provider]}_footprints.json"), "r") as footprint_file: footprints = json.load(footprint_file) - optimizer_footprint = None + processor_final_pass_footprint = None for _, footprint in footprints.items(): - if footprint["from_pass"] == "OrtTransformersOptimization": - optimizer_footprint = footprint + if footprint["from_pass"] == olive_config["passes"][olive_config["pass_flows"][-1][-1]]["type"]: + processor_final_pass_footprint = footprint - assert optimizer_footprint, "Failed to optimize model" + assert processor_final_pass_footprint, "Failed to optimize model" optimized_model_paths[submodel] = ONNXModel( - **optimizer_footprint["model_config"]["config"] + **processor_final_pass_footprint["model_config"]["config"] ).model_path - log.info(f"Optimized {submodel}") + log.info(f"Processed {submodel}") for submodel in self.submodels: src_path = optimized_model_paths[submodel]