From ba66bf79001a5f3679eb5e20e5d5b3dbe071ac43 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sat, 4 Nov 2023 15:25:09 +0900 Subject: [PATCH] add sdxl configs --- ...text_encoder.json => sd_text_encoder.json} | 0 .../olive/{config_unet.json => sd_unet.json} | 0 ...g_vae_decoder.json => sd_vae_decoder.json} | 0 ...g_vae_encoder.json => sd_vae_encoder.json} | 0 configs/olive/sdxl_text_encoder.json | 97 +++++++++++++++ configs/olive/sdxl_text_encoder_2.json | 97 +++++++++++++++ configs/olive/sdxl_unet.json | 117 ++++++++++++++++++ configs/olive/sdxl_vae_decoder.json | 104 ++++++++++++++++ configs/olive/sdxl_vae_encoder.json | 104 ++++++++++++++++ modules/olive.py | 2 +- 10 files changed, 520 insertions(+), 1 deletion(-) rename configs/olive/{config_text_encoder.json => sd_text_encoder.json} (100%) rename configs/olive/{config_unet.json => sd_unet.json} (100%) rename configs/olive/{config_vae_decoder.json => sd_vae_decoder.json} (100%) rename configs/olive/{config_vae_encoder.json => sd_vae_encoder.json} (100%) create mode 100644 configs/olive/sdxl_text_encoder.json create mode 100644 configs/olive/sdxl_text_encoder_2.json create mode 100644 configs/olive/sdxl_unet.json create mode 100644 configs/olive/sdxl_vae_decoder.json create mode 100644 configs/olive/sdxl_vae_encoder.json diff --git a/configs/olive/config_text_encoder.json b/configs/olive/sd_text_encoder.json similarity index 100% rename from configs/olive/config_text_encoder.json rename to configs/olive/sd_text_encoder.json diff --git a/configs/olive/config_unet.json b/configs/olive/sd_unet.json similarity index 100% rename from configs/olive/config_unet.json rename to configs/olive/sd_unet.json diff --git a/configs/olive/config_vae_decoder.json b/configs/olive/sd_vae_decoder.json similarity index 100% rename from configs/olive/config_vae_decoder.json rename to configs/olive/sd_vae_decoder.json diff --git a/configs/olive/config_vae_encoder.json b/configs/olive/sd_vae_encoder.json similarity index 100% rename from configs/olive/config_vae_encoder.json rename to configs/olive/sd_vae_encoder.json diff --git a/configs/olive/sdxl_text_encoder.json b/configs/olive/sdxl_text_encoder.json new file mode 100644 index 000000000..e4ae6fcb1 --- /dev/null +++ b/configs/olive/sdxl_text_encoder.json @@ -0,0 +1,97 @@ +{ + "input_model": { + "type": "PyTorchModel", + "config": { + "model_path": "", + "model_loader": "text_encoder_load", + "model_script": "modules/olive.py", + "io_config": { + "input_names": ["input_ids"], + "output_names": ["last_hidden_state", "pooler_output"], + "dynamic_axes": { "input_ids": { "0": "batch", "1": "sequence" } } + }, + "dummy_inputs_func": "text_encoder_conversion_inputs" + } + }, + "systems": { + "local_system": { + "type": "LocalSystem", + "config": { + "accelerators": ["gpu"] + } + } + }, + "evaluators": { + "common_evaluator": { + "metrics": [ + { + "name": "latency", + "type": "latency", + "sub_types": [{ "name": "avg" }], + "user_config": { + "user_script": "modules/olive.py", + "dataloader_func": "text_encoder_data_loader", + "batch_size": 1 + } + } + ] + } + }, + "passes": { + "convert": { + "type": "OnnxConversion", + "config": { + "target_opset": 14 + } + }, + "optimize": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "clip", + "float16": true, + "use_gpu": true, + "keep_io_types": false, + "optimization_options": { + "enable_gelu": true, + "enable_layer_norm": true, + "enable_attention": true, + "use_multi_head_attention": true, + "enable_skip_layer_norm": false, + "enable_embed_layer_norm": true, + "enable_bias_skip_layer_norm": false, + "enable_bias_gelu": true, + "enable_gelu_approximation": false, + "enable_qordered_matmul": false, + "enable_shape_inference": true, + "enable_gemm_fast_gelu": false, + "enable_nhwc_conv": false, + "enable_group_norm": true, + "enable_bias_splitgelu": false, + "enable_packed_qkv": true, + "enable_packed_kv": true, + "enable_bias_add": false, + "group_norm_channels_last": false, + "force_fp16_inputs": { + "GroupNorm": [0, 1, 2] + } + }, + "force_fp32_ops": ["RandomNormalLike"] + } + } + }, + "engine": { + "search_strategy": { + "execution_order": "joint", + "search_algorithm": "exhaustive" + }, + "evaluator": "common_evaluator", + "evaluate_input_model": false, + "host": "local_system", + "target": "local_system", + "cache_dir": "cache", + "output_name": "text_encoder", + "output_dir": "footprints", + "execution_providers": ["DmlExecutionProvider"] + } +} diff --git a/configs/olive/sdxl_text_encoder_2.json b/configs/olive/sdxl_text_encoder_2.json new file mode 100644 index 000000000..e4ae6fcb1 --- /dev/null +++ b/configs/olive/sdxl_text_encoder_2.json @@ -0,0 +1,97 @@ +{ + "input_model": { + "type": "PyTorchModel", + "config": { + "model_path": "", + "model_loader": "text_encoder_load", + "model_script": "modules/olive.py", + "io_config": { + "input_names": ["input_ids"], + "output_names": ["last_hidden_state", "pooler_output"], + "dynamic_axes": { "input_ids": { "0": "batch", "1": "sequence" } } + }, + "dummy_inputs_func": "text_encoder_conversion_inputs" + } + }, + "systems": { + "local_system": { + "type": "LocalSystem", + "config": { + "accelerators": ["gpu"] + } + } + }, + "evaluators": { + "common_evaluator": { + "metrics": [ + { + "name": "latency", + "type": "latency", + "sub_types": [{ "name": "avg" }], + "user_config": { + "user_script": "modules/olive.py", + "dataloader_func": "text_encoder_data_loader", + "batch_size": 1 + } + } + ] + } + }, + "passes": { + "convert": { + "type": "OnnxConversion", + "config": { + "target_opset": 14 + } + }, + "optimize": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "clip", + "float16": true, + "use_gpu": true, + "keep_io_types": false, + "optimization_options": { + "enable_gelu": true, + "enable_layer_norm": true, + "enable_attention": true, + "use_multi_head_attention": true, + "enable_skip_layer_norm": false, + "enable_embed_layer_norm": true, + "enable_bias_skip_layer_norm": false, + "enable_bias_gelu": true, + "enable_gelu_approximation": false, + "enable_qordered_matmul": false, + "enable_shape_inference": true, + "enable_gemm_fast_gelu": false, + "enable_nhwc_conv": false, + "enable_group_norm": true, + "enable_bias_splitgelu": false, + "enable_packed_qkv": true, + "enable_packed_kv": true, + "enable_bias_add": false, + "group_norm_channels_last": false, + "force_fp16_inputs": { + "GroupNorm": [0, 1, 2] + } + }, + "force_fp32_ops": ["RandomNormalLike"] + } + } + }, + "engine": { + "search_strategy": { + "execution_order": "joint", + "search_algorithm": "exhaustive" + }, + "evaluator": "common_evaluator", + "evaluate_input_model": false, + "host": "local_system", + "target": "local_system", + "cache_dir": "cache", + "output_name": "text_encoder", + "output_dir": "footprints", + "execution_providers": ["DmlExecutionProvider"] + } +} diff --git a/configs/olive/sdxl_unet.json b/configs/olive/sdxl_unet.json new file mode 100644 index 000000000..d370ccf56 --- /dev/null +++ b/configs/olive/sdxl_unet.json @@ -0,0 +1,117 @@ +{ + "input_model": { + "type": "PyTorchModel", + "config": { + "model_path": "", + "model_loader": "unet_load", + "model_script": "modules/olive.py", + "io_config": { + "input_names": [ + "sample", + "timestep", + "encoder_hidden_states", + "return_dict" + ], + "output_names": ["out_sample"], + "dynamic_axes": { + "sample": { + "0": "unet_sample_batch", + "1": "unet_sample_channels", + "2": "unet_sample_height", + "3": "unet_sample_width" + }, + "timestep": { "0": "unet_time_batch" }, + "encoder_hidden_states": { + "0": "unet_hidden_batch", + "1": "unet_hidden_sequence" + } + } + }, + "dummy_inputs_func": "unet_conversion_inputs" + } + }, + "systems": { + "local_system": { + "type": "LocalSystem", + "config": { + "accelerators": ["gpu"] + } + } + }, + "evaluators": { + "common_evaluator": { + "metrics": [ + { + "name": "latency", + "type": "latency", + "sub_types": [{ "name": "avg" }], + "user_config": { + "user_script": "modules/olive.py", + "dataloader_func": "unet_data_loader", + "batch_size": 2 + } + } + ] + } + }, + "passes": { + "convert": { + "type": "OnnxConversion", + "config": { + "target_opset": 14, + "save_as_external_data": true, + "all_tensors_to_one_file": true, + "external_data_name": "weights.pb" + } + }, + "optimize": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "unet", + "float16": true, + "use_gpu": true, + "keep_io_types": false, + "optimization_options": { + "enable_gelu": true, + "enable_layer_norm": true, + "enable_attention": true, + "use_multi_head_attention": true, + "enable_skip_layer_norm": false, + "enable_embed_layer_norm": true, + "enable_bias_skip_layer_norm": false, + "enable_bias_gelu": true, + "enable_gelu_approximation": false, + "enable_qordered_matmul": false, + "enable_shape_inference": true, + "enable_gemm_fast_gelu": false, + "enable_nhwc_conv": false, + "enable_group_norm": true, + "enable_bias_splitgelu": false, + "enable_packed_qkv": true, + "enable_packed_kv": true, + "enable_bias_add": false, + "group_norm_channels_last": false, + "force_fp16_inputs": { + "GroupNorm": [0, 1, 2] + } + }, + "force_fp32_ops": ["RandomNormalLike"] + } + } + }, + "engine": { + "search_strategy": { + "execution_order": "joint", + "search_algorithm": "exhaustive" + }, + "evaluator": "common_evaluator", + "evaluate_input_model": false, + "host": "local_system", + "target": "local_system", + "cache_dir": "cache", + "output_name": "unet", + "output_dir": "footprints", + "execution_providers": ["DmlExecutionProvider"] + } +} diff --git a/configs/olive/sdxl_vae_decoder.json b/configs/olive/sdxl_vae_decoder.json new file mode 100644 index 000000000..86095098f --- /dev/null +++ b/configs/olive/sdxl_vae_decoder.json @@ -0,0 +1,104 @@ +{ + "input_model": { + "type": "PyTorchModel", + "config": { + "model_path": "", + "model_loader": "vae_decoder_load", + "model_script": "modules/olive.py", + "io_config": { + "input_names": ["latent_sample", "return_dict"], + "output_names": ["sample"], + "dynamic_axes": { + "latent_sample": { + "0": "batch", + "1": "channels", + "2": "height", + "3": "width" + } + } + }, + "dummy_inputs_func": "vae_decoder_conversion_inputs" + } + }, + "systems": { + "local_system": { + "type": "LocalSystem", + "config": { + "accelerators": ["gpu"] + } + } + }, + "evaluators": { + "common_evaluator": { + "metrics": [ + { + "name": "latency", + "type": "latency", + "sub_types": [{ "name": "avg" }], + "user_config": { + "user_script": "modules/olive.py", + "dataloader_func": "vae_decoder_data_loader", + "batch_size": 1 + } + } + ] + } + }, + "passes": { + "convert": { + "type": "OnnxConversion", + "config": { + "target_opset": 14 + } + }, + "optimize": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "vae", + "float16": true, + "use_gpu": true, + "keep_io_types": false, + "optimization_options": { + "enable_gelu": true, + "enable_layer_norm": true, + "enable_attention": true, + "use_multi_head_attention": true, + "enable_skip_layer_norm": false, + "enable_embed_layer_norm": true, + "enable_bias_skip_layer_norm": false, + "enable_bias_gelu": true, + "enable_gelu_approximation": false, + "enable_qordered_matmul": false, + "enable_shape_inference": true, + "enable_gemm_fast_gelu": false, + "enable_nhwc_conv": false, + "enable_group_norm": true, + "enable_bias_splitgelu": false, + "enable_packed_qkv": true, + "enable_packed_kv": true, + "enable_bias_add": false, + "group_norm_channels_last": false, + "force_fp16_inputs": { + "GroupNorm": [0, 1, 2] + } + }, + "force_fp32_ops": ["RandomNormalLike"] + } + } + }, + "engine": { + "search_strategy": { + "execution_order": "joint", + "search_algorithm": "exhaustive" + }, + "evaluator": "common_evaluator", + "evaluate_input_model": false, + "host": "local_system", + "target": "local_system", + "cache_dir": "cache", + "output_name": "vae_decoder", + "output_dir": "footprints", + "execution_providers": ["DmlExecutionProvider"] + } +} diff --git a/configs/olive/sdxl_vae_encoder.json b/configs/olive/sdxl_vae_encoder.json new file mode 100644 index 000000000..e00dc01f6 --- /dev/null +++ b/configs/olive/sdxl_vae_encoder.json @@ -0,0 +1,104 @@ +{ + "input_model": { + "type": "PyTorchModel", + "config": { + "model_path": "", + "model_loader": "vae_encoder_load", + "model_script": "modules/olive.py", + "io_config": { + "input_names": ["sample", "return_dict"], + "output_names": ["latent_sample"], + "dynamic_axes": { + "sample": { + "0": "batch", + "1": "channels", + "2": "height", + "3": "width" + } + } + }, + "dummy_inputs_func": "vae_encoder_conversion_inputs" + } + }, + "systems": { + "local_system": { + "type": "LocalSystem", + "config": { + "accelerators": ["gpu"] + } + } + }, + "evaluators": { + "common_evaluator": { + "metrics": [ + { + "name": "latency", + "type": "latency", + "sub_types": [{ "name": "avg" }], + "user_config": { + "user_script": "modules/olive.py", + "dataloader_func": "vae_encoder_data_loader", + "batch_size": 1 + } + } + ] + } + }, + "passes": { + "convert": { + "type": "OnnxConversion", + "config": { + "target_opset": 14 + } + }, + "optimize": { + "type": "OrtTransformersOptimization", + "disable_search": true, + "config": { + "model_type": "vae", + "float16": true, + "use_gpu": true, + "keep_io_types": false, + "optimization_options": { + "enable_gelu": true, + "enable_layer_norm": true, + "enable_attention": true, + "use_multi_head_attention": true, + "enable_skip_layer_norm": false, + "enable_embed_layer_norm": true, + "enable_bias_skip_layer_norm": false, + "enable_bias_gelu": true, + "enable_gelu_approximation": false, + "enable_qordered_matmul": false, + "enable_shape_inference": true, + "enable_gemm_fast_gelu": false, + "enable_nhwc_conv": false, + "enable_group_norm": true, + "enable_bias_splitgelu": false, + "enable_packed_qkv": true, + "enable_packed_kv": true, + "enable_bias_add": false, + "group_norm_channels_last": false, + "force_fp16_inputs": { + "GroupNorm": [0, 1, 2] + } + }, + "force_fp32_ops": ["RandomNormalLike"] + } + } + }, + "engine": { + "search_strategy": { + "execution_order": "joint", + "search_algorithm": "exhaustive" + }, + "evaluator": "common_evaluator", + "evaluate_input_model": false, + "host": "local_system", + "target": "local_system", + "cache_dir": "cache", + "output_name": "vae_encoder", + "output_dir": "footprints", + "execution_providers": ["DmlExecutionProvider"] + } +} diff --git a/modules/olive.py b/modules/olive.py index 5dfa87bae..64a9d2387 100644 --- a/modules/olive.py +++ b/modules/olive.py @@ -111,7 +111,7 @@ class OlivePipeline(diffusers.DiffusionPipeline): for submodel in submodels: log.info(f"\nOptimizing {submodel}") - with open(os.path.join(sd_configs_path, "olive", f"config_{submodel}.json"), "r") as config_file: + with open(os.path.join(sd_configs_path, "olive", f"sd_{submodel}.json"), "r") as config_file: olive_config = json.load(config_file) olive_config["passes"]["optimize"]["config"]["float16"] = shared.opts.onnx_olive_float16 if (submodel == "unet" or "vae" in submodel) and (shared.opts.onnx_execution_provider == ExecutionProvider.CUDA or shared.opts.onnx_execution_provider == ExecutionProvider.ROCm):