diff --git a/configs/olive/sd/text_encoder.json b/configs/olive/sd/text_encoder.json new file mode 100644 index 000000000..a6aa757b0 --- /dev/null +++ b/configs/olive/sd/text_encoder.json @@ -0,0 +1 @@ +{"input_model": {"type": "PyTorchModel", "config": {"model_path": "", "model_loader": "text_encoder_load", "model_script": "modules/olive.py", "io_config": {"input_names": ["input_ids"], "output_names": ["last_hidden_state", "pooler_output"], "dynamic_axes": {"input_ids": {"0": "batch", "1": "sequence"}}}, "dummy_inputs_func": "text_encoder_conversion_inputs"}}, "systems": {"local_system": {"type": "LocalSystem", "config": {"accelerators": ["gpu"]}}}, "evaluators": {"common_evaluator": {"metrics": [{"name": "latency", "type": "latency", "sub_types": [{"name": "avg"}], "user_config": {"user_script": "modules/olive.py", "dataloader_func": "text_encoder_data_loader", "batch_size": 1}}]}}, "passes": {"optimize_DmlExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "clip", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false, "optimization_options": {"enable_gelu": true, "enable_layer_norm": true, "enable_attention": true, "use_multi_head_attention": true, "enable_skip_layer_norm": false, "enable_embed_layer_norm": true, "enable_bias_skip_layer_norm": false, "enable_bias_gelu": true, "enable_gelu_approximation": false, "enable_qordered_matmul": false, "enable_shape_inference": true, "enable_gemm_fast_gelu": false, "enable_nhwc_conv": false, "enable_group_norm": true, "enable_bias_splitgelu": false, "enable_packed_qkv": true, "enable_packed_kv": true, "enable_bias_add": false, "group_norm_channels_last": false}, "force_fp32_ops": ["RandomNormalLike"], "force_fp16_inputs": {"GroupNorm": [0, 1, 2]}}}, "optimize_CUDAExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "clip", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}, "optimize_ROCMExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "clip", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}}, "pass_flows": [[]], "engine": {"search_strategy": {"execution_order": "joint", "search_algorithm": "exhaustive"}, "evaluator": "common_evaluator", "evaluate_input_model": false, "host": "local_system", "target": "local_system", "cache_dir": "cache", "output_name": "text_encoder", "output_dir": "footprints", "execution_providers": ["DmlExecutionProvider"]}} \ No newline at end of file diff --git a/configs/olive/sd/unet.json b/configs/olive/sd/unet.json new file mode 100644 index 000000000..5b02f5535 --- /dev/null +++ b/configs/olive/sd/unet.json @@ -0,0 +1 @@ +{"input_model": {"type": "PyTorchModel", "config": {"model_path": "", "model_loader": "unet_load", "model_script": "modules/olive.py", "io_config": {"input_names": ["sample", "timestep", "encoder_hidden_states", "return_dict"], "output_names": ["out_sample"], "dynamic_axes": {"sample": {"0": "unet_sample_batch", "1": "unet_sample_channels", "2": "unet_sample_height", "3": "unet_sample_width"}, "timestep": {"0": "unet_time_batch"}, "encoder_hidden_states": {"0": "unet_hidden_batch", "1": "unet_hidden_sequence"}}}, "dummy_inputs_func": "unet_conversion_inputs"}}, "systems": {"local_system": {"type": "LocalSystem", "config": {"accelerators": ["gpu"]}}}, "evaluators": {"common_evaluator": {"metrics": [{"name": "latency", "type": "latency", "sub_types": [{"name": "avg"}], "user_config": {"user_script": "modules/olive.py", "dataloader_func": "unet_data_loader", "batch_size": 2}}]}}, "passes": {"optimize_DmlExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "unet", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false, "optimization_options": {"enable_gelu": true, "enable_layer_norm": true, "enable_attention": true, "use_multi_head_attention": true, "enable_skip_layer_norm": false, "enable_embed_layer_norm": true, "enable_bias_skip_layer_norm": false, "enable_bias_gelu": true, "enable_gelu_approximation": false, "enable_qordered_matmul": false, "enable_shape_inference": true, "enable_gemm_fast_gelu": false, "enable_nhwc_conv": false, "enable_group_norm": true, "enable_bias_splitgelu": false, "enable_packed_qkv": true, "enable_packed_kv": true, "enable_bias_add": false, "group_norm_channels_last": false}, "force_fp32_ops": ["RandomNormalLike"], "force_fp16_inputs": {"GroupNorm": [0, 1, 2]}}}, "optimize_CUDAExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "unet", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}, "optimize_ROCMExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "unet", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}}, "pass_flows": [[]], "engine": {"search_strategy": {"execution_order": "joint", "search_algorithm": "exhaustive"}, "evaluator": "common_evaluator", "evaluate_input_model": false, "host": "local_system", "target": "local_system", "cache_dir": "cache", "output_name": "unet", "output_dir": "footprints", "execution_providers": ["DmlExecutionProvider"]}} \ No newline at end of file diff --git a/configs/olive/sd/vae_decoder.json b/configs/olive/sd/vae_decoder.json new file mode 100644 index 000000000..0601f40ad --- /dev/null +++ b/configs/olive/sd/vae_decoder.json @@ -0,0 +1 @@ +{"input_model": {"type": "PyTorchModel", "config": {"model_path": "", "model_loader": "vae_decoder_load", "model_script": "modules/olive.py", "io_config": {"input_names": ["latent_sample", "return_dict"], "output_names": ["sample"], "dynamic_axes": {"latent_sample": {"0": "batch", "1": "channels", "2": "height", "3": "width"}}}, "dummy_inputs_func": "vae_decoder_conversion_inputs"}}, "systems": {"local_system": {"type": "LocalSystem", "config": {"accelerators": ["gpu"]}}}, "evaluators": {"common_evaluator": {"metrics": [{"name": "latency", "type": "latency", "sub_types": [{"name": "avg"}], "user_config": {"user_script": "modules/olive.py", "dataloader_func": "vae_decoder_data_loader", "batch_size": 1}}]}}, "passes": {"optimize_DmlExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "vae", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false, "optimization_options": {"enable_gelu": true, "enable_layer_norm": true, "enable_attention": true, "use_multi_head_attention": true, "enable_skip_layer_norm": false, "enable_embed_layer_norm": true, "enable_bias_skip_layer_norm": false, "enable_bias_gelu": true, "enable_gelu_approximation": false, "enable_qordered_matmul": false, "enable_shape_inference": true, "enable_gemm_fast_gelu": false, "enable_nhwc_conv": false, "enable_group_norm": true, "enable_bias_splitgelu": false, "enable_packed_qkv": true, "enable_packed_kv": true, "enable_bias_add": false, "group_norm_channels_last": false}, "force_fp32_ops": ["RandomNormalLike"], "force_fp16_inputs": {"GroupNorm": [0, 1, 2]}}}, "optimize_CUDAExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "vae", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}, "optimize_ROCMExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "vae", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}}, "pass_flows": [[]], "engine": {"search_strategy": {"execution_order": "joint", "search_algorithm": "exhaustive"}, "evaluator": "common_evaluator", "evaluate_input_model": false, "host": "local_system", "target": "local_system", "cache_dir": "cache", "output_name": "vae_decoder", "output_dir": "footprints", "execution_providers": ["DmlExecutionProvider"]}} \ No newline at end of file diff --git a/configs/olive/sd/vae_encoder.json b/configs/olive/sd/vae_encoder.json new file mode 100644 index 000000000..30ff18bd0 --- /dev/null +++ b/configs/olive/sd/vae_encoder.json @@ -0,0 +1 @@ +{"input_model": {"type": "PyTorchModel", "config": {"model_path": "", "model_loader": "vae_encoder_load", "model_script": "modules/olive.py", "io_config": {"input_names": ["sample", "return_dict"], "output_names": ["latent_sample"], "dynamic_axes": {"sample": {"0": "batch", "1": "channels", "2": "height", "3": "width"}}}, "dummy_inputs_func": "vae_encoder_conversion_inputs"}}, "systems": {"local_system": {"type": "LocalSystem", "config": {"accelerators": ["gpu"]}}}, "evaluators": {"common_evaluator": {"metrics": [{"name": "latency", "type": "latency", "sub_types": [{"name": "avg"}], "user_config": {"user_script": "modules/olive.py", "dataloader_func": "vae_encoder_data_loader", "batch_size": 1}}]}}, "passes": {"optimize_DmlExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "vae", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false, "optimization_options": {"enable_gelu": true, "enable_layer_norm": true, "enable_attention": true, "use_multi_head_attention": true, "enable_skip_layer_norm": false, "enable_embed_layer_norm": true, "enable_bias_skip_layer_norm": false, "enable_bias_gelu": true, "enable_gelu_approximation": false, "enable_qordered_matmul": false, "enable_shape_inference": true, "enable_gemm_fast_gelu": false, "enable_nhwc_conv": false, "enable_group_norm": true, "enable_bias_splitgelu": false, "enable_packed_qkv": true, "enable_packed_kv": true, "enable_bias_add": false, "group_norm_channels_last": false}, "force_fp32_ops": ["RandomNormalLike"], "force_fp16_inputs": {"GroupNorm": [0, 1, 2]}}}, "optimize_CUDAExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "vae", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}, "optimize_ROCMExecutionProvider": {"type": "OrtTransformersOptimization", "disable_search": true, "config": {"model_type": "vae", "opt_level": 0, "float16": true, "use_gpu": true, "keep_io_types": false}}}, "pass_flows": [[]], "engine": {"search_strategy": {"execution_order": "joint", "search_algorithm": "exhaustive"}, "evaluator": "common_evaluator", "evaluate_input_model": false, "host": "local_system", "target": "local_system", "cache_dir": "cache", "output_name": "vae_encoder", "output_dir": "footprints", "execution_providers": ["DmlExecutionProvider"]}} \ No newline at end of file diff --git a/configs/olive/sd_text_encoder.json b/configs/olive/sd_text_encoder.json deleted file mode 100644 index 639d58ddf..000000000 --- a/configs/olive/sd_text_encoder.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "input_model": { - "type": "PyTorchModel", - "config": { - "model_path": "", - "model_loader": "text_encoder_load", - "model_script": "modules/olive.py", - "io_config": { - "input_names": ["input_ids"], - "output_names": ["last_hidden_state", "pooler_output"], - "dynamic_axes": { "input_ids": { "0": "batch", "1": "sequence" } } - }, - "dummy_inputs_func": "text_encoder_conversion_inputs" - } - }, - "systems": { - "local_system": { - "type": "LocalSystem", - "config": { - "accelerators": ["gpu"] - } - } - }, - "evaluators": { - "common_evaluator": { - "metrics": [ - { - "name": "latency", - "type": "latency", - "sub_types": [{ "name": "avg" }], - "user_config": { - "user_script": "modules/olive.py", - "dataloader_func": "text_encoder_data_loader", - "batch_size": 1 - } - } - ] - } - }, - "passes": { - "optimize_DmlExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "clip", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false, - "optimization_options": { - "enable_gelu": true, - "enable_layer_norm": true, - "enable_attention": true, - "use_multi_head_attention": true, - "enable_skip_layer_norm": false, - "enable_embed_layer_norm": true, - "enable_bias_skip_layer_norm": false, - "enable_bias_gelu": true, - "enable_gelu_approximation": false, - "enable_qordered_matmul": false, - "enable_shape_inference": true, - "enable_gemm_fast_gelu": false, - "enable_nhwc_conv": false, - "enable_group_norm": true, - "enable_bias_splitgelu": false, - "enable_packed_qkv": true, - "enable_packed_kv": true, - "enable_bias_add": false, - "group_norm_channels_last": false - }, - "force_fp32_ops": ["RandomNormalLike"], - "force_fp16_inputs": { - "GroupNorm": [0, 1, 2] - } - } - }, - "optimize_CUDAExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "clip", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - }, - "optimize_ROCMExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "clip", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - } - }, - "pass_flows": [[]], - "engine": { - "search_strategy": { - "execution_order": "joint", - "search_algorithm": "exhaustive" - }, - "evaluator": "common_evaluator", - "evaluate_input_model": false, - "host": "local_system", - "target": "local_system", - "cache_dir": "cache", - "output_name": "text_encoder", - "output_dir": "footprints", - "execution_providers": ["DmlExecutionProvider"] - } -} diff --git a/configs/olive/sd_unet.json b/configs/olive/sd_unet.json deleted file mode 100644 index c34b8987c..000000000 --- a/configs/olive/sd_unet.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "input_model": { - "type": "PyTorchModel", - "config": { - "model_path": "", - "model_loader": "unet_load", - "model_script": "modules/olive.py", - "io_config": { - "input_names": [ - "sample", - "timestep", - "encoder_hidden_states", - "return_dict" - ], - "output_names": ["out_sample"], - "dynamic_axes": { - "sample": { - "0": "unet_sample_batch", - "1": "unet_sample_channels", - "2": "unet_sample_height", - "3": "unet_sample_width" - }, - "timestep": { "0": "unet_time_batch" }, - "encoder_hidden_states": { - "0": "unet_hidden_batch", - "1": "unet_hidden_sequence" - } - } - }, - "dummy_inputs_func": "unet_conversion_inputs" - } - }, - "systems": { - "local_system": { - "type": "LocalSystem", - "config": { - "accelerators": ["gpu"] - } - } - }, - "evaluators": { - "common_evaluator": { - "metrics": [ - { - "name": "latency", - "type": "latency", - "sub_types": [{ "name": "avg" }], - "user_config": { - "user_script": "modules/olive.py", - "dataloader_func": "unet_data_loader", - "batch_size": 2 - } - } - ] - } - }, - "passes": { - "optimize_DmlExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "unet", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false, - "optimization_options": { - "enable_gelu": true, - "enable_layer_norm": true, - "enable_attention": true, - "use_multi_head_attention": true, - "enable_skip_layer_norm": false, - "enable_embed_layer_norm": true, - "enable_bias_skip_layer_norm": false, - "enable_bias_gelu": true, - "enable_gelu_approximation": false, - "enable_qordered_matmul": false, - "enable_shape_inference": true, - "enable_gemm_fast_gelu": false, - "enable_nhwc_conv": false, - "enable_group_norm": true, - "enable_bias_splitgelu": false, - "enable_packed_qkv": true, - "enable_packed_kv": true, - "enable_bias_add": false, - "group_norm_channels_last": false - }, - "force_fp32_ops": ["RandomNormalLike"], - "force_fp16_inputs": { - "GroupNorm": [0, 1, 2] - } - } - }, - "optimize_CUDAExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "unet", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - }, - "optimize_ROCMExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "unet", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - } - }, - "pass_flows": [[]], - "engine": { - "search_strategy": { - "execution_order": "joint", - "search_algorithm": "exhaustive" - }, - "evaluator": "common_evaluator", - "evaluate_input_model": false, - "host": "local_system", - "target": "local_system", - "cache_dir": "cache", - "output_name": "unet", - "output_dir": "footprints", - "execution_providers": ["DmlExecutionProvider"] - } -} diff --git a/configs/olive/sd_vae_decoder.json b/configs/olive/sd_vae_decoder.json deleted file mode 100644 index e30c8e037..000000000 --- a/configs/olive/sd_vae_decoder.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "input_model": { - "type": "PyTorchModel", - "config": { - "model_path": "", - "model_loader": "vae_decoder_load", - "model_script": "modules/olive.py", - "io_config": { - "input_names": ["latent_sample", "return_dict"], - "output_names": ["sample"], - "dynamic_axes": { - "latent_sample": { - "0": "batch", - "1": "channels", - "2": "height", - "3": "width" - } - } - }, - "dummy_inputs_func": "vae_decoder_conversion_inputs" - } - }, - "systems": { - "local_system": { - "type": "LocalSystem", - "config": { - "accelerators": ["gpu"] - } - } - }, - "evaluators": { - "common_evaluator": { - "metrics": [ - { - "name": "latency", - "type": "latency", - "sub_types": [{ "name": "avg" }], - "user_config": { - "user_script": "modules/olive.py", - "dataloader_func": "vae_decoder_data_loader", - "batch_size": 1 - } - } - ] - } - }, - "passes": { - "optimize_DmlExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "vae", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false, - "optimization_options": { - "enable_gelu": true, - "enable_layer_norm": true, - "enable_attention": true, - "use_multi_head_attention": true, - "enable_skip_layer_norm": false, - "enable_embed_layer_norm": true, - "enable_bias_skip_layer_norm": false, - "enable_bias_gelu": true, - "enable_gelu_approximation": false, - "enable_qordered_matmul": false, - "enable_shape_inference": true, - "enable_gemm_fast_gelu": false, - "enable_nhwc_conv": false, - "enable_group_norm": true, - "enable_bias_splitgelu": false, - "enable_packed_qkv": true, - "enable_packed_kv": true, - "enable_bias_add": false, - "group_norm_channels_last": false - }, - "force_fp32_ops": ["RandomNormalLike"], - "force_fp16_inputs": { - "GroupNorm": [0, 1, 2] - } - } - }, - "optimize_CUDAExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "vae", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - }, - "optimize_ROCMExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "vae", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - } - }, - "pass_flows": [[]], - "engine": { - "search_strategy": { - "execution_order": "joint", - "search_algorithm": "exhaustive" - }, - "evaluator": "common_evaluator", - "evaluate_input_model": false, - "host": "local_system", - "target": "local_system", - "cache_dir": "cache", - "output_name": "vae_decoder", - "output_dir": "footprints", - "execution_providers": ["DmlExecutionProvider"] - } -} diff --git a/configs/olive/sd_vae_encoder.json b/configs/olive/sd_vae_encoder.json deleted file mode 100644 index 7f29ca720..000000000 --- a/configs/olive/sd_vae_encoder.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "input_model": { - "type": "PyTorchModel", - "config": { - "model_path": "", - "model_loader": "vae_encoder_load", - "model_script": "modules/olive.py", - "io_config": { - "input_names": ["sample", "return_dict"], - "output_names": ["latent_sample"], - "dynamic_axes": { - "sample": { - "0": "batch", - "1": "channels", - "2": "height", - "3": "width" - } - } - }, - "dummy_inputs_func": "vae_encoder_conversion_inputs" - } - }, - "systems": { - "local_system": { - "type": "LocalSystem", - "config": { - "accelerators": ["gpu"] - } - } - }, - "evaluators": { - "common_evaluator": { - "metrics": [ - { - "name": "latency", - "type": "latency", - "sub_types": [{ "name": "avg" }], - "user_config": { - "user_script": "modules/olive.py", - "dataloader_func": "vae_encoder_data_loader", - "batch_size": 1 - } - } - ] - } - }, - "passes": { - "optimize_DmlExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "vae", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false, - "optimization_options": { - "enable_gelu": true, - "enable_layer_norm": true, - "enable_attention": true, - "use_multi_head_attention": true, - "enable_skip_layer_norm": false, - "enable_embed_layer_norm": true, - "enable_bias_skip_layer_norm": false, - "enable_bias_gelu": true, - "enable_gelu_approximation": false, - "enable_qordered_matmul": false, - "enable_shape_inference": true, - "enable_gemm_fast_gelu": false, - "enable_nhwc_conv": false, - "enable_group_norm": true, - "enable_bias_splitgelu": false, - "enable_packed_qkv": true, - "enable_packed_kv": true, - "enable_bias_add": false, - "group_norm_channels_last": false - }, - "force_fp32_ops": ["RandomNormalLike"], - "force_fp16_inputs": { - "GroupNorm": [0, 1, 2] - } - } - }, - "optimize_CUDAExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "vae", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - }, - "optimize_ROCMExecutionProvider": { - "type": "OrtTransformersOptimization", - "disable_search": true, - "config": { - "model_type": "vae", - "opt_level": 0, - "float16": true, - "use_gpu": true, - "keep_io_types": false - } - } - }, - "pass_flows": [[]], - "engine": { - "search_strategy": { - "execution_order": "joint", - "search_algorithm": "exhaustive" - }, - "evaluator": "common_evaluator", - "evaluate_input_model": false, - "host": "local_system", - "target": "local_system", - "cache_dir": "cache", - "output_name": "vae_encoder", - "output_dir": "footprints", - "execution_providers": ["DmlExecutionProvider"] - } -} diff --git a/configs/olive/sdxl_text_encoder.json b/configs/olive/sdxl/text_encoder.json similarity index 100% rename from configs/olive/sdxl_text_encoder.json rename to configs/olive/sdxl/text_encoder.json diff --git a/configs/olive/sdxl_text_encoder_2.json b/configs/olive/sdxl/text_encoder_2.json similarity index 100% rename from configs/olive/sdxl_text_encoder_2.json rename to configs/olive/sdxl/text_encoder_2.json diff --git a/configs/olive/sdxl_unet.json b/configs/olive/sdxl/unet.json similarity index 100% rename from configs/olive/sdxl_unet.json rename to configs/olive/sdxl/unet.json diff --git a/configs/olive/sdxl_vae_decoder.json b/configs/olive/sdxl/vae_decoder.json similarity index 100% rename from configs/olive/sdxl_vae_decoder.json rename to configs/olive/sdxl/vae_decoder.json diff --git a/configs/olive/sdxl_vae_encoder.json b/configs/olive/sdxl/vae_encoder.json similarity index 100% rename from configs/olive/sdxl_vae_encoder.json rename to configs/olive/sdxl/vae_encoder.json diff --git a/installer.py b/installer.py index 14a16847d..3c9c880c3 100644 --- a/installer.py +++ b/installer.py @@ -379,8 +379,7 @@ def check_torch(): log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}') torch_command = os.environ.get('TORCH_COMMAND', '') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') - if not installed('onnxruntime', quiet=True) and not installed('onnxruntime-gpu', quiet=True): # allow either - install('onnxruntime', 'onnxruntime', ignore=True) + install('onnxruntime', 'onnxruntime', ignore=True) if torch_command != '': pass elif allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): diff --git a/modules/onnx_ep.py b/modules/onnx_ep.py index 3d0e40f91..9028c6a33 100644 --- a/modules/onnx_ep.py +++ b/modules/onnx_ep.py @@ -1,3 +1,4 @@ +import sys from enum import Enum from typing import Tuple, List import onnxruntime as ort @@ -69,3 +70,45 @@ def get_provider() -> Tuple: from modules.shared import opts return (opts.onnx_execution_provider, get_execution_provider_options(),) + + +def install_execution_provider(ep: ExecutionProvider): + from installer import pip, uninstall, installed + from modules.shared import log + + if installed("onnxruntime"): + uninstall("onnxruntime") + if installed("onnxruntime-directml"): + uninstall("onnxruntime-directml") + if installed("onnxruntime-gpu"): + uninstall("onnxruntime-gpu") + if installed("onnxruntime-training"): + uninstall("onnxruntime-training") + if installed("onnxruntime-openvino"): + uninstall("onnxruntime-openvino") + + packages = ["onnxruntime"] # Failed to load olive: cannot import name '__version__' from 'onnxruntime' + + if ep == ExecutionProvider.DirectML: + packages.append("onnxruntime-directml") + elif ep == ExecutionProvider.CUDA: + packages.append("onnxruntime-gpu") + elif ep == ExecutionProvider.ROCm: + if "linux" not in sys.platform: + log.warn("ROCMExecutionProvider is not supported on Windows.") + return + + try: + major, minor = sys.version_info + cp_str = f"{major}{minor}" + packages.append(f"https://download.onnxruntime.ai/onnxruntime_training-1.16.3%2Brocm56-cp{cp_str}-cp{cp_str}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") + except Exception: + log.warn("Failed to install onnxruntime for ROCm.") + elif ep == ExecutionProvider.OpenVINO: + if installed("openvino"): + uninstall("openvino") + packages.append("openvino") + packages.append("onnxruntime-openvino") + + pip(f"install --upgrade {' '.join(packages)}") + log.info("Please restart SD.Next.") diff --git a/modules/onnx_pipelines.py b/modules/onnx_pipelines.py index 6bcbc85d8..3339aae49 100644 --- a/modules/onnx_pipelines.py +++ b/modules/onnx_pipelines.py @@ -72,23 +72,17 @@ class OnnxRawPipeline(OnnxPipelineBase): def __init__(self, constructor: Type[OnnxPipelineBase], path: os.PathLike): self.model_type = constructor.__name__ self._is_sdxl = check_pipeline_sdxl(constructor) - self.is_refiner = self._is_sdxl and "Img2Img" in diffusers.DiffusionPipeline.load_config(path)["_class_name"] self.from_huggingface_cache = shared.opts.diffusers_dir in os.path.abspath(path) self.path = path self.original_filename = os.path.basename(path) - self.constructor = construct_refiner_pipeline if self.is_refiner else constructor - self.submodels = (submodels_sdxl_refiner if self.is_refiner else submodels_sdxl) if self._is_sdxl else submodels_sd if os.path.isdir(path): + self.is_refiner = self._is_sdxl and "Img2Img" in diffusers.DiffusionPipeline.load_config(path)["_class_name"] self.init_dict = load_init_dict(constructor, path) self.scheduler = load_submodel(self.path, None, "scheduler", self.init_dict["scheduler"]) else: try: - cls = None - if self._is_sdxl: - cls = diffusers.StableDiffusionXLPipeline - else: - cls = diffusers.StableDiffusionPipeline + cls = diffusers.StableDiffusionXLPipeline if self._is_sdxl else diffusers.StableDiffusionPipeline pipeline = cls.from_single_file(path) self.scheduler = pipeline.scheduler if os.path.isdir(shared.opts.onnx_temp_dir): @@ -96,12 +90,16 @@ class OnnxRawPipeline(OnnxPipelineBase): os.mkdir(shared.opts.onnx_temp_dir) pipeline.save_pretrained(shared.opts.onnx_temp_dir) del pipeline + self.is_refiner = self._is_sdxl and "Img2Img" in diffusers.DiffusionPipeline.load_config(shared.opts.onnx_temp_dir)["_class_name"] self.init_dict = load_init_dict(constructor, shared.opts.onnx_temp_dir) except Exception: log.error('Failed to load pipeline to optimize.') if "vae" in self.init_dict: del self.init_dict["vae"] + self.constructor = construct_refiner_pipeline if self.is_refiner else constructor + self.submodels = (submodels_sdxl_refiner if self.is_refiner else submodels_sdxl) if self._is_sdxl else submodels_sd + def derive_properties(self, pipeline: diffusers.DiffusionPipeline): pipeline.sd_model_hash = self.sd_model_hash pipeline.sd_checkpoint_info = self.sd_checkpoint_info @@ -237,9 +235,9 @@ class OnnxRawPipeline(OnnxPipelineBase): optimized_model_paths = {} for submodel in self.submodels: - log.info(f"\nOptimizing {submodel}") + log.info(f"\nProcessing {submodel}") - with open(os.path.join(sd_configs_path, "olive", f"{'sdxl' if self._is_sdxl else 'sd'}_{submodel}.json"), "r") as config_file: + with open(os.path.join(sd_configs_path, "olive", 'sdxl' if self._is_sdxl else 'sd', f"{submodel}.json"), "r") as config_file: olive_config = json.load(config_file) pass_key = f"optimize_{shared.opts.onnx_execution_provider}" olive_config["pass_flows"] = [[pass_key]] diff --git a/modules/ui.py b/modules/ui.py index b6d99ed78..6b94fafcb 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -376,6 +376,12 @@ def create_ui(startup_timer = None): interfaces += [(extensions_interface, "Extensions", "extensions")] timer.startup.record("ui-extensions") + if shared.opts.diffusers_pipeline.startswith("ONNX"): + from modules import ui_onnx + + onnx_interface = ui_onnx.create_ui() + interfaces += [(onnx_interface, "ONNX", "onnx")] + shared.tab_names = [] for _interface, label, _ifid in interfaces: shared.tab_names.append(label) diff --git a/modules/ui_onnx.py b/modules/ui_onnx.py new file mode 100644 index 000000000..e5603e807 --- /dev/null +++ b/modules/ui_onnx.py @@ -0,0 +1,126 @@ +import os +import json +from typing import Dict, List, Union +import gradio as gr +from olive.passes import REGISTRY + + +def get_recursively(d: Union[Dict, List], *args): + if len(args) == 0: + return d + return get_recursively(d.get(args[0]), *args[1:]) + + +def create_ui(): + from modules.ui_components import DropdownMulti + from modules.shared import log, opts, cmd_opts + from modules.paths import sd_configs_path + from modules.onnx_ep import ExecutionProvider, install_execution_provider + + with gr.Blocks(analytics_enabled=False) as ui: + with gr.Row(): + with gr.Tabs(elem_id="tabs_onnx"): + with gr.TabItem("Manage execution providers", id="onnxep"): + choices = [] + + for ep in ExecutionProvider: + choices.append(ep) + + ep_default = None + if cmd_opts.use_directml: + ep_default = ExecutionProvider.DirectML + elif cmd_opts.use_cuda: + ep_default = ExecutionProvider.CUDA + elif cmd_opts.use_rocm: + ep_default = ExecutionProvider.ROCm + elif cmd_opts.use_openvino: + ep_default = ExecutionProvider.OpenVINO + + ep_checkbox = gr.Radio(label="Execution provider", value=ep_default, choices=choices) + ep_install = gr.Button(value="Install") + gr.Text("Warning! If you are trying to reinstall, it may not work due to permission issue.") + + ep_install.click(fn=install_execution_provider, inputs=ep_checkbox) + + if opts.cuda_compile_backend == "olive-ai": + with gr.Tabs(elem_id="tabs_olive"): + with gr.TabItem("Customize pass flow", id="pass_flow"): + with gr.Tabs(elem_id="tabs_model_type"): + with gr.TabItem("Stable Diffusion", id="sd"): + sd_config_path = os.path.join(sd_configs_path, "olive", "sd") + sd_submodels = os.listdir(sd_config_path) + sd_configs: Dict[str, Dict] = {} + + with gr.Tabs(elem_id="tabs_sd_submodel"): + def sd_create_change_listener(*args): + def listener(v: Dict): + get_recursively(sd_configs, *args[:-1])[args[-1]] = v + return listener + + for submodel in sd_submodels: + config: Dict = None + + with open(os.path.join(sd_config_path, submodel), "r") as file: + config = json.load(file) + sd_configs[submodel] = config + + submodel_name = submodel[:-5] + with gr.TabItem(submodel_name, id=f"sd_{submodel_name}"): + pass_flows = DropdownMulti(label="Pass flow", value=sd_configs[submodel]["pass_flows"][0], choices=sd_configs[submodel]["passes"].keys()) + pass_flows.change(fn=sd_create_change_listener(submodel, "pass_flows", 0), inputs=pass_flows) + + with gr.Tabs(elem_id=f"tabs_sd_{submodel_name}_pass"): + for k in sd_configs[submodel]["passes"]: + with gr.TabItem(k, id=f"sd_{submodel_name}_pass_{k}"): + pass_type = gr.Dropdown(label="Type", value=sd_configs[submodel]["passes"][k]["type"], choices=(x.__name__ for x in tuple(REGISTRY.values()))) + + pass_type.change(fn=sd_create_change_listener(submodel, "passes", k, "type"), inputs=pass_type) + + def sd_save(): + for k, v in sd_configs.items(): + with open(os.path.join(sd_config_path, k), "w") as file: + json.dump(v, file) + log.info("Olive: config for SD was saved.") + + sd_save_button = gr.Button(value="Save") + sd_save_button.click(fn=sd_save) + + with gr.TabItem("Stable Diffusion XL", id="sdxl"): + sdxl_config_path = os.path.join(sd_configs_path, "olive", "sdxl") + sdxl_submodels = os.listdir(sdxl_config_path) + sdxl_configs: Dict[str, Dict] = {} + + with gr.Tabs(elem_id="tabs_sdxl_submodel"): + def sdxl_create_change_listener(*args): + def listener(v: Dict): + get_recursively(sdxl_configs, *args[:-1])[args[-1]] = v + return listener + + for submodel in sdxl_submodels: + config: Dict = None + + with open(os.path.join(sdxl_config_path, submodel), "r") as file: + config = json.load(file) + sdxl_configs[submodel] = config + + submodel_name = submodel[:-5] + with gr.TabItem(submodel_name, id=f"sdxl_{submodel_name}"): + pass_flows = DropdownMulti(label="Pass flow", value=sdxl_configs[submodel]["pass_flows"][0], choices=sdxl_configs[submodel]["passes"].keys()) + pass_flows.change(fn=sdxl_create_change_listener(submodel, "pass_flows", 0), inputs=pass_flows) + + with gr.Tabs(elem_id=f"tabs_sdxl_{submodel_name}_pass"): + for k in sdxl_configs[submodel]["passes"]: + with gr.TabItem(k, id=f"sdxl_{submodel_name}_pass_{k}"): + pass_type = gr.Dropdown(label="Type", value=sdxl_configs[submodel]["passes"][k]["type"], choices=(x.__name__ for x in tuple(REGISTRY.values()))) + + pass_type.change(fn=sdxl_create_change_listener(submodel, "passes", k, "type"), inputs=pass_type) + + def sdxl_save(): + for k, v in sdxl_configs.items(): + with open(os.path.join(sdxl_config_path, k), "w") as file: + json.dump(v, file) + log.info("Olive: config for SDXL was saved.") + + sdxl_save_button = gr.Button(value="Save") + sdxl_save_button.click(fn=sdxl_save) + return ui