From d92f85474d00ce3ee8c2a9e929e77331f100f7eb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 27 Jan 2024 18:33:41 +0300 Subject: [PATCH] OpenVINO reverse NNCF order --- modules/intel/openvino/__init__.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 8c3fc7a5c..6771a7691 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -233,11 +233,7 @@ def openvino_compile(gm: GraphModule, *example_inputs, model_hash_str: str = Non om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) om.validate_nodes_and_infer_types() - if shared.opts.nncf_compress_weights and not dont_use_nncf: - if dont_use_4bit_nncf or shared.opts.nncf_compress_weights_mode == "INT8": - om = nncf.compress_weights(om) - else: - om = nncf.compress_weights(om, mode=getattr(nncf.CompressWeightsMode, shared.opts.nncf_compress_weights_mode), group_size=8, ratio=shared.opts.nncf_compress_weights_raito) + if shared.opts.nncf_quantize and not dont_use_quant: new_inputs = [] for idx, _ in enumerate(example_inputs): @@ -250,6 +246,13 @@ def openvino_compile(gm: GraphModule, *example_inputs, model_hash_str: str = Non advanced_parameters=nncf.quantization.advanced_parameters.AdvancedQuantizationParameters( overflow_fix=nncf.quantization.advanced_parameters.OverflowFix.DISABLE, backend_params=None)) + if shared.opts.nncf_compress_weights and not dont_use_nncf: + if dont_use_4bit_nncf or shared.opts.nncf_compress_weights_mode == "INT8": + om = nncf.compress_weights(om) + else: + om = nncf.compress_weights(om, mode=getattr(nncf.CompressWeightsMode, shared.opts.nncf_compress_weights_mode), group_size=8, ratio=shared.opts.nncf_compress_weights_raito) + + if model_hash_str is not None: core.set_property({'CACHE_DIR': cache_root + '/blob'}) dont_use_nncf = False @@ -282,11 +285,7 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs): om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) om.validate_nodes_and_infer_types() - if shared.opts.nncf_compress_weights and not dont_use_nncf: - if dont_use_4bit_nncf or shared.opts.nncf_compress_weights_mode == "INT8": - om = nncf.compress_weights(om) - else: - om = nncf.compress_weights(om, mode=getattr(nncf.CompressWeightsMode, shared.opts.nncf_compress_weights_mode), group_size=8, ratio=shared.opts.nncf_compress_weights_raito) + if shared.opts.nncf_quantize and not dont_use_quant: new_inputs = [] for idx, _ in enumerate(example_inputs): @@ -299,6 +298,12 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs): advanced_parameters=nncf.quantization.advanced_parameters.AdvancedQuantizationParameters( overflow_fix=nncf.quantization.advanced_parameters.OverflowFix.DISABLE, backend_params=None)) + if shared.opts.nncf_compress_weights and not dont_use_nncf: + if dont_use_4bit_nncf or shared.opts.nncf_compress_weights_mode == "INT8": + om = nncf.compress_weights(om) + else: + om = nncf.compress_weights(om, mode=getattr(nncf.CompressWeightsMode, shared.opts.nncf_compress_weights_mode), group_size=8, ratio=shared.opts.nncf_compress_weights_raito) + core.set_property({'CACHE_DIR': shared.opts.openvino_cache_path + '/blob'}) dont_use_nncf = False dont_use_quant = False