动态量化
# Dynamic quantization 动态量化
from onnxruntime.quantization import quantize_dynamic, QuantType
model_input = 'yolov8n-detect.onnx'
model_output = 'yolov8n-detect_dynamic_int8.onnx'
quantized_model = quantize_dynamic(
model_input, # 输入模型
model_output # 输出模型
)
def quantize_dynamic 函数定义
def quantize_dynamic(
model_input: Path,
model_output: Path,
op_types_to_quantize=None,
per_channel=False,
reduce_range=False,
weight_type=QuantType.QInt8,
nodes_to_quantize=None,
nodes_to_exclude=None,
optimize_model=True,
use_external_data_format=False,
extra_options=None,
):
"""Given an onnx model, create a quantized onnx model and save it into a file
Args:
model_input: file path of model to quantize
model_output: file path of quantized model
op_types_to_quantize:
specify the types of operators to quantize, like ['Conv'] to quantize Conv only.
It quantizes all supported operators by default.
per_channel: quantize weights per channel
reduce_range:
quantize weights with 7-bits. It may improve the accuracy for some models running on non-VNNI machine,
especially for per-channel mode
weight_type:
quantization data type of weight. Please refer to
https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
nodes_to_quantize:
List of nodes names to quantize. When this list is not None only the nodes in this list
are quantized.
example:
[
'Conv__224',
'Conv__252'
]
nodes_to_exclude:
List of nodes names to exclude. The nodes in this list will be excluded from quantization
when it is not None.
optimize_model: Deprecating Soon! Optimize model before quantization. NOT recommended, optimization will
change the computation graph, making debugging of quantization loss difficult.
use_external_data_format: option used for large size (>2GB) model. Set to False by default.
extra_options:
key value pair dictionary for various options in different case. Current used:
extra.Sigmoid.nnapi = True/False (Default is False)
ActivationSymmetric = True/False: symmetrize calibration data for activations (default is False).
WeightSymmetric = True/False: symmetrize calibration data for weights (default is True).
EnableSubgraph = True/False :
Default is False. If enabled, subgraph will be quantized. Dynamic mode currently is supported. Will
support more in the future.
ForceQuantizeNoInputCheck = True/False :
By default, some latent operators like maxpool, transpose, do not quantize if their input is not
quantized already. Setting to True to force such operator always quantize input and so generate
quantized output. Also the True behavior could be disabled per node using the nodes_to_exclude.
MatMulConstBOnly = True/False:
Default is True for dynamic mode. If enabled, only MatMul with const B will be quantized.
"""
静态量化
import os
import numpy as np
import time
from PIL import Image
import onnxruntime
from onnxruntime.quantization import quantize_static, CalibrationDataReader, QuantFormat, QuantType
class DataReader(CalibrationDataReader):
def __init__(self, calibration_image_folder, augmented_model_path=None):
self.image_folder = calibration_image_folder
self.augmented_model_path = augmented_model_path
self.preprocess_flag = True
self.enum_data_dicts = []
self.datasize = 0
def get_next(self):
if self.preprocess_flag:
self.preprocess_flag = False
session = onnxruntime.InferenceSession(self.augmented_model_path, None)
(_,_,height, width) = session.get_inputs()[0].shape
nhwc_data_list = proprocess_func(self.image_folder, height, width, size_limit=0)
input_name = session.get_inputs()[0].name
self.datasize = len(nhwc_data_list)
self.enum_data_dicts = iter([{input_name: nhwc_data} for nhwc_data in nhwc_data_list])
return next(self.enum_data_dicts, None)
def proprocess_func(images_folder, height, width, size_limit=0):
image_names = os.listdir(images_folder)
if size_limit > 0 and len(image_names) >= size_limit:
batch_filenames = [image_names[i] for i in range(size_limit)]
else:
batch_filenames = image_names
unconcatenated_batch_data = []
for image_name in batch_filenames:
# print(image_name)
image_filepath = images_folder + '/' + image_name
pillow_img = Image.new("RGB", (width, height))
pillow_img.paste(Image.open(image_filepath).resize((width, height)))
pillow_img.resize((640,640), 0)
input_data = np.float32(pillow_img)/255.
input_data = input_data[np.newaxis, :, :]
input_data=input_data.transpose(0,3,1,2)
input_data = np.array(input_data, dtype=np.float32)
unconcatenated_batch_data.append(input_data)
batch_data = np.concatenate(np.expand_dims(unconcatenated_batch_data, axis=0), axis=0)
return batch_data
def benchmark(model_path):
"""
用于测试速度
"""
session = onnxruntime.InferenceSession(model_path)
input_name = session.get_inputs()[0].name
total = 0.0
runs = 10
input_data = np.zeros((1,3,640,640), np.float32) # 随便输入一个假数据
# warming up
_ = session.run([], {input_name: input_data})
for i in range(runs):
start = time.perf_counter()
_ = session.run([], {input_name: input_data})
end = (time.perf_counter() - start) * 1000
total += end
print(f"{end:.2f}ms")
total /= runs
print(f"Avg: {total:.2f}ms")
def main():
input_model_path = 'yolov8n-detect.onnx' # 输入onnx模型
output_model_path = 'yolov8n-detect_static_int8.onnx' # 输出模型名
calibration_dataset_path = 'C:/MyPro/datasets/coco128/images/train2017' # 校准数据集图像地址
# 用于校准数据加载,注意这个方法里面需要做图像一些操作,与pytorch训练的时候加载数据操作一致
dr = DataReader(calibration_dataset_path, input_model_path)
# 开始量化
quantize_static(input_model_path,
output_model_path,
dr,
quant_format=QuantFormat.QDQ,
activation_type=QuantType.QUInt8,
weight_type=QuantType.QUInt8)
print("量化完成")
print("float32测试")
benchmark(input_model_path)
print("int8测试")
benchmark(output_model_path)
if __name__ == "__main__":
main()
def quantize_static 函数定义
def quantize_static(
model_input,
model_output,
calibration_data_reader: CalibrationDataReader,
quant_format=QuantFormat.QDQ,
op_types_to_quantize=None,
per_channel=False,
reduce_range=False,
activation_type=QuantType.QInt8,
weight_type=QuantType.QInt8,
nodes_to_quantize=None,
nodes_to_exclude=None,
optimize_model=True,
use_external_data_format=False,
calibrate_method=CalibrationMethod.MinMax,
extra_options=None,
):
"""
Given an onnx model and calibration data reader, create a quantized onnx model and save it into a file
It is recommended to use QuantFormat.QDQ format from 1.11 with activation_type = QuantType.QInt8 and weight_type
= QuantType.QInt8. If model is targeted to GPU/TRT, symmetric activation and weight are required. If model is
targeted to CPU, asymmetric activation and symmetric weight are recommended for balance of performance and
accuracy.
Args:
model_input: file path of model to quantize
model_output: file path of quantized model
calibration_data_reader: a calibration data reader. It
enumerates calibration data and generates inputs for the
original model.
quant_format: QuantFormat{QOperator, QDQ}.
QOperator format quantizes the model with quantized operators directly.
QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
activation_type:
quantization data type of activation. Please refer to
https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
calibrate_method:
Current calibration methods supported are MinMax and Entropy.
Please use CalibrationMethod.MinMax or CalibrationMethod.Entropy as options.
op_types_to_quantize:
specify the types of operators to quantize, like ['Conv'] to quantize Conv only.
It quantizes all supported operators by default.
per_channel: quantize weights per channel
reduce_range:
quantize weights with 7-bits. It may improve the accuracy for some models running on non-VNNI machine,
especially for per-channel mode
weight_type:
quantization data type of weight. Please refer to
https://onnxruntime.ai/docs/performance/quantization.html for more details on data type selection
nodes_to_quantize:
List of nodes names to quantize. When this list is not None only the nodes in this list
are quantized.
example:
[
'Conv__224',
'Conv__252'
]
nodes_to_exclude:
List of nodes names to exclude. The nodes in this list will be excluded from quantization
when it is not None.
optimize_model: Deprecating Soon! Optimize model before quantization. NOT recommended, optimization will
change the computation graph, making debugging of quantization loss difficult.
use_external_data_format: option used for large size (>2GB) model. Set to False by default.
extra_options:
key value pair dictionary for various options in different case. Current used:
extra.Sigmoid.nnapi = True/False (Default is False)
ActivationSymmetric = True/False: symmetrize calibration data for activations (default is False).
WeightSymmetric = True/False: symmetrize calibration data for weights (default is True).
EnableSubgraph = True/False : Default is False. If enabled, subgraph will be quantized.
Dyanmic mode currently is supported. Will support more in the future.
ForceQuantizeNoInputCheck = True/False :
By default, some latent operators like maxpool, transpose, do not quantize if their input is not
quantized already. Setting to True to force such operator always quantize input and so generate
quantized output. Also, the True behavior could be disabled per node using the nodes_to_exclude.
MatMulConstBOnly = True/False:
Default is False for static mode. If enabled, only MatMul with const B will be quantized.
AddQDQPairToWeight = True/False :
Default is False which quantizes floating-point weight and feeds it to solely inserted
DeQuantizeLinear node. If True, it remains floating-point weight and inserts both
QuantizeLinear/DeQuantizeLinear nodes to weight.
OpTypesToExcludeOutputQuantization = list of op type :
Default is []. If any op type is specified, it won't quantize the output of ops with this
specific op types.
DedicatedQDQPair = True/False :
Default is False. When inserting QDQ pair, multiple nodes can share a single QDQ pair as their
inputs. If True, it will create identical and dedicated QDQ pair for each node.
QDQOpTypePerChannelSupportToAxis = dictionary :
Default is {}. Set channel axis for specific op type, for example: {'MatMul': 1}, and it's
effective only when per channel quantization is supported and per_channel is True. If specific
op type supports per channel quantization but not explicitly specified with channel axis,
default channel axis will be used.
CalibTensorRangeSymmetric = True/False :
Default is False. If enabled, the final range of tensor during calibration will be explicitly
set to symmetric to central point "0".
CalibMovingAverage = True/False :
Default is False. If enabled, the moving average of the minimum and maximum values will be
computed when the calibration method selected is MinMax.
CalibMovingAverageConstant = float :
Default is 0.01. Constant smoothing factor to use when computing the moving average of the
minimum and maximum values. Effective only when the calibration method selected is MinMax and
when CalibMovingAverage is set to True.
"""
