|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +/*! |
| 21 | + * \file mkldnn_quantized_batch_norm.cc |
| 22 | + * \brief |
| 23 | + * \author Yixin Bao |
| 24 | +*/ |
| 25 | + |
| 26 | +#if MXNET_USE_MKLDNN == 1 |
| 27 | +#include "../../nn/mkldnn/mkldnn_batch_norm-inl.h" |
| 28 | +#include "../quantization_utils.h" |
| 29 | + |
| 30 | +namespace mxnet { |
| 31 | +namespace op { |
| 32 | + |
| 33 | +static void MKLDNNQuantizedBatchNormForward(const nnvm::NodeAttrs &attrs, const OpContext &ctx, |
| 34 | + const std::vector<NDArray> &in_data, |
| 35 | + const std::vector<OpReqType> &req, |
| 36 | + const std::vector<NDArray> &outputs) { |
| 37 | + CHECK_EQ(in_data.size(), 7U); |
| 38 | + CHECK_EQ(outputs.size(), 3U); |
| 39 | + |
| 40 | + TmpMemMgr::Get()->Init(ctx.requested[batchnorm::kTempSpace]); |
| 41 | + const BatchNormParam ¶m = nnvm::get<BatchNormParam>(attrs.parsed); |
| 42 | + const NDArray &data = in_data[quantized_batchnorm::kData]; |
| 43 | + const size_t channelAxis = static_cast<size_t>( |
| 44 | + param.axis < 0 ? static_cast<int>(data.shape().ndim()) + param.axis : param.axis); |
| 45 | + const int channel_count = data.shape()[channelAxis]; |
| 46 | + const float min_data = in_data[quantized_batchnorm::kDataMin].data().dptr<float>()[0]; |
| 47 | + const float max_data = in_data[quantized_batchnorm::kDataMax].data().dptr<float>()[0]; |
| 48 | + const float max_abs_data = std::max(std::abs(min_data), std::abs(max_data)); |
| 49 | + |
| 50 | + float *min_output_ptr = outputs[quantized_batchnorm::kOutMin].data().dptr<float>(); |
| 51 | + float *max_output_ptr = outputs[quantized_batchnorm::kOutMax].data().dptr<float>(); |
| 52 | + if (param.min_calib_range.has_value() && param.max_calib_range.has_value()) { |
| 53 | + *max_output_ptr = param.max_calib_range.value(); |
| 54 | + *min_output_ptr = param.min_calib_range.value(); |
| 55 | + } else { |
| 56 | + LOG(FATAL) << "min_calib_range or max_calib_range is not available. Quantized BN currently " |
| 57 | + "don't support calib_mode=None"; |
| 58 | + } |
| 59 | + const float max_abs_output = std::max(std::abs(*min_output_ptr), std::abs(*max_output_ptr)); |
| 60 | + |
| 61 | + unsigned flags = mkldnn::use_global_stats | mkldnn::use_scale_shift; |
| 62 | + auto &fwd = GetBNForward<float>(param, ctx, data, flags); |
| 63 | + const mkldnn::memory &weight_mem = fwd.GetWeight(); |
| 64 | + CHECK_EQ(weight_mem.get_primitive_desc().get_size(), channel_count * sizeof(float) * 2); |
| 65 | + float *weight_buf = reinterpret_cast<float *>(weight_mem.get_data_handle()); |
| 66 | + |
| 67 | + float *gamma_ptr = in_data[quantized_batchnorm::kGamma].data().dptr<float>(); |
| 68 | + float *beta_ptr = in_data[quantized_batchnorm::kBeta].data().dptr<float>(); |
| 69 | + |
| 70 | + const NDArray &moving_mean = in_data[quantized_batchnorm::kInMovingMean]; |
| 71 | + const NDArray &moving_var = in_data[quantized_batchnorm::kInMovingVar]; |
| 72 | + float *moving_mean_ptr = moving_mean.data().dptr<float>(); |
| 73 | + float *moving_var_ptr = moving_var.data().dptr<float>(); |
| 74 | + |
| 75 | + // rescale gamma and beta, to make mean=0 and var=1 |
| 76 | + auto rescaled_mean_mem = |
| 77 | + TmpMemMgr::Get()->Alloc(moving_mean.GetMKLDNNData()->get_primitive_desc()); |
| 78 | + auto rescaled_var_mem = TmpMemMgr::Get()->Alloc(moving_var.GetMKLDNNData()->get_primitive_desc()); |
| 79 | + float *rescaled_mean_ptr = reinterpret_cast<float *>(rescaled_mean_mem->get_data_handle()); |
| 80 | + float *rescaled_var_ptr = reinterpret_cast<float *>(rescaled_var_mem->get_data_handle()); |
| 81 | + |
| 82 | +#pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount()) |
| 83 | + for (int channel = 0; channel < channel_count; ++channel) { |
| 84 | + float invstd = 1.0 / std::sqrt(moving_var_ptr[channel] + param.eps); |
| 85 | + weight_buf[channel] = gamma_ptr[channel] * invstd * max_abs_data / max_abs_output; |
| 86 | + weight_buf[channel_count + channel] = |
| 87 | + (beta_ptr[channel] - moving_mean_ptr[channel] * gamma_ptr[channel] * invstd) * kInt8Range / |
| 88 | + max_abs_output; |
| 89 | + rescaled_mean_ptr[channel] = 0.0f; |
| 90 | + rescaled_var_ptr[channel] = 1.0f; |
| 91 | + } |
| 92 | + |
| 93 | + auto out_mem = CreateMKLDNNMem(outputs[batchnorm::kOut], |
| 94 | + fwd.GetPd().dst_primitive_desc(), req[batchnorm::kOut], &data); |
| 95 | + fwd.SetDataHandle(data, rescaled_mean_mem, rescaled_var_mem, out_mem.second); |
| 96 | + |
| 97 | + MKLDNNStream::Get()->RegisterPrim(fwd.GetFwd()); |
| 98 | + MKLDNNStream::Get()->Submit(); |
| 99 | +} |
| 100 | + |
| 101 | +inline static bool QuantizedBatchNormStorageType(const nnvm::NodeAttrs &attrs, const int dev_mask, |
| 102 | + DispatchMode *dispatch_mode, |
| 103 | + std::vector<int> *in_attrs, |
| 104 | + std::vector<int> *out_attrs) { |
| 105 | + bool dispatched = false; |
| 106 | + if (!dispatched) { |
| 107 | + dispatched = MKLDNNStorageType(attrs, dev_mask, true, dispatch_mode, in_attrs, out_attrs); |
| 108 | + } |
| 109 | + return dispatched; |
| 110 | +} |
| 111 | + |
| 112 | +NNVM_REGISTER_OP(_contrib_quantized_batch_norm) |
| 113 | +.set_attr<FInferStorageType>("FInferStorageType", QuantizedBatchNormStorageType) |
| 114 | +.set_attr<FComputeEx>("FComputeEx<cpu>", MKLDNNQuantizedBatchNormForward) |
| 115 | +.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& n) { |
| 116 | + return std::vector<ResourceRequest>{ResourceRequest::kTempSpace}; |
| 117 | +}) |
| 118 | +.set_attr<bool>("TIsMKLDNN", true); |
| 119 | + |
| 120 | +} // namespace op |
| 121 | +} // namespace mxnet |
| 122 | + |
| 123 | +#endif // MXNET_USE_MKLDNN == 1 |
0 commit comments