UnknownObject/HyperLPR-Windows-Shared
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新到HyperLPR3版本

Browse Source
This commit is contained in:
tunmx
2023-02-27 15:47:55 +08:00
parent 7ae4d385e1
commit 0864e05f76
912 changed files with 8160 additions and 221461 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cpp/src/inference_helper_module/inference_helper.cpp
+430
View File
@@ -0,0 +1,430 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/*** Include ***/
/* for general */
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
/* for My modules */
#include "inference_helper_log.h"
#include "inference_helper.h"
#ifdef INFERENCE_HELPER_ENABLE_OPENCV
#include "inference_helper_opencv.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_TFLITE) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_EDGETPU)
#include "inference_helper_tensorflow_lite.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORRT
#include "inference_helper_tensorrt.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_NCNN
#include "inference_helper_ncnn.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_MNN
#include "inference_helper_mnn.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_SNPE
#include "inference_helper_snpe.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_ARMNN
#include "inference_helper_armnn.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_NNABLA) || defined(INFERENCE_HELPER_ENABLE_NNABLA_CUDA)
#include "inference_helper_nnabla.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME) || defined(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA)
#include "inference_helper_onnx_runtime.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_LIBTORCH) || defined(INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA)
#include "inference_helper_libtorch.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_TENSORFLOW) || defined(INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU)
#include "inference_helper_tensorflow.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_SAMPLE
#include "inference_helper_sample.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_RKNN
#include "inference_helper_rknn.h"
#endif
/*** Macro ***/
#define TAG "InferenceHelper"
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceHelper* InferenceHelper::Create(const InferenceHelper::HelperType helper_type)
{
InferenceHelper* p = nullptr;
switch (helper_type) {
#ifdef INFERENCE_HELPER_ENABLE_OPENCV
case kOpencv:
case kOpencvGpu:
PRINT("Use OpenCV \n");
p = new InferenceHelperOpenCV();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE
case kTensorflowLite:
PRINT("Use TensorflowLite\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK
case kTensorflowLiteXnnpack:
PRINT("Use TensorflowLite XNNPACK Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU
case kTensorflowLiteGpu:
PRINT("Use TensorflowLite GPU Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_EDGETPU
case kTensorflowLiteEdgetpu:
PRINT("Use TensorflowLite EdgeTPU Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_NNAPI
case kTensorflowLiteNnapi:
PRINT("Use TensorflowLite NNAPI Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORRT
case kTensorrt:
PRINT("Use TensorRT \n");
p = new InferenceHelperTensorRt();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_NCNN
case kNcnn:
case kNcnnVulkan:
PRINT("Use NCNN\n");
p = new InferenceHelperNcnn();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_MNN
case kMnn:
PRINT("Use MNN\n");
p = new InferenceHelperMnn();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_SNPE
case kSnpe:
PRINT("Use SNPE\n");
p = new InferenceHelperSnpe();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_ARMNN
case kArmnn:
PRINT("Use ARMNN\n");
p = new InferenceHelperArmnn();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_NNABLA
case kNnabla:
PRINT("Use NNabla\n");
p = new InferenceHelperNnabla();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_NNABLA_CUDA
case kNnablaCuda:
PRINT("Use NNabla_CUDA\n");
p = new InferenceHelperNnabla();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_ONNX_RUNTIME
case kOnnxRuntime:
PRINT("Use ONNX Runtime\n");
p = new InferenceHelperOnnxRuntime();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA
case kOnnxRuntimeCuda:
PRINT("Use ONNX Runtime_CUDA\n");
p = new InferenceHelperOnnxRuntime();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_LIBTORCH
case kLibtorch:
PRINT("Use LibTorch\n");
p = new InferenceHelperLibtorch();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA
case kLibtorchCuda:
PRINT("Use LibTorch CUDA\n");
p = new InferenceHelperLibtorch();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORFLOW
case kTensorflow:
PRINT("Use TensorFlow\n");
p = new InferenceHelperTensorflow();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU
case kTensorflowGpu:
PRINT("Use TensorFlow GPU\n");
p = new InferenceHelperTensorflow();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_SAMPLE
case kSample:
PRINT("Do not use this. this is just a reference code\n");
p = new InferenceHelperSample();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_RKNN
case kRknn:
PRINT("Use Rknn\n")
p = new InferenceHelperRKNN();
break;
#endif
default:
PRINT_E("Unsupported inference helper type (%d)\n", helper_type);
break;
}
if (p == nullptr) {
PRINT_E("Failed to create inference helper\n");
} else {
p->helper_type_ = helper_type;
}
return p;
}
#ifdef INFERENCE_HELPER_ENABLE_PRE_PROCESS_BY_OPENCV
#include <opencv2/opencv.hpp>
void InferenceHelper::PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob)
{
/* Generate mat from original data */
cv::Mat img_src = cv::Mat(cv::Size(input_tensor_info.image_info.width, input_tensor_info.image_info.height), (input_tensor_info.image_info.channel == 3) ? CV_8UC3 : CV_8UC1, input_tensor_info.data);
/* Crop image */
if (input_tensor_info.image_info.width == input_tensor_info.image_info.crop_width && input_tensor_info.image_info.height == input_tensor_info.image_info.crop_height) {
/* do nothing */
} else {
img_src = img_src(cv::Rect(input_tensor_info.image_info.crop_x, input_tensor_info.image_info.crop_y, input_tensor_info.image_info.crop_width, input_tensor_info.image_info.crop_height));
}
/* Resize image */
if (input_tensor_info.image_info.crop_width == input_tensor_info.GetWidth() && input_tensor_info.image_info.crop_height == input_tensor_info.GetHeight()) {
/* do nothing */
} else {
cv::resize(img_src, img_src, cv::Size(input_tensor_info.GetWidth(), input_tensor_info.GetHeight()));
}
/* Convert color type */
if (input_tensor_info.image_info.channel == input_tensor_info.GetChannel()) {
if (input_tensor_info.image_info.channel == 3 && input_tensor_info.image_info.swap_color) {
cv::cvtColor(img_src, img_src, cv::COLOR_BGR2RGB);
}
} else if (input_tensor_info.image_info.channel == 3 && input_tensor_info.GetChannel() == 1) {
cv::cvtColor(img_src, img_src, (input_tensor_info.image_info.is_bgr) ? cv::COLOR_BGR2GRAY : cv::COLOR_RGB2GRAY);
} else if (input_tensor_info.image_info.channel == 1 && input_tensor_info.GetChannel() == 3) {
cv::cvtColor(img_src, img_src, cv::COLOR_GRAY2BGR);
}
if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) {
/* Normalize image */
if (input_tensor_info.GetChannel() == 3) {
#if 1
img_src.convertTo(img_src, CV_32FC3);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.mean)), img_src);
cv::multiply(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.norm)), img_src);
#else
img_src.convertTo(img_src, CV_32FC3, 1.0 / 255);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.mean)), img_src);
cv::divide(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.norm)), img_src);
#endif
} else {
#if 1
img_src.convertTo(img_src, CV_32FC1);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.mean)), img_src);
cv::multiply(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.norm)), img_src);
#else
img_src.convertTo(img_src, CV_32FC1, 1.0 / 255);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.mean)), img_src);
cv::divide(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.norm)), img_src);
#endif
}
} else {
/* do nothing */
}
if (is_nchw) {
/* Convert to 4-dimensional Mat in NCHW */
img_src = cv::dnn::blobFromImage(img_src);
}
img_blob = img_src;
//memcpy(blobData, img_src.data, img_src.cols * img_src.rows * img_src.channels());
}
#else
/* For the environment where OpenCV is not supported */
void InferenceHelper::PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob)
{
PRINT_E("[PreProcessByOpenCV] Unsupported function called\n");
exit(-1);
}
#endif
void InferenceHelper::ConvertNormalizeParameters(InputTensorInfo& tensor_info)
{
if (tensor_info.data_type != InputTensorInfo::kDataTypeImage) return;
#if 0
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = src * 1 / (255 * norm) - (mean / norm) */
for (int32_t i = 0; i < 3; i++) {
tensor_info.normalize.mean[i] /= tensor_info.normalize.norm[i];
tensor_info.normalize.norm[i] *= 255.0f;
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
}
#endif
#if 1
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = (src - (mean * 255)) * (1 / (255 * norm)) */
for (int32_t i = 0; i < 3; i++) {
tensor_info.normalize.mean[i] *= 255.0f;
tensor_info.normalize.norm[i] *= 255.0f;
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
}
#endif
}
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
}
}
} else {
/* convert NHWC to NHWC */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
#if 1
dst[i * img_channel + c] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
#else
dst[i * img_channel + c] = (src[i * img_channel + c] / 255.0f - input_tensor_info.normalize.mean[c]) / input_tensor_info.normalize.norm[c];
#endif
}
}
}
}
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = src[i * img_channel + c];
}
}
} else {
/* convert NHWC to NHWC */
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
}
}
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = src[i * img_channel + c] - 128;
}
}
} else {
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = src[i * img_channel + c] - 128;
}
}
}
}
template<typename T>
void InferenceHelper::PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
T* src = static_cast<T*>(input_tensor_info.data);
if ((input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw && input_tensor_info.is_nchw) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc && !input_tensor_info.is_nchw)) {
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
} else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) {
/* NCHW -> NHWC */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = src[c * (img_width * img_height) + i];
}
}
} else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) {
/* NHWC -> NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[c * (img_width * img_height) + i] = src[i * img_channel + c];
}
}
}
}
template void InferenceHelper::PreProcessBlob<float>(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
template void InferenceHelper::PreProcessBlob<int32_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int32_t* dst);
template void InferenceHelper::PreProcessBlob<int64_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int64_t* dst);
template void InferenceHelper::PreProcessBlob<uint8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
template void InferenceHelper::PreProcessBlob<int8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);
cpp/src/inference_helper_module/inference_helper.h
+280
View File
@@ -0,0 +1,280 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef INFERENCE_HELPER_
#define INFERENCE_HELPER_
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
class TensorInfo {
public:
enum {
kTensorTypeNone,
kTensorTypeUint8,
kTensorTypeInt8,
kTensorTypeFp32,
kTensorTypeInt32,
kTensorTypeInt64,
};
public:
TensorInfo()
: name("")
, id(-1)
, tensor_type(kTensorTypeNone)
, is_nchw(true)
{}
~TensorInfo() {}
int32_t GetElementNum() const
{
int32_t element_num = 1;
for (const auto& dim : tensor_dims) {
element_num *= dim;
}
return element_num;
}
int32_t GetBatch() const
{
if (tensor_dims.size() <= 0) return -1;
return tensor_dims[0];
}
int32_t GetChannel() const
{
if (is_nchw) {
if (tensor_dims.size() <= 1) return -1;
return tensor_dims[1];
} else {
if (tensor_dims.size() <= 3) return -1;
return tensor_dims[3];
}
}
int32_t GetHeight() const
{
if (is_nchw) {
if (tensor_dims.size() <= 2) return -1;
return tensor_dims[2];
} else {
if (tensor_dims.size() <= 1) return -1;
return tensor_dims[1];
}
}
int32_t GetWidth() const
{
if (is_nchw) {
if (tensor_dims.size() <= 3) return -1;
return tensor_dims[3];
} else {
if (tensor_dims.size() <= 2) return -1;
return tensor_dims[2];
}
}
public:
std::string name; // [In] Set the name_ of tensor
int32_t id; // [Out] Do not modify (Used in InferenceHelper)
int32_t tensor_type; // [In] The type of tensor (e.g. kTensorTypeFp32)
std::vector<int32_t> tensor_dims; // InputTensorInfo: [In] The dimentions of tensor. (If empty at initialize, the size is updated from model info.)
// OutputTensorInfo: [Out] The dimentions of tensor is set from model information
bool is_nchw; // [IN] NCHW or NHWC
};
class InputTensorInfo : public TensorInfo {
public:
enum {
kDataTypeImage,
kDataTypeBlobNhwc, // data_ which already finished preprocess(color conversion, resize, normalize_, etc.)
kDataTypeBlobNchw,
};
public:
InputTensorInfo()
: data(nullptr)
, data_type(kDataTypeImage)
, image_info({ -1, -1, -1, -1, -1, -1, -1, true, false })
, normalize({ 0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f })
{}
InputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true)
: InputTensorInfo()
{
name = name_;
tensor_type = tensor_type_;
is_nchw = is_nchw_;
}
~InputTensorInfo() {}
public:
void* data; // [In] Set the pointer to image/blob
int32_t data_type; // [In] Set the type of data_ (e.g. kDataTypeImage)
struct {
int32_t width;
int32_t height;
int32_t channel;
int32_t crop_x;
int32_t crop_y;
int32_t crop_width;
int32_t crop_height;
bool is_bgr; // used when channel == 3 (true: BGR, false: RGB)
bool swap_color;
} image_info; // [In] used when data_type_ == kDataTypeImage
struct {
float mean[3];
float norm[3];
} normalize; // [In] used when data_type_ == kDataTypeImage
};
class OutputTensorInfo : public TensorInfo {
public:
OutputTensorInfo()
: data(nullptr)
, quant({ 1.0f, 0 })
, data_fp32_(nullptr)
{}
OutputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true)
: OutputTensorInfo()
{
name = name_;
tensor_type = tensor_type_;
is_nchw = is_nchw_;
}
~OutputTensorInfo() {
if (data_fp32_ != nullptr) {
delete[] data_fp32_;
}
}
float* GetDataAsFloat() { /* Returned pointer should be with const, but returning pointer without const is convenient to create cv::Mat */
if (tensor_type == kTensorTypeUint8 || tensor_type == kTensorTypeInt8) {
if (data_fp32_ == nullptr) {
data_fp32_ = new float[GetElementNum()];
}
if (tensor_type == kTensorTypeUint8) {
#pragma omp parallel
for (int32_t i = 0; i < GetElementNum(); i++) {
const uint8_t* val_uint8 = static_cast<const uint8_t*>(data);
float val_float = (val_uint8[i] - quant.zero_point) * quant.scale;
data_fp32_[i] = val_float;
}
} else {
#pragma omp parallel
for (int32_t i = 0; i < GetElementNum(); i++) {
const int8_t* val_int8 = static_cast<const int8_t*>(data);
float val_float = (val_int8[i] - quant.zero_point) * quant.scale;
data_fp32_[i] = val_float;
}
}
return data_fp32_;
} else if (tensor_type == kTensorTypeFp32) {
return static_cast<float*>(data);
} else {
return nullptr;
}
}
public:
void* data; // [Out] Pointer to the output data_
struct {
float scale;
int32_t zero_point;
} quant; // [Out] Parameters for dequantization (convert uint8 to float)
private:
float* data_fp32_;
};
namespace cv {
class Mat;
};
class InferenceHelper {
public:
enum {
kRetOk = 0,
kRetErr = -1,
};
typedef enum {
kOpencv,
kOpencvGpu,
kTensorflowLite,
kTensorflowLiteXnnpack,
kTensorflowLiteGpu,
kTensorflowLiteEdgetpu,
kTensorflowLiteNnapi,
kTensorrt,
kNcnn,
kNcnnVulkan,
kMnn,
kSnpe,
kArmnn,
kNnabla,
kNnablaCuda,
kOnnxRuntime,
kOnnxRuntimeCuda,
kLibtorch,
kLibtorchCuda,
kTensorflow,
kTensorflowGpu,
kSample,
kRknn,
} HelperType;
public:
static InferenceHelper* Create(const HelperType helper_type);
static void PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob); // use this if the selected inference engine doesn't support pre-process
public:
virtual ~InferenceHelper() {}
virtual int32_t SetNumThreads(const int32_t num_threads) = 0;
virtual int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) = 0;
virtual int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t Finalize(void) = 0;
virtual int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) = 0;
virtual int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
protected:
void ConvertNormalizeParameters(InputTensorInfo& tensor_info);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);
template<typename T>
void PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T *dst);
protected:
HelperType helper_type_;
};
#endif
cpp/src/inference_helper_module/inference_helper_log.h
+45
View File
@@ -0,0 +1,45 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef INFERENCE_HELPER_LOG_
#define INFERENCE_HELPER_LOG_
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#if defined(ANDROID) || defined(__ANDROID__)
#define CV_COLOR_IS_RGB
#include <android/log.h>
#define INFERENCE_HELPER_LOG_NDK_TAG "HyperLPR3-Native-Inference"
#define INFERENCE_HELPER_LOG_PRINT_(...) __android_log_print(ANDROID_LOG_INFO, INFERENCE_HELPER_LOG_NDK_TAG, __VA_ARGS__)
#else
#define INFERENCE_HELPER_LOG_PRINT_(...) printf(__VA_ARGS__)
#endif
#define INFERENCE_HELPER_LOG_PRINT(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \
INFERENCE_HELPER_LOG_PRINT_("[" INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \
} while(0);
#define INFERENCE_HELPER_LOG_PRINT_E(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \
INFERENCE_HELPER_LOG_PRINT_("[ERR: " INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \
} while(0);
#endif
cpp/src/inference_helper_module/inference_helper_mnn.cpp
+317
View File
@@ -0,0 +1,317 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/*** Include ***/
/* for general */
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
/* for MNN */
#include <MNN/ImageProcess.hpp>
#include <MNN/Interpreter.hpp>
#include <MNN/AutoTime.hpp>
/* for My modules */
#include "inference_helper_log.h"
#include "inference_helper_mnn.h"
/*** Macro ***/
#define TAG "InferenceHelperMnn"
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
/*** Function ***/
InferenceHelperMnn::InferenceHelperMnn()
{
num_threads_ = 1;
}
InferenceHelperMnn::~InferenceHelperMnn()
{
}
int32_t InferenceHelperMnn::SetNumThreads(const int32_t num_threads)
{
num_threads_ = num_threads;
return kRetOk;
}
int32_t InferenceHelperMnn::SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops)
{
PRINT("[WARNING] This method is not supported\n");
return kRetOk;
}
int32_t InferenceHelperMnn::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
/* Check tensor info fits the info from model */
for (auto& input_tensor_info : input_tensor_info_list) {
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
if (input_tensor == nullptr) {
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return kRetErr;
}
if ((input_tensor->getType().code == halide_type_float) && (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32)) {
/* OK */
} else if ((input_tensor->getType().code == halide_type_uint) && (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8)) {
/* OK */
} else {
PRINT_E("Incorrect input tensor type (%d, %d)\n", input_tensor->getType().code, input_tensor_info.tensor_type);
return kRetErr;
}
if ((input_tensor->channel() != -1) && (input_tensor->height() != -1) && (input_tensor->width() != -1)) {
if (input_tensor_info.GetChannel() != -1) {
if ((input_tensor->channel() == input_tensor_info.GetChannel()) && (input_tensor->height() == input_tensor_info.GetHeight()) && (input_tensor->width() == input_tensor_info.GetWidth())) {
/* OK */
} else {
PRINT_E("W: %d != %d\n", input_tensor->width() , input_tensor_info.GetWidth());
PRINT_E("H: %d != %d\n", input_tensor->height() , input_tensor_info.GetHeight());
PRINT_E("C: %d != %d\n", input_tensor->channel() , input_tensor_info.GetChannel());
PRINT_E("Incorrect input tensor size\n");
return kRetErr;
}
} else {
PRINT("Input tensor size is set from the model\n");
input_tensor_info.tensor_dims.clear();
for (int32_t dim = 0; dim < input_tensor->dimensions(); dim++) {
input_tensor_info.tensor_dims.push_back(input_tensor->length(dim));
}
}
} else {
if (input_tensor_info.GetChannel() != -1) {
PRINT("Input tensor size is resized\n");
/* In case the input size is not fixed */
net_->resizeTensor(input_tensor, { 1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth() });
net_->resizeSession(session_);
} else {
PRINT_E("Model input size is not set\n");
return kRetErr;
}
}
}
for (const auto& output_tensor_info : output_tensor_info_list) {
auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
if (output_tensor == nullptr) {
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
return kRetErr;
}
/* Output size is set when run inference later */
}
/* Convert normalize parameter to speed up */
for (auto& input_tensor_info : input_tensor_info_list) {
ConvertNormalizeParameters(input_tensor_info);
}
/* Check if tensor info is set */
for (const auto& input_tensor_info : input_tensor_info_list) {
for (const auto& dim : input_tensor_info.tensor_dims) {
if (dim <= 0) {
PRINT_E("Invalid tensor size\n");
return kRetErr;
}
}
}
//for (const auto& output_tensor_info : output_tensor_info_list) {
// for (const auto& dim : output_tensor_info.tensor_dims) {
// if (dim <= 0) {
// PRINT_E("Invalid tensor size\n");
// return kRetErr;
// }
// }
//}
return kRetOk;
}
int32_t InferenceHelperMnn::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
/*** Create network ***/
net_.reset(MNN::Interpreter::createFromBuffer(model_buffer, model_size));
if (!net_) {
PRINT_E("Failed to load model model buffer\n");
return kRetErr;
}
MNN::ScheduleConfig scheduleConfig;
scheduleConfig.type = MNN_FORWARD_CPU;
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
// MNN::BackendConfig bnconfig;
// bnconfig.power = MNN::BackendConfig::Power_High;
// bnconfig.precision = MNN::BackendConfig::Precision_Low;
// scheduleConfig.backendConfig = &bnconfig;
session_ = net_->createSession(scheduleConfig);
if (!session_) {
PRINT_E("Failed to create session\n");
return kRetErr;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceHelperMnn::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list)
{
/*** Create network ***/
net_.reset(MNN::Interpreter::createFromFile(model_filename.c_str()));
if (!net_) {
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str());
return kRetErr;
}
MNN::ScheduleConfig scheduleConfig;
scheduleConfig.type = MNN_FORWARD_AUTO;
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
// MNN::BackendConfig bnconfig;
// bnconfig.power = MNN::BackendConfig::Power_High;
// bnconfig.precision = MNN::BackendConfig::Precision_Low;
// scheduleConfig.backendConfig = &bnconfig;
session_ = net_->createSession(scheduleConfig);
if (!session_) {
PRINT_E("Failed to create session\n");
return kRetErr;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
};
int32_t InferenceHelperMnn::Finalize(void)
{
net_->releaseSession(session_);
net_->releaseModel();
net_.reset();
out_mat_list_.clear();
return kRetOk;
}
int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list)
{
for (const auto& input_tensor_info : input_tensor_info_list) {
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
if (input_tensor == nullptr) {
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return kRetErr;
}
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) {
/* Crop */
if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) || (input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) {
PRINT_E("Crop is not supported\n");
return kRetErr;
}
MNN::CV::ImageProcess::Config image_processconfig;
/* Convert color type */
if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
if (input_tensor_info.image_info.swap_color) {
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::RGB : MNN::CV::BGR;
} else {
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
}
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::GRAY;
} else if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
image_processconfig.destFormat = MNN::CV::GRAY;
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::BGR;
} else {
PRINT_E("Unsupported color conversion (%d, %d)\n", input_tensor_info.image_info.channel, input_tensor_info.GetChannel());
return kRetErr;
}
/* Normalize image */
std::memcpy(image_processconfig.mean, input_tensor_info.normalize.mean, sizeof(image_processconfig.mean));
std::memcpy(image_processconfig.normal, input_tensor_info.normalize.norm, sizeof(image_processconfig.normal));
/* Resize image */
image_processconfig.filterType = MNN::CV::BILINEAR;
MNN::CV::Matrix trans;
trans.setScale(static_cast<float>(input_tensor_info.image_info.crop_width) / input_tensor_info.GetWidth(), static_cast<float>(input_tensor_info.image_info.crop_height) / input_tensor_info.GetHeight());
/* Do pre-process */
std::shared_ptr<MNN::CV::ImageProcess> pretreat(MNN::CV::ImageProcess::create(image_processconfig));
pretreat->setMatrix(trans);
pretreat->convert(static_cast<uint8_t*>(input_tensor_info.data), input_tensor_info.image_info.crop_width, input_tensor_info.image_info.crop_height, 0, input_tensor);
} else if ( (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) ) {
std::unique_ptr<MNN::Tensor> tensor;
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) {
tensor.reset(new MNN::Tensor(input_tensor, MNN::Tensor::TENSORFLOW));
} else {
tensor.reset(new MNN::Tensor(input_tensor, MNN::Tensor::CAFFE));
}
if (tensor->getType().code == halide_type_float) {
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
tensor->host<float>()[i] = static_cast<float*>(input_tensor_info.data)[i];
}
} else {
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
tensor->host<uint8_t>()[i] = static_cast<uint8_t*>(input_tensor_info.data)[i];
}
}
input_tensor->copyFromHostTensor(tensor.get());
} else {
PRINT_E("Unsupported data type (%d)\n", input_tensor_info.data_type);
return kRetErr;
}
}
return kRetOk;
}
int32_t InferenceHelperMnn::Process(std::vector<OutputTensorInfo>& output_tensor_info_list)
{
net_->runSession(session_);
out_mat_list_.clear();
for (auto& output_tensor_info : output_tensor_info_list) {
auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
if (output_tensor == nullptr) {
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
return kRetErr;
}
auto dimType = output_tensor->getDimensionType();
std::unique_ptr<MNN::Tensor> outputUser(new MNN::Tensor(output_tensor, dimType));
output_tensor->copyToHostTensor(outputUser.get());
auto type = outputUser->getType();
if (type.code == halide_type_float) {
output_tensor_info.tensor_type = TensorInfo::kTensorTypeFp32;
output_tensor_info.data = outputUser->host<float>();
} else if (type.code == halide_type_uint && type.bytes() == 1) {
output_tensor_info.tensor_type = TensorInfo::kTensorTypeUint8;
output_tensor_info.data = outputUser->host<uint8_t>();
} else {
PRINT_E("Unexpected data type\n");
return kRetErr;
}
output_tensor_info.tensor_dims.clear();
for (int32_t dim = 0; dim < outputUser->dimensions(); dim++) {
output_tensor_info.tensor_dims.push_back(outputUser->length(dim));
}
out_mat_list_.push_back(std::move(outputUser)); // store data in member variable so that data keep exist
}
return kRetOk;
}
cpp/src/inference_helper_module/inference_helper_mnn.h
+53
View File
@@ -0,0 +1,53 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef INFERENCE_HELPER_MNN_
#define INFERENCE_HELPER_MNN_
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
/* for MNN */
#include <MNN/ImageProcess.hpp>
#include <MNN/Interpreter.hpp>
#include <MNN/AutoTime.hpp>
/* for My modules */
#include "inference_helper.h"
class InferenceHelperMnn : public InferenceHelper {
public:
InferenceHelperMnn();
~InferenceHelperMnn() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
private:
std::unique_ptr<MNN::Interpreter> net_;
MNN::Session* session_;
std::vector<std::unique_ptr<MNN::Tensor>> out_mat_list_;
int32_t num_threads_;
};
#endif