更新到HyperLPR3版本
This commit is contained in:
tunmx
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_BASIC_TYPES_H
|
||||
#define ZEPHYRLPR_BASIC_TYPES_H
|
||||
#include "opencv2/opencv.hpp"
|
||||
#include "log.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
typedef std::vector<size_t> IndexList;
|
||||
|
||||
typedef cv::Size InputSize;
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif // ZEPHYRLPR_BASIC_TYPES_H
|
||||
@@ -0,0 +1,10 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_CAMERA_BUFFER_ALL_H
|
||||
#define ZEPHYRLPR_CAMERA_BUFFER_ALL_H
|
||||
|
||||
#include "camera_buffer.h"
|
||||
|
||||
#endif //ZEPHYRLPR_CAMERA_BUFFER_ALL_H
|
||||
@@ -0,0 +1,236 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/4/24.
|
||||
//
|
||||
|
||||
#include "camera_buffer.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
CameraBuffer::CameraBuffer() {
|
||||
config_.sourceFormat = MNN::CV::YUV_NV21;
|
||||
config_.destFormat = MNN::CV::BGR;
|
||||
config_.filterType = MNN::CV::NEAREST;
|
||||
config_.wrap = MNN::CV::ZERO;
|
||||
rotation_mode_ = ROTATION_0;
|
||||
}
|
||||
|
||||
void CameraBuffer::SetDataBuffer(const uint8_t *data_buffer, int height, int width) {
|
||||
this->buffer_ = data_buffer;
|
||||
this->height_ = height;
|
||||
this->width_ = width;
|
||||
}
|
||||
|
||||
void CameraBuffer::SetRotationMode(ROTATION_MODE mode) {
|
||||
rotation_mode_ = mode;
|
||||
}
|
||||
|
||||
void CameraBuffer::SetDataFormat(DATA_FORMAT data_format) {
|
||||
if (data_format == NV21) {
|
||||
config_.sourceFormat = MNN::CV::YUV_NV21;
|
||||
}
|
||||
if (data_format == NV12) {
|
||||
config_.sourceFormat = MNN::CV::YUV_NV12;
|
||||
}
|
||||
if (data_format == RGBA) {
|
||||
config_.sourceFormat = MNN::CV::RGBA;
|
||||
}
|
||||
if (data_format == RGB) {
|
||||
config_.sourceFormat = MNN::CV::RGB;
|
||||
}
|
||||
if (data_format == BGR) {
|
||||
config_.sourceFormat = MNN::CV::BGR;
|
||||
}
|
||||
if (data_format == BGRA) {
|
||||
config_.sourceFormat = MNN::CV::BGRA;
|
||||
}
|
||||
if (data_format == YCrCb) {
|
||||
config_.sourceFormat = MNN::CV::YCrCb;
|
||||
}
|
||||
}
|
||||
|
||||
cv::Mat CameraBuffer::GetAffineRGBImage(const cv::Mat &affine_matrix, const int width_out, const int height_out) const {
|
||||
int sw = width_;
|
||||
int sh = height_;
|
||||
int rot_sw = sw;
|
||||
int rot_sh = sh;
|
||||
MNN::CV::Matrix tr;
|
||||
assert(affine_matrix.rows == 2);
|
||||
assert(affine_matrix.cols == 3);
|
||||
assert(affine_matrix.type() == CV_64F);
|
||||
cv::Mat trans_matrix;
|
||||
affine_matrix.convertTo(trans_matrix, CV_32F);
|
||||
std::vector<float> tr_cv({1, 0, 0, 0, 1, 0, 0, 0, 1});
|
||||
memcpy(tr_cv.data(), trans_matrix.data, sizeof(float) * 6);
|
||||
tr.set9(tr_cv.data());
|
||||
MNN::CV::Matrix tr_inv;
|
||||
tr.invert(&tr_inv);
|
||||
std::shared_ptr<MNN::CV::ImageProcess> process(
|
||||
MNN::CV::ImageProcess::create(config_));
|
||||
process->setMatrix(tr_inv);
|
||||
cv::Mat img_out(height_out, width_out, CV_8UC3);
|
||||
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
|
||||
std::vector<int>{1, height_out, width_out, 3}, img_out.data));
|
||||
process->convert(buffer_, sw, sh, 0, tensor.get());
|
||||
return img_out;
|
||||
}
|
||||
|
||||
cv::Mat CameraBuffer::GetScaledImage(const float scale, bool with_rotation) {
|
||||
int sw = width_;
|
||||
int sh = height_;
|
||||
int rot_sw = sw;
|
||||
int rot_sh = sh;
|
||||
// MNN::CV::Matrix tr;
|
||||
std::shared_ptr<MNN::CV::ImageProcess> process(
|
||||
MNN::CV::ImageProcess::create(config_));
|
||||
if (rotation_mode_ == ROTATION_270 && with_rotation) {
|
||||
float srcPoints[] = {
|
||||
0.0f,
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ - 1),
|
||||
(float)(width_ - 1),
|
||||
0.0f,
|
||||
(float)(width_ - 1),
|
||||
(float)(height_ - 1),
|
||||
};
|
||||
float dstPoints[] = {(float)(height_ * scale - 1),
|
||||
0.0f,
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ * scale - 1),
|
||||
(float)(width_ * scale - 1),
|
||||
0.0f,
|
||||
(float)(width_ * scale - 1)};
|
||||
|
||||
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
|
||||
(MNN::CV::Point *)srcPoints, 4);
|
||||
process->setMatrix(tr_);
|
||||
int scaled_height = static_cast<int>(width_ * scale);
|
||||
int scaled_width = static_cast<int>(height_ * scale);
|
||||
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
|
||||
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
|
||||
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
|
||||
process->convert(buffer_, sw, sh, 0, tensor.get());
|
||||
return img_out;
|
||||
} else if (rotation_mode_ == ROTATION_90 && with_rotation) {
|
||||
float srcPoints[] = {
|
||||
0.0f,
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ - 1),
|
||||
(float)(width_ - 1),
|
||||
0.0f,
|
||||
(float)(width_ - 1),
|
||||
(float)(height_ - 1),
|
||||
};
|
||||
float dstPoints[] = {
|
||||
0.0f,
|
||||
(float)(width_ * scale - 1),
|
||||
(float)(height_ * scale - 1),
|
||||
(float)(width_ * scale - 1),
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ * scale - 1),
|
||||
0.0f,
|
||||
};
|
||||
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
|
||||
(MNN::CV::Point *)srcPoints, 4);
|
||||
process->setMatrix(tr_);
|
||||
int scaled_height = static_cast<int>(width_ * scale);
|
||||
int scaled_width = static_cast<int>(height_ * scale);
|
||||
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
|
||||
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
|
||||
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
|
||||
process->convert(buffer_, sw, sh, 0, tensor.get());
|
||||
return img_out;
|
||||
} else if (rotation_mode_ == ROTATION_180 && with_rotation) {
|
||||
float srcPoints[] = {
|
||||
0.0f,
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ - 1),
|
||||
(float)(width_ - 1),
|
||||
0.0f,
|
||||
(float)(width_ - 1),
|
||||
(float)(height_ - 1),
|
||||
};
|
||||
float dstPoints[] = {
|
||||
(float)(width_ * scale - 1),
|
||||
(float)(height_ * scale - 1),
|
||||
(float)(width_ * scale - 1),
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ * scale - 1),
|
||||
0.0f,
|
||||
0.0f,
|
||||
};
|
||||
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
|
||||
(MNN::CV::Point *)srcPoints, 4);
|
||||
process->setMatrix(tr_);
|
||||
int scaled_height = static_cast<int>(height_ * scale);
|
||||
int scaled_width = static_cast<int>(width_ * scale);
|
||||
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
|
||||
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
|
||||
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
|
||||
process->convert(buffer_, sw, sh, 0, tensor.get());
|
||||
return img_out;
|
||||
} else {
|
||||
float srcPoints[] = {
|
||||
0.0f,
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ - 1),
|
||||
(float)(width_ - 1),
|
||||
0.0f,
|
||||
(float)(width_ - 1),
|
||||
(float)(height_ - 1),
|
||||
};
|
||||
float dstPoints[] = {
|
||||
0.0f,
|
||||
0.0f,
|
||||
0.0f,
|
||||
(float)(height_ * scale - 1),
|
||||
(float)(width_ * scale - 1),
|
||||
0.0f,
|
||||
(float)(width_ * scale - 1),
|
||||
(float)(height_ * scale - 1),
|
||||
};
|
||||
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
|
||||
(MNN::CV::Point *)srcPoints, 4);
|
||||
process->setMatrix(tr_);
|
||||
int scaled_height = static_cast<int>(height_ * scale);
|
||||
int scaled_width = static_cast<int>(width_ * scale);
|
||||
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
|
||||
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
|
||||
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
|
||||
process->convert(buffer_, sw, sh, 0, tensor.get());
|
||||
return img_out;
|
||||
}
|
||||
}
|
||||
|
||||
cv::Mat CameraBuffer::GetAffineMatrix() const {
|
||||
cv::Mat affine_matrix(3, 3, CV_32F);
|
||||
tr_.get9((float *)affine_matrix.data);
|
||||
cv::Mat affine = affine_matrix.rowRange(0, 2);
|
||||
cv::Mat affine_64;
|
||||
affine.convertTo(affine_64, CV_64F);
|
||||
assert(affine_64.rows == 2);
|
||||
assert(affine_64.cols == 3);
|
||||
assert(affine_64.type() == CV_64F);
|
||||
return affine_64;
|
||||
}
|
||||
|
||||
int CameraBuffer::GetHeight() const {
|
||||
return height_;
|
||||
}
|
||||
|
||||
int CameraBuffer::GetWidth() const {
|
||||
return width_;
|
||||
}
|
||||
|
||||
|
||||
int CameraBuffer::GetRotationMode() const{
|
||||
return rotation_mode_;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/4/24.
|
||||
//
|
||||
|
||||
#ifndef SOLEXCV_CAMERA_BUFFER_H
|
||||
#define SOLEXCV_CAMERA_BUFFER_H
|
||||
|
||||
#include <opencv2/opencv.hpp>
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
#include <memory>
|
||||
|
||||
namespace hyper {
|
||||
|
||||
enum ROTATION_MODE {
|
||||
ROTATION_0 = 0,
|
||||
ROTATION_90 = 1,
|
||||
ROTATION_180 = 2,
|
||||
ROTATION_270 = 3
|
||||
};
|
||||
|
||||
enum DATA_FORMAT {
|
||||
NV21 = 0, NV12 = 1, RGBA = 2, RGB = 3, BGR = 4, BGRA = 5, YCrCb = 6
|
||||
};
|
||||
|
||||
class CameraBuffer {
|
||||
public:
|
||||
|
||||
CameraBuffer();
|
||||
|
||||
|
||||
void SetDataBuffer(const uint8_t *data_buffer, int height, int width);
|
||||
|
||||
|
||||
void SetRotationMode(ROTATION_MODE mode);
|
||||
|
||||
|
||||
void SetDataFormat(DATA_FORMAT data_format);
|
||||
|
||||
|
||||
cv::Mat GetAffineRGBImage(const cv::Mat &affine_matrix, const int width_out,
|
||||
const int height_out) const;
|
||||
|
||||
|
||||
cv::Mat GetScaledImage(const float scale, bool with_rotation);
|
||||
|
||||
|
||||
cv::Mat GetAffineMatrix() const;
|
||||
|
||||
int GetHeight() const;
|
||||
|
||||
int GetWidth() const;
|
||||
|
||||
int GetRotationMode() const;
|
||||
|
||||
private:
|
||||
const uint8_t *buffer_;
|
||||
int buffer_size_;
|
||||
std::vector<float> rotation_matrix;
|
||||
int height_;
|
||||
int width_;
|
||||
MNN::CV::Matrix tr_;
|
||||
ROTATION_MODE rotation_mode_;
|
||||
MNN::CV::ImageProcess::Config config_;
|
||||
std::shared_ptr <MNN::CV::ImageProcess> process_;
|
||||
};
|
||||
|
||||
}
|
||||
#endif //SOLEXCV_CAMERA_BUFFER_H
|
||||
@@ -0,0 +1,29 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/21.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_CONFIGURATION_H
|
||||
#define ZEPHYRLPR_CONFIGURATION_H
|
||||
|
||||
#include "basic_types.h"
|
||||
#include <iostream>
|
||||
#include "log.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
const std::string DETECT_LOW_BACKBONE_FILENAME = "b320_backbone_h.mnn";
|
||||
const std::string DETECT_LOW_HEAD_FILENAME = "b320_header_h.mnn";
|
||||
const std::string DETECT_HIGH_BACKBONE_FILENAME = "b640x_backbone_h.mnn";
|
||||
const std::string DETECT_HIGH_HEAD_FILENAME = "b640x_head_h.mnn";
|
||||
const std::string CLS_MODEL_FILENAME = "litemodel_cls_96xh.mnn";
|
||||
const std::string REC_MODEL_FILENAME = "rpv3_mdict_160_r3.mnn";
|
||||
|
||||
const InputSize REC_INPUT_SIZE = {160, 48};
|
||||
const InputSize CLS_INPUT_SIZE = {96, 96};
|
||||
|
||||
const int REC_MAX_CHAR_NUM = 20;
|
||||
const int REC_CHAR_CLASS_NUM = 78;
|
||||
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_CONFIGURATION_H
|
||||
@@ -0,0 +1,11 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_HYPER_LPR_CONTEXT_ALL_H
|
||||
#define ZEPHYRLPR_HYPER_LPR_CONTEXT_ALL_H
|
||||
|
||||
#include "hyper_lpr_context.h"
|
||||
#include "hyper_lpr_common.h"
|
||||
|
||||
#endif //ZEPHYRLPR_HYPER_LPR_CONTEXT_ALL_H
|
||||
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_HYPER_LPR_COMMON_H
|
||||
#define ZEPHYRLPR_HYPER_LPR_COMMON_H
|
||||
|
||||
#include "nn_implementation_module/all.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
enum PlateType {
|
||||
UNKNOWN = -1, ///< 未知车牌
|
||||
BLUE = 0, ///< 蓝牌
|
||||
YELLOW_SINGLE = 1, ///< 黄牌单层
|
||||
WHILE_SINGLE = 2, ///< 白牌单层
|
||||
GREEN = 3, ///< 绿牌新能源
|
||||
BLACK_HK_MACAO = 4, ///< 黑牌港澳
|
||||
HK_SINGLE = 5, ///< 香港单层
|
||||
HK_DOUBLE = 6, ///< 香港双层
|
||||
MACAO_SINGLE = 7, ///< 澳门单层
|
||||
MACAO_DOUBLE = 8, ///< 澳门双层
|
||||
YELLOW_DOUBLE = 9, ///< 黄牌双层
|
||||
};
|
||||
|
||||
enum DetectLevel {
|
||||
DETECT_LEVEL_LOW = 0,
|
||||
DETECT_LEVEL_HIGH,
|
||||
};
|
||||
|
||||
typedef struct PlateObject {
|
||||
PlateLocation location{};
|
||||
TextLine line{};
|
||||
PlateColor type{};
|
||||
|
||||
} PlateObject;
|
||||
|
||||
typedef struct PlateResult {
|
||||
float x1;
|
||||
float y1;
|
||||
float x2;
|
||||
float y2;
|
||||
PlateType type;
|
||||
float text_confidence;
|
||||
char code[128];
|
||||
} PlateResult;
|
||||
|
||||
typedef std::vector<PlateObject> PlateObjectList;
|
||||
|
||||
typedef std::vector<PlateResult> PlateResultList;
|
||||
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_HYPER_LPR_COMMON_H
|
||||
@@ -0,0 +1,223 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include "hyper_lpr_context.h"
|
||||
#include "configuration.h"
|
||||
#include "utils.h"
|
||||
#include "log.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
HyperLPRContext::HyperLPRContext() = default;
|
||||
|
||||
void HyperLPRContext::operator()(CameraBuffer &buffer) {
|
||||
cv::Mat process_image;
|
||||
process_image = buffer.GetScaledImage(1.0f, true);
|
||||
m_plate_detector_->Detection(process_image, true, 1.0f);
|
||||
PlateResultList().swap(m_object_results_);
|
||||
auto &detect_results = m_plate_detector_->m_results_;
|
||||
// cv::imwrite("/storage/emulated/0/Android/data/com.hyperai.hyperlpr_sdk_demo/files/bug.jpg", process_image);
|
||||
|
||||
// std::sort(detect_results.begin(), detect_results.end(),
|
||||
// [](PlateLocation a, PlateLocation b) { return xyxyArea(a.x1, a.y1, a.x2, a.y2) > xyxyArea(b.x1, b.y1, b.x2, b.y2); });
|
||||
//
|
||||
auto iteration_num = detect_results.size() > m_rec_max_num_ ? m_rec_max_num_: detect_results.size();
|
||||
for (size_t i = 0; i < iteration_num; ++i) {
|
||||
auto &loc = detect_results[i];
|
||||
PlateResult obj;
|
||||
obj.x1 = loc.x1;
|
||||
obj.y1 = loc.y1;
|
||||
obj.x2 = loc.x2;
|
||||
obj.y2 = loc.y2;
|
||||
cv::Mat align_image;
|
||||
getRotateCropAndAlignPad(process_image, align_image, loc.kps);
|
||||
// std::cout << align_image_pad.size << std::endl;
|
||||
// cv::imshow("align_image_pad", align_image_pad);
|
||||
// cv::imshow("align_image", align_image);
|
||||
// cv::waitKey(0);
|
||||
|
||||
|
||||
TextLine text_line;
|
||||
if (loc.layers == LayersNum::DOUBLE) {
|
||||
int line = (int )((float )align_image.rows * 0.4f);
|
||||
int bottom_h = align_image.rows - line;
|
||||
cv::Rect_<int> top_rect(0, 0, align_image.cols, line);
|
||||
cv::Rect_<int> bottom_rect(0, line, align_image.cols, bottom_h);
|
||||
cv::Mat top_crop = align_image(top_rect);
|
||||
cv::Mat bottom_crop = align_image(bottom_rect);
|
||||
|
||||
std::vector<cv::Mat> candidate = {top_crop, bottom_crop};
|
||||
text_line.code = "";
|
||||
text_line.average_score = 0.0f;
|
||||
for (int j = 0; j < candidate.size(); ++j) {
|
||||
cv::Mat &align = candidate[j];
|
||||
cv::Mat align_pad;
|
||||
float wh_ratio = (float) align.cols / align.rows;
|
||||
imagePadding(align, align_pad, wh_ratio, m_plate_recognition_->getMInputImageSize());
|
||||
TextLine candidate_text;
|
||||
// cv::imshow("align_pad", align_pad);
|
||||
// cv::waitKey(0);
|
||||
// if (j == 0)
|
||||
// cv::imwrite("a.jpg", align_pad);
|
||||
m_plate_recognition_->Inference(align_pad, candidate_text);
|
||||
text_line.code += candidate_text.code;
|
||||
text_line.average_score += candidate_text.average_score;
|
||||
}
|
||||
text_line.average_score /= candidate.size();
|
||||
// cv::imshow("top_crop", top_crop);
|
||||
// cv::imshow("bottom_crop", bottom_crop);
|
||||
// cv::waitKey(0);
|
||||
} else {
|
||||
cv::Mat align_image_pad;
|
||||
float wh_ratio = (float) align_image.cols / align_image.rows;
|
||||
imagePadding(align_image, align_image_pad, wh_ratio, m_plate_recognition_->getMInputImageSize());
|
||||
m_plate_recognition_->Inference(align_image_pad, text_line);
|
||||
}
|
||||
cv::resize(align_image, align_image, m_plate_classification_->getMInputImageSize());
|
||||
// SLOG_CRITICAL("cfg: {}", text_line.average_score);
|
||||
// SLOG_CRITICAL("code: {}", text_line.code);
|
||||
if (text_line.average_score < m_plate_recognition_->getMConfidenceThreshold()) {
|
||||
continue;
|
||||
}
|
||||
// cv::imshow("resize", align_image_pad);
|
||||
// cv::imshow("align_image", align_image);
|
||||
// cv::waitKey(0);
|
||||
// obj.color_classify = PlateColor::GREEN;
|
||||
// obj.layers = loc.layers;
|
||||
// LOGD("size %d", text_line.code.size());
|
||||
if (text_line.code.size() >= 7) {
|
||||
obj.text_confidence = text_line.average_score;
|
||||
auto type = PreGetPlateType(text_line.code);
|
||||
if (type == PlateType::UNKNOWN) {
|
||||
m_plate_classification_->Inference(align_image);
|
||||
auto color_type = m_plate_classification_->getMOutputColor();
|
||||
// obj.color_classify = color_type;
|
||||
// obj.layers = loc.layers;
|
||||
if (color_type == PlateColor::YELLOW) {
|
||||
// 黄牌
|
||||
if (loc.layers == LayersNum::DOUBLE) {
|
||||
// 双层黄牌
|
||||
type = PlateType::YELLOW_DOUBLE;
|
||||
} else {
|
||||
// 单层黄牌
|
||||
type = PlateType::YELLOW_SINGLE;
|
||||
}
|
||||
} else if (color_type == PlateColor::BLUE) {
|
||||
type = PlateType::BLUE;
|
||||
} else if (color_type == PlateColor::GREEN) {
|
||||
type = PlateType::GREEN;
|
||||
}
|
||||
}
|
||||
obj.type = type;
|
||||
|
||||
strcpy(obj.code, text_line.code.c_str());
|
||||
// obj.code = text_line.code;
|
||||
|
||||
// cv::imshow("align_image_pad", align_image_pad);
|
||||
// cv::waitKey(0);
|
||||
m_object_results_.push_back(obj);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int32_t HyperLPRContext::Initialize(const std::string& models_folder_path, int max_num, DetectLevel detect_level,
|
||||
int threads, bool use_half, float box_conf_threshold, float nms_threshold, float rec_confidence_threshold) {
|
||||
int32_t ret;
|
||||
std::string det_backbone_model = models_folder_path + "/" + hyper::DETECT_LOW_BACKBONE_FILENAME;
|
||||
if (!exists(det_backbone_model)) {
|
||||
LOGE("file: %s does not exist.", hyper::DETECT_LOW_BACKBONE_FILENAME.c_str());
|
||||
return hRetErr;
|
||||
}
|
||||
std::string det_header_model = models_folder_path + "/" + hyper::DETECT_LOW_HEAD_FILENAME;
|
||||
if (!exists(det_header_model)) {
|
||||
LOGE("file: %s does not exist.", hyper::DETECT_LOW_HEAD_FILENAME.c_str());
|
||||
return hRetErr;
|
||||
}
|
||||
if (detect_level == DETECT_LEVEL_HIGH) {
|
||||
det_backbone_model = models_folder_path + "/" + hyper::DETECT_HIGH_BACKBONE_FILENAME;
|
||||
if (!exists(det_backbone_model)) {
|
||||
LOGE("file: %s does not exist.", hyper::DETECT_HIGH_BACKBONE_FILENAME.c_str());
|
||||
return hRetErr;
|
||||
}
|
||||
det_header_model = models_folder_path + "/" + hyper::DETECT_HIGH_HEAD_FILENAME;
|
||||
if (!exists(det_header_model)) {
|
||||
LOGE("file: %s does not exist.", hyper::DETECT_HIGH_HEAD_FILENAME.c_str());
|
||||
return hRetErr;
|
||||
}
|
||||
m_pre_image_size_ = 640;
|
||||
}
|
||||
m_plate_detector_ = std::make_shared<DetArch>();
|
||||
ret = m_plate_detector_->Initialize(det_backbone_model, det_header_model, m_pre_image_size_, threads, box_conf_threshold, nms_threshold,
|
||||
use_half);
|
||||
if (ret != hRetOk) {
|
||||
LOGE("Detect model loading errors.");
|
||||
return hRetErr;
|
||||
}
|
||||
// init classification
|
||||
std::string classification_model = models_folder_path + "/" + hyper::CLS_MODEL_FILENAME;
|
||||
if (!exists(classification_model)) {
|
||||
LOGE("file: %s does not exist.", hyper::CLS_MODEL_FILENAME.c_str());
|
||||
return hRetErr;
|
||||
}
|
||||
m_plate_classification_ = std::make_shared<ClassificationEngine>();
|
||||
ret = m_plate_classification_->Initialize(classification_model, CLS_INPUT_SIZE, threads, use_half);
|
||||
if (ret != hRetOk) {
|
||||
LOGE("Cls model loading errors.");
|
||||
return hRetErr;
|
||||
}
|
||||
// init recognition
|
||||
std::string recognition_model = models_folder_path + "/" + hyper::REC_MODEL_FILENAME;
|
||||
if (!exists(recognition_model)) {
|
||||
LOGE("file: %s does not exist.", hyper::REC_MODEL_FILENAME.c_str());
|
||||
return hRetErr;
|
||||
}
|
||||
m_plate_recognition_ = std::make_shared<RecognitionEngine>();
|
||||
ret = m_plate_recognition_->Initialize(recognition_model, REC_INPUT_SIZE, threads, rec_confidence_threshold,
|
||||
use_half);
|
||||
if (ret != hRetOk) {
|
||||
LOGE("Rec model loading errors.");
|
||||
return hRetErr;
|
||||
}
|
||||
m_rec_max_num_ = std::max(1, max_num); // Can't be less than 1
|
||||
|
||||
m_init_status_ = hRetOk;
|
||||
|
||||
return hRetOk;
|
||||
}
|
||||
|
||||
PlateResultList &HyperLPRContext::getMObjectResults() {
|
||||
return m_object_results_;
|
||||
}
|
||||
|
||||
PlateType HyperLPRContext::PreGetPlateType(std::string& code) {
|
||||
PlateType type = PlateType::UNKNOWN;
|
||||
if (code[0] == 'W' && code[1] == 'J'){
|
||||
type = PlateType::WHILE_SINGLE;
|
||||
} else if (code.size() == 10) {
|
||||
type = PlateType::GREEN;
|
||||
} else if (code.find("学") != -1) {
|
||||
type = PlateType::BLUE;
|
||||
} else if (code.find("港") != -1) {
|
||||
type = PlateType::BLACK_HK_MACAO;
|
||||
} else if (code.find("澳") != -1) {
|
||||
type = PlateType::BLACK_HK_MACAO;
|
||||
} else if (code.find("警") != -1) {
|
||||
type = PlateType::WHILE_SINGLE;
|
||||
} else if (code.find("粤Z") != -1) {
|
||||
type = PlateType::BLACK_HK_MACAO;
|
||||
}
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
int32_t HyperLPRContext::getMInitStatus() const {
|
||||
return m_init_status_;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_HYPER_LPR_CONTEXT_H
|
||||
#define ZEPHYRLPR_HYPER_LPR_CONTEXT_H
|
||||
|
||||
#include <iostream>
|
||||
#include <opencv2/opencv.hpp>
|
||||
#include "nn_implementation_module/all.h"
|
||||
#include "buffer_module/all.h"
|
||||
#include "hyper_lpr_common.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
enum {
|
||||
hRetOk = InferenceHelper::kRetOk, ///< 成功
|
||||
hRetErr = InferenceHelper::kRetErr, ///< 失败
|
||||
};
|
||||
|
||||
|
||||
class HyperLPRContext {
|
||||
public:
|
||||
|
||||
HyperLPRContext(const HyperLPRContext &) = delete;
|
||||
|
||||
HyperLPRContext &operator=(const HyperLPRContext &) = delete;
|
||||
|
||||
explicit HyperLPRContext();
|
||||
|
||||
void operator()(CameraBuffer &buffer);
|
||||
|
||||
/**
|
||||
* 手动初始化并实例化内部模型对象
|
||||
* @param models_folder_path 存放模型文件夹的路径地址
|
||||
* @param max_num 最大识别车牌数量
|
||||
* @param detect_level 检测器等级 low速度快，high速度慢检出率略高于low
|
||||
* @param threads 推理线程数量 (暂时无效)
|
||||
* @param use_half 是否开启半精度推理 (暂时无效)
|
||||
* @param box_conf_threshold 检测框置信度阈值
|
||||
* @param nms_threshold 非极大值抑制阈值
|
||||
* @param rec_confidence_threshold 车牌字符识别置信度阈值
|
||||
* @return 初始化状态
|
||||
*/
|
||||
int32_t Initialize(const std::string& models_folder_path, int max_num = 1, DetectLevel detect_level = DETECT_LEVEL_LOW, int threads = 1, bool use_half = false, float box_conf_threshold = 0.3f,
|
||||
float nms_threshold = 0.5f,
|
||||
float rec_confidence_threshold = 0.75f);
|
||||
|
||||
PlateResultList &getMObjectResults();
|
||||
|
||||
static PlateType PreGetPlateType(std::string& code);
|
||||
|
||||
int32_t getMInitStatus() const;
|
||||
|
||||
private:
|
||||
std::shared_ptr<DetArch> m_plate_detector_;
|
||||
|
||||
std::shared_ptr<ClassificationEngine> m_plate_classification_;
|
||||
|
||||
std::shared_ptr<RecognitionEngine> m_plate_recognition_;
|
||||
|
||||
int m_pre_image_size_ = 320;
|
||||
|
||||
PlateResultList m_object_results_;
|
||||
|
||||
int m_rec_max_num_ = 1;
|
||||
|
||||
int32_t m_init_status_ = hRetErr;
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
|
||||
#endif //ZEPHYRLPR_HYPER_LPR_CONTEXT_H
|
||||
@@ -0,0 +1,8 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/12/29.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_DOC_H
|
||||
#define ZEPHYRLPR_DOC_H
|
||||
|
||||
#endif //ZEPHYRLPR_DOC_H
|
||||
@@ -0,0 +1,430 @@
|
||||
/* Copyright 2021 iwatake2222
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
/*** Include ***/
|
||||
/* for general */
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
|
||||
/* for My modules */
|
||||
#include "inference_helper_log.h"
|
||||
#include "inference_helper.h"
|
||||
|
||||
#ifdef INFERENCE_HELPER_ENABLE_OPENCV
|
||||
#include "inference_helper_opencv.h"
|
||||
#endif
|
||||
#if defined(INFERENCE_HELPER_ENABLE_TFLITE) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_EDGETPU)
|
||||
#include "inference_helper_tensorflow_lite.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TENSORRT
|
||||
#include "inference_helper_tensorrt.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_NCNN
|
||||
#include "inference_helper_ncnn.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_MNN
|
||||
#include "inference_helper_mnn.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_SNPE
|
||||
#include "inference_helper_snpe.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_ARMNN
|
||||
#include "inference_helper_armnn.h"
|
||||
#endif
|
||||
#if defined(INFERENCE_HELPER_ENABLE_NNABLA) || defined(INFERENCE_HELPER_ENABLE_NNABLA_CUDA)
|
||||
#include "inference_helper_nnabla.h"
|
||||
#endif
|
||||
#if defined(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME) || defined(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA)
|
||||
#include "inference_helper_onnx_runtime.h"
|
||||
#endif
|
||||
#if defined(INFERENCE_HELPER_ENABLE_LIBTORCH) || defined(INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA)
|
||||
#include "inference_helper_libtorch.h"
|
||||
#endif
|
||||
#if defined(INFERENCE_HELPER_ENABLE_TENSORFLOW) || defined(INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU)
|
||||
#include "inference_helper_tensorflow.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_SAMPLE
|
||||
#include "inference_helper_sample.h"
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_RKNN
|
||||
#include "inference_helper_rknn.h"
|
||||
#endif
|
||||
|
||||
/*** Macro ***/
|
||||
#define TAG "InferenceHelper"
|
||||
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
|
||||
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
|
||||
|
||||
|
||||
InferenceHelper* InferenceHelper::Create(const InferenceHelper::HelperType helper_type)
|
||||
{
|
||||
InferenceHelper* p = nullptr;
|
||||
switch (helper_type) {
|
||||
#ifdef INFERENCE_HELPER_ENABLE_OPENCV
|
||||
case kOpencv:
|
||||
case kOpencvGpu:
|
||||
PRINT("Use OpenCV \n");
|
||||
p = new InferenceHelperOpenCV();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TFLITE
|
||||
case kTensorflowLite:
|
||||
PRINT("Use TensorflowLite\n");
|
||||
p = new InferenceHelperTensorflowLite();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK
|
||||
case kTensorflowLiteXnnpack:
|
||||
PRINT("Use TensorflowLite XNNPACK Delegate\n");
|
||||
p = new InferenceHelperTensorflowLite();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU
|
||||
case kTensorflowLiteGpu:
|
||||
PRINT("Use TensorflowLite GPU Delegate\n");
|
||||
p = new InferenceHelperTensorflowLite();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_EDGETPU
|
||||
case kTensorflowLiteEdgetpu:
|
||||
PRINT("Use TensorflowLite EdgeTPU Delegate\n");
|
||||
p = new InferenceHelperTensorflowLite();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_NNAPI
|
||||
case kTensorflowLiteNnapi:
|
||||
PRINT("Use TensorflowLite NNAPI Delegate\n");
|
||||
p = new InferenceHelperTensorflowLite();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TENSORRT
|
||||
case kTensorrt:
|
||||
PRINT("Use TensorRT \n");
|
||||
p = new InferenceHelperTensorRt();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_NCNN
|
||||
case kNcnn:
|
||||
case kNcnnVulkan:
|
||||
PRINT("Use NCNN\n");
|
||||
p = new InferenceHelperNcnn();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_MNN
|
||||
case kMnn:
|
||||
PRINT("Use MNN\n");
|
||||
p = new InferenceHelperMnn();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_SNPE
|
||||
case kSnpe:
|
||||
PRINT("Use SNPE\n");
|
||||
p = new InferenceHelperSnpe();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_ARMNN
|
||||
case kArmnn:
|
||||
PRINT("Use ARMNN\n");
|
||||
p = new InferenceHelperArmnn();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_NNABLA
|
||||
case kNnabla:
|
||||
PRINT("Use NNabla\n");
|
||||
p = new InferenceHelperNnabla();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_NNABLA_CUDA
|
||||
case kNnablaCuda:
|
||||
PRINT("Use NNabla_CUDA\n");
|
||||
p = new InferenceHelperNnabla();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_ONNX_RUNTIME
|
||||
case kOnnxRuntime:
|
||||
PRINT("Use ONNX Runtime\n");
|
||||
p = new InferenceHelperOnnxRuntime();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA
|
||||
case kOnnxRuntimeCuda:
|
||||
PRINT("Use ONNX Runtime_CUDA\n");
|
||||
p = new InferenceHelperOnnxRuntime();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_LIBTORCH
|
||||
case kLibtorch:
|
||||
PRINT("Use LibTorch\n");
|
||||
p = new InferenceHelperLibtorch();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA
|
||||
case kLibtorchCuda:
|
||||
PRINT("Use LibTorch CUDA\n");
|
||||
p = new InferenceHelperLibtorch();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TENSORFLOW
|
||||
case kTensorflow:
|
||||
PRINT("Use TensorFlow\n");
|
||||
p = new InferenceHelperTensorflow();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU
|
||||
case kTensorflowGpu:
|
||||
PRINT("Use TensorFlow GPU\n");
|
||||
p = new InferenceHelperTensorflow();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_SAMPLE
|
||||
case kSample:
|
||||
PRINT("Do not use this. this is just a reference code\n");
|
||||
p = new InferenceHelperSample();
|
||||
break;
|
||||
#endif
|
||||
#ifdef INFERENCE_HELPER_ENABLE_RKNN
|
||||
case kRknn:
|
||||
PRINT("Use Rknn\n")
|
||||
p = new InferenceHelperRKNN();
|
||||
break;
|
||||
|
||||
#endif
|
||||
default:
|
||||
PRINT_E("Unsupported inference helper type (%d)\n", helper_type);
|
||||
break;
|
||||
}
|
||||
if (p == nullptr) {
|
||||
PRINT_E("Failed to create inference helper\n");
|
||||
} else {
|
||||
p->helper_type_ = helper_type;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
#ifdef INFERENCE_HELPER_ENABLE_PRE_PROCESS_BY_OPENCV
|
||||
#include <opencv2/opencv.hpp>
|
||||
void InferenceHelper::PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob)
|
||||
{
|
||||
/* Generate mat from original data */
|
||||
cv::Mat img_src = cv::Mat(cv::Size(input_tensor_info.image_info.width, input_tensor_info.image_info.height), (input_tensor_info.image_info.channel == 3) ? CV_8UC3 : CV_8UC1, input_tensor_info.data);
|
||||
|
||||
/* Crop image */
|
||||
if (input_tensor_info.image_info.width == input_tensor_info.image_info.crop_width && input_tensor_info.image_info.height == input_tensor_info.image_info.crop_height) {
|
||||
/* do nothing */
|
||||
} else {
|
||||
img_src = img_src(cv::Rect(input_tensor_info.image_info.crop_x, input_tensor_info.image_info.crop_y, input_tensor_info.image_info.crop_width, input_tensor_info.image_info.crop_height));
|
||||
}
|
||||
|
||||
/* Resize image */
|
||||
if (input_tensor_info.image_info.crop_width == input_tensor_info.GetWidth() && input_tensor_info.image_info.crop_height == input_tensor_info.GetHeight()) {
|
||||
/* do nothing */
|
||||
} else {
|
||||
cv::resize(img_src, img_src, cv::Size(input_tensor_info.GetWidth(), input_tensor_info.GetHeight()));
|
||||
}
|
||||
|
||||
/* Convert color type */
|
||||
if (input_tensor_info.image_info.channel == input_tensor_info.GetChannel()) {
|
||||
if (input_tensor_info.image_info.channel == 3 && input_tensor_info.image_info.swap_color) {
|
||||
cv::cvtColor(img_src, img_src, cv::COLOR_BGR2RGB);
|
||||
}
|
||||
} else if (input_tensor_info.image_info.channel == 3 && input_tensor_info.GetChannel() == 1) {
|
||||
cv::cvtColor(img_src, img_src, (input_tensor_info.image_info.is_bgr) ? cv::COLOR_BGR2GRAY : cv::COLOR_RGB2GRAY);
|
||||
} else if (input_tensor_info.image_info.channel == 1 && input_tensor_info.GetChannel() == 3) {
|
||||
cv::cvtColor(img_src, img_src, cv::COLOR_GRAY2BGR);
|
||||
}
|
||||
|
||||
if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) {
|
||||
/* Normalize image */
|
||||
if (input_tensor_info.GetChannel() == 3) {
|
||||
#if 1
|
||||
img_src.convertTo(img_src, CV_32FC3);
|
||||
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.mean)), img_src);
|
||||
cv::multiply(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.norm)), img_src);
|
||||
#else
|
||||
img_src.convertTo(img_src, CV_32FC3, 1.0 / 255);
|
||||
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.mean)), img_src);
|
||||
cv::divide(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.norm)), img_src);
|
||||
#endif
|
||||
} else {
|
||||
#if 1
|
||||
img_src.convertTo(img_src, CV_32FC1);
|
||||
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.mean)), img_src);
|
||||
cv::multiply(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.norm)), img_src);
|
||||
#else
|
||||
img_src.convertTo(img_src, CV_32FC1, 1.0 / 255);
|
||||
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.mean)), img_src);
|
||||
cv::divide(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.norm)), img_src);
|
||||
#endif
|
||||
}
|
||||
} else {
|
||||
/* do nothing */
|
||||
}
|
||||
|
||||
if (is_nchw) {
|
||||
/* Convert to 4-dimensional Mat in NCHW */
|
||||
img_src = cv::dnn::blobFromImage(img_src);
|
||||
}
|
||||
|
||||
img_blob = img_src;
|
||||
//memcpy(blobData, img_src.data, img_src.cols * img_src.rows * img_src.channels());
|
||||
|
||||
}
|
||||
|
||||
#else
|
||||
/* For the environment where OpenCV is not supported */
|
||||
void InferenceHelper::PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob)
|
||||
{
|
||||
PRINT_E("[PreProcessByOpenCV] Unsupported function called\n");
|
||||
exit(-1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
void InferenceHelper::ConvertNormalizeParameters(InputTensorInfo& tensor_info)
|
||||
{
|
||||
if (tensor_info.data_type != InputTensorInfo::kDataTypeImage) return;
|
||||
|
||||
#if 0
|
||||
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = src * 1 / (255 * norm) - (mean / norm) */
|
||||
for (int32_t i = 0; i < 3; i++) {
|
||||
tensor_info.normalize.mean[i] /= tensor_info.normalize.norm[i];
|
||||
tensor_info.normalize.norm[i] *= 255.0f;
|
||||
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
|
||||
}
|
||||
#endif
|
||||
#if 1
|
||||
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = (src - (mean * 255)) * (1 / (255 * norm)) */
|
||||
for (int32_t i = 0; i < 3; i++) {
|
||||
tensor_info.normalize.mean[i] *= 255.0f;
|
||||
tensor_info.normalize.norm[i] *= 255.0f;
|
||||
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst)
|
||||
{
|
||||
const int32_t img_width = input_tensor_info.GetWidth();
|
||||
const int32_t img_height = input_tensor_info.GetHeight();
|
||||
const int32_t img_channel = input_tensor_info.GetChannel();
|
||||
uint8_t* src = (uint8_t*)(input_tensor_info.data);
|
||||
if (input_tensor_info.is_nchw == true) {
|
||||
/* convert NHWC to NCHW */
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
dst[c * img_width * img_height + i] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/* convert NHWC to NHWC */
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
#if 1
|
||||
dst[i * img_channel + c] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
|
||||
#else
|
||||
dst[i * img_channel + c] = (src[i * img_channel + c] / 255.0f - input_tensor_info.normalize.mean[c]) / input_tensor_info.normalize.norm[c];
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst)
|
||||
{
|
||||
const int32_t img_width = input_tensor_info.GetWidth();
|
||||
const int32_t img_height = input_tensor_info.GetHeight();
|
||||
const int32_t img_channel = input_tensor_info.GetChannel();
|
||||
uint8_t* src = (uint8_t*)(input_tensor_info.data);
|
||||
if (input_tensor_info.is_nchw == true) {
|
||||
/* convert NHWC to NCHW */
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
dst[c * img_width * img_height + i] = src[i * img_channel + c];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/* convert NHWC to NHWC */
|
||||
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
|
||||
}
|
||||
}
|
||||
|
||||
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst)
|
||||
{
|
||||
const int32_t img_width = input_tensor_info.GetWidth();
|
||||
const int32_t img_height = input_tensor_info.GetHeight();
|
||||
const int32_t img_channel = input_tensor_info.GetChannel();
|
||||
uint8_t* src = (uint8_t*)(input_tensor_info.data);
|
||||
if (input_tensor_info.is_nchw == true) {
|
||||
/* convert NHWC to NCHW */
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
dst[c * img_width * img_height + i] = src[i * img_channel + c] - 128;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
dst[i * img_channel + c] = src[i * img_channel + c] - 128;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void InferenceHelper::PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T* dst)
|
||||
{
|
||||
const int32_t img_width = input_tensor_info.GetWidth();
|
||||
const int32_t img_height = input_tensor_info.GetHeight();
|
||||
const int32_t img_channel = input_tensor_info.GetChannel();
|
||||
T* src = static_cast<T*>(input_tensor_info.data);
|
||||
if ((input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw && input_tensor_info.is_nchw) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc && !input_tensor_info.is_nchw)) {
|
||||
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
|
||||
} else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) {
|
||||
/* NCHW -> NHWC */
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
dst[i * img_channel + c] = src[c * (img_width * img_height) + i];
|
||||
}
|
||||
}
|
||||
} else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) {
|
||||
/* NHWC -> NCHW */
|
||||
#pragma omp parallel for num_threads(num_thread)
|
||||
for (int32_t i = 0; i < img_width * img_height; i++) {
|
||||
for (int32_t c = 0; c < img_channel; c++) {
|
||||
dst[c * (img_width * img_height) + i] = src[i * img_channel + c];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template void InferenceHelper::PreProcessBlob<float>(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
|
||||
template void InferenceHelper::PreProcessBlob<int32_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int32_t* dst);
|
||||
template void InferenceHelper::PreProcessBlob<int64_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int64_t* dst);
|
||||
template void InferenceHelper::PreProcessBlob<uint8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
|
||||
template void InferenceHelper::PreProcessBlob<int8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);
|
||||
@@ -0,0 +1,280 @@
|
||||
/* Copyright 2021 iwatake2222
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef INFERENCE_HELPER_
|
||||
#define INFERENCE_HELPER_
|
||||
|
||||
/* for general */
|
||||
#include <cstdint>
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <memory>
|
||||
|
||||
class TensorInfo {
|
||||
public:
|
||||
enum {
|
||||
kTensorTypeNone,
|
||||
kTensorTypeUint8,
|
||||
kTensorTypeInt8,
|
||||
kTensorTypeFp32,
|
||||
kTensorTypeInt32,
|
||||
kTensorTypeInt64,
|
||||
};
|
||||
|
||||
public:
|
||||
TensorInfo()
|
||||
: name("")
|
||||
, id(-1)
|
||||
, tensor_type(kTensorTypeNone)
|
||||
, is_nchw(true)
|
||||
{}
|
||||
~TensorInfo() {}
|
||||
|
||||
int32_t GetElementNum() const
|
||||
{
|
||||
int32_t element_num = 1;
|
||||
for (const auto& dim : tensor_dims) {
|
||||
element_num *= dim;
|
||||
}
|
||||
return element_num;
|
||||
}
|
||||
|
||||
int32_t GetBatch() const
|
||||
{
|
||||
if (tensor_dims.size() <= 0) return -1;
|
||||
return tensor_dims[0];
|
||||
}
|
||||
|
||||
int32_t GetChannel() const
|
||||
{
|
||||
if (is_nchw) {
|
||||
if (tensor_dims.size() <= 1) return -1;
|
||||
return tensor_dims[1];
|
||||
} else {
|
||||
if (tensor_dims.size() <= 3) return -1;
|
||||
return tensor_dims[3];
|
||||
}
|
||||
}
|
||||
|
||||
int32_t GetHeight() const
|
||||
{
|
||||
if (is_nchw) {
|
||||
if (tensor_dims.size() <= 2) return -1;
|
||||
return tensor_dims[2];
|
||||
} else {
|
||||
if (tensor_dims.size() <= 1) return -1;
|
||||
return tensor_dims[1];
|
||||
}
|
||||
}
|
||||
|
||||
int32_t GetWidth() const
|
||||
{
|
||||
if (is_nchw) {
|
||||
if (tensor_dims.size() <= 3) return -1;
|
||||
return tensor_dims[3];
|
||||
} else {
|
||||
if (tensor_dims.size() <= 2) return -1;
|
||||
return tensor_dims[2];
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
std::string name; // [In] Set the name_ of tensor
|
||||
int32_t id; // [Out] Do not modify (Used in InferenceHelper)
|
||||
int32_t tensor_type; // [In] The type of tensor (e.g. kTensorTypeFp32)
|
||||
std::vector<int32_t> tensor_dims; // InputTensorInfo: [In] The dimentions of tensor. (If empty at initialize, the size is updated from model info.)
|
||||
// OutputTensorInfo: [Out] The dimentions of tensor is set from model information
|
||||
bool is_nchw; // [IN] NCHW or NHWC
|
||||
};
|
||||
|
||||
class InputTensorInfo : public TensorInfo {
|
||||
public:
|
||||
enum {
|
||||
kDataTypeImage,
|
||||
kDataTypeBlobNhwc, // data_ which already finished preprocess(color conversion, resize, normalize_, etc.)
|
||||
kDataTypeBlobNchw,
|
||||
};
|
||||
|
||||
public:
|
||||
InputTensorInfo()
|
||||
: data(nullptr)
|
||||
, data_type(kDataTypeImage)
|
||||
, image_info({ -1, -1, -1, -1, -1, -1, -1, true, false })
|
||||
, normalize({ 0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f })
|
||||
{}
|
||||
|
||||
InputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true)
|
||||
: InputTensorInfo()
|
||||
{
|
||||
name = name_;
|
||||
tensor_type = tensor_type_;
|
||||
is_nchw = is_nchw_;
|
||||
}
|
||||
|
||||
~InputTensorInfo() {}
|
||||
|
||||
public:
|
||||
void* data; // [In] Set the pointer to image/blob
|
||||
int32_t data_type; // [In] Set the type of data_ (e.g. kDataTypeImage)
|
||||
|
||||
struct {
|
||||
int32_t width;
|
||||
int32_t height;
|
||||
int32_t channel;
|
||||
int32_t crop_x;
|
||||
int32_t crop_y;
|
||||
int32_t crop_width;
|
||||
int32_t crop_height;
|
||||
bool is_bgr; // used when channel == 3 (true: BGR, false: RGB)
|
||||
bool swap_color;
|
||||
} image_info; // [In] used when data_type_ == kDataTypeImage
|
||||
|
||||
struct {
|
||||
float mean[3];
|
||||
float norm[3];
|
||||
} normalize; // [In] used when data_type_ == kDataTypeImage
|
||||
};
|
||||
|
||||
|
||||
class OutputTensorInfo : public TensorInfo {
|
||||
public:
|
||||
OutputTensorInfo()
|
||||
: data(nullptr)
|
||||
, quant({ 1.0f, 0 })
|
||||
, data_fp32_(nullptr)
|
||||
{}
|
||||
|
||||
OutputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true)
|
||||
: OutputTensorInfo()
|
||||
{
|
||||
name = name_;
|
||||
tensor_type = tensor_type_;
|
||||
is_nchw = is_nchw_;
|
||||
}
|
||||
|
||||
~OutputTensorInfo() {
|
||||
if (data_fp32_ != nullptr) {
|
||||
delete[] data_fp32_;
|
||||
}
|
||||
}
|
||||
|
||||
float* GetDataAsFloat() { /* Returned pointer should be with const, but returning pointer without const is convenient to create cv::Mat */
|
||||
if (tensor_type == kTensorTypeUint8 || tensor_type == kTensorTypeInt8) {
|
||||
if (data_fp32_ == nullptr) {
|
||||
data_fp32_ = new float[GetElementNum()];
|
||||
}
|
||||
if (tensor_type == kTensorTypeUint8) {
|
||||
#pragma omp parallel
|
||||
for (int32_t i = 0; i < GetElementNum(); i++) {
|
||||
const uint8_t* val_uint8 = static_cast<const uint8_t*>(data);
|
||||
float val_float = (val_uint8[i] - quant.zero_point) * quant.scale;
|
||||
data_fp32_[i] = val_float;
|
||||
}
|
||||
} else {
|
||||
#pragma omp parallel
|
||||
for (int32_t i = 0; i < GetElementNum(); i++) {
|
||||
const int8_t* val_int8 = static_cast<const int8_t*>(data);
|
||||
float val_float = (val_int8[i] - quant.zero_point) * quant.scale;
|
||||
data_fp32_[i] = val_float;
|
||||
}
|
||||
}
|
||||
return data_fp32_;
|
||||
} else if (tensor_type == kTensorTypeFp32) {
|
||||
return static_cast<float*>(data);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
void* data; // [Out] Pointer to the output data_
|
||||
struct {
|
||||
float scale;
|
||||
int32_t zero_point;
|
||||
} quant; // [Out] Parameters for dequantization (convert uint8 to float)
|
||||
|
||||
private:
|
||||
float* data_fp32_;
|
||||
};
|
||||
|
||||
|
||||
namespace cv {
|
||||
class Mat;
|
||||
};
|
||||
|
||||
class InferenceHelper {
|
||||
public:
|
||||
enum {
|
||||
kRetOk = 0,
|
||||
kRetErr = -1,
|
||||
};
|
||||
|
||||
typedef enum {
|
||||
kOpencv,
|
||||
kOpencvGpu,
|
||||
kTensorflowLite,
|
||||
kTensorflowLiteXnnpack,
|
||||
kTensorflowLiteGpu,
|
||||
kTensorflowLiteEdgetpu,
|
||||
kTensorflowLiteNnapi,
|
||||
kTensorrt,
|
||||
kNcnn,
|
||||
kNcnnVulkan,
|
||||
kMnn,
|
||||
kSnpe,
|
||||
kArmnn,
|
||||
kNnabla,
|
||||
kNnablaCuda,
|
||||
kOnnxRuntime,
|
||||
kOnnxRuntimeCuda,
|
||||
kLibtorch,
|
||||
kLibtorchCuda,
|
||||
kTensorflow,
|
||||
kTensorflowGpu,
|
||||
kSample,
|
||||
kRknn,
|
||||
} HelperType;
|
||||
|
||||
public:
|
||||
static InferenceHelper* Create(const HelperType helper_type);
|
||||
static void PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob); // use this if the selected inference engine doesn't support pre-process
|
||||
|
||||
public:
|
||||
virtual ~InferenceHelper() {}
|
||||
virtual int32_t SetNumThreads(const int32_t num_threads) = 0;
|
||||
virtual int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) = 0;
|
||||
virtual int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
|
||||
virtual int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
|
||||
virtual int32_t Finalize(void) = 0;
|
||||
virtual int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) = 0;
|
||||
virtual int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
|
||||
virtual int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
|
||||
|
||||
protected:
|
||||
void ConvertNormalizeParameters(InputTensorInfo& tensor_info);
|
||||
|
||||
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
|
||||
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
|
||||
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);
|
||||
|
||||
template<typename T>
|
||||
void PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T *dst);
|
||||
|
||||
protected:
|
||||
HelperType helper_type_;
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,45 @@
|
||||
/* Copyright 2021 iwatake2222
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef INFERENCE_HELPER_LOG_
|
||||
#define INFERENCE_HELPER_LOG_
|
||||
|
||||
/* for general */
|
||||
#include <cstdint>
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
|
||||
|
||||
#if defined(ANDROID) || defined(__ANDROID__)
|
||||
#define CV_COLOR_IS_RGB
|
||||
#include <android/log.h>
|
||||
#define INFERENCE_HELPER_LOG_NDK_TAG "HyperLPR3-Native-Inference"
|
||||
#define INFERENCE_HELPER_LOG_PRINT_(...) __android_log_print(ANDROID_LOG_INFO, INFERENCE_HELPER_LOG_NDK_TAG, __VA_ARGS__)
|
||||
#else
|
||||
#define INFERENCE_HELPER_LOG_PRINT_(...) printf(__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
#define INFERENCE_HELPER_LOG_PRINT(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \
|
||||
INFERENCE_HELPER_LOG_PRINT_("[" INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
|
||||
INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \
|
||||
} while(0);
|
||||
|
||||
#define INFERENCE_HELPER_LOG_PRINT_E(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \
|
||||
INFERENCE_HELPER_LOG_PRINT_("[ERR: " INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
|
||||
INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \
|
||||
} while(0);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,317 @@
|
||||
/* Copyright 2021 iwatake2222
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
/*** Include ***/
|
||||
/* for general */
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
|
||||
/* for MNN */
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
#include <MNN/Interpreter.hpp>
|
||||
#include <MNN/AutoTime.hpp>
|
||||
|
||||
/* for My modules */
|
||||
#include "inference_helper_log.h"
|
||||
#include "inference_helper_mnn.h"
|
||||
|
||||
/*** Macro ***/
|
||||
#define TAG "InferenceHelperMnn"
|
||||
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
|
||||
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
|
||||
|
||||
/*** Function ***/
|
||||
InferenceHelperMnn::InferenceHelperMnn()
|
||||
{
|
||||
num_threads_ = 1;
|
||||
}
|
||||
|
||||
InferenceHelperMnn::~InferenceHelperMnn()
|
||||
{
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::SetNumThreads(const int32_t num_threads)
|
||||
{
|
||||
num_threads_ = num_threads;
|
||||
return kRetOk;
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops)
|
||||
{
|
||||
PRINT("[WARNING] This method is not supported\n");
|
||||
return kRetOk;
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
|
||||
/* Check tensor info fits the info from model */
|
||||
for (auto& input_tensor_info : input_tensor_info_list) {
|
||||
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
|
||||
if (input_tensor == nullptr) {
|
||||
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
|
||||
return kRetErr;
|
||||
}
|
||||
if ((input_tensor->getType().code == halide_type_float) && (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32)) {
|
||||
/* OK */
|
||||
} else if ((input_tensor->getType().code == halide_type_uint) && (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8)) {
|
||||
/* OK */
|
||||
} else {
|
||||
PRINT_E("Incorrect input tensor type (%d, %d)\n", input_tensor->getType().code, input_tensor_info.tensor_type);
|
||||
return kRetErr;
|
||||
}
|
||||
if ((input_tensor->channel() != -1) && (input_tensor->height() != -1) && (input_tensor->width() != -1)) {
|
||||
if (input_tensor_info.GetChannel() != -1) {
|
||||
if ((input_tensor->channel() == input_tensor_info.GetChannel()) && (input_tensor->height() == input_tensor_info.GetHeight()) && (input_tensor->width() == input_tensor_info.GetWidth())) {
|
||||
/* OK */
|
||||
} else {
|
||||
PRINT_E("W: %d != %d\n", input_tensor->width() , input_tensor_info.GetWidth());
|
||||
PRINT_E("H: %d != %d\n", input_tensor->height() , input_tensor_info.GetHeight());
|
||||
PRINT_E("C: %d != %d\n", input_tensor->channel() , input_tensor_info.GetChannel());
|
||||
PRINT_E("Incorrect input tensor size\n");
|
||||
return kRetErr;
|
||||
}
|
||||
} else {
|
||||
PRINT("Input tensor size is set from the model\n");
|
||||
input_tensor_info.tensor_dims.clear();
|
||||
for (int32_t dim = 0; dim < input_tensor->dimensions(); dim++) {
|
||||
input_tensor_info.tensor_dims.push_back(input_tensor->length(dim));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (input_tensor_info.GetChannel() != -1) {
|
||||
PRINT("Input tensor size is resized\n");
|
||||
/* In case the input size is not fixed */
|
||||
net_->resizeTensor(input_tensor, { 1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth() });
|
||||
net_->resizeSession(session_);
|
||||
} else {
|
||||
PRINT_E("Model input size is not set\n");
|
||||
return kRetErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const auto& output_tensor_info : output_tensor_info_list) {
|
||||
auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
|
||||
if (output_tensor == nullptr) {
|
||||
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
|
||||
return kRetErr;
|
||||
}
|
||||
/* Output size is set when run inference later */
|
||||
}
|
||||
|
||||
/* Convert normalize parameter to speed up */
|
||||
for (auto& input_tensor_info : input_tensor_info_list) {
|
||||
ConvertNormalizeParameters(input_tensor_info);
|
||||
}
|
||||
|
||||
|
||||
/* Check if tensor info is set */
|
||||
for (const auto& input_tensor_info : input_tensor_info_list) {
|
||||
for (const auto& dim : input_tensor_info.tensor_dims) {
|
||||
if (dim <= 0) {
|
||||
PRINT_E("Invalid tensor size\n");
|
||||
return kRetErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
//for (const auto& output_tensor_info : output_tensor_info_list) {
|
||||
// for (const auto& dim : output_tensor_info.tensor_dims) {
|
||||
// if (dim <= 0) {
|
||||
// PRINT_E("Invalid tensor size\n");
|
||||
// return kRetErr;
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
return kRetOk;
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
|
||||
/*** Create network ***/
|
||||
net_.reset(MNN::Interpreter::createFromBuffer(model_buffer, model_size));
|
||||
if (!net_) {
|
||||
PRINT_E("Failed to load model model buffer\n");
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
MNN::ScheduleConfig scheduleConfig;
|
||||
scheduleConfig.type = MNN_FORWARD_CPU;
|
||||
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
|
||||
// MNN::BackendConfig bnconfig;
|
||||
// bnconfig.power = MNN::BackendConfig::Power_High;
|
||||
// bnconfig.precision = MNN::BackendConfig::Precision_Low;
|
||||
// scheduleConfig.backendConfig = &bnconfig;
|
||||
session_ = net_->createSession(scheduleConfig);
|
||||
if (!session_) {
|
||||
PRINT_E("Failed to create session\n");
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list)
|
||||
{
|
||||
/*** Create network ***/
|
||||
net_.reset(MNN::Interpreter::createFromFile(model_filename.c_str()));
|
||||
if (!net_) {
|
||||
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str());
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
MNN::ScheduleConfig scheduleConfig;
|
||||
scheduleConfig.type = MNN_FORWARD_AUTO;
|
||||
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
|
||||
// MNN::BackendConfig bnconfig;
|
||||
// bnconfig.power = MNN::BackendConfig::Power_High;
|
||||
// bnconfig.precision = MNN::BackendConfig::Precision_Low;
|
||||
// scheduleConfig.backendConfig = &bnconfig;
|
||||
session_ = net_->createSession(scheduleConfig);
|
||||
if (!session_) {
|
||||
PRINT_E("Failed to create session\n");
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
|
||||
|
||||
};
|
||||
|
||||
|
||||
int32_t InferenceHelperMnn::Finalize(void)
|
||||
{
|
||||
net_->releaseSession(session_);
|
||||
net_->releaseModel();
|
||||
net_.reset();
|
||||
out_mat_list_.clear();
|
||||
return kRetOk;
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list)
|
||||
{
|
||||
for (const auto& input_tensor_info : input_tensor_info_list) {
|
||||
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
|
||||
if (input_tensor == nullptr) {
|
||||
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
|
||||
return kRetErr;
|
||||
}
|
||||
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) {
|
||||
/* Crop */
|
||||
if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) || (input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) {
|
||||
PRINT_E("Crop is not supported\n");
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
MNN::CV::ImageProcess::Config image_processconfig;
|
||||
/* Convert color type */
|
||||
if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) {
|
||||
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
|
||||
if (input_tensor_info.image_info.swap_color) {
|
||||
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::RGB : MNN::CV::BGR;
|
||||
} else {
|
||||
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
|
||||
}
|
||||
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 1)) {
|
||||
image_processconfig.sourceFormat = MNN::CV::GRAY;
|
||||
image_processconfig.destFormat = MNN::CV::GRAY;
|
||||
} else if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 1)) {
|
||||
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
|
||||
image_processconfig.destFormat = MNN::CV::GRAY;
|
||||
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 3)) {
|
||||
image_processconfig.sourceFormat = MNN::CV::GRAY;
|
||||
image_processconfig.destFormat = MNN::CV::BGR;
|
||||
} else {
|
||||
PRINT_E("Unsupported color conversion (%d, %d)\n", input_tensor_info.image_info.channel, input_tensor_info.GetChannel());
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
/* Normalize image */
|
||||
std::memcpy(image_processconfig.mean, input_tensor_info.normalize.mean, sizeof(image_processconfig.mean));
|
||||
std::memcpy(image_processconfig.normal, input_tensor_info.normalize.norm, sizeof(image_processconfig.normal));
|
||||
|
||||
/* Resize image */
|
||||
image_processconfig.filterType = MNN::CV::BILINEAR;
|
||||
MNN::CV::Matrix trans;
|
||||
trans.setScale(static_cast<float>(input_tensor_info.image_info.crop_width) / input_tensor_info.GetWidth(), static_cast<float>(input_tensor_info.image_info.crop_height) / input_tensor_info.GetHeight());
|
||||
|
||||
/* Do pre-process */
|
||||
std::shared_ptr<MNN::CV::ImageProcess> pretreat(MNN::CV::ImageProcess::create(image_processconfig));
|
||||
pretreat->setMatrix(trans);
|
||||
pretreat->convert(static_cast<uint8_t*>(input_tensor_info.data), input_tensor_info.image_info.crop_width, input_tensor_info.image_info.crop_height, 0, input_tensor);
|
||||
} else if ( (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) ) {
|
||||
std::unique_ptr<MNN::Tensor> tensor;
|
||||
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) {
|
||||
tensor.reset(new MNN::Tensor(input_tensor, MNN::Tensor::TENSORFLOW));
|
||||
} else {
|
||||
tensor.reset(new MNN::Tensor(input_tensor, MNN::Tensor::CAFFE));
|
||||
}
|
||||
if (tensor->getType().code == halide_type_float) {
|
||||
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
|
||||
tensor->host<float>()[i] = static_cast<float*>(input_tensor_info.data)[i];
|
||||
}
|
||||
} else {
|
||||
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
|
||||
tensor->host<uint8_t>()[i] = static_cast<uint8_t*>(input_tensor_info.data)[i];
|
||||
}
|
||||
}
|
||||
input_tensor->copyFromHostTensor(tensor.get());
|
||||
} else {
|
||||
PRINT_E("Unsupported data type (%d)\n", input_tensor_info.data_type);
|
||||
return kRetErr;
|
||||
}
|
||||
}
|
||||
return kRetOk;
|
||||
}
|
||||
|
||||
int32_t InferenceHelperMnn::Process(std::vector<OutputTensorInfo>& output_tensor_info_list)
|
||||
{
|
||||
net_->runSession(session_);
|
||||
|
||||
out_mat_list_.clear();
|
||||
for (auto& output_tensor_info : output_tensor_info_list) {
|
||||
auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
|
||||
if (output_tensor == nullptr) {
|
||||
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
auto dimType = output_tensor->getDimensionType();
|
||||
std::unique_ptr<MNN::Tensor> outputUser(new MNN::Tensor(output_tensor, dimType));
|
||||
output_tensor->copyToHostTensor(outputUser.get());
|
||||
auto type = outputUser->getType();
|
||||
if (type.code == halide_type_float) {
|
||||
output_tensor_info.tensor_type = TensorInfo::kTensorTypeFp32;
|
||||
output_tensor_info.data = outputUser->host<float>();
|
||||
} else if (type.code == halide_type_uint && type.bytes() == 1) {
|
||||
output_tensor_info.tensor_type = TensorInfo::kTensorTypeUint8;
|
||||
output_tensor_info.data = outputUser->host<uint8_t>();
|
||||
} else {
|
||||
PRINT_E("Unexpected data type\n");
|
||||
return kRetErr;
|
||||
}
|
||||
|
||||
output_tensor_info.tensor_dims.clear();
|
||||
for (int32_t dim = 0; dim < outputUser->dimensions(); dim++) {
|
||||
output_tensor_info.tensor_dims.push_back(outputUser->length(dim));
|
||||
}
|
||||
|
||||
out_mat_list_.push_back(std::move(outputUser)); // store data in member variable so that data keep exist
|
||||
}
|
||||
|
||||
return kRetOk;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/* Copyright 2021 iwatake2222
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef INFERENCE_HELPER_MNN_
|
||||
#define INFERENCE_HELPER_MNN_
|
||||
|
||||
/* for general */
|
||||
#include <cstdint>
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <memory>
|
||||
|
||||
/* for MNN */
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
#include <MNN/Interpreter.hpp>
|
||||
#include <MNN/AutoTime.hpp>
|
||||
|
||||
/* for My modules */
|
||||
#include "inference_helper.h"
|
||||
|
||||
class InferenceHelperMnn : public InferenceHelper {
|
||||
public:
|
||||
InferenceHelperMnn();
|
||||
~InferenceHelperMnn() override;
|
||||
int32_t SetNumThreads(const int32_t num_threads) override;
|
||||
int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) override;
|
||||
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
|
||||
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
|
||||
int32_t Finalize(void) override;
|
||||
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
|
||||
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
|
||||
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
|
||||
private:
|
||||
std::unique_ptr<MNN::Interpreter> net_;
|
||||
MNN::Session* session_;
|
||||
std::vector<std::unique_ptr<MNN::Tensor>> out_mat_list_;
|
||||
int32_t num_threads_;
|
||||
};
|
||||
|
||||
#endif
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#ifndef _LOG_UTILS_H_
|
||||
#define _LOG_UTILS_H_
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
//#define DEBUG
|
||||
|
||||
#define __FILENAME__ (strrchr(__FILE__, '/') + 1)
|
||||
|
||||
//#ifdef DEBUG
|
||||
|
||||
#ifdef ANDROID
|
||||
|
||||
#include <android/log.h>
|
||||
#define TAG "HyperLPR3-Native" // 这个是自定义的LOG的标识
|
||||
#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG,TAG ,__VA_ARGS__) // 定义LOGD类型
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,TAG ,__VA_ARGS__) // 定义LOGI类型
|
||||
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN,TAG ,__VA_ARGS__) // 定义LOGW类型
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,TAG ,__VA_ARGS__) // 定义LOGE类型
|
||||
#define LOGF(...) __android_log_print(ANDROID_LOG_FATAL,TAG ,__VA_ARGS__) // 定义LOGF类型
|
||||
|
||||
#else
|
||||
|
||||
// color setting
|
||||
#define ANSI_COLOR_RED "\x1b[31m"
|
||||
#define ANSI_COLOR_GREEN "\x1b[32m"
|
||||
#define ANSI_COLOR_YELLOW "\x1b[33m"
|
||||
#define ANSI_COLOR_BLUE "\x1b[34m"
|
||||
#define ANSI_COLOR_MAGENTA "\x1b[35m"
|
||||
#define ANSI_COLOR_CYAN "\x1b[36m"
|
||||
#define ANSI_COLOR_WHITE "\x1b[37m"
|
||||
#define ANSI_COLOR_RESET "\x1b[0m"
|
||||
|
||||
#define LOGD(format, ...) \
|
||||
printf("[%s][%s][%d]: " format "\n", __FILENAME__, __FUNCTION__, __LINE__, \
|
||||
##__VA_ARGS__)
|
||||
|
||||
#define LOGE(format, ...) \
|
||||
printf("%s[%s][%s][%d]: " format "\n%s", ANSI_COLOR_RED, __FILENAME__, __FUNCTION__, __LINE__, \
|
||||
##__VA_ARGS__, ANSI_COLOR_RESET)
|
||||
|
||||
|
||||
#endif //ANDROID
|
||||
|
||||
//#else
|
||||
//#define LOGD(format, ...)
|
||||
//#endif
|
||||
|
||||
#endif // _LOG_UTILS_H_
|
||||
@@ -0,0 +1,12 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_NN_IMPLEMENTATION_ALL_H
|
||||
#define ZEPHYRLPR_NN_IMPLEMENTATION_ALL_H
|
||||
|
||||
#include "recognition/all.h"
|
||||
#include "detect/all.h"
|
||||
#include "classification/all.h"
|
||||
|
||||
#endif //ZEPHYRLPR_NN_IMPLEMENTATION_ALL_H
|
||||
@@ -0,0 +1,12 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/22.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_CLASSIFICATION_ALL_H
|
||||
#define ZEPHYRLPR_CLASSIFICATION_ALL_H
|
||||
|
||||
#include "plate_classification.h"
|
||||
#include "plate_cls_common.h"
|
||||
#include "classification_engine.h"
|
||||
|
||||
#endif //ZEPHYRLPR_CLASSIFICATION_ALL_H
|
||||
@@ -0,0 +1,107 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/9.
|
||||
//
|
||||
|
||||
#include "classification_engine.h"
|
||||
#include "utils.h"
|
||||
|
||||
|
||||
namespace hyper {
|
||||
|
||||
|
||||
ClassificationEngine::ClassificationEngine() {
|
||||
|
||||
}
|
||||
|
||||
ClassificationEngine::~ClassificationEngine() {
|
||||
|
||||
}
|
||||
|
||||
int32_t ClassificationEngine::Initialize(const std::string& model_filename, cv::Size input_size, int threads, bool use_half) {
|
||||
m_input_image_size_ = std::move(input_size);
|
||||
|
||||
/* Set output tensor info */
|
||||
m_output_tensor_info_list_.clear();
|
||||
m_output_tensor_info_list_.push_back(OutputTensorInfo("output", TensorInfo::kTensorTypeFp32));
|
||||
|
||||
|
||||
m_nn_infer_.reset(InferenceHelper::Create(InferenceHelper::kMnn));
|
||||
|
||||
if (m_nn_infer_->SetNumThreads(threads) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
if (m_nn_infer_->Initialize(model_filename, m_input_tensor_info_list_, m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
m_input_tensor_info_list_.clear();
|
||||
|
||||
InputTensorInfo input_tensor_info("data", TensorInfo::kTensorTypeFp32, true);
|
||||
|
||||
input_tensor_info.tensor_dims = { 1, 3, m_input_image_size_.height, m_input_image_size_.width };
|
||||
input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
|
||||
|
||||
input_tensor_info.normalize.mean[0] = 0.0f;
|
||||
input_tensor_info.normalize.mean[1] = 0.0f;
|
||||
input_tensor_info.normalize.mean[2] = 0.0f;
|
||||
input_tensor_info.normalize.norm[0] = 0.003921568627f;
|
||||
input_tensor_info.normalize.norm[1] = 0.003921568627f;
|
||||
input_tensor_info.normalize.norm[2] = 0.003921568627f;
|
||||
|
||||
m_input_tensor_info_list_.push_back(input_tensor_info);
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
int32_t ClassificationEngine::Inference(const cv::Mat &bgr_pad) {
|
||||
InputTensorInfo& input_tensor_info = m_input_tensor_info_list_[0];
|
||||
|
||||
input_tensor_info.data = bgr_pad.data;
|
||||
input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
|
||||
input_tensor_info.image_info.width = bgr_pad.cols;
|
||||
input_tensor_info.image_info.height = bgr_pad.rows;
|
||||
input_tensor_info.image_info.channel = bgr_pad.channels();
|
||||
input_tensor_info.image_info.crop_x = 0;
|
||||
input_tensor_info.image_info.crop_y = 0;
|
||||
input_tensor_info.image_info.crop_width = bgr_pad.cols;
|
||||
input_tensor_info.image_info.crop_height = bgr_pad.rows;
|
||||
input_tensor_info.image_info.is_bgr = true;
|
||||
input_tensor_info.image_info.swap_color = false;
|
||||
|
||||
if (m_nn_infer_->PreProcess(m_input_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
if (m_nn_infer_->Process(m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
std::vector<float> output_score_raw_list(m_output_tensor_info_list_[0].GetDataAsFloat(),
|
||||
m_output_tensor_info_list_[0].GetDataAsFloat() +
|
||||
m_output_tensor_info_list_[0].GetElementNum());
|
||||
|
||||
|
||||
auto index = argmax(output_score_raw_list.begin(), output_score_raw_list.end());
|
||||
m_output_max_confidence_ = output_score_raw_list[index];
|
||||
m_output_color_ = PlateColor(index);
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
|
||||
PlateColor ClassificationEngine::getMOutputColor() const {
|
||||
return m_output_color_;
|
||||
}
|
||||
|
||||
const cv::Size &ClassificationEngine::getMInputImageSize() const {
|
||||
return m_input_image_size_;
|
||||
}
|
||||
|
||||
float ClassificationEngine::getMOutputMaxConfidence() const {
|
||||
return m_output_max_confidence_;
|
||||
}
|
||||
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/9.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_CLASSIFICATION_ENGINE_H
|
||||
#define ZEPHYRLPR_CLASSIFICATION_ENGINE_H
|
||||
|
||||
#include "inference_helper_module/inference_helper.h"
|
||||
#include "configuration.h"
|
||||
#include "plate_cls_common.h"
|
||||
#include "basic_types.h"
|
||||
#include "opencv2/opencv.hpp"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class ClassificationEngine {
|
||||
public:
|
||||
ClassificationEngine(const ClassificationEngine &) = delete;
|
||||
|
||||
ClassificationEngine &operator=(const ClassificationEngine &) = delete;
|
||||
|
||||
explicit ClassificationEngine();
|
||||
|
||||
~ClassificationEngine();
|
||||
|
||||
int32_t Initialize(const std::string& model_filename, cv::Size input_size = CLS_INPUT_SIZE, int threads = 1, bool use_half = false);
|
||||
|
||||
int32_t Inference(const cv::Mat &bgr_pad);
|
||||
|
||||
PlateColor getMOutputColor() const;
|
||||
|
||||
const cv::Size &getMInputImageSize() const;
|
||||
|
||||
float getMOutputMaxConfidence() const;
|
||||
|
||||
private:
|
||||
|
||||
cv::Size m_input_image_size_{}; // 输入图像宽高
|
||||
|
||||
std::unique_ptr<InferenceHelper> m_nn_infer_; // 推理模块
|
||||
|
||||
std::vector<InputTensorInfo> m_input_tensor_info_list_;
|
||||
|
||||
std::vector<OutputTensorInfo> m_output_tensor_info_list_;
|
||||
|
||||
PlateColor m_output_color_;
|
||||
|
||||
float m_output_max_confidence_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
#endif //ZEPHYRLPR_CLASSIFICATION_ENGINE_H
|
||||
@@ -0,0 +1,34 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/22.
|
||||
//
|
||||
|
||||
#include "plate_classification.h"
|
||||
#include "utils.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
PlateClassification::PlateClassification() = default;
|
||||
|
||||
int PlateClassification::Initialize(const std::string& model_filename, cv::Size input_size, int threads, bool use_half) {
|
||||
m_input_image_size_ = std::move(input_size);
|
||||
float mean[] = {0.0f, 0.0f, 0.0f};
|
||||
float normal[] = {0.003921568627f, 0.003921568627f, 0.003921568627f};
|
||||
m_nn_adapter_ = std::make_shared<MNNAdapterInference>(model_filename, threads, mean, normal, use_half);
|
||||
m_nn_adapter_->Initialization("data", "output", m_input_image_size_.width, m_input_image_size_.height);
|
||||
return 0;
|
||||
}
|
||||
|
||||
PlateColor PlateClassification::Inference(const cv::Mat &bgr_pad) {
|
||||
std::vector<float> output = m_nn_adapter_->Invoking(bgr_pad);
|
||||
// for (int i = 0; i < output.size(); ++i) {
|
||||
// std::cout << output[i] << std::endl;
|
||||
// }
|
||||
auto index = argmax(output.begin(), output.end());
|
||||
return PlateColor(index);
|
||||
}
|
||||
|
||||
const cv::Size &PlateClassification::getMInputImageSize() const {
|
||||
return m_input_image_size_;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,39 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/22.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_PLATE_CLASSIFICATION_H
|
||||
#define ZEPHYRLPR_PLATE_CLASSIFICATION_H
|
||||
|
||||
#include "nn_module/mnn_adapter.h"
|
||||
#include "basic_types.h"
|
||||
#include "plate_cls_common.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class PlateClassification {
|
||||
public:
|
||||
|
||||
PlateClassification(const PlateClassification &) = delete;
|
||||
|
||||
PlateClassification &operator=(const PlateClassification &) = delete;
|
||||
|
||||
explicit PlateClassification();
|
||||
|
||||
int Initialize(const std::string& model_filename, cv::Size input_size = cv::Size(96, 96), int threads = 1, bool use_half = false);
|
||||
|
||||
PlateColor Inference(const cv::Mat &bgr_pad);
|
||||
|
||||
const cv::Size &getMInputImageSize() const;
|
||||
|
||||
private:
|
||||
|
||||
cv::Size m_input_image_size_{};
|
||||
|
||||
std::shared_ptr<MNNAdapterInference> m_nn_adapter_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
#endif //ZEPHYRLPR_PLATE_CLASSIFICATION_H
|
||||
@@ -0,0 +1,14 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/22.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_PLATE_CLS_COMMON_H
|
||||
#define ZEPHYRLPR_PLATE_CLS_COMMON_H
|
||||
|
||||
typedef enum {
|
||||
BLUE, ///< 蓝牌
|
||||
GREEN, ///< 绿牌
|
||||
YELLOW, ///< 黄牌
|
||||
} PlateColor;
|
||||
|
||||
#endif //ZEPHYRLPR_PLATE_CLS_COMMON_H
|
||||
@@ -0,0 +1,16 @@
|
||||
//
|
||||
// Created by tunm on 2022/12/29.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_DETECT_ALL_H
|
||||
#define ZEPHYRLPR_DETECT_ALL_H
|
||||
|
||||
#include "plate_det_common.h"
|
||||
#include "plate_detector.h"
|
||||
#include "det_backbone.h"
|
||||
#include "det_header.h"
|
||||
#include "det_arch.h"
|
||||
|
||||
|
||||
|
||||
#endif //ZEPHYRLPR_DETECT_ALL_H
|
||||
@@ -0,0 +1,143 @@
|
||||
//
|
||||
// Created by tunm on 2023/2/8.
|
||||
//
|
||||
|
||||
#include "det_arch.h"
|
||||
#include "log.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
|
||||
DetArch::DetArch() = default;
|
||||
|
||||
|
||||
int32_t DetArch::Initialize(const std::string& backbone_path, const std::string& head_path, int input_size, int threads,
|
||||
float box_conf_threshold, float nms_threshold, bool use_half) {
|
||||
int32_t ret;
|
||||
m_input_size_ = input_size;
|
||||
m_box_conf_threshold_ = box_conf_threshold;
|
||||
m_nms_threshold_ = nms_threshold;
|
||||
m_backbone_net_ = std::make_shared<DetBackbone>();
|
||||
m_header_net_ = std::make_shared<DetHeader>();
|
||||
ret = m_backbone_net_->Initialize(backbone_path, cv::Size(input_size, input_size), use_half);
|
||||
if (ret != InferenceHelper::kRetOk) {
|
||||
LOGE("Backbone model error.");
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
ret = m_header_net_->Initialize(head_path, input_size, use_half);
|
||||
if (ret != InferenceHelper::kRetOk) {
|
||||
LOGE("Head model error.");
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
if (input_size == 320) {
|
||||
m_grid_num_ = 6300;
|
||||
} else if (input_size == 640) {
|
||||
m_grid_num_ = 25200;
|
||||
} else {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
void DetArch::Decode(T features, std::vector<PlateLocation> &outputs, float scale, float conf_threshold,
|
||||
float nms_threshold) {
|
||||
|
||||
int item_num = 15;
|
||||
for (int index = 0; index < m_grid_num_; ++index) {
|
||||
auto det_score = features[index * item_num + 4];
|
||||
if (det_score > conf_threshold) {
|
||||
// std::cout << det_score << std::endl;
|
||||
PlateLocation plate;
|
||||
plate.det_confidence = det_score;
|
||||
auto class_score_1 = features[index * item_num + 13] * det_score;
|
||||
auto class_score_2 = features[index * item_num + 14] * det_score;
|
||||
LayersNum type = class_score_1 > class_score_2 ? MONO : DOUBLE;
|
||||
plate.layers = type;
|
||||
auto x = features[index * item_num + 0];
|
||||
auto y = features[index * item_num + 1];
|
||||
auto w = features[index * item_num + 2];
|
||||
auto h = features[index * item_num + 3];
|
||||
std::vector<float> xyxy = xywh2xyxy(x, y, w, h);
|
||||
plate.x1 = xyxy[0];
|
||||
plate.y1 = xyxy[1];
|
||||
plate.x2 = xyxy[2];
|
||||
plate.y2 = xyxy[3];
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
plate.kps[i] = features[index * item_num + i + 5];
|
||||
}
|
||||
outputs.push_back(plate);
|
||||
}
|
||||
|
||||
}
|
||||
nms(outputs, nms_threshold);
|
||||
|
||||
for (auto &plate : outputs) {
|
||||
plate.x1 = plate.x1 / scale;
|
||||
plate.y1 = plate.y1 / scale;
|
||||
plate.x2 = plate.x2 / scale;
|
||||
plate.y2 = plate.y2 / scale;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
plate.kps[i * 2 + 0] = plate.kps[i * 2 + 0] / scale;
|
||||
plate.kps[i * 2 + 1] = plate.kps[i * 2 + 1] / scale;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
void DetArch::Detection(const cv::Mat &bgr, bool is_resize, float scale) {
|
||||
cv::Mat pad;
|
||||
if (is_resize) {
|
||||
int ori_w = bgr.cols;
|
||||
int ori_h = bgr.rows;
|
||||
|
||||
int w, h;
|
||||
if (ori_w > ori_h) {
|
||||
scale = (float) m_input_size_ / ori_w;
|
||||
w = m_input_size_;
|
||||
h = ori_h * scale;
|
||||
} else {
|
||||
scale = (float) m_input_size_ / ori_h;
|
||||
h = m_input_size_;
|
||||
w = ori_w * scale;
|
||||
}
|
||||
int wpad = m_input_size_ - w;
|
||||
int hpad = m_input_size_ - h;
|
||||
cv::Mat resized_img;
|
||||
cv::resize(bgr, resized_img, cv::Size(w, h));
|
||||
cv::copyMakeBorder(resized_img, pad, 0, hpad, 0, wpad, cv::BORDER_CONSTANT, cv::Scalar(127.5, 127.5, 127.5));
|
||||
// cv::imwrite("i.jpg", pad);
|
||||
// cv::imshow("pad_border", pad);
|
||||
// cv::waitKey(0);
|
||||
} else {
|
||||
pad = bgr;
|
||||
}
|
||||
double time;
|
||||
|
||||
time = (double)cv::getTickCount();
|
||||
m_backbone_net_->Inference(pad);
|
||||
time = ((double)cv::getTickCount() - time) / cv::getTickFrequency();
|
||||
|
||||
time = (double)cv::getTickCount();
|
||||
m_header_net_->Inference(m_backbone_net_->getMOutputTensorInfoList()[0].GetDataAsFloat(),
|
||||
m_backbone_net_->getMOutputTensorInfoList()[1].GetDataAsFloat(),
|
||||
m_backbone_net_->getMOutputTensorInfoList()[2].GetDataAsFloat());
|
||||
time = ((double)cv::getTickCount() - time) / cv::getTickFrequency();
|
||||
|
||||
// std::vector<float> tensor(m_header_net_->getMOutputTensorInfoList()[0].GetDataAsFloat(),
|
||||
// m_header_net_->getMOutputTensorInfoList()[0].GetDataAsFloat() +
|
||||
// m_header_net_->getMOutputTensorInfoList()[0].GetElementNum());
|
||||
|
||||
|
||||
|
||||
std::vector<PlateLocation>().swap(m_results_);
|
||||
Decode(m_header_net_->getMOutputTensorInfoList()[0].GetDataAsFloat(), m_results_, scale, m_box_conf_threshold_, m_nms_threshold_);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// Created by tunm on 2023/2/8.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_DET_ARCH_H
|
||||
#define ZEPHYRLPR_DET_ARCH_H
|
||||
|
||||
#include "det_backbone.h"
|
||||
#include "det_header.h"
|
||||
#include "plate_det_common.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class DetArch {
|
||||
public:
|
||||
DetArch(const DetArch &) = delete;
|
||||
|
||||
DetArch &operator=(const DetArch &) = delete;
|
||||
|
||||
explicit DetArch();
|
||||
|
||||
int32_t Initialize(const std::string& backbone_path, const std::string& head_path, int input_size = 320, int threads=1, float box_conf_threshold = 0.3f, float nms_threshold = 0.6f, bool use_half = false);
|
||||
|
||||
void Detection(const cv::Mat& bgr, bool is_resize = false, float scale = 1.0f);
|
||||
|
||||
public:
|
||||
std::vector<PlateLocation> m_results_; // 存放检测结果
|
||||
|
||||
private:
|
||||
template<class T>
|
||||
void Decode(T features, std::vector<PlateLocation> &outputs, float scale,
|
||||
float conf_threshold = 0.5, float nms_threshold = 0.5);
|
||||
|
||||
private:
|
||||
int m_input_size_{};
|
||||
|
||||
float m_box_conf_threshold_{};
|
||||
|
||||
float m_nms_threshold_{};
|
||||
|
||||
std::shared_ptr<DetBackbone> m_backbone_net_;
|
||||
|
||||
std::shared_ptr<DetHeader> m_header_net_;
|
||||
|
||||
int m_grid_num_ = 6300;
|
||||
|
||||
};
|
||||
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
|
||||
#endif //ZEPHYRLPR_DET_ARCH_H
|
||||
@@ -0,0 +1,96 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/8.
|
||||
//
|
||||
|
||||
#include <iostream>
|
||||
#include "det_backbone.h"
|
||||
#include "log.h"
|
||||
#include <utility>
|
||||
|
||||
namespace hyper {
|
||||
|
||||
DetBackbone::~DetBackbone() {
|
||||
if (!m_nn_infer_) {
|
||||
LOGD("[DetBackbone]Inference helper is not created\n");
|
||||
return;
|
||||
}
|
||||
m_nn_infer_->Finalize();
|
||||
m_nn_infer_.reset();
|
||||
}
|
||||
|
||||
int32_t DetBackbone::Initialize(const std::string& model_filename, cv::Size input_size, bool use_half) {
|
||||
m_input_image_size_ = std::move(input_size);
|
||||
|
||||
|
||||
m_nn_infer_.reset(InferenceHelper::Create(InferenceHelper::kMnn));
|
||||
|
||||
if (m_nn_infer_->SetNumThreads(1) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
/* Set output tensor info */
|
||||
m_output_tensor_info_list_.clear();
|
||||
|
||||
m_output_tensor_info_list_ = {OutputTensorInfo("948", TensorInfo::kTensorTypeFp32),
|
||||
OutputTensorInfo("1061", TensorInfo::kTensorTypeFp32),
|
||||
OutputTensorInfo("1174", TensorInfo::kTensorTypeFp32)};
|
||||
if (m_nn_infer_->Initialize(model_filename, m_input_tensor_info_list_, m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
m_input_tensor_info_list_.clear();
|
||||
|
||||
InputTensorInfo input_tensor_info("input", TensorInfo::kTensorTypeFp32, true);
|
||||
|
||||
input_tensor_info.tensor_dims = { 1, 3, m_input_image_size_.height, m_input_image_size_.width };
|
||||
input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
|
||||
|
||||
input_tensor_info.normalize.mean[0] = 0.0f;
|
||||
input_tensor_info.normalize.mean[1] = 0.0f;
|
||||
input_tensor_info.normalize.mean[2] = 0.0f;
|
||||
input_tensor_info.normalize.norm[0] = 0.003921568627f;
|
||||
input_tensor_info.normalize.norm[1] = 0.003921568627f;
|
||||
input_tensor_info.normalize.norm[2] = 0.003921568627f;
|
||||
m_input_tensor_info_list_.push_back(input_tensor_info);
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
int32_t DetBackbone::Inference(const cv::Mat &bgr_pad) {
|
||||
InputTensorInfo& input_tensor_info = m_input_tensor_info_list_[0];
|
||||
input_tensor_info.data = bgr_pad.data;
|
||||
input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
|
||||
input_tensor_info.image_info.width = bgr_pad.cols;
|
||||
input_tensor_info.image_info.height = bgr_pad.rows;
|
||||
input_tensor_info.image_info.channel = bgr_pad.channels();
|
||||
input_tensor_info.image_info.crop_x = 0;
|
||||
input_tensor_info.image_info.crop_y = 0;
|
||||
input_tensor_info.image_info.crop_width = bgr_pad.cols;
|
||||
input_tensor_info.image_info.crop_height = bgr_pad.rows;
|
||||
input_tensor_info.image_info.is_bgr = true;
|
||||
input_tensor_info.image_info.swap_color = false;
|
||||
|
||||
if (m_nn_infer_->PreProcess(m_input_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
if (m_nn_infer_->Process(m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
|
||||
DetBackbone::DetBackbone() {
|
||||
|
||||
}
|
||||
|
||||
std::vector<OutputTensorInfo> &DetBackbone::getMOutputTensorInfoList() {
|
||||
return m_output_tensor_info_list_;
|
||||
}
|
||||
|
||||
|
||||
} // namespace;
|
||||
@@ -0,0 +1,43 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/8.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_DET_BACKBONE_H
|
||||
#define ZEPHYRLPR_DET_BACKBONE_H
|
||||
|
||||
#include "inference_helper_module/inference_helper.h"
|
||||
#include "plate_det_common.h"
|
||||
#include "opencv2/opencv.hpp"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class DetBackbone {
|
||||
public:
|
||||
|
||||
DetBackbone();
|
||||
|
||||
~DetBackbone();
|
||||
|
||||
int32_t Initialize(const std::string& model_filename, cv::Size input_size = cv::Size(320, 320), bool use_half = false);
|
||||
|
||||
int32_t Inference(const cv::Mat &bgr_pad);
|
||||
|
||||
public:
|
||||
|
||||
std::vector<OutputTensorInfo> &getMOutputTensorInfoList();
|
||||
|
||||
private:
|
||||
|
||||
cv::Size m_input_image_size_{}; // 输入图像宽高
|
||||
|
||||
std::shared_ptr<InferenceHelper> m_nn_infer_; // 推理模块
|
||||
|
||||
std::vector<InputTensorInfo> m_input_tensor_info_list_;
|
||||
|
||||
std::vector<OutputTensorInfo> m_output_tensor_info_list_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace;
|
||||
|
||||
#endif //ZEPHYRLPR_DET_BACKBONE_H
|
||||
@@ -0,0 +1,100 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/8.
|
||||
//
|
||||
|
||||
#include "det_header.h"
|
||||
#include "log.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
#define INPUTS_NUM 3
|
||||
#define OUTPUT "output"
|
||||
|
||||
DetHeader::DetHeader() = default;
|
||||
|
||||
DetHeader::~DetHeader() {
|
||||
if (!m_nn_infer_) {
|
||||
LOGD("Inference helper is not created\n");
|
||||
return;
|
||||
}
|
||||
m_nn_infer_->Finalize();
|
||||
m_nn_infer_.reset();
|
||||
}
|
||||
|
||||
int32_t DetHeader::Initialize(const std::string& model_filename, int input_size, bool use_half) {
|
||||
|
||||
m_nn_infer_.reset(InferenceHelper::Create(InferenceHelper::kMnn));
|
||||
if (m_nn_infer_->SetNumThreads(1) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
if (input_size == 320) {
|
||||
m_header_list_ = {
|
||||
{"948", { 1, 45, 40, 40 }},
|
||||
{"1061", { 1, 45, 20, 20 }},
|
||||
{"1174", { 1, 45, 10, 10 }},};
|
||||
} else {
|
||||
m_header_list_ = {
|
||||
{"948", { 1, 45, 80, 80 }},
|
||||
{"1061", { 1, 45, 40, 40 }},
|
||||
{"1174", { 1, 45, 20, 20 }},};
|
||||
}
|
||||
m_output_tensor_info_list_.clear();
|
||||
m_output_tensor_info_list_.emplace_back(OUTPUT, TensorInfo::kTensorTypeFp32);
|
||||
|
||||
if (m_nn_infer_->Initialize(model_filename, m_input_tensor_info_list_, m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
m_input_tensor_info_list_.clear();
|
||||
/* Set input tensor info */
|
||||
for (int i = 0; i < INPUTS_NUM; ++i) {
|
||||
auto &head = m_header_list_[i];
|
||||
InputTensorInfo input_tensor_info(head.input_name, TensorInfo::kTensorTypeFp32, false);
|
||||
input_tensor_info.tensor_dims = head.dims;
|
||||
input_tensor_info.data_type = InputTensorInfo::kTensorTypeFp32;
|
||||
input_tensor_info.normalize.mean[0] = 0.0f;
|
||||
input_tensor_info.normalize.mean[1] = 0.0f;
|
||||
input_tensor_info.normalize.mean[2] = 0.0f;
|
||||
input_tensor_info.normalize.norm[0] = 1.0f;
|
||||
input_tensor_info.normalize.norm[1] = 1.0f;
|
||||
input_tensor_info.normalize.norm[2] = 1.0f;
|
||||
m_input_tensor_info_list_.push_back(input_tensor_info);
|
||||
}
|
||||
|
||||
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
int32_t DetHeader::Inference(float* ptr3, float* ptr2, float* ptr1) {
|
||||
|
||||
m_input_tensor_info_list_[0].data_type = InputTensorInfo::kDataTypeBlobNchw;
|
||||
m_input_tensor_info_list_[0].data = ptr3;
|
||||
|
||||
m_input_tensor_info_list_[1].data_type = InputTensorInfo::kDataTypeBlobNchw;
|
||||
m_input_tensor_info_list_[1].data = ptr2;
|
||||
|
||||
m_input_tensor_info_list_[2].data_type = InputTensorInfo::kDataTypeBlobNchw;
|
||||
m_input_tensor_info_list_[2].data = ptr1;
|
||||
|
||||
if (m_nn_infer_->PreProcess(m_input_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
if (m_nn_infer_->Process(m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
|
||||
std::vector<OutputTensorInfo> &DetHeader::getMOutputTensorInfoList() {
|
||||
return m_output_tensor_info_list_;
|
||||
}
|
||||
|
||||
void DetHeader::setMOutputTensorInfoList(const std::vector<OutputTensorInfo> &mOutputTensorInfoList) {
|
||||
m_output_tensor_info_list_ = mOutputTensorInfoList;
|
||||
}
|
||||
} // namespace
|
||||
@@ -0,0 +1,53 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2023/2/8.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_DET_HEADER_H
|
||||
#define ZEPHYRLPR_DET_HEADER_H
|
||||
|
||||
#include "inference_helper_module/inference_helper.h"
|
||||
#include "plate_det_common.h"
|
||||
#include "opencv2/opencv.hpp"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
typedef struct {
|
||||
std::string input_name;
|
||||
std::vector<int> dims;
|
||||
} Header;
|
||||
|
||||
typedef std::vector<Header> HeaderList;
|
||||
|
||||
class DetHeader {
|
||||
public:
|
||||
DetHeader(const DetHeader &) = delete;
|
||||
|
||||
DetHeader &operator=(const DetHeader &) = delete;
|
||||
|
||||
explicit DetHeader();
|
||||
|
||||
~DetHeader();
|
||||
|
||||
int32_t Initialize(const std::string& model_filename, int input_size = 320, bool use_half = false);
|
||||
|
||||
int32_t Inference(float* ptr3, float* ptr2, float* ptr1);
|
||||
|
||||
std::vector<OutputTensorInfo> &getMOutputTensorInfoList();
|
||||
|
||||
void setMOutputTensorInfoList(const std::vector<OutputTensorInfo> &mOutputTensorInfoList);
|
||||
|
||||
private:
|
||||
|
||||
HeaderList m_header_list_;
|
||||
|
||||
std::unique_ptr<InferenceHelper> m_nn_infer_; // 推理模块
|
||||
|
||||
std::vector<InputTensorInfo> m_input_tensor_info_list_;
|
||||
|
||||
std::vector<OutputTensorInfo> m_output_tensor_info_list_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
#endif //ZEPHYRLPR_DET_HEADER_H
|
||||
@@ -0,0 +1,67 @@
|
||||
//
|
||||
// Created by tunm on 2022/12/29.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_PLATE_DET_COMMON_H
|
||||
#define ZEPHYRLPR_PLATE_DET_COMMON_H
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace hyper {
|
||||
|
||||
enum LayersNum {
|
||||
MONO, ///< 单层车牌
|
||||
DOUBLE, ///< 双层车牌
|
||||
};
|
||||
|
||||
typedef struct PlateLocation {
|
||||
float x1; ///< 左上角点x坐标
|
||||
float y1; ///< 左上角点y坐标
|
||||
float x2; ///< 右下角点x坐标
|
||||
float y2; ///< 右下角点y坐标
|
||||
float det_confidence; ///< 目标检测的置信度
|
||||
float kps[8]; ///< 四个角的角点信息
|
||||
LayersNum layers; ///< 车牌层级类型
|
||||
};
|
||||
|
||||
|
||||
inline void nms(std::vector<PlateLocation> &input_plates, float nms_threshold) {
|
||||
std::sort(input_plates.begin(), input_plates.end(),
|
||||
[](PlateLocation a, PlateLocation b) { return a.det_confidence > b.det_confidence; });
|
||||
std::vector<float> area(input_plates.size());
|
||||
for (int i = 0; i < int(input_plates.size()); ++i) {
|
||||
area[i] =
|
||||
(input_plates.at(i).x2 - input_plates.at(i).x1 + 1) *
|
||||
(input_plates.at(i).y2 - input_plates.at(i).y1 + 1);
|
||||
}
|
||||
for (int i = 0; i < int(input_plates.size()); ++i) {
|
||||
for (int j = i + 1; j < int(input_plates.size());) {
|
||||
float xx1 = (std::max)(input_plates[i].x1, input_plates[j].x1);
|
||||
float yy1 = (std::max)(input_plates[i].y1, input_plates[j].y1);
|
||||
float xx2 = (std::min)(input_plates[i].x2, input_plates[j].x2);
|
||||
float yy2 = (std::min)(input_plates[i].y2, input_plates[j].y2);
|
||||
float w = (std::max)(float(0), xx2 - xx1 + 1);
|
||||
float h = (std::max)(float(0), yy2 - yy1 + 1);
|
||||
float inter = w * h;
|
||||
float ovr = inter / (area[i] + area[j] - inter);
|
||||
if (ovr >= nms_threshold) {
|
||||
input_plates.erase(input_plates.begin() + j);
|
||||
area.erase(area.begin() + j);
|
||||
} else {
|
||||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline std::vector<float> xywh2xyxy(float x, float y, float w, float h) {
|
||||
float x1 = x - w / 2;
|
||||
float y1 = y - h / 2;
|
||||
float x2 = x + w / 2;
|
||||
float y2 = y + h / 2;
|
||||
return {x1, y1, x2, y2};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_PLATE_DET_COMMON_H
|
||||
@@ -0,0 +1,107 @@
|
||||
//
|
||||
// Created by tunm on 2022/12/29.
|
||||
//
|
||||
|
||||
#include "plate_detector.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
PlateDetector::PlateDetector() {
|
||||
|
||||
}
|
||||
|
||||
int PlateDetector::Initialize(const char *model_path,
|
||||
int input_size,
|
||||
int threads,
|
||||
float box_conf_threshold,
|
||||
float nms_threshold,
|
||||
bool use_half){
|
||||
m_input_size_ = input_size;
|
||||
m_box_conf_threshold_ = box_conf_threshold;
|
||||
m_nms_threshold_ = nms_threshold;
|
||||
float mean[] = {0.0f, 0.0f, 0.0f};
|
||||
float normal[] = {0.003921568627f, 0.003921568627f, 0.003921568627f};
|
||||
m_nn_adapter_ = std::make_shared<MNNAdapterInference>(model_path, threads, mean, normal, use_half);
|
||||
m_nn_adapter_->Initialization("input", "output", m_input_size_, m_input_size_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void PlateDetector::Decode(const std::vector<float> &tensor, std::vector<PlateLocation> &outputs, float scale,
|
||||
float conf_threshold, float nms_threshold) {
|
||||
int grid_num = 6300;
|
||||
int item_num = 15;
|
||||
for (int index = 0; index < grid_num; ++index) {
|
||||
auto det_score = tensor[index * item_num + 4];
|
||||
if (det_score > conf_threshold) {
|
||||
PlateLocation plate;
|
||||
plate.det_confidence = det_score;
|
||||
auto class_score_1 = tensor[index * item_num + 13] * det_score;
|
||||
auto class_score_2 = tensor[index * item_num + 14] * det_score;
|
||||
LayersNum type = class_score_1 > class_score_2 ? MONO : DOUBLE;
|
||||
plate.layers = type;
|
||||
auto x = tensor[index * item_num + 0];
|
||||
auto y = tensor[index * item_num + 1];
|
||||
auto w = tensor[index * item_num + 2];
|
||||
auto h = tensor[index * item_num + 3];
|
||||
std::vector<float> xyxy = xywh2xyxy(x, y, w, h);
|
||||
plate.x1 = xyxy[0];
|
||||
plate.y1 = xyxy[1];
|
||||
plate.x2 = xyxy[2];
|
||||
plate.y2 = xyxy[3];
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
plate.kps[i] = tensor[index * item_num + i + 5];
|
||||
}
|
||||
outputs.push_back(plate);
|
||||
}
|
||||
|
||||
}
|
||||
nms(outputs, nms_threshold);
|
||||
|
||||
for (auto &plate : outputs) {
|
||||
plate.x1 = plate.x1 / scale;
|
||||
plate.y1 = plate.y1 / scale;
|
||||
plate.x2 = plate.x2 / scale;
|
||||
plate.y2 = plate.y2 / scale;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
plate.kps[i * 2 + 0] = plate.kps[i * 2 + 0] / scale;
|
||||
plate.kps[i * 2 + 1] = plate.kps[i * 2 + 1] / scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void PlateDetector::Detection(const cv::Mat &bgr, bool is_resize, float scale) {
|
||||
cv::Mat pad;
|
||||
if (is_resize) {
|
||||
int ori_w = bgr.cols;
|
||||
int ori_h = bgr.rows;
|
||||
|
||||
int w, h;
|
||||
if (ori_w > ori_h) {
|
||||
scale = (float) m_input_size_ / ori_w;
|
||||
w = m_input_size_;
|
||||
h = ori_h * scale;
|
||||
} else {
|
||||
scale = (float) m_input_size_ / ori_h;
|
||||
h = m_input_size_;
|
||||
w = ori_w * scale;
|
||||
}
|
||||
int wpad = m_input_size_ - w;
|
||||
int hpad = m_input_size_ - h;
|
||||
cv::Mat resized_img;
|
||||
cv::resize(bgr, resized_img, cv::Size(w, h));
|
||||
cv::copyMakeBorder(resized_img, pad, 0, hpad, 0, wpad, cv::BORDER_CONSTANT, 0.0f);
|
||||
} else {
|
||||
pad = bgr;
|
||||
}
|
||||
// cv::imshow("a", pad);
|
||||
auto output = m_nn_adapter_->Invoking(pad);
|
||||
std::vector<PlateLocation>().swap(m_results_);
|
||||
Decode(output, m_results_, scale, m_box_conf_threshold_, m_nms_threshold_);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//
|
||||
// Created by tunm on 2022/12/29.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_PLATE_DETECTOR_H
|
||||
#define ZEPHYRLPR_PLATE_DETECTOR_H
|
||||
#include "nn_module/mnn_adapter.h"
|
||||
#include "plate_det_common.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class PlateDetector {
|
||||
public:
|
||||
|
||||
PlateDetector(const PlateDetector &) = delete;
|
||||
|
||||
PlateDetector &operator=(const PlateDetector &) = delete;
|
||||
|
||||
explicit PlateDetector();
|
||||
|
||||
int Initialize(const char *model_path, int input_size = 320, int threads=1, float box_conf_threshold = 0.3, float nms_threshold = 0.6, bool use_half = false);
|
||||
|
||||
void Detection(const cv::Mat& bgr, bool is_resize = false, float scale = 1.0f);
|
||||
|
||||
private:
|
||||
void Decode(const std::vector<float> &tensor, std::vector<PlateLocation> &outputs, float scale,
|
||||
float conf_threshold = 0.5, float nms_threshold = 0.5);
|
||||
|
||||
public:
|
||||
std::vector<PlateLocation> m_results_;
|
||||
|
||||
private:
|
||||
|
||||
int m_input_size_{};
|
||||
|
||||
float m_box_conf_threshold_{};
|
||||
|
||||
float m_nms_threshold_{};
|
||||
|
||||
std::shared_ptr<MNNAdapterInference> m_nn_adapter_;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_PLATE_DETECTOR_H
|
||||
@@ -0,0 +1,12 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/12/30.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_RECOGNITION_ALL_H
|
||||
#define ZEPHYRLPR_RECOGNITION_ALL_H
|
||||
|
||||
#include "plate_recognition.h"
|
||||
#include "recognition_commom.h"
|
||||
#include "recognition_engine.h"
|
||||
|
||||
#endif //ZEPHYRLPR_RECOGNITION_ALL_H
|
||||
@@ -0,0 +1,82 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/12/30.
|
||||
//
|
||||
|
||||
#include "plate_recognition.h"
|
||||
#include <utility>
|
||||
#include "utils.h"
|
||||
#include "plate_recognition_tokenize.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
PlateRecognition::PlateRecognition() = default;
|
||||
|
||||
int PlateRecognition::Initialize(const std::string& model_filename, cv::Size input_size, int threads, float confidence_threshold,
|
||||
bool use_half) {
|
||||
m_input_image_size_ = std::move(input_size);
|
||||
m_confidence_threshold_ = confidence_threshold;
|
||||
float mean[] = {127.5f, 127.5f, 127.5f};
|
||||
float normal[] = {1.0 / 127.5f, 1.0 / 127.5f, 1.0 / 127.5f};
|
||||
m_nn_adapter_ = std::make_shared<MNNAdapterInference>(model_filename, threads, mean, normal, use_half);
|
||||
m_nn_adapter_->Initialization("data", "output", m_input_image_size_.width, m_input_image_size_.height);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
TextLine PlateRecognition::Inference(const cv::Mat &bgr_pad) {
|
||||
std::vector<float> output = m_nn_adapter_->Invoking(bgr_pad);
|
||||
// SLOG_INFO("out: {}", output.size());
|
||||
auto line = decode(output);
|
||||
return line;
|
||||
}
|
||||
|
||||
TextLine PlateRecognition::decode(const std::vector<float>& tensor) {
|
||||
int text_total_num = 24;
|
||||
int classify_num = 75;
|
||||
IndexList index_list;
|
||||
std::vector<float> max_list;
|
||||
for (int index = 0; index < text_total_num; ++index) {
|
||||
std::vector<float> mapping;
|
||||
mapping.reserve(classify_num);
|
||||
for (int i = 0; i < classify_num; ++i) {
|
||||
mapping.push_back(tensor[index * classify_num + i]);
|
||||
}
|
||||
auto max_index = argmax(mapping.begin(), mapping.end());
|
||||
float max_value = mapping[max_index];
|
||||
index_list.push_back(max_index);
|
||||
max_list.push_back(max_value);
|
||||
}
|
||||
|
||||
TextLine line;
|
||||
line.code = "";
|
||||
float total_ = 0.0f;
|
||||
for (int i = 0; i < index_list.size(); ++i) {
|
||||
auto &idx = index_list[i];
|
||||
IndexList::const_iterator result = std::find(m_ignored_tokens_.begin(), m_ignored_tokens_.end(), idx);
|
||||
if (result == m_ignored_tokens_.end()) {
|
||||
if ((i > 0) && index_list[i - 1] == idx) {
|
||||
continue; // remove_duplicate
|
||||
}
|
||||
// std::cout << "b " << idx << std::endl;
|
||||
line.char_index.push_back(idx);
|
||||
line.char_scores.push_back(max_list[i]);
|
||||
line.code += TOKENIZE[idx];
|
||||
// SLOG_INFO("{}", TOKENIZE[idx]);
|
||||
total_ += max_list[i];
|
||||
}
|
||||
}
|
||||
line.average_score = total_ / line.char_scores.size();
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
const cv::Size &PlateRecognition::getMInputImageSize() const {
|
||||
return m_input_image_size_;
|
||||
}
|
||||
|
||||
float PlateRecognition::getMConfidenceThreshold() const {
|
||||
return m_confidence_threshold_;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/12/30.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_PLATE_RECOGNITION_H
|
||||
#define ZEPHYRLPR_PLATE_RECOGNITION_H
|
||||
|
||||
#include "nn_module/mnn_adapter.h"
|
||||
#include "configuration.h"
|
||||
#include "recognition_commom.h"
|
||||
#include "basic_types.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class PlateRecognition {
|
||||
public:
|
||||
|
||||
PlateRecognition(const PlateRecognition &) = delete;
|
||||
|
||||
PlateRecognition &operator=(const PlateRecognition &) = delete;
|
||||
|
||||
explicit PlateRecognition();
|
||||
|
||||
int Initialize(const std::string& model_filename, cv::Size input_size = REC_INPUT_SIZE, int threads = 1,
|
||||
float confidence_threshold = 0.5, bool use_half = false);
|
||||
|
||||
TextLine Inference(const cv::Mat &bgr_pad);
|
||||
|
||||
const cv::Size &getMInputImageSize() const;
|
||||
|
||||
float getMConfidenceThreshold() const;
|
||||
|
||||
private:
|
||||
|
||||
TextLine decode(const std::vector<float>& tensor);
|
||||
|
||||
private:
|
||||
|
||||
cv::Size m_input_image_size_{}; // 输入图像宽高
|
||||
|
||||
float m_confidence_threshold_{}; // 置信度阈值
|
||||
|
||||
std::shared_ptr<MNNAdapterInference> m_nn_adapter_; // 推理模块
|
||||
|
||||
IndexList m_ignored_tokens_ = {0, }; // 需要被忽略的索引
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_PLATE_RECOGNITION_H
|
||||
@@ -0,0 +1,17 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#ifndef ZEPHYRLPR_PLATE_RECOGNITION_TOKENIZE_H
|
||||
#define ZEPHYRLPR_PLATE_RECOGNITION_TOKENIZE_H
|
||||
|
||||
#include <iostream>
|
||||
|
||||
static const std::vector<std::string> TOKENIZE = {
|
||||
"blank", "'", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "G", "H", "J",
|
||||
"K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "云", "京", "冀", "吉", "学", "宁",
|
||||
"川", "挂", "新", "晋", "桂", "民", "沪", "津", "浙", "渝", "港", "湘", "琼", "甘", "皖", "粤", "航", "苏", "蒙", "藏", "警", "豫",
|
||||
"贵", "赣", "辽", "鄂", "闽", "陕", "青", "鲁", "黑", "领", "使", "澳"};
|
||||
|
||||
#endif //ZEPHYRLPR_PLATE_RECOGNITION_TOKENIZE_H
|
||||
@@ -0,0 +1,23 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/12/30.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_RECOGNITION_COMMOM_H
|
||||
#define ZEPHYRLPR_RECOGNITION_COMMOM_H
|
||||
|
||||
#include "basic_types.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
typedef struct TextLine {
|
||||
IndexList char_index; ///< 索引列表
|
||||
std::vector<float> char_scores; ///< 置信度列表
|
||||
float average_score; ///< 平均置信度
|
||||
std::string code; ///< 车牌字符串
|
||||
};
|
||||
|
||||
typedef std::vector<TextLine> TextLines; ///< 多行文本表达
|
||||
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_RECOGNITION_COMMOM_H
|
||||
@@ -0,0 +1,151 @@
|
||||
//
|
||||
// Created by tunm on 2023/2/6.
|
||||
//
|
||||
|
||||
#include "recognition_engine.h"
|
||||
#include <utility>
|
||||
#include "utils.h"
|
||||
#include "plate_recognition_tokenize.h"
|
||||
|
||||
#define INPUT_NAME "data"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
|
||||
RecognitionEngine::RecognitionEngine() = default;
|
||||
|
||||
|
||||
int32_t RecognitionEngine::Initialize(const std::string& model_filename, cv::Size input_size, int threads, float confidence_threshold,
|
||||
bool use_half) {
|
||||
|
||||
m_input_image_size_ = input_size;
|
||||
m_confidence_threshold_ = confidence_threshold;
|
||||
|
||||
|
||||
/* Set output tensor info */
|
||||
m_output_tensor_info_list_.clear();
|
||||
m_output_tensor_info_list_.push_back(OutputTensorInfo("output", TensorInfo::kTensorTypeFp32));
|
||||
|
||||
|
||||
m_nn_infer_.reset(InferenceHelper::Create(InferenceHelper::kMnn));
|
||||
|
||||
if (m_nn_infer_->SetNumThreads(threads) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
if (m_nn_infer_->Initialize(model_filename, m_input_tensor_info_list_, m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
m_nn_infer_.reset();
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
m_input_tensor_info_list_.clear();
|
||||
|
||||
InputTensorInfo input_tensor_info(INPUT_NAME, TensorInfo::kTensorTypeFp32, true);
|
||||
|
||||
input_tensor_info.tensor_dims = { 1, 3, m_input_image_size_.height, m_input_image_size_.width };
|
||||
input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
|
||||
|
||||
input_tensor_info.normalize.mean[0] = 127.5f;
|
||||
input_tensor_info.normalize.mean[1] = 127.5f;
|
||||
input_tensor_info.normalize.mean[2] = 127.5f;
|
||||
input_tensor_info.normalize.norm[0] = 0.007843137255f;
|
||||
input_tensor_info.normalize.norm[1] = 0.007843137255f;
|
||||
input_tensor_info.normalize.norm[2] = 0.007843137255f;
|
||||
|
||||
m_input_tensor_info_list_.push_back(input_tensor_info);
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
int32_t RecognitionEngine::Inference(const cv::Mat &bgr_pad, TextLine &line) {
|
||||
InputTensorInfo& input_tensor_info = m_input_tensor_info_list_[0];
|
||||
|
||||
input_tensor_info.data = bgr_pad.data;
|
||||
input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
|
||||
input_tensor_info.image_info.width = bgr_pad.cols;
|
||||
input_tensor_info.image_info.height = bgr_pad.rows;
|
||||
input_tensor_info.image_info.channel = bgr_pad.channels();
|
||||
input_tensor_info.image_info.crop_x = 0;
|
||||
input_tensor_info.image_info.crop_y = 0;
|
||||
input_tensor_info.image_info.crop_width = bgr_pad.cols;
|
||||
input_tensor_info.image_info.crop_height = bgr_pad.rows;
|
||||
input_tensor_info.image_info.is_bgr = true;
|
||||
input_tensor_info.image_info.swap_color = false;
|
||||
|
||||
if (m_nn_infer_->PreProcess(m_input_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
if (m_nn_infer_->Process(m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
|
||||
return InferenceHelper::kRetErr;
|
||||
}
|
||||
|
||||
std::vector<float> output_score_raw_list(m_output_tensor_info_list_[0].GetDataAsFloat(),
|
||||
m_output_tensor_info_list_[0].GetDataAsFloat() +
|
||||
m_output_tensor_info_list_[0].GetElementNum());
|
||||
|
||||
decode(output_score_raw_list, line);
|
||||
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
const cv::Size &RecognitionEngine::getMInputImageSize() const {
|
||||
return m_input_image_size_;
|
||||
}
|
||||
|
||||
|
||||
float RecognitionEngine::getMConfidenceThreshold() const {
|
||||
return m_confidence_threshold_;
|
||||
}
|
||||
|
||||
|
||||
int32_t RecognitionEngine::decode(const std::vector<float> &tensor, TextLine &line) {
|
||||
int text_total_num = REC_MAX_CHAR_NUM;
|
||||
int classify_num = REC_CHAR_CLASS_NUM;
|
||||
IndexList index_list;
|
||||
std::vector<float> max_list;
|
||||
for (int index = 0; index < text_total_num; ++index) {
|
||||
std::vector<float> mapping;
|
||||
mapping.reserve(classify_num);
|
||||
for (int i = 0; i < classify_num; ++i) {
|
||||
mapping.push_back(tensor[index * classify_num + i]);
|
||||
}
|
||||
auto max_index = argmax(mapping.begin(), mapping.end());
|
||||
float max_value = mapping[max_index];
|
||||
index_list.push_back(max_index);
|
||||
max_list.push_back(max_value);
|
||||
}
|
||||
|
||||
line.code = "";
|
||||
float total_ = 0.0f;
|
||||
for (int i = 0; i < index_list.size(); ++i) {
|
||||
auto &idx = index_list[i];
|
||||
IndexList::const_iterator result = std::find(m_ignored_tokens_.begin(), m_ignored_tokens_.end(), idx);
|
||||
if (result == m_ignored_tokens_.end()) {
|
||||
if ((i > 0) && index_list[i - 1] == idx) {
|
||||
continue; // remove_duplicate
|
||||
}
|
||||
// std::cout << "b " << idx << std::endl;
|
||||
line.char_index.push_back(idx);
|
||||
line.char_scores.push_back(max_list[i]);
|
||||
line.code += TOKENIZE[idx];
|
||||
// SLOG_INFO("{}", TOKENIZE[idx]);
|
||||
total_ += max_list[i];
|
||||
}
|
||||
}
|
||||
line.average_score = total_ / line.char_scores.size();
|
||||
|
||||
return InferenceHelper::kRetOk;
|
||||
}
|
||||
|
||||
RecognitionEngine::~RecognitionEngine() {
|
||||
if (!m_nn_infer_) {
|
||||
LOGD("Inference helper is not created\n");
|
||||
return;
|
||||
}
|
||||
m_nn_infer_->Finalize();
|
||||
m_nn_infer_.reset();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,58 @@
|
||||
//
|
||||
// Created by tunm on 2023/2/6.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_RECOGNITION_ENGINE_H
|
||||
#define ZEPHYRLPR_RECOGNITION_ENGINE_H
|
||||
|
||||
#include "inference_helper_module/inference_helper.h"
|
||||
#include "configuration.h"
|
||||
#include "recognition_commom.h"
|
||||
#include "basic_types.h"
|
||||
#include "opencv2/opencv.hpp"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class RecognitionEngine {
|
||||
public:
|
||||
|
||||
RecognitionEngine(const RecognitionEngine &) = delete;
|
||||
|
||||
RecognitionEngine &operator=(const RecognitionEngine &) = delete;
|
||||
|
||||
explicit RecognitionEngine();
|
||||
|
||||
~RecognitionEngine();
|
||||
|
||||
int32_t Initialize(const std::string& model_filename, cv::Size input_size = REC_INPUT_SIZE, int threads = 1,
|
||||
float confidence_threshold = 0.5, bool use_half = false);
|
||||
|
||||
int32_t Inference(const cv::Mat &bgr_pad, TextLine &line);
|
||||
|
||||
const cv::Size &getMInputImageSize() const;
|
||||
|
||||
float getMConfidenceThreshold() const;
|
||||
|
||||
private:
|
||||
|
||||
int32_t decode(const std::vector<float>& tensor, TextLine &line);
|
||||
|
||||
private:
|
||||
cv::Size m_input_image_size_{}; // 输入图像宽高
|
||||
|
||||
float m_confidence_threshold_{}; // 置信度阈值
|
||||
|
||||
std::unique_ptr<InferenceHelper> m_nn_infer_; // 推理模块
|
||||
|
||||
IndexList m_ignored_tokens_ = {0, }; // 需要被忽略的索引
|
||||
|
||||
std::vector<InputTensorInfo> m_input_tensor_info_list_;
|
||||
|
||||
std::vector<OutputTensorInfo> m_output_tensor_info_list_;
|
||||
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
#endif //ZEPHYRLPR_RECOGNITION_ENGINE_H
|
||||
@@ -0,0 +1,10 @@
|
||||
//
|
||||
// Created by tunm on 2023/1/25.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_MNN_ADAPTER_ALL_H
|
||||
#define ZEPHYRLPR_MNN_ADAPTER_ALL_H
|
||||
|
||||
#include "mnn_adapter.h"
|
||||
|
||||
#endif //ZEPHYRLPR_MNN_ADAPTER_ALL_H
|
||||
@@ -0,0 +1,107 @@
|
||||
//
|
||||
// Created by tunm on 2022/4/24.
|
||||
//
|
||||
|
||||
#include "mnn_adapter.h"
|
||||
|
||||
namespace hyper {
|
||||
|
||||
|
||||
MNNAdapterInference::MNNAdapterInference(const std::string &model, int thread, const float *mean,
|
||||
const float *normal, bool is_use_half,
|
||||
bool use_model_bin, bool is_use_cuda) {
|
||||
|
||||
if (is_use_cuda) {
|
||||
backend_ = MNN_FORWARD_CUDA;
|
||||
} else {
|
||||
backend_ = MNN_FORWARD_CPU;
|
||||
}
|
||||
if (use_model_bin) {
|
||||
raw_model_ = std::shared_ptr<MNN::Interpreter>(
|
||||
MNN::Interpreter::createFromBuffer(model.c_str(), model.size()));
|
||||
} else {
|
||||
raw_model_ = std::shared_ptr<MNN::Interpreter>(
|
||||
MNN::Interpreter::createFromFile(model.c_str()));
|
||||
}
|
||||
|
||||
_config.type = backend_;
|
||||
//_config.layers = MNN_FORWARD_CUDA;
|
||||
_config.numThread = 2;
|
||||
//_config.numThread = thread;
|
||||
MNN::BackendConfig backendConfig;
|
||||
if (is_use_half) {
|
||||
backendConfig.precision = MNN::BackendConfig::Precision_Low;
|
||||
} else {
|
||||
backendConfig.precision = MNN::BackendConfig::Precision_High;
|
||||
}
|
||||
backendConfig.power = MNN::BackendConfig::Power_High;
|
||||
_config.backendConfig = &backendConfig;
|
||||
for (int i = 0; i < 3; i++) {
|
||||
this->mean[i] = mean[i];
|
||||
this->normal[i] = normal[i];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
MNNAdapterInference::~MNNAdapterInference() {
|
||||
raw_model_->releaseModel();
|
||||
raw_model_->releaseSession(sess);
|
||||
}
|
||||
|
||||
void
|
||||
MNNAdapterInference::Initialization(const std::string &input, const std::string &output, int width, int height) {
|
||||
sess = raw_model_->createSession(_config);
|
||||
tensor_shape_.resize(4);
|
||||
tensor_shape_ = {1, 3, height, width};
|
||||
input_ = raw_model_->getSessionInput(sess, input.c_str());
|
||||
output_ = raw_model_->getSessionOutput(sess, output.c_str());
|
||||
width_ = width;
|
||||
height_ = height;
|
||||
}
|
||||
|
||||
std::vector<float> MNNAdapterInference::Invoking(const cv::Mat &mat) {
|
||||
assert(mat.rows == height_);
|
||||
assert(mat.cols == width_);
|
||||
MNN::CV::ImageProcess::Config config;
|
||||
config.destFormat = image_format_dst_;
|
||||
config.sourceFormat = image_format_src_;
|
||||
for (int i = 0; i < 3; i++) {
|
||||
config.mean[i] = mean[i];
|
||||
config.normal[i] = normal[i];
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<MNN::CV::ImageProcess> process(
|
||||
MNN::CV::ImageProcess::create(config));
|
||||
process->convert(mat.data, mat.cols, mat.rows, (int) mat.step1(), input_);
|
||||
raw_model_->runSession(sess);
|
||||
auto dimType = input_->getDimensionType();
|
||||
|
||||
if (output_->getType().code != halide_type_float) {
|
||||
dimType = MNN::Tensor::TENSORFLOW;
|
||||
}
|
||||
std::shared_ptr<MNN::Tensor> outputUser(new MNN::Tensor(output_, dimType));
|
||||
output_->copyToHostTensor(outputUser.get());
|
||||
auto type = outputUser->getType();
|
||||
auto size = outputUser->elementSize();
|
||||
std::vector<float> tempValues(size);
|
||||
if (type.code == halide_type_float) {
|
||||
auto values = outputUser->host<float>();
|
||||
for (int i = 0; i < size; ++i) {
|
||||
tempValues[i] = values[i];
|
||||
}
|
||||
}
|
||||
return tempValues;
|
||||
}
|
||||
|
||||
void MNNAdapterInference::setImageFormatSrc(MNN::CV::ImageFormat imageFormatSrc) {
|
||||
image_format_src_ = imageFormatSrc;
|
||||
}
|
||||
|
||||
void MNNAdapterInference::setImageFormatDst(MNN::CV::ImageFormat imageFormatDst) {
|
||||
image_format_dst_ = imageFormatDst;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// Created by tunm on 2022/4/24.
|
||||
//
|
||||
|
||||
#ifndef ZEPHYRLPR_MNN_ADAPTER_H
|
||||
#define ZEPHYRLPR_MNN_ADAPTER_H
|
||||
|
||||
#include <opencv2/opencv.hpp>
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
#include <MNN/Interpreter.hpp>
|
||||
#include <MNN/MNNDefine.h>
|
||||
#include <MNN/Tensor.hpp>
|
||||
#include <MNN/MNNForwardType.h>
|
||||
|
||||
namespace hyper {
|
||||
|
||||
class MNNAdapterInference {
|
||||
public:
|
||||
|
||||
MNNAdapterInference(const std::string &model, int thread, const float mean[], const float normal[], bool is_use_half = false,
|
||||
bool use_model_bin = false, bool is_use_cuda = false);
|
||||
|
||||
~MNNAdapterInference();
|
||||
|
||||
|
||||
void Initialization(const std::string &input, const std::string &output, int width, int height);
|
||||
|
||||
|
||||
std::vector<float> Invoking(const cv::Mat &mat);
|
||||
|
||||
void setImageFormatSrc(MNN::CV::ImageFormat imageFormatSrc);
|
||||
|
||||
void setImageFormatDst(MNN::CV::ImageFormat imageFormatDst);
|
||||
|
||||
private:
|
||||
float mean[3]{};
|
||||
float normal[3]{};
|
||||
std::shared_ptr<MNN::Interpreter> raw_model_;
|
||||
MNN::Tensor *input_{};
|
||||
MNN::Tensor *output_{};
|
||||
MNN::Session *sess{};
|
||||
std::vector<int> tensor_shape_;
|
||||
MNN::ScheduleConfig _config;
|
||||
MNNForwardType backend_;
|
||||
int width_{};
|
||||
int height_{};
|
||||
|
||||
MNN::CV::ImageFormat image_format_src_{MNN::CV::ImageFormat::BGR};
|
||||
MNN::CV::ImageFormat image_format_dst_{MNN::CV::ImageFormat::BGR};
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
#endif //ZEPHYRLPR_MNN_ADAPTER_H
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
//
|
||||
// Created by Tunm-Air13 on 2022/12/30.
|
||||
//
|
||||
#pragma once
|
||||
#ifndef ZEPHYRLPR_UTILS_H
|
||||
#define ZEPHYRLPR_UTILS_H
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
inline bool exists(const std::string &file_path) {
|
||||
return (access(file_path.c_str(), F_OK) != -1);
|
||||
}
|
||||
|
||||
template<class ForwardIterator>
|
||||
inline size_t argmin(ForwardIterator first, ForwardIterator last) {
|
||||
return std::distance(first, std::min_element(first, last));
|
||||
}
|
||||
|
||||
template<class ForwardIterator>
|
||||
inline size_t argmax(ForwardIterator first, ForwardIterator last) {
|
||||
return std::distance(first, std::max_element(first, last));
|
||||
}
|
||||
|
||||
inline double L2NormFor2Points(const cv::Point2f &p1, const cv::Point2f &p2) {
|
||||
auto r = pow((p1.x - p2.x), 2) + pow((p1.y - p2.y), 2);
|
||||
|
||||
return sqrt(r);
|
||||
}
|
||||
|
||||
inline cv::Mat getRotateCropAndAlignMatrix(int img_crop_width, int img_crop_height,
|
||||
float *points2d, int point_num = 4) {
|
||||
assert(point_num == 4);
|
||||
float dst_array[] = {0, 0,
|
||||
(float) img_crop_width, 0,
|
||||
(float) img_crop_width, (float) img_crop_height,
|
||||
0, (float) img_crop_height};
|
||||
cv::Mat src(4, 2, CV_32F, points2d);
|
||||
cv::Mat dst(4, 2, CV_32F);
|
||||
dst.data = (uchar *) dst_array;
|
||||
|
||||
cv::Mat transform_matrix = cv::getPerspectiveTransform(src, dst);
|
||||
|
||||
return transform_matrix;
|
||||
}
|
||||
|
||||
inline void getRotateCropAndAlignPad(const cv::Mat &image, cv::Mat &out, float *points2d, int point_num = 4) {
|
||||
assert(point_num == 4);
|
||||
std::vector<cv::Point2f> vertex;
|
||||
for (int i = 0; i < point_num; ++i) {
|
||||
auto point = cv::Point2f(points2d[i * 2 + 0], points2d[i * 2 + 1]);
|
||||
vertex.push_back(point);
|
||||
}
|
||||
int img_crop_width = (int) std::max(L2NormFor2Points(vertex[0], vertex[1]), L2NormFor2Points(vertex[2], vertex[3]));
|
||||
int img_crop_height = (int) std::max(L2NormFor2Points(vertex[0], vertex[3]),
|
||||
L2NormFor2Points(vertex[1], vertex[2]));
|
||||
|
||||
cv::Mat transform_matrix = getRotateCropAndAlignMatrix(img_crop_width, img_crop_height, points2d, point_num);
|
||||
cv::warpPerspective(image, out, transform_matrix, cv::Size(img_crop_width, img_crop_height), cv::INTER_CUBIC,
|
||||
cv::BORDER_REPLICATE);
|
||||
|
||||
if ((float) img_crop_height / (float) img_crop_width >= 1.5) {
|
||||
cv::rotate(out, out, cv::ROTATE_90_CLOCKWISE);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
inline void
|
||||
imagePadding(const cv::Mat &image, cv::Mat &out, cv::Size target_size) {
|
||||
int ori_w = image.cols;
|
||||
int ori_h = image.rows;
|
||||
|
||||
int w, h;
|
||||
float scale;
|
||||
if (ori_w > ori_h) {
|
||||
scale = (float) target_size.width / ori_w;
|
||||
w = target_size.width;
|
||||
h = ori_h * scale;
|
||||
} else {
|
||||
scale = (float) target_size.height / ori_h;
|
||||
h = target_size.height;
|
||||
w = ori_w * scale;
|
||||
}
|
||||
int wpad = std::max(target_size.width - w, 0);
|
||||
int hpad = std::max(target_size.height - h, 0);
|
||||
cv::Mat resized_img;
|
||||
cv::resize(image, resized_img, cv::Size(w, h));
|
||||
cv::copyMakeBorder(resized_img, out, 0, hpad, 0, wpad, cv::BORDER_CONSTANT, cv::Scalar(127.5f, 127.5f, 127.5f));
|
||||
}
|
||||
|
||||
inline void
|
||||
imagePadding(const cv::Mat &image, cv::Mat &out, float max_wh_ratio, cv::Size target_size,
|
||||
int limited_max_width = 160,
|
||||
int limited_min_width = 48) {
|
||||
// cv::imshow("w", image);
|
||||
// cv::waitKey(0);
|
||||
// std::cout << image.size << std::endl;
|
||||
int target_h = target_size.height;
|
||||
int target_w = target_size.width;
|
||||
max_wh_ratio = std::max(max_wh_ratio, (float) target_w / (float) target_h);
|
||||
target_w = (int) (max_wh_ratio * target_h);
|
||||
target_w = std::max(std::min(target_w, limited_max_width), limited_min_width);
|
||||
auto image_h = image.rows;
|
||||
auto image_w = image.cols;
|
||||
auto ratio = (float) image_w / image_h;
|
||||
int ratio_img_h = ceil(target_h * ratio);
|
||||
ratio_img_h = std::max(ratio_img_h, limited_min_width);
|
||||
int resized_w;
|
||||
if (ratio_img_h > target_w) {
|
||||
resized_w = target_w;
|
||||
} else {
|
||||
resized_w = int(ratio_img_h);
|
||||
}
|
||||
cv::Mat resized_image;
|
||||
cv::resize(image, resized_image, cv::Size(resized_w, target_h));
|
||||
|
||||
|
||||
int wpad = std::max(target_size.width - resized_w, 0);
|
||||
int hpad = std::max(target_size.height - target_h, 0);
|
||||
// std::cout << resized_w << "," << target_h << std::endl;
|
||||
cv::copyMakeBorder(resized_image, out, 0, hpad, 0, wpad, cv::BORDER_CONSTANT, cv::Scalar(127.5f, 127.5f, 127.5f));
|
||||
|
||||
|
||||
}
|
||||
|
||||
template<class T>
|
||||
inline float xyxyArea(T x1, T y1, T x2, T y2) {
|
||||
auto w = x2 - x1;
|
||||
auto h = y2 - y1;
|
||||
|
||||
return (float) (w * h);
|
||||
}
|
||||
|
||||
template<class T>
|
||||
inline float boundBoxOverlap(const cv::Rect_<T> &box1, const cv::Rect_<T> &box2) {
|
||||
if (box1.x > box2.x + box2.width) { return 0.0; }
|
||||
if (box1.y > box2.y + box2.height) { return 0.0; }
|
||||
if (box1.x + box1.width < box2.x) { return 0.0; }
|
||||
if (box1.y + box1.height < box2.y) { return 0.0; }
|
||||
T colInt = std::min(box1.x + box1.width, box2.x + box2.width) - std::max(box1.x, box2.x);
|
||||
T rowInt = std::min(box1.y + box1.height, box2.y + box2.height) - std::max(box1.y, box2.y);
|
||||
T intersection = colInt * rowInt;
|
||||
T area1 = box1.width * box1.height;
|
||||
T area2 = box2.width * box2.height;
|
||||
|
||||
return intersection / (area1 + area2 - intersection);
|
||||
}
|
||||
|
||||
#endif //ZEPHYRLPR_UTILS_H
|
||||
Reference in New Issue
Block a user