更新到HyperLPR3版本

cpp/src/nn_implementation_module/recognition/all.h

@@ -0,0 +1,12 @@
+//
+// Created by Tunm-Air13 on 2022/12/30.
+//
+
+#ifndef ZEPHYRLPR_RECOGNITION_ALL_H
+#define ZEPHYRLPR_RECOGNITION_ALL_H
+
+#include "plate_recognition.h"
+#include "recognition_commom.h"
+#include "recognition_engine.h"
+
+#endif //ZEPHYRLPR_RECOGNITION_ALL_H

cpp/src/nn_implementation_module/recognition/plate_recognition.cpp

@@ -0,0 +1,82 @@
+//
+// Created by Tunm-Air13 on 2022/12/30.
+//
+
+#include "plate_recognition.h"
+#include <utility>
+#include "utils.h"
+#include "plate_recognition_tokenize.h"
+
+namespace hyper {
+
+PlateRecognition::PlateRecognition() = default;
+
+int PlateRecognition::Initialize(const std::string& model_filename, cv::Size input_size, int threads, float confidence_threshold,
+                                 bool use_half) {
+    m_input_image_size_ = std::move(input_size);
+    m_confidence_threshold_ = confidence_threshold;
+    float mean[] = {127.5f, 127.5f, 127.5f};
+    float normal[] = {1.0 / 127.5f, 1.0 / 127.5f, 1.0 / 127.5f};
+    m_nn_adapter_ = std::make_shared<MNNAdapterInference>(model_filename, threads, mean, normal, use_half);
+    m_nn_adapter_->Initialization("data", "output", m_input_image_size_.width, m_input_image_size_.height);
+
+    return 0;
+}
+
+TextLine PlateRecognition::Inference(const cv::Mat &bgr_pad) {
+    std::vector<float> output = m_nn_adapter_->Invoking(bgr_pad);
+//    SLOG_INFO("out: {}", output.size());
+    auto line = decode(output);
+    return line;
+}
+
+TextLine PlateRecognition::decode(const std::vector<float>& tensor) {
+    int text_total_num = 24;
+    int classify_num = 75;
+    IndexList index_list;
+    std::vector<float> max_list;
+    for (int index = 0; index < text_total_num; ++index) {
+        std::vector<float> mapping;
+        mapping.reserve(classify_num);
+        for (int i = 0; i < classify_num; ++i) {
+            mapping.push_back(tensor[index * classify_num + i]);
+        }
+        auto max_index = argmax(mapping.begin(), mapping.end());
+        float max_value = mapping[max_index];
+        index_list.push_back(max_index);
+        max_list.push_back(max_value);
+    }
+
+    TextLine line;
+    line.code = "";
+    float total_ = 0.0f;
+    for (int i = 0; i < index_list.size(); ++i) {
+        auto &idx = index_list[i];
+        IndexList::const_iterator result = std::find(m_ignored_tokens_.begin(), m_ignored_tokens_.end(), idx);
+        if (result == m_ignored_tokens_.end()) {
+            if ((i > 0) && index_list[i - 1] == idx) {
+                continue;   // remove_duplicate
+            }
+//            std::cout << "b " << idx << std::endl;
+            line.char_index.push_back(idx);
+            line.char_scores.push_back(max_list[i]);
+            line.code += TOKENIZE[idx];
+//            SLOG_INFO("{}", TOKENIZE[idx]);
+            total_ += max_list[i];
+        }
+    }
+    line.average_score = total_ / line.char_scores.size();
+
+    return line;
+}
+
+const cv::Size &PlateRecognition::getMInputImageSize() const {
+    return m_input_image_size_;
+}
+
+float PlateRecognition::getMConfidenceThreshold() const {
+    return m_confidence_threshold_;
+}
+
+
+}

cpp/src/nn_implementation_module/recognition/plate_recognition.h

@@ -0,0 +1,51 @@
+//
+// Created by Tunm-Air13 on 2022/12/30.
+//
+#pragma once
+#ifndef ZEPHYRLPR_PLATE_RECOGNITION_H
+#define ZEPHYRLPR_PLATE_RECOGNITION_H
+
+#include "nn_module/mnn_adapter.h"
+#include "configuration.h"
+#include "recognition_commom.h"
+#include "basic_types.h"
+
+namespace hyper {
+
+class PlateRecognition {
+public:
+
+    PlateRecognition(const PlateRecognition &) = delete;
+
+    PlateRecognition &operator=(const PlateRecognition &) = delete;
+
+    explicit PlateRecognition();
+
+    int Initialize(const std::string& model_filename, cv::Size input_size = REC_INPUT_SIZE, int threads = 1,
+                   float confidence_threshold = 0.5, bool use_half = false);
+
+    TextLine Inference(const cv::Mat &bgr_pad);
+
+    const cv::Size &getMInputImageSize() const;
+
+    float getMConfidenceThreshold() const;
+
+private:
+
+    TextLine decode(const std::vector<float>& tensor);
+
+private:
+
+    cv::Size m_input_image_size_{};                            // 输入图像宽高
+
+    float m_confidence_threshold_{};                             // 置信度阈值
+
+    std::shared_ptr<MNNAdapterInference> m_nn_adapter_;        // 推理模块
+
+    IndexList m_ignored_tokens_ = {0, };                       // 需要被忽略的索引
+
+};
+
+}
+
+#endif //ZEPHYRLPR_PLATE_RECOGNITION_H

cpp/src/nn_implementation_module/recognition/plate_recognition_tokenize.h

@@ -0,0 +1,17 @@
+//
+// Created by tunm on 2023/1/25.
+//
+#pragma once
+
+#ifndef ZEPHYRLPR_PLATE_RECOGNITION_TOKENIZE_H
+#define ZEPHYRLPR_PLATE_RECOGNITION_TOKENIZE_H
+
+#include <iostream>
+
+static const std::vector<std::string> TOKENIZE = {
+        "blank", "'", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "G", "H", "J",
+        "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "云", "京", "冀", "吉", "学", "宁",
+        "川", "挂", "新", "晋", "桂", "民", "沪", "津", "浙", "渝", "港", "湘", "琼", "甘", "皖", "粤", "航", "苏", "蒙", "藏", "警", "豫",
+        "贵", "赣", "辽", "鄂", "闽", "陕", "青", "鲁", "黑", "领", "使", "澳"};
+
+#endif //ZEPHYRLPR_PLATE_RECOGNITION_TOKENIZE_H

cpp/src/nn_implementation_module/recognition/recognition_commom.h

@@ -0,0 +1,23 @@
+//
+// Created by Tunm-Air13 on 2022/12/30.
+//
+
+#ifndef ZEPHYRLPR_RECOGNITION_COMMOM_H
+#define ZEPHYRLPR_RECOGNITION_COMMOM_H
+
+#include "basic_types.h"
+
+namespace hyper {
+
+typedef struct TextLine {
+    IndexList char_index;                             ///< 索引列表
+    std::vector<float> char_scores;                   ///< 置信度列表
+    float average_score;                              ///< 平均置信度
+    std::string code;                                 ///< 车牌字符串
+};
+
+typedef std::vector<TextLine> TextLines;             ///< 多行文本表达
+
+}
+
+#endif //ZEPHYRLPR_RECOGNITION_COMMOM_H

cpp/src/nn_implementation_module/recognition/recognition_engine.cpp

@@ -0,0 +1,151 @@
+//
+// Created by tunm on 2023/2/6.
+//
+
+#include "recognition_engine.h"
+#include <utility>
+#include "utils.h"
+#include "plate_recognition_tokenize.h"
+
+#define INPUT_NAME "data"
+
+namespace hyper {
+
+
+RecognitionEngine::RecognitionEngine() = default;
+
+
+int32_t RecognitionEngine::Initialize(const std::string& model_filename, cv::Size input_size, int threads, float confidence_threshold,
+                                      bool use_half) {
+
+    m_input_image_size_ = input_size;
+    m_confidence_threshold_ = confidence_threshold;
+
+
+    /* Set output tensor info */
+    m_output_tensor_info_list_.clear();
+    m_output_tensor_info_list_.push_back(OutputTensorInfo("output", TensorInfo::kTensorTypeFp32));
+
+
+    m_nn_infer_.reset(InferenceHelper::Create(InferenceHelper::kMnn));
+
+    if (m_nn_infer_->SetNumThreads(threads) != InferenceHelper::kRetOk) {
+        m_nn_infer_.reset();
+        return InferenceHelper::kRetErr;
+    }
+
+    if (m_nn_infer_->Initialize(model_filename, m_input_tensor_info_list_, m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
+        m_nn_infer_.reset();
+        return InferenceHelper::kRetErr;
+    }
+
+    m_input_tensor_info_list_.clear();
+
+    InputTensorInfo input_tensor_info(INPUT_NAME, TensorInfo::kTensorTypeFp32, true);
+
+    input_tensor_info.tensor_dims =  { 1, 3, m_input_image_size_.height, m_input_image_size_.width };
+    input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
+
+    input_tensor_info.normalize.mean[0] = 127.5f;
+    input_tensor_info.normalize.mean[1] = 127.5f;
+    input_tensor_info.normalize.mean[2] = 127.5f;
+    input_tensor_info.normalize.norm[0] = 0.007843137255f;
+    input_tensor_info.normalize.norm[1] = 0.007843137255f;
+    input_tensor_info.normalize.norm[2] = 0.007843137255f;
+
+    m_input_tensor_info_list_.push_back(input_tensor_info);
+
+    return InferenceHelper::kRetOk;
+}
+
+int32_t RecognitionEngine::Inference(const cv::Mat &bgr_pad, TextLine &line) {
+    InputTensorInfo& input_tensor_info = m_input_tensor_info_list_[0];
+
+    input_tensor_info.data = bgr_pad.data;
+    input_tensor_info.data_type = InputTensorInfo::kDataTypeImage;
+    input_tensor_info.image_info.width = bgr_pad.cols;
+    input_tensor_info.image_info.height = bgr_pad.rows;
+    input_tensor_info.image_info.channel = bgr_pad.channels();
+    input_tensor_info.image_info.crop_x = 0;
+    input_tensor_info.image_info.crop_y = 0;
+    input_tensor_info.image_info.crop_width = bgr_pad.cols;
+    input_tensor_info.image_info.crop_height = bgr_pad.rows;
+    input_tensor_info.image_info.is_bgr = true;
+    input_tensor_info.image_info.swap_color = false;
+
+    if (m_nn_infer_->PreProcess(m_input_tensor_info_list_) != InferenceHelper::kRetOk) {
+        return InferenceHelper::kRetErr;
+    }
+    if (m_nn_infer_->Process(m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
+        return InferenceHelper::kRetErr;
+    }
+
+    std::vector<float> output_score_raw_list(m_output_tensor_info_list_[0].GetDataAsFloat(),
+                                             m_output_tensor_info_list_[0].GetDataAsFloat() +
+                                             m_output_tensor_info_list_[0].GetElementNum());
+
+    decode(output_score_raw_list, line);
+
+
+    return InferenceHelper::kRetOk;
+}
+
+const cv::Size &RecognitionEngine::getMInputImageSize() const {
+    return m_input_image_size_;
+}
+
+
+float RecognitionEngine::getMConfidenceThreshold() const {
+    return m_confidence_threshold_;
+}
+
+
+int32_t RecognitionEngine::decode(const std::vector<float> &tensor, TextLine &line) {
+    int text_total_num = REC_MAX_CHAR_NUM;
+    int classify_num = REC_CHAR_CLASS_NUM;
+    IndexList index_list;
+    std::vector<float> max_list;
+    for (int index = 0; index < text_total_num; ++index) {
+        std::vector<float> mapping;
+        mapping.reserve(classify_num);
+        for (int i = 0; i < classify_num; ++i) {
+            mapping.push_back(tensor[index * classify_num + i]);
+        }
+        auto max_index = argmax(mapping.begin(), mapping.end());
+        float max_value = mapping[max_index];
+        index_list.push_back(max_index);
+        max_list.push_back(max_value);
+    }
+
+    line.code = "";
+    float total_ = 0.0f;
+    for (int i = 0; i < index_list.size(); ++i) {
+        auto &idx = index_list[i];
+        IndexList::const_iterator result = std::find(m_ignored_tokens_.begin(), m_ignored_tokens_.end(), idx);
+        if (result == m_ignored_tokens_.end()) {
+            if ((i > 0) && index_list[i - 1] == idx) {
+                continue;   // remove_duplicate
+            }
+//            std::cout << "b " << idx << std::endl;
+            line.char_index.push_back(idx);
+            line.char_scores.push_back(max_list[i]);
+            line.code += TOKENIZE[idx];
+//            SLOG_INFO("{}", TOKENIZE[idx]);
+            total_ += max_list[i];
+        }
+    }
+    line.average_score = total_ / line.char_scores.size();
+
+    return InferenceHelper::kRetOk;
+}
+
+RecognitionEngine::~RecognitionEngine() {
+    if (!m_nn_infer_) {
+        LOGD("Inference helper is not created\n");
+        return;
+    }
+    m_nn_infer_->Finalize();
+    m_nn_infer_.reset();
+}
+
+} // namespace

cpp/src/nn_implementation_module/recognition/recognition_engine.h

@@ -0,0 +1,58 @@
+//
+// Created by tunm on 2023/2/6.
+//
+#pragma once
+#ifndef ZEPHYRLPR_RECOGNITION_ENGINE_H
+#define ZEPHYRLPR_RECOGNITION_ENGINE_H
+
+#include "inference_helper_module/inference_helper.h"
+#include "configuration.h"
+#include "recognition_commom.h"
+#include "basic_types.h"
+#include "opencv2/opencv.hpp"
+
+namespace hyper {
+
+class RecognitionEngine {
+public:
+
+    RecognitionEngine(const RecognitionEngine &) = delete;
+
+    RecognitionEngine &operator=(const RecognitionEngine &) = delete;
+
+    explicit RecognitionEngine();
+
+    ~RecognitionEngine();
+
+    int32_t Initialize(const std::string& model_filename, cv::Size input_size = REC_INPUT_SIZE, int threads = 1,
+                       float confidence_threshold = 0.5, bool use_half = false);
+
+    int32_t Inference(const cv::Mat &bgr_pad, TextLine &line);
+
+    const cv::Size &getMInputImageSize() const;
+
+    float getMConfidenceThreshold() const;
+
+private:
+
+    int32_t decode(const std::vector<float>& tensor, TextLine &line);
+
+private:
+    cv::Size m_input_image_size_{};                             // 输入图像宽高
+
+    float m_confidence_threshold_{};                            // 置信度阈值
+
+    std::unique_ptr<InferenceHelper> m_nn_infer_;               // 推理模块
+
+    IndexList m_ignored_tokens_ = {0, };                        // 需要被忽略的索引
+
+    std::vector<InputTensorInfo> m_input_tensor_info_list_;
+
+    std::vector<OutputTensorInfo> m_output_tensor_info_list_;
+
+
+};
+
+} // namespace
+
+#endif //ZEPHYRLPR_RECOGNITION_ENGINE_H