更新到HyperLPR3版本

Prj-Python/hyperlpr3/inference/base/base.py

@@ -0,0 +1,27 @@
+from abc import ABCMeta, abstractmethod
+
+
+class HamburgerABC(metaclass=ABCMeta):
+
+    def __init__(self, input_size: tuple = None, *args,
+                 **kwargs):
+        self.input_size = input_size
+
+    @abstractmethod
+    def _run_session(self, data):
+        pass
+
+    @abstractmethod
+    def _postprocess(self, data):
+        pass
+
+    @abstractmethod
+    def _preprocess(self, image):
+        pass
+
+    def __call__(self, image):
+        flow = self._preprocess(image)
+        flow = self._run_session(flow)
+        result = self._postprocess(flow)
+
+        return result

Prj-Python/hyperlpr3/inference/classification.py

@@ -0,0 +1,49 @@
+import cv2
+import numpy as np
+from .base.base import HamburgerABC
+from hyperlpr3.common.tools_process import cost
+
+
+def encode_images(image: np.ndarray):
+    image_encode = image / 255.0
+    if len(image_encode.shape) == 4:
+        image_encode = image_encode.transpose(0, 3, 1, 2)
+    else:
+        image_encode = image_encode.transpose(2, 0, 1)
+    image_encode = image_encode.astype(np.float32)
+
+    return image_encode
+
+
+class ClassificationORT(HamburgerABC):
+
+    def __init__(self, onnx_path, *args, **kwargs):
+        import onnxruntime as ort
+        super().__init__(*args, **kwargs)
+        self.session = ort.InferenceSession(onnx_path, None)
+        self.input_config = self.session.get_inputs()[0]
+        self.output_config = self.session.get_outputs()[0]
+        self.input_size = tuple(self.input_config.shape[2:])
+
+    # @cost('Cls')
+    def _run_session(self, data) -> np.ndarray:
+        result = self.session.run([self.output_config.name], {self.input_config.name: data})
+
+        return result[0]
+
+    def _postprocess(self, data) -> np.ndarray:
+        return data
+
+    def _preprocess(self, image) -> np.ndarray:
+        assert len(
+            image.shape) == 3, "The dimensions of the input image object do not match. The input supports a single " \
+                               "image. "
+        # print(self.input_size)
+        image_resize = cv2.resize(image, self.input_size)
+        encode = encode_images(image_resize)
+        encode = encode.astype(np.float32)
+        input_tensor = np.expand_dims(encode, 0)
+
+        return input_tensor
+
+

Prj-Python/hyperlpr3/inference/detect.py

@@ -0,0 +1,200 @@
+from hyperlpr3.common.tools_process import *
+from .base.base import HamburgerABC
+
+ANCHORS_MAP = {
+    320: [[9.38281, 3.08398], [15.53125, 4.93750], [19.98438, 7.78906],
+          [31.10938, 10.35156], [45.21875, 14.14844], [32.34375, 21.04688],
+          [65.62500, 19.57812], [76.12500, 46.12500], [253.25000, 137.50000]],
+    640: [[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
+          [59, 119], [116, 90], [156, 198], [373, 326]]
+}
+
+
+def image_to_input_tensor(image):
+    data = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    data = data.transpose(2, 0, 1) / 255.0
+    data = np.expand_dims(data, 0)
+    data = data.astype(np.float32)
+
+    return data
+
+
+class Y5rkDetectorMNN(HamburgerABC):
+    def __init__(self, mnn_path, box_threshold: float = 0.5, nms_threshold: float = 0.6, *args, **kwargs):
+        from .common.mnn_adapt import MNNAdapter
+        super().__init__(*args, **kwargs)
+        self.box_threshold = box_threshold
+        self.nms_threshold = nms_threshold
+        self.input_shape = (1, 3, self.input_size[0], self.input_size[1])
+        self.tensor_shape = ((1, 18, 40, 40), (1, 18, 20, 20), (1, 18, 10, 10))
+        self.session = MNNAdapter(mnn_path, self.input_shape, outputs_name=['output', '335', '336'],
+                                  outputs_shape=self.tensor_shape)
+        assert self.input_size[0] == self.input_size[1]
+        self.anchors = ANCHORS_MAP[self.input_size[0]]
+
+    def _run_session(self, data):
+        outputs = self.session.inference(data)
+        result = list()
+        for idx, output in enumerate(outputs):
+            result.append(output.reshape(self.tensor_shape[idx]))
+
+        return result
+
+    def _postprocess(self, data):
+        ratio, (dw, dh) = self.temp_pack
+        input0_data = data[0]
+        input1_data = data[1]
+        input2_data = data[2]
+        input0_data = input0_data.reshape([3, -1] + list(input0_data.shape[-2:]))
+        input1_data = input1_data.reshape([3, -1] + list(input1_data.shape[-2:]))
+        input2_data = input2_data.reshape([3, -1] + list(input2_data.shape[-2:]))
+        input_data = list()
+        input_data.append(np.transpose(input0_data, (2, 3, 0, 1)))
+        input_data.append(np.transpose(input1_data, (2, 3, 0, 1)))
+        input_data.append(np.transpose(input2_data, (2, 3, 0, 1)))
+
+        boxes, classes, scores = self.decode_outputs(input_data, self.input_size)
+
+        boxes = restore_bound_box(boxes, ratio, (dw, dh))
+
+        return boxes, classes, scores
+
+    def _preprocess(self, image):
+        h, w, _ = image.shape
+        resize_img, ratio, (dw, dh) = letterbox(image, new_shape=(self.input_size[1], self.input_size[0]))
+        data = image_to_input_tensor(resize_img)
+        self.temp_pack = ratio, (dw, dh)
+
+        return data
+
+    def decode_outputs(self, input_data, size):
+        masks = [[0, 1, 2], [3, 4, 5], [6, 7, 8]]
+        anchors = self.anchors
+
+        boxes, classes, scores = [], [], []
+        for input, mask in zip(input_data, masks):
+            b, c, s = process(input, mask, anchors, size)
+            b, c, s = filter_boxes(b, c, s, self.box_threshold, self.nms_threshold)
+            boxes.append(b)
+            classes.append(c)
+            scores.append(s)
+
+        boxes = np.concatenate(boxes)
+        boxes = xywh2xyxy(boxes)
+        classes = np.concatenate(classes)
+        scores = np.concatenate(scores)
+
+        nboxes, nclasses, nscores = [], [], []
+        for c in set(classes):
+            inds = np.where(classes == c)
+            b = boxes[inds]
+            c = classes[inds]
+            s = scores[inds]
+
+            keep = nms_boxes(b, s, self.nms_threshold)
+
+            nboxes.append(b[keep])
+            nclasses.append(c[keep])
+            nscores.append(s[keep])
+
+        if not nclasses and not nscores:
+            return None, None, None
+
+        boxes = np.concatenate(nboxes)
+        classes = np.concatenate(nclasses)
+        scores = np.concatenate(nscores)
+
+        return boxes, classes, scores
+
+
+class Y5rkDetectorORT(HamburgerABC):
+
+    def __init__(self, onnx_path, box_threshold: float = 0.5, nms_threshold: float = 0.6, *args, **kwargs):
+        import onnxruntime as ort
+        super().__init__(*args, **kwargs)
+        self.box_threshold = box_threshold
+        self.nms_threshold = nms_threshold
+        self.session = ort.InferenceSession(onnx_path, None)
+        self.inputs_option = self.session.get_inputs()
+        self.outputs_option = self.session.get_outputs()
+        input_option = self.inputs_option[0]
+        input_size_ = tuple(input_option.shape[2:])
+        self.input_size = tuple(self.input_size)
+        if not self.input_size:
+            self.input_size = input_size_
+        assert self.input_size == input_size_, 'The dimensions of the input do not match the model expectations.'
+        assert self.input_size[0] == self.input_size[1]
+        self.input_name = input_option.name
+        self.anchors = ANCHORS_MAP[self.input_size[0]]
+
+    def decode_outputs(self, input_data, size):
+        masks = [[0, 1, 2], [3, 4, 5], [6, 7, 8]]
+        anchors = self.anchors
+
+        boxes, classes, scores = [], [], []
+        for input, mask in zip(input_data, masks):
+            b, c, s = process(input, mask, anchors, size)
+            b, c, s = filter_boxes(b, c, s, self.box_threshold, self.nms_threshold)
+            boxes.append(b)
+            classes.append(c)
+            scores.append(s)
+
+        boxes = np.concatenate(boxes)
+        boxes = xywh2xyxy(boxes)
+        classes = np.concatenate(classes)
+        scores = np.concatenate(scores)
+
+        nboxes, nclasses, nscores = [], [], []
+        for c in set(classes):
+            inds = np.where(classes == c)
+            b = boxes[inds]
+            c = classes[inds]
+            s = scores[inds]
+
+            keep = nms_boxes(b, s, self.nms_threshold)
+
+            nboxes.append(b[keep])
+            nclasses.append(c[keep])
+            nscores.append(s[keep])
+
+        if not nclasses and not nscores:
+            return None, None, None
+
+        boxes = np.concatenate(nboxes)
+        classes = np.concatenate(nclasses)
+        scores = np.concatenate(nscores)
+
+        return boxes, classes, scores
+
+    @cost("Detect")
+    def _run_session(self, data):
+        outputs = self.session.run([], {"images": data})
+
+        return outputs
+
+    def _postprocess(self, data):
+        ratio, (dw, dh) = self.temp_pack
+        input0_data = data[0]
+        input1_data = data[1]
+        input2_data = data[2]
+        input0_data = input0_data.reshape([3, -1] + list(input0_data.shape[-2:]))
+        input1_data = input1_data.reshape([3, -1] + list(input1_data.shape[-2:]))
+        input2_data = input2_data.reshape([3, -1] + list(input2_data.shape[-2:]))
+        input_data = list()
+        input_data.append(np.transpose(input0_data, (2, 3, 0, 1)))
+        input_data.append(np.transpose(input1_data, (2, 3, 0, 1)))
+        input_data.append(np.transpose(input2_data, (2, 3, 0, 1)))
+
+        boxes, classes, scores = self.decode_outputs(input_data, self.input_size)
+
+        boxes = restore_bound_box(boxes, ratio, (dw, dh))
+
+        return boxes, classes, scores
+
+    def _preprocess(self, image):
+        h, w, _ = image.shape
+        resize_img, ratio, (dw, dh) = letterbox(image, new_shape=(self.input_size[1], self.input_size[0]))
+        data = image_to_input_tensor(resize_img)
+        self.temp_pack = ratio, (dw, dh)
+
+        return data

Prj-Python/hyperlpr3/inference/multitask_detect.py

@@ -0,0 +1,184 @@
+import numpy as np
+import cv2
+import copy
+from .base.base import HamburgerABC
+
+
+def xywh2xyxy(boxes):
+    xywh = copy.deepcopy(boxes)
+    xywh[:, 0] = boxes[:, 0] - boxes[:, 2] / 2
+    xywh[:, 1] = boxes[:, 1] - boxes[:, 3] / 2
+    xywh[:, 2] = boxes[:, 0] + boxes[:, 2] / 2
+    xywh[:, 3] = boxes[:, 1] + boxes[:, 3] / 2
+    return xywh
+
+
+def nms(boxes, iou_thresh):  # nms
+    index = np.argsort(boxes[:, 4])[::-1]
+    keep = []
+    while index.size > 0:
+        i = index[0]
+        keep.append(i)
+        x1 = np.maximum(boxes[i, 0], boxes[index[1:], 0])
+        y1 = np.maximum(boxes[i, 1], boxes[index[1:], 1])
+        x2 = np.minimum(boxes[i, 2], boxes[index[1:], 2])
+        y2 = np.minimum(boxes[i, 3], boxes[index[1:], 3])
+
+        w = np.maximum(0, x2 - x1)
+        h = np.maximum(0, y2 - y1)
+
+        inter_area = w * h
+        union_area = (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1]) + (
+                boxes[index[1:], 2] - boxes[index[1:], 0]) * (boxes[index[1:], 3] - boxes[index[1:], 1])
+        iou = inter_area / (union_area - inter_area)
+        idx = np.where(iou <= iou_thresh)[0]
+        index = index[idx + 1]
+    return keep
+
+
+def restore_box(boxes, r, left, top):
+    boxes[:, [0, 2, 5, 7, 9, 11]] -= left
+    boxes[:, [1, 3, 6, 8, 10, 12]] -= top
+
+    boxes[:, [0, 2, 5, 7, 9, 11]] /= r
+    boxes[:, [1, 3, 6, 8, 10, 12]] /= r
+    return boxes
+
+
+def detect_pre_precessing(img, img_size):
+    img, r, left, top = letter_box(img, img_size)
+    img = img[:, :, ::-1].transpose(2, 0, 1).copy().astype(np.float32)
+    img = img / 255
+    img = img.reshape(1, *img.shape)
+    return img, r, left, top
+
+
+def post_precessing(dets, r, left, top, conf_thresh=0.25, iou_thresh=0.5):
+    choice = dets[:, :, 4] > conf_thresh
+    dets = dets[choice]
+    dets[:, 13:15] *= dets[:, 4:5]
+    box = dets[:, :4]
+    boxes = xywh2xyxy(box)
+    score = np.max(dets[:, 13:15], axis=-1, keepdims=True)
+    index = np.argmax(dets[:, 13:15], axis=-1).reshape(-1, 1)
+    output = np.concatenate((boxes, score, dets[:, 5:13], index), axis=1)
+    reserve_ = nms(output, iou_thresh)
+    output = output[reserve_]
+    output = restore_box(output, r, left, top)
+    return output
+
+
+def letter_box(img, size=(640, 640)):
+    h, w, c = img.shape
+    r = min(size[0] / h, size[1] / w)
+    new_h, new_w = int(h * r), int(w * r)
+    top = int((size[0] - new_h) / 2)
+    left = int((size[1] - new_w) / 2)
+
+    bottom = size[0] - new_h - top
+    right = size[1] - new_w - left
+    img_resize = cv2.resize(img, (new_w, new_h))
+    img = cv2.copyMakeBorder(img_resize, top, bottom, left, right, borderType=cv2.BORDER_CONSTANT,
+                             value=(0, 0, 0))
+    return img, r, left, top
+
+
+class MultiTaskDetectorMNN(HamburgerABC):
+
+    def __init__(self, mnn_path, box_threshold: float = 0.5, nms_threshold: float = 0.6, *args, **kwargs):
+        from hyperlpr3.common.mnn_adapt import MNNAdapter
+        super().__init__(*args, **kwargs)
+        assert self.input_size[0] == self.input_size[1]
+        self.box_threshold = box_threshold
+        self.nms_threshold = nms_threshold
+        self.input_shape = (1, 3, self.input_size[0], self.input_size[1])
+        if self.input_size[0] == 320:
+            self.tensor_shape = [(1, 6300, 15)]
+        elif self.input_size[0] == 640:
+            self.tensor_shape = [(1, 25200, 15)]
+        self.session = MNNAdapter(mnn_path, self.input_shape, outputs_name=['output', ],
+                                  outputs_shape=self.tensor_shape)
+
+    def _run_session(self, data):
+        outputs = self.session.inference(data)
+        result = list()
+        for idx, output in enumerate(outputs):
+            result.append(output.reshape(self.tensor_shape[idx]))
+        result = np.asarray(result)
+
+        return result[0]
+
+    def _postprocess(self, data):
+        r, left, top = self.tmp_pack
+
+        return post_precessing(data, r, left, top)
+
+    def _preprocess(self, image):
+        img, r, left, top = detect_pre_precessing(image, self.input_size)
+        self.tmp_pack = r, left, top
+
+        return img
+
+
+class MultiTaskDetectorDNN(HamburgerABC):
+
+    def __init__(self, onnx_path, box_threshold: float = 0.5, nms_threshold: float = 0.6, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.box_threshold = box_threshold
+        self.nms_threshold = nms_threshold
+        self.session = cv2.dnn.readNetFromONNX(onnx_path)
+        self.input_shape = (1, 3, self.input_size[0], self.input_size[1])
+        self.tensor_shape = [(1, 6300, 15)]
+
+    def _run_session(self, data):
+        self.session.setInput(data)
+        outputs = self.session.forward()
+
+        return outputs
+
+    def _postprocess(self, data):
+
+        r, left, top = self.tmp_pack
+
+        return post_precessing(data, r, left, top)
+
+    def _preprocess(self, image):
+        img, r, left, top = detect_pre_precessing(image, self.input_size)
+        self.tmp_pack = r, left, top
+
+        return img
+
+
+class MultiTaskDetectorORT(HamburgerABC):
+
+    def __init__(self, onnx_path, box_threshold: float = 0.5, nms_threshold: float = 0.6, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        import onnxruntime as ort
+        self.box_threshold = box_threshold
+        self.nms_threshold = nms_threshold
+        self.session = ort.InferenceSession(onnx_path, providers=['CPUExecutionProvider'])
+        self.inputs_option = self.session.get_inputs()
+        self.outputs_option = self.session.get_outputs()
+        input_option = self.inputs_option[0]
+        input_size_ = tuple(input_option.shape[2:])
+        self.input_size = tuple(self.input_size)
+        if not self.input_size:
+            self.input_size = input_size_
+        assert self.input_size == input_size_, 'The dimensions of the input do not match the model expectations.'
+        assert self.input_size[0] == self.input_size[1]
+        self.input_name = input_option.name
+
+    def _run_session(self, data):
+        result = self.session.run([self.outputs_option[0].name], {self.input_name: data})[0]
+
+        return result
+
+    def _postprocess(self, data):
+        r, left, top = self.tmp_pack
+        return post_precessing(data, r, left, top)
+
+    def _preprocess(self, image):
+        img, r, left, top = detect_pre_precessing(image, self.input_size)
+        self.tmp_pack = r, left, top
+
+        return img

Prj-Python/hyperlpr3/inference/pipeline.py

@@ -0,0 +1,117 @@
+import numpy as np
+
+from hyperlpr3.common.typedef import *
+from hyperlpr3.common.tools_process import *
+
+
+class LPRMultiTaskPipeline(object):
+
+    def __init__(self, detector, recognizer, classifier):
+        self.detector = detector
+        self.recognizer = recognizer
+        self.classifier = classifier
+
+    def run(self, image: np.ndarray) -> list:
+        result = list()
+        assert len(image.shape) == 3, "Input image must be 3 channels."
+        assert image is not None, "Input image cannot be empty."
+        outputs = self.detector(image)
+        for out in outputs:
+            rect = out[:4].astype(int)
+            score = out[4]
+            land_marks = out[5:13].reshape(4, 2).astype(int)
+            layer_num = int(out[13])
+            print(layer_num)
+            pad = get_rotate_crop_image(image, land_marks)
+            if layer_num == DOUBLE:
+                # double
+                h, w, _ = pad.shape
+                line = int(h * 0.4)
+                top = pad[:line, :, ]
+                bottom = pad[line:, :]
+                top_code, top_confidence = self.recognizer(top)
+                bottom_code, bottom_confidence = self.recognizer(bottom)
+                plate_code = top_code + bottom_code
+                rec_confidence = (top_confidence + bottom_confidence) / 2
+                # cv2.imshow("top", top)
+                # cv2.imshow("bottom", bottom)
+                # cv2.waitKey(0)
+            else:
+                plate_code, rec_confidence = self.recognizer(pad)
+            if plate_code == '':
+                continue
+            if len(plate_code) >= 7:
+                plate_type = code_filter(plate_code)
+                if plate_type == UNKNOWN:
+                    cls = self.classifier(pad)
+                    idx = int(np.argmax(cls))
+                    if idx == PLATE_TYPE_YELLOW:
+                        if layer_num == DOUBLE:
+                            plate_type = YELLOW_DOUBLE
+                        else:
+                            plate_type = YELLOW_SINGLE
+                    elif idx == PLATE_TYPE_BLUE:
+                        plate_type = BLUE
+                    elif idx == PLATE_TYPE_GREEN:
+                        plate_type = GREEN
+                plate = Plate(vertex=land_marks, plate_code=plate_code, det_bound_box=np.asarray(rect),
+                              rec_confidence=rec_confidence, dex_bound_confidence=score, plate_type=plate_type)
+                result.append(plate.to_result())
+
+        return result
+
+    def __call__(self, image: np.ndarray, *args, **kwargs):
+        return self.run(image)
+
+
+class LPRPipeline(object):
+
+    def __init__(self, detector, vertex_predictor, recognizer, ):
+        self.detector = detector
+        self.vertex_predictor = vertex_predictor
+        self.recognizer = recognizer
+
+    # @cost("PipelineTotalCost")
+    def run(self, image: np.ndarray) -> list:
+        result = list()
+        boxes, classes, scores = self.detector(image)
+        fp_boxes_index = find_the_adjacent_boxes(boxes)
+        image_blacks = list()
+        if len(fp_boxes_index) > 0:
+            for idx in fp_boxes_index:
+                image_black = np.zeros_like(image)
+                box = boxes[idx]
+                x1, y1, x2, y2 = np.asarray(box).astype(int)
+                image_black[y1:y2, x1:x2] = image[y1:y2, x1:x2]
+                image_blacks.append(image_black)
+        if boxes:
+            fp = 0
+            for idx, box in enumerate(boxes):
+                det_confidence = scores[idx]
+                if idx in fp_boxes_index:
+                    warped, p, mat = align_box(image_blacks[fp], box, scale_factor=1.2, size=96)
+                    fp += 1
+                else:
+                    warped, p, mat = align_box(image, box, scale_factor=1.2, size=96)
+                kps = self.vertex_predictor(warped)
+                polyline = list()
+                for point in kps:
+                    polyline.append([point[0], point[1], 1])
+                polyline = np.asarray(polyline)
+                inv = cv2.invertAffineTransform(mat)
+                trans_points = np.dot(inv, polyline.T).T
+                pad = get_rotate_crop_image(image, trans_points)
+                # print(pad.shape)
+                # cv2.imshow("pad", pad)
+                # cv2.waitKey(0)
+                plate_code, rec_confidence = self.recognizer(pad)
+                if plate_code == '':
+                    continue
+                plate = Plate(vertex=trans_points, plate_code=plate_code, det_bound_box=np.asarray(box),
+                              rec_confidence=rec_confidence, dex_bound_confidence=det_confidence)
+                result.append(plate.to_dict())
+
+        return result
+
+    def __call__(self, image: np.ndarray, *args, **kwargs):
+        return self.run(image)

Prj-Python/hyperlpr3/inference/recognition.py

@@ -0,0 +1,233 @@
+import cv2
+import numpy as np
+from .base.base import HamburgerABC
+from hyperlpr3.common.tools_process import cost
+import math
+from hyperlpr3.common.tokenize import token
+
+
+def encode_images(image: np.ndarray, max_wh_ratio, target_shape, limited_max_width=160, limited_min_width=48):
+    imgC = 3
+    imgH, imgW = target_shape
+    # cv2.imshow("image", image)
+    # cv2.waitKey(0)
+    assert imgC == image.shape[2]
+    max_wh_ratio = max(max_wh_ratio, imgW / imgH)
+    imgW = int((imgH * max_wh_ratio))
+    imgW = max(min(imgW, limited_max_width), limited_min_width)
+    h, w = image.shape[:2]
+    ratio = w / float(h)
+    ratio_imgH = math.ceil(imgH * ratio)
+    ratio_imgH = max(ratio_imgH, limited_min_width)
+    if ratio_imgH > imgW:
+        resized_w = imgW
+    else:
+        resized_w = int(ratio_imgH)
+    resized_image = cv2.resize(image, (resized_w, imgH))
+    # print((resized_w, imgH))
+    # padding_im1 = np.ones((imgH, imgW, imgC), dtype=np.uint8) * 128
+    # padding_im1[:, 0:resized_w, :] = resized_image
+    # cv2.imwrite("pad.jpg", padding_im1)
+
+    resized_image = resized_image.astype('float32')
+    resized_image = (resized_image.transpose((2, 0, 1)) - 127.5) / 127.5
+    # resized_image -= 0.5
+    # resized_image *= 2
+    padding_im = np.zeros((imgC, imgH, imgW), dtype=np.float32)
+    padding_im[:, :, 0:resized_w] = resized_image
+
+    # np.save('fk.npy', padding_im)
+
+    return padding_im
+
+
+def get_ignored_tokens():
+    return [0]  # for ctc blank
+
+
+class PPRCNNRecognitionMNN(HamburgerABC):
+
+    def __init__(self, mnn_path, character_file, *args, **kwargs):
+        from hyperlpr3.common.mnn_adapt import MNNAdapter
+        super().__init__(*args, **kwargs)
+        self.input_shape = (1, 3, self.input_size[0], self.input_size[1])
+        self.session = MNNAdapter(mnn_path, input_shape=self.input_shape, outputs_name=['output'])
+        self.character_list = token
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] in ignored_tokens:
+                    continue
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[batch_idx][idx]:
+                        continue
+                char_list.append(self.character_list[int(text_index[batch_idx][idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list)))
+        return result_list
+
+    def _run_session(self, data):
+        output = self.session.inference(data)
+        output = output.reshape(40, 6625)
+        # print(output[:, 0])
+        output = np.expand_dims([output], 0)
+        return output
+
+    def _postprocess(self, data):
+        prod = data[0]
+        argmax = np.argmax(prod, axis=2)
+        # print(argmax)
+        rmax = np.max(prod, axis=2)
+        # print(rmax)
+        result = self.decode(argmax, rmax, is_remove_duplicate=True)
+
+        return result[0]
+
+    def _preprocess(self, image):
+        assert len(
+            image.shape) == 3, "The dimensions of the input image object do not match. The input supports a single " \
+                               "image. "
+        h, w, _ = image.shape
+        wh_ratio = w * 1.0 / h
+        data = encode_images(image, wh_ratio, self.input_size, )
+        data = np.expand_dims(data, 0)
+
+        return data
+
+
+class PPRCNNRecognitionORT(HamburgerABC):
+
+    def __init__(self, onnx_path, token_dict=token, *args, **kwargs):
+        import onnxruntime as ort
+        super().__init__(*args, **kwargs)
+        self.session = ort.InferenceSession(onnx_path, None)
+        self.input_config = self.session.get_inputs()[0]
+        self.output_config = self.session.get_outputs()[0]
+        self.input_size = self.input_config.shape[2:]
+        # print(self.input_size)
+        self.character_list = token_dict
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] in ignored_tokens:
+                    continue
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[batch_idx][idx]:
+                        continue
+                # print(int(text_index[batch_idx][idx]))
+                char_list.append(self.character_list[int(text_index[batch_idx][idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list)))
+        return result_list
+
+    # @cost("Recognition")
+    def _run_session(self, data) -> np.ndarray:
+        result = self.session.run([self.output_config.name], {self.input_config.name: data})
+
+        return result
+
+    def _postprocess(self, data) -> tuple:
+        if data:
+            prod = data[0]
+            argmax = np.argmax(prod, axis=2)
+            rmax = np.max(prod, axis=2)
+            result = self.decode(argmax, rmax, is_remove_duplicate=True)
+
+            return result[0]
+        else:
+            return '', 0.0
+
+    def _preprocess(self, image) -> np.ndarray:
+        assert len(
+            image.shape) == 3, "The dimensions of the input image object do not match. The input supports a single " \
+                               "image. "
+        h, w, _ = image.shape
+        wh_ratio = w * 1.0 / h
+        data = encode_images(image, wh_ratio, self.input_size, )
+        data = np.expand_dims(data, 0)
+        # print(data.shape)
+
+        return data
+
+
+class PPRCNNRecognitionDNN(HamburgerABC):
+
+    def __init__(self, onnx_path, character_file, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.session = cv2.dnn.readNetFromONNX(onnx_path)
+        self.input_shape = (1, 3, self.input_size[0], self.input_size[1])
+        self.character_list = token
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        result_list = []
+        ignored_tokens = get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] in ignored_tokens:
+                    continue
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[batch_idx][idx]:
+                        continue
+                char_list.append(self.character_list[int(text_index[batch_idx][idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list)))
+        return result_list
+
+    def _run_session(self, data):
+        self.session.setInput(data)
+        outputs = self.session.forward()
+        outputs = np.expand_dims(outputs, 0)
+        # print(outputs.shape)
+
+        return outputs
+
+    def _postprocess(self, data):
+        prod = data[0]
+        argmax = np.argmax(prod, axis=2)
+        rmax = np.max(prod, axis=2)
+        result = self.decode(argmax, rmax, is_remove_duplicate=True)
+
+        return result[0]
+
+    def _preprocess(self, image):
+        assert len(
+            image.shape) == 3, "The dimensions of the input image object do not match. The input supports a single " \
+                               "image. "
+        h, w, _ = image.shape
+        wh_ratio = w * 1.0 / h
+        data = encode_images(image, wh_ratio, self.input_size, )
+        data = np.expand_dims(data, 0)
+
+        return data

Prj-Python/hyperlpr3/inference/vertex.py

@@ -0,0 +1,83 @@
+import cv2
+import numpy as np
+from .base.base import HamburgerABC
+from hyperlpr3.common.tools_process import cost
+
+
+def encode_images(image: np.ndarray):
+    image_encode = image / 255.0
+    if len(image_encode.shape) == 4:
+        image_encode = image_encode.transpose(0, 3, 1, 2)
+    else:
+        image_encode = image_encode.transpose(2, 0, 1)
+    image_encode = image_encode.astype(np.float32)
+
+    return image_encode
+
+
+class BVTVertexMNN(HamburgerABC):
+
+    def __init__(self, mnn_path, *args, **kwargs):
+        from .common.mnn_adapt import MNNAdapter
+        super().__init__(*args, **kwargs)
+        self.input_shape = (1, 3, self.input_size[0], self.input_size[1])
+        self.session = MNNAdapter(mnn_path, self.input_shape)
+
+    def _run_session(self, data):
+        outputs = self.session.inference(data)
+        return outputs
+
+    def _postprocess(self, data):
+        assert data.shape[0] == 1
+        data = np.asarray(data).reshape(-1, 4, 2)
+        data[:, :, 0] *= self.input_size[1]
+        data[:, :, 1] *= self.input_size[0]
+
+        return data[0]
+
+    def _preprocess(self, image):
+        assert len(
+            image.shape) == 3, "The dimensions of the input image object do not match. The input supports a single " \
+                               "image. "
+        image_resize = cv2.resize(image, self.input_size)
+        encode = encode_images(image_resize)
+        encode = encode.astype(np.float32)
+        input_tensor = np.expand_dims(encode, 0)
+
+        return input_tensor
+
+
+class BVTVertexORT(HamburgerABC):
+
+    def __init__(self, onnx_path, *args, **kwargs):
+        import onnxruntime as ort
+        super().__init__(*args, **kwargs)
+        self.session = ort.InferenceSession(onnx_path, None)
+        self.input_config = self.session.get_inputs()[0]
+        self.output_config = self.session.get_outputs()[0]
+        self.input_size = self.input_config.shape[2:]
+
+    # @cost('Vertex')
+    def _run_session(self, data) -> np.ndarray:
+        result = self.session.run([self.output_config.name], {self.input_config.name: data})
+
+        return result[0]
+
+    def _postprocess(self, data) -> np.ndarray:
+        assert data.shape[0] == 1
+        data = np.asarray(data).reshape(-1, 4, 2)
+        data[:, :, 0] *= self.input_size[1]
+        data[:, :, 1] *= self.input_size[0]
+
+        return data[0]
+
+    def _preprocess(self, image) -> np.ndarray:
+        assert len(
+            image.shape) == 3, "The dimensions of the input image object do not match. The input supports a single " \
+                               "image. "
+        image_resize = cv2.resize(image, self.input_size)
+        encode = encode_images(image_resize)
+        encode = encode.astype(np.float32)
+        input_tensor = np.expand_dims(encode, 0)
+
+        return input_tensor