You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
405 lines
13 KiB
C++
405 lines
13 KiB
C++
//
|
|
// Created by 庾金科 on 16/10/2017.
|
|
//
|
|
|
|
#include "PlateSegmentation.h"
|
|
#include "niBlackThreshold.h"
|
|
|
|
|
|
//#define DEBUG
|
|
namespace pr{
|
|
|
|
PlateSegmentation::PlateSegmentation(std::string prototxt,std::string caffemodel) {
|
|
net = cv::dnn::readNetFromCaffe(prototxt, caffemodel);
|
|
}
|
|
cv::Mat PlateSegmentation::classifyResponse(const cv::Mat &cropped){
|
|
cv::Mat inputBlob = cv::dnn::blobFromImage(cropped, 1/255.0, cv::Size(22,22), cv::Scalar(0,0,0),false);
|
|
net.setInput(inputBlob,"data");
|
|
return net.forward();
|
|
}
|
|
|
|
void drawHist(float* seq,int size,const char* name){
|
|
cv::Mat image(300,size,CV_8U);
|
|
image.setTo(0);
|
|
float* start =seq;
|
|
float* end = seq+size;
|
|
float l = *std::max_element(start,end);
|
|
for(int i = 0;i<size;i++)
|
|
{
|
|
int p = int(float(seq[i])/l*300);
|
|
cv::line(image,cv::Point(i,300),cv::Point(i,300-p),cv::Scalar(255,255,255));
|
|
}
|
|
cv::resize(image,image,cv::Size(600,100));
|
|
cv::imshow(name,image);
|
|
}
|
|
|
|
inline void computeSafeMargin(int &val,const int &rows){
|
|
val = std::min(val,rows);
|
|
val = std::max(val,0);
|
|
}
|
|
|
|
cv::Rect boxFromCenter(const cv::Point center,int left,int right,int top,int bottom,cv::Size bdSize)
|
|
{
|
|
cv::Point p1(center.x - left ,center.y - top);
|
|
cv::Point p2( center.x + right, center.y + bottom);
|
|
p1.x = std::max(0,p1.x);
|
|
p1.y = std::max(0,p1.y);
|
|
p2.x = std::min(p2.x,bdSize.width-1);
|
|
p2.y = std::min(p2.y,bdSize.height-1);
|
|
cv::Rect rect(p1,p2);
|
|
return rect;
|
|
}
|
|
|
|
cv::Rect boxPadding(cv::Rect rect,int left,int right,int top,int bottom,cv::Size bdSize)
|
|
{
|
|
|
|
cv::Point center(rect.x+(rect.width>>1),rect.y + (rect.height>>1));
|
|
int rebuildLeft = (rect.width>>1 )+ left;
|
|
int rebuildRight = (rect.width>>1 )+ right;
|
|
int rebuildTop = (rect.height>>1 )+ top;
|
|
int rebuildBottom = (rect.height>>1 )+ bottom;
|
|
return boxFromCenter(center,rebuildLeft,rebuildRight,rebuildTop,rebuildBottom,bdSize);
|
|
|
|
}
|
|
|
|
|
|
|
|
void PlateSegmentation:: refineRegion(cv::Mat &plateImage,const std::vector<int> &candidatePts,const int padding,std::vector<cv::Rect> &rects){
|
|
int w = candidatePts[5] - candidatePts[4];
|
|
int cols = plateImage.cols;
|
|
int rows = plateImage.rows;
|
|
for(int i = 0 ; i < candidatePts.size() ; i++)
|
|
{
|
|
int left = 0;
|
|
int right = 0 ;
|
|
|
|
if(i == 0 ){
|
|
left= candidatePts[i];
|
|
right = left+w+padding;
|
|
}
|
|
else {
|
|
left = candidatePts[i] - padding;
|
|
right = left + w + padding * 2;
|
|
}
|
|
|
|
computeSafeMargin(right,cols);
|
|
computeSafeMargin(left,cols);
|
|
cv::Rect roi(left,0,right - left,rows-1);
|
|
cv::Mat roiImage;
|
|
plateImage(roi).copyTo(roiImage);
|
|
|
|
if (i>=1)
|
|
{
|
|
|
|
cv::Mat roi_thres;
|
|
// cv::threshold(roiImage,roi_thres,0,255,cv::THRESH_OTSU|cv::THRESH_BINARY);
|
|
|
|
niBlackThreshold(roiImage,roi_thres,255,cv::THRESH_BINARY,15,0.27,BINARIZATION_NIBLACK);
|
|
|
|
std::vector<std::vector<cv::Point>> contours;
|
|
cv::findContours(roi_thres,contours,cv::RETR_LIST,cv::CHAIN_APPROX_SIMPLE);
|
|
cv::Point boxCenter(roiImage.cols>>1,roiImage.rows>>1);
|
|
|
|
cv::Rect final_bdbox;
|
|
cv::Point final_center;
|
|
int final_dist = INT_MAX;
|
|
|
|
|
|
for(auto contour:contours)
|
|
{
|
|
cv::Rect bdbox = cv::boundingRect(contour);
|
|
cv::Point center(bdbox.x+(bdbox.width>>1),bdbox.y + (bdbox.height>>1));
|
|
int dist = (center.x - boxCenter.x)*(center.x - boxCenter.x);
|
|
if(dist<final_dist and bdbox.height > rows>>1)
|
|
{ final_dist =dist;
|
|
final_center = center;
|
|
final_bdbox = bdbox;
|
|
}
|
|
}
|
|
|
|
//rebuild box
|
|
if(final_bdbox.height/ static_cast<float>(final_bdbox.width) > 3.5 && final_bdbox.width*final_bdbox.height<10)
|
|
final_bdbox = boxFromCenter(final_center,8,8,(rows>>1)-3 , (rows>>1) - 2,roiImage.size());
|
|
else {
|
|
if(i == candidatePts.size()-1)
|
|
final_bdbox = boxPadding(final_bdbox, padding/2, padding, padding/2, padding/2, roiImage.size());
|
|
else
|
|
final_bdbox = boxPadding(final_bdbox, padding, padding, padding, padding, roiImage.size());
|
|
|
|
|
|
// std::cout<<final_bdbox<<std::endl;
|
|
// std::cout<<roiImage.size()<<std::endl;
|
|
#ifdef DEBUG
|
|
cv::imshow("char_thres",roi_thres);
|
|
|
|
cv::imshow("char",roiImage(final_bdbox));
|
|
cv::waitKey(0);
|
|
#endif
|
|
|
|
|
|
}
|
|
|
|
|
|
final_bdbox.x += left;
|
|
|
|
rects.push_back(final_bdbox);
|
|
//
|
|
|
|
}
|
|
else
|
|
{
|
|
rects.push_back(roi);
|
|
}
|
|
|
|
// else
|
|
// {
|
|
//
|
|
// }
|
|
|
|
// cv::GaussianBlur(roiImage,roiImage,cv::Size(7,7),3);
|
|
//
|
|
// cv::imshow("image",roiImage);
|
|
// cv::waitKey(0);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
void avgfilter(float *angle_list,int size,int windowsSize) {
|
|
float *filterd = new float[size];
|
|
for(int i = 0 ; i < size ; i++) filterd [i] = angle_list[i];
|
|
// memcpy(filterd,angle_list,size);
|
|
|
|
cv::Mat kernal_gaussian = cv::getGaussianKernel(windowsSize,3,CV_32F);
|
|
float *kernal = (float*)kernal_gaussian.data;
|
|
// kernal+=windowsSize;
|
|
int r = windowsSize/2;
|
|
|
|
|
|
|
|
|
|
for (int i = 0; i < size; i++) {
|
|
float avg = 0.00f;
|
|
for (int j = 0; j < windowsSize; j++) {
|
|
if(i+j-r>0&&i+j+r<size-1)
|
|
avg += filterd[i + j-r]*kernal[j];
|
|
}
|
|
// avg = avg / windowsSize;
|
|
angle_list[i] = avg;
|
|
|
|
}
|
|
|
|
delete filterd;
|
|
}
|
|
|
|
void PlateSegmentation::templateMatchFinding(const cv::Mat &respones,int windowsWidth,std::pair<float,std::vector<int>> &candidatePts){
|
|
int rows = respones.rows;
|
|
int cols = respones.cols;
|
|
|
|
|
|
|
|
float *data = (float*)respones.data;
|
|
float *engNum_prob = data;
|
|
float *false_prob = data+cols;
|
|
float *ch_prob = data+cols*2;
|
|
|
|
avgfilter(engNum_prob,cols,5);
|
|
avgfilter(false_prob,cols,5);
|
|
// avgfilter(ch_prob,cols,5);
|
|
std::vector<int> candidate_pts(7);
|
|
#ifdef DEBUG
|
|
drawHist(engNum_prob,cols,"engNum_prob");
|
|
drawHist(false_prob,cols,"false_prob");
|
|
drawHist(ch_prob,cols,"ch_prob");
|
|
cv::waitKey(0);
|
|
#endif
|
|
|
|
|
|
|
|
|
|
int cp_list[7];
|
|
float loss_selected = -10;
|
|
|
|
for(int start = 0 ; start < 20 ; start+=2)
|
|
for(int width = windowsWidth-5; width < windowsWidth+5 ; width++ ){
|
|
for(int interval = windowsWidth/2; interval < windowsWidth; interval++)
|
|
{
|
|
int cp1_ch = start;
|
|
int cp2_p0 = cp1_ch+ width;
|
|
int cp3_p1 = cp2_p0+ width + interval;
|
|
int cp4_p2 = cp3_p1 + width;
|
|
int cp5_p3 = cp4_p2 + width+1;
|
|
int cp6_p4 = cp5_p3 + width+2;
|
|
int cp7_p5= cp6_p4+ width+2;
|
|
|
|
int md1 = (cp1_ch+cp2_p0)>>1;
|
|
int md2 = (cp2_p0+cp3_p1)>>1;
|
|
int md3 = (cp3_p1+cp4_p2)>>1;
|
|
int md4 = (cp4_p2+cp5_p3)>>1;
|
|
int md5 = (cp5_p3+cp6_p4)>>1;
|
|
int md6 = (cp6_p4+cp7_p5)>>1;
|
|
|
|
|
|
|
|
|
|
if(cp7_p5>=cols)
|
|
continue;
|
|
// float loss = ch_prob[cp1_ch]+
|
|
// engNum_prob[cp2_p0] +engNum_prob[cp3_p1]+engNum_prob[cp4_p2]+engNum_prob[cp5_p3]+engNum_prob[cp6_p4] +engNum_prob[cp7_p5]
|
|
// + (false_prob[md2]+false_prob[md3]+false_prob[md4]+false_prob[md5]+false_prob[md5] + false_prob[md6]);
|
|
float loss = ch_prob[cp1_ch]*3 -(false_prob[cp3_p1]+false_prob[cp4_p2]+false_prob[cp5_p3]+false_prob[cp6_p4]+false_prob[cp7_p5]);
|
|
|
|
if(loss>loss_selected)
|
|
{
|
|
loss_selected = loss;
|
|
cp_list[0]= cp1_ch;
|
|
cp_list[1]= cp2_p0;
|
|
cp_list[2]= cp3_p1;
|
|
cp_list[3]= cp4_p2;
|
|
cp_list[4]= cp5_p3;
|
|
cp_list[5]= cp6_p4;
|
|
cp_list[6]= cp7_p5;
|
|
}
|
|
}
|
|
}
|
|
candidate_pts[0] = cp_list[0];
|
|
candidate_pts[1] = cp_list[1];
|
|
candidate_pts[2] = cp_list[2];
|
|
candidate_pts[3] = cp_list[3];
|
|
candidate_pts[4] = cp_list[4];
|
|
candidate_pts[5] = cp_list[5];
|
|
candidate_pts[6] = cp_list[6];
|
|
|
|
candidatePts.first = loss_selected;
|
|
candidatePts.second = candidate_pts;
|
|
|
|
};
|
|
|
|
|
|
void PlateSegmentation::segmentPlateBySlidingWindows(cv::Mat &plateImage,int windowsWidth,int stride,cv::Mat &respones){
|
|
|
|
|
|
// cv::resize(plateImage,plateImage,cv::Size(136,36));
|
|
|
|
cv::Mat plateImageGray;
|
|
cv::cvtColor(plateImage,plateImageGray,cv::COLOR_BGR2GRAY);
|
|
int padding = plateImage.cols-136 ;
|
|
// int padding = 0 ;
|
|
int height = plateImage.rows - 1;
|
|
int width = plateImage.cols - 1 - padding;
|
|
for(int i = 0 ; i < width - windowsWidth +1 ; i +=stride)
|
|
{
|
|
cv::Rect roi(i,0,windowsWidth,height);
|
|
cv::Mat roiImage = plateImageGray(roi);
|
|
cv::Mat response = classifyResponse(roiImage);
|
|
respones.push_back(response);
|
|
}
|
|
|
|
|
|
|
|
|
|
respones = respones.t();
|
|
// std::pair<float,std::vector<int>> images ;
|
|
//
|
|
//
|
|
// std::cout<<images.first<<" ";
|
|
// for(int i = 0 ; i < images.second.size() ; i++)
|
|
// {
|
|
// std::cout<<images.second[i]<<" ";
|
|
//// cv::line(plateImageGray,cv::Point(images.second[i],0),cv::Point(images.second[i],36),cv::Scalar(255,255,255),1); //DEBUG
|
|
// }
|
|
|
|
// int w = images.second[5] - images.second[4];
|
|
|
|
// cv::line(plateImageGray,cv::Point(images.second[5]+w,0),cv::Point(images.second[5]+w,36),cv::Scalar(255,255,255),1); //DEBUG
|
|
// cv::line(plateImageGray,cv::Point(images.second[5]+2*w,0),cv::Point(images.second[5]+2*w,36),cv::Scalar(255,255,255),1); //DEBUG
|
|
|
|
|
|
// RefineRegion(plateImageGray,images.second,5);
|
|
|
|
// std::cout<<w<<std::endl;
|
|
|
|
// std::cout<<<<std::endl;
|
|
|
|
// cv::resize(plateImageGray,plateImageGray,cv::Size(600,100));
|
|
|
|
|
|
|
|
}
|
|
|
|
// void filterGaussian(cv::Mat &respones,float sigma){
|
|
//
|
|
// }
|
|
|
|
|
|
void PlateSegmentation::segmentPlatePipline(PlateInfo &plateInfo,int stride,std::vector<cv::Rect> &Char_rects){
|
|
cv::Mat plateImage = plateInfo.getPlateImage(); // get src image .
|
|
cv::Mat plateImageGray;
|
|
cv::cvtColor(plateImage,plateImageGray,cv::COLOR_BGR2GRAY);
|
|
//do binarzation
|
|
//
|
|
std::pair<float,std::vector<int>> sections ; // segment points variables .
|
|
|
|
cv::Mat respones; //three response of every sub region from origin image .
|
|
segmentPlateBySlidingWindows(plateImage,DEFAULT_WIDTH,1,respones);
|
|
templateMatchFinding(respones,DEFAULT_WIDTH/stride,sections);
|
|
for(int i = 0; i < sections.second.size() ; i++)
|
|
{
|
|
sections.second[i]*=stride;
|
|
|
|
}
|
|
|
|
// std::cout<<sections<<std::endl;
|
|
|
|
refineRegion(plateImageGray,sections.second,5,Char_rects);
|
|
#ifdef DEBUG
|
|
for(int i = 0 ; i < sections.second.size() ; i++)
|
|
{
|
|
std::cout<<sections.second[i]<<" ";
|
|
cv::line(plateImageGray,cv::Point(sections.second[i],0),cv::Point(sections.second[i],36),cv::Scalar(255,255,255),1); //DEBUG
|
|
}
|
|
cv::imshow("plate",plateImageGray);
|
|
cv::waitKey(0);
|
|
#endif
|
|
// cv::waitKey(0);
|
|
|
|
}
|
|
|
|
void PlateSegmentation::ExtractRegions(PlateInfo &plateInfo,std::vector<cv::Rect> &rects){
|
|
cv::Mat plateImage = plateInfo.getPlateImage();
|
|
for(int i = 0 ; i < rects.size() ; i++){
|
|
cv::Mat charImage;
|
|
plateImage(rects[i]).copyTo(charImage);
|
|
if(charImage.channels())
|
|
cv::cvtColor(charImage,charImage,cv::COLOR_BGR2GRAY);
|
|
// cv::imshow("image",charImage);
|
|
// cv::waitKey(0);
|
|
cv::equalizeHist(charImage,charImage);
|
|
//
|
|
|
|
//
|
|
|
|
|
|
std::pair<CharType,cv::Mat> char_instance;
|
|
if(i == 0 ){
|
|
|
|
char_instance.first = CHINESE;
|
|
|
|
|
|
} else if(i == 1){
|
|
char_instance.first = LETTER;
|
|
}
|
|
else{
|
|
char_instance.first = LETTER_NUMS;
|
|
}
|
|
char_instance.second = charImage;
|
|
plateInfo.appendPlateChar(char_instance);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}//namespace pr
|