Modified resize and boxes coordinates to float, to achieve same darknet accuracy

Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>

include/tkDNN/Yolo3Detection.h

@@ -13,6 +13,7 @@ private:
     int num = 0;
     int nMasks = 0;
     int nDets = 0;
+    bool letterbox = false;
     tk::dnn::Yolo::detection *dets = nullptr;
     tk::dnn::Yolo* yolo[3];
 
@@ -21,7 +22,7 @@ private:
     cv::Mat bgr_h;
     
 public:
-    Yolo3Detection() {};
+    Yolo3Detection(const bool letter_box=false) :letterbox(letter_box){}
     ~Yolo3Detection() {}; 
 
     bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);

src/Yolo3Detection.cpp

@@ -52,28 +52,80 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
     return true;
 } 
 
-void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
+cv::Mat resize_image(cv::Mat im, int w, int h)
-#ifdef OPENCV_CUDACONTRIB
+{
-    cv::cuda::GpuMat orig_img, img_resized;
+    cv::Mat resized = cv::Mat(cv::Size(w,h), CV_32FC3, cv::Scalar(0) );
-    orig_img = cv::cuda::GpuMat(frame);
+    cv::Mat part = cv::Mat(cv::Size(w,im.rows), CV_32FC3, cv::Scalar(0) );
-    cv::cuda::resize(orig_img, img_resized, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+    int r, c, k;
+    float w_scale = (float)(im.cols - 1) / (w - 1);
+    float h_scale = (float)(im.rows - 1) / (h - 1);
     
-    img_resized.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
+    for(k = 0; k < im.channels(); ++k){
-
+        for(r = 0; r < im.rows; ++r){
-    //split channels
+            for(c = 0; c < w; ++c){
-    cv::cuda::split(imagePreproc,bgr);//split source
+                float val = 0;
-
+                if(c == w-1 || im.cols == 1){
-    //write channels
+                    val = im.at<cv::Vec3f>(r, im.cols-1)[k];
-    for(int i=0; i<netRT->input_dim.c; i++) {
+                } else {
-        int size = imagePreproc.rows * imagePreproc.cols;
+                    float sx = c*w_scale;
-        int ch = netRT->input_dim.c-1 -i;
+                    int ix = (int) sx;
-        bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
+                    float dx = sx - ix;
-        checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
+                    val = (1 - dx) * im.at<cv::Vec3f>(r, ix)[k] + dx * im.at<cv::Vec3f>(r,ix+1)[k];
                 }
-#else
+                part.at<cv::Vec3f>(r,c)[k] = val;
-    cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+            }
+        }
+    }
+    for(k = 0; k < im.channels(); ++k){
+        for(r = 0; r < h; ++r){
+            float sy = r*h_scale;
+            int iy = (int) sy;
+            float dy = sy - iy;
+            for(c = 0; c < w; ++c){
+                float val = (1-dy) * part.at<cv::Vec3f>(iy, c)[k];
+                resized.at<cv::Vec3f>(r, c)[k] = val;
+            }
+            if(r == h-1 || im.rows == 1) continue;
+            for(c = 0; c < w; ++c){
+                float val = dy * part.at<cv::Vec3f>(iy+1, c)[k];
+                resized.at<cv::Vec3f>(r,c)[k] += val;
+            }
+        }
+    }
+
+    return resized;
+}
+
+void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
     frame.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
 
+    if(letterbox){
+        int im_w = frame.cols;
+        int im_h = frame.rows;
+        int net_w = netRT->input_dim.w;
+        int net_h = netRT->input_dim.h;
+        if(net_w == net_h && letterbox){
+            float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
+
+            int new_h = im_h/ratio;
+            int new_w = im_w/ratio;
+
+            imagePreproc = resize_image(imagePreproc, new_w, new_h);
+
+            cv::Mat borders;
+            int top = (net_h - new_h)/2;
+            int bottom = (net_h - new_h) - top;
+            int left = (net_w - new_w)/2;
+            int right = (net_w - new_w) - left;
+
+            cv::copyMakeBorder(imagePreproc,imagePreproc, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0.5,0.5,0.5));
+        }
+        else
+            FatalError("letterbox not spported with h!=w");
+    }
+    else
+        imagePreproc = resize_image(imagePreproc, netRT->input_dim.w, netRT->input_dim.h);
+
     //split channels
     cv::split(imagePreproc,bgr);//split source
 
@@ -84,7 +136,6 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
         memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));     
     }
     checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
-#endif
 }
 
 void Yolo3Detection::postprocess(const int bi, const bool mAP){
@@ -105,14 +156,45 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
     }
     tk::dnn::Yolo::mergeDetections(dets, nDets, classes);
 
+    int im_w = originalSize[bi].width;
+    int im_h = originalSize[bi].height;
+    int net_w = netRT->input_dim.w;
+    int net_h = netRT->input_dim.h;
+    int new_h, new_w;
+
+    int top = 0, left = 0;
+
+    if(letterbox){
+        float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
+        x_ratio = ratio;
+        y_ratio = ratio;
+        std::cout<<ratio<<std::endl;
+
+        int new_h = im_h/ratio;
+        int new_w = im_w/ratio;
+
+        top = (net_h - new_h)/2;
+        left = (net_w - new_w)/2;
+    }
+    else{
+        new_h = net_h;
+        new_w = net_w;
+    }
+    
+    float deltaw = net_w - new_w;
+    float deltah = net_h - new_h;
+    float ratiow = (float)new_w / net_w;
+    float ratioh = (float)new_h / net_h;
+
     // fill detected
     detected.clear();
     for(int j=0; j<nDets; j++) {
         tk::dnn::Yolo::box b = dets[j].bbox;
-        int x0   = (b.x-b.w/2.);
+
-        int x1   = (b.x+b.w/2.);
+        float x0   = (b.x - left - b.w/2.);
-        int y0   = (b.y-b.h/2.);
+        float x1   = (b.x - left + b.w/2.);
-        int y1   = (b.y+b.h/2.);
+        float y0   = (b.y - top - b.h/2.);
+        float y1   = (b.y - top + b.h/2.);
 
         // convert to image coords
         x0 = x_ratio*x0;
@@ -133,15 +215,9 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
                 res.w = x1 - x0;
                 res.h = y1 - y0;
 
-                // FIXME: this shuld be useless
-                // if(mAP)
-                //     for(int c=0; c<classes; c++) 
-                //         res.probs.push_back(dets[j].prob[c]);
-
                 detected.push_back(res);
             }
         }
-
     }
     batchDetected.push_back(detected);
 }

src/evaluation.cpp

@@ -342,14 +342,16 @@ void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path,
                 //min threshold confidence is set in DetectionNN.h
                 if (bbox[i].probs[j] > 0) {
 
-                    *out_file <<    "{\"image_id\":" << image_id <<
+                    *out_file << std::fixed  << std::setprecision(6) <<
+                                    "{\"image_id\":" << image_id <<
                                     ", \"category_id\":" << coco_ids[j] << 
                                     ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
                                     "], \"score\":" << bbox[i].probs[j] << "},\n";
                 }
             }
         else
-            *out_file <<    "{\"image_id\":" << image_id <<
+            *out_file <<     std::fixed  << std::setprecision(6) <<
+                            "{\"image_id\":" << image_id <<
                             ", \"category_id\":" << coco_ids[bbox[i].cl] << 
                             ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
                             "], \"score\":" << bbox[i].prob << "},\n";