Modified resize and boxes coordinates to float, to achieve same darknet accuracy

Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>

include/tkDNN/Yolo3Detection.h

@@ -13,6 +13,7 @@ private:
     int num = 0;
     int nMasks = 0;
     int nDets = 0;
+    bool letterbox = false;
     tk::dnn::Yolo::detection *dets = nullptr;
     tk::dnn::Yolo* yolo[3];
 
@@ -21,7 +22,7 @@ private:
     cv::Mat bgr_h;
     
 public:
-    Yolo3Detection() {};
+    Yolo3Detection(const bool letter_box=false) :letterbox(letter_box){}
     ~Yolo3Detection() {}; 
 
     bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);

src/Yolo3Detection.cpp

@@ -52,28 +52,80 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
     return true;
 } 
 
-void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
+cv::Mat resize_image(cv::Mat im, int w, int h)
-#ifdef OPENCV_CUDACONTRIB
+{
-    cv::cuda::GpuMat orig_img, img_resized;
+    cv::Mat resized = cv::Mat(cv::Size(w,h), CV_32FC3, cv::Scalar(0) );
-    orig_img = cv::cuda::GpuMat(frame);
+    cv::Mat part = cv::Mat(cv::Size(w,im.rows), CV_32FC3, cv::Scalar(0) );
-    cv::cuda::resize(orig_img, img_resized, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+    int r, c, k;
+    float w_scale = (float)(im.cols - 1) / (w - 1);
+    float h_scale = (float)(im.rows - 1) / (h - 1);
     
-    img_resized.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
+    for(k = 0; k < im.channels(); ++k){
-
+        for(r = 0; r < im.rows; ++r){
-    //split channels
+            for(c = 0; c < w; ++c){
-    cv::cuda::split(imagePreproc,bgr);//split source
+                float val = 0;
-
+                if(c == w-1 || im.cols == 1){
-    //write channels
+                    val = im.at<cv::Vec3f>(r, im.cols-1)[k];
-    for(int i=0; i<netRT->input_dim.c; i++) {
+                } else {
-        int size = imagePreproc.rows * imagePreproc.cols;
+                    float sx = c*w_scale;
-        int ch = netRT->input_dim.c-1 -i;
+                    int ix = (int) sx;
-        bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
+                    float dx = sx - ix;
-        checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
+                    val = (1 - dx) * im.at<cv::Vec3f>(r, ix)[k] + dx * im.at<cv::Vec3f>(r,ix+1)[k];
+                }
+                part.at<cv::Vec3f>(r,c)[k] = val;
+            }
+        }
     }
-#else
+    for(k = 0; k < im.channels(); ++k){
-    cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+        for(r = 0; r < h; ++r){
+            float sy = r*h_scale;
+            int iy = (int) sy;
+            float dy = sy - iy;
+            for(c = 0; c < w; ++c){
+                float val = (1-dy) * part.at<cv::Vec3f>(iy, c)[k];
+                resized.at<cv::Vec3f>(r, c)[k] = val;
+            }
+            if(r == h-1 || im.rows == 1) continue;
+            for(c = 0; c < w; ++c){
+                float val = dy * part.at<cv::Vec3f>(iy+1, c)[k];
+                resized.at<cv::Vec3f>(r,c)[k] += val;
+            }
+        }
+    }
+
+    return resized;
+}
+
+void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
     frame.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
 
+    if(letterbox){
+        int im_w = frame.cols;
+        int im_h = frame.rows;
+        int net_w = netRT->input_dim.w;
+        int net_h = netRT->input_dim.h;
+        if(net_w == net_h && letterbox){
+            float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
+
+            int new_h = im_h/ratio;
+            int new_w = im_w/ratio;
+
+            imagePreproc = resize_image(imagePreproc, new_w, new_h);
+
+            cv::Mat borders;
+            int top = (net_h - new_h)/2;
+            int bottom = (net_h - new_h) - top;
+            int left = (net_w - new_w)/2;
+            int right = (net_w - new_w) - left;
+
+            cv::copyMakeBorder(imagePreproc,imagePreproc, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0.5,0.5,0.5));
+        }
+        else
+            FatalError("letterbox not spported with h!=w");
+    }
+    else
+        imagePreproc = resize_image(imagePreproc, netRT->input_dim.w, netRT->input_dim.h);
+
     //split channels
     cv::split(imagePreproc,bgr);//split source
 
@@ -84,7 +136,6 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
         memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));     
     }
     checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
-#endif
 }
 
 void Yolo3Detection::postprocess(const int bi, const bool mAP){
@@ -105,14 +156,45 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
     }
     tk::dnn::Yolo::mergeDetections(dets, nDets, classes);
 
+    int im_w = originalSize[bi].width;
+    int im_h = originalSize[bi].height;
+    int net_w = netRT->input_dim.w;
+    int net_h = netRT->input_dim.h;
+    int new_h, new_w;
+
+    int top = 0, left = 0;
+
+    if(letterbox){
+        float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
+        x_ratio = ratio;
+        y_ratio = ratio;
+        std::cout<<ratio<<std::endl;
+
+        int new_h = im_h/ratio;
+        int new_w = im_w/ratio;
+
+        top = (net_h - new_h)/2;
+        left = (net_w - new_w)/2;
+    }
+    else{
+        new_h = net_h;
+        new_w = net_w;
+    }
+    
+    float deltaw = net_w - new_w;
+    float deltah = net_h - new_h;
+    float ratiow = (float)new_w / net_w;
+    float ratioh = (float)new_h / net_h;
+
     // fill detected
     detected.clear();
     for(int j=0; j<nDets; j++) {
         tk::dnn::Yolo::box b = dets[j].bbox;
-        int x0   = (b.x-b.w/2.);
+
-        int x1   = (b.x+b.w/2.);
+        float x0   = (b.x - left - b.w/2.);
-        int y0   = (b.y-b.h/2.);
+        float x1   = (b.x - left + b.w/2.);
-        int y1   = (b.y+b.h/2.);
+        float y0   = (b.y - top - b.h/2.);
+        float y1   = (b.y - top + b.h/2.);
 
         // convert to image coords
         x0 = x_ratio*x0;
@@ -133,15 +215,9 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
                 res.w = x1 - x0;
                 res.h = y1 - y0;
 
-                // FIXME: this shuld be useless
-                // if(mAP)
-                //     for(int c=0; c<classes; c++) 
-                //         res.probs.push_back(dets[j].prob[c]);
-
                 detected.push_back(res);
             }
         }
-
     }
     batchDetected.push_back(detected);
 }

src/evaluation.cpp

@@ -342,14 +342,16 @@ void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path,
                 //min threshold confidence is set in DetectionNN.h
                 if (bbox[i].probs[j] > 0) {
 
-                    *out_file <<    "{\"image_id\":" << image_id <<
+                    *out_file << std::fixed  << std::setprecision(6) <<
+                                    "{\"image_id\":" << image_id <<
                                     ", \"category_id\":" << coco_ids[j] << 
                                     ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
                                     "], \"score\":" << bbox[i].probs[j] << "},\n";
                 }
             }
         else
-            *out_file <<    "{\"image_id\":" << image_id <<
+            *out_file <<     std::fixed  << std::setprecision(6) <<
+                            "{\"image_id\":" << image_id <<
                             ", \"category_id\":" << coco_ids[bbox[i].cl] << 
                             ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
                             "], \"score\":" << bbox[i].prob << "},\n";

tests/darknet/cfg/yolo3_voc.cfg

@@ -1,785 +0,0 @@
-[net]
-# Testing
-#  batch=1
-#  subdivisions=1
-# Training
-batch=64
-subdivisions=16
-width=416
-height=416
-channels=3
-momentum=0.9
-decay=0.0005
-angle=0
-saturation = 1.5
-exposure = 1.5
-hue=.1
-
-learning_rate=0.001
-burn_in=1000
-max_batches = 50200
-policy=steps
-steps=40000,45000
-scales=.1,.1
-
-
-
-[convolutional]
-batch_normalize=1
-filters=32
-size=3
-stride=1
-pad=1
-activation=leaky
-
-# Downsample
-
-[convolutional]
-batch_normalize=1
-filters=64
-size=3
-stride=2
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=32
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=64
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-# Downsample
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=3
-stride=2
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=64
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=64
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-# Downsample
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=2
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-# Downsample
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=2
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-# Downsample
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=2
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[shortcut]
-from=-3
-activation=linear
-
-######################
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[convolutional]
-size=1
-stride=1
-pad=1
-filters=75
-activation=linear
-
-[yolo]
-mask = 6,7,8
-anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
-classes=20
-num=9
-jitter=.3
-ignore_thresh = .5
-truth_thresh = 1
-random=1
-
-[route]
-layers = -4
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[upsample]
-stride=2
-
-[route]
-layers = -1, 61
-
-
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=512
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=512
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=512
-activation=leaky
-
-[convolutional]
-size=1
-stride=1
-pad=1
-filters=75
-activation=linear
-
-[yolo]
-mask = 3,4,5
-anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
-classes=20
-num=9
-jitter=.3
-ignore_thresh = .5
-truth_thresh = 1
-random=1
-
-[route]
-layers = -4
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[upsample]
-stride=2
-
-[route]
-layers = -1, 36
-
-
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=256
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=256
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=256
-activation=leaky
-
-[convolutional]
-size=1
-stride=1
-pad=1
-filters=75
-activation=linear
-
-[yolo]
-mask = 0,1,2
-anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
-classes=20
-num=9
-jitter=.3
-ignore_thresh = .5
-truth_thresh = 1
-random=1
-

tests/darknet/yolo3_voc.cpp

@@ -1,35 +0,0 @@
-#include<iostream>
-#include<vector>
-#include "tkdnn.h"
-#include "test.h"
-#include "DarknetParser.h"
-
-int main() {
-    std::string bin_path  = "yolo3_voc";
-    std::vector<std::string> input_bins = { 
-        bin_path + "/layers/input.bin"
-    };
-    std::vector<std::string> output_bins = {
-	bin_path + "/debug/layer82_out.bin",
-        bin_path + "/debug/layer94_out.bin",
-        bin_path + "/debug/layer106_out.bin"
-    };
-    std::string wgs_path  = bin_path + "/layers";
-    std::string cfg_path  = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo3_voc.cfg";
-    std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/voc.names";
-    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/mDJwCBgADc2xL4M/download");
-
-    // parse darknet network
-    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
-    net->print();
-
-    //convert network to tensorRT
-    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
-    
-    int ret = testInference(input_bins, output_bins, net, netRT);
-    net->releaseLayers();
-    delete net;
-    delete netRT;
-    return ret;
-}
-