Modified resize and boxes coordinates to float, to achieve same darknet accuracy

Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>

README.md

@@ -1,9 +1,9 @@
 # tkDNN
-tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU.
+tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier, Nano and several discrete GPUs.
 The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. 
 
 
-If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ .
+If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ .
 
 ```
 Accepted paper @ IRC 2020, will soon be published.
@@ -175,15 +175,25 @@ All models from darknet are now parsed directly from cfg, you still need to expo
   mish
 </details>
 
-## Run the demo
+## Run the demo 
+This is an example using yolov4.
 
-To run the an object detection demo follow these steps (example with yolov3):
+To run the an object detection first create the .rt file by running:
 ```
-rm yolo3_fp32.rt        # be sure to delete(or move) old tensorRT files
+rm yolo4_fp32.rt        # be sure to delete(or move) old tensorRT files
-./test_yolo3            # run the yolo test (is slow)
+./test_yolo4            # run the yolo test (is slow)
-./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y
 ```
-In general the demo program takes 4 parameters:
+If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
+```
+cmake .. -DDEBUG=True
+make
+```
+
+Once you have succesfully created your rt file, run the demo: 
+```
+./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y
+```
+In general the demo program takes 6 parameters:
 ```
 ./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag>
 ```
@@ -197,6 +207,7 @@ where
 
 N.b. By default it is used FP32 inference
 
+
 ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
 
 ### FP16 inference

include/tkDNN/Yolo3Detection.h

@@ -13,6 +13,7 @@ private:
     int num = 0;
     int nMasks = 0;
     int nDets = 0;
+    bool letterbox = false;
     tk::dnn::Yolo::detection *dets = nullptr;
     tk::dnn::Yolo* yolo[3];
 
@@ -21,7 +22,7 @@ private:
     cv::Mat bgr_h;
     
 public:
-    Yolo3Detection() {};
+    Yolo3Detection(const bool letter_box=false) :letterbox(letter_box){}
     ~Yolo3Detection() {}; 
 
     bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);

src/Conv2d.cpp

@@ -62,9 +62,10 @@ void Conv2d::initCUDNN(bool back) {
     // init workspace
     workSpace = NULL;
     ws_sizeInBytes = 0;
+    int algo_count = 0;
     if(back) {
         checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm_v7(net->cudnnHandle,
-                                                                filterDesc, dstTensor, convDesc, srcTensor, 1, 0, &bwAlgo) );
+                                                                filterDesc, dstTensor, convDesc, srcTensor, 1, &algo_count, &bwAlgo) );
         checkCUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(net->cudnnHandle,
                                                                 filterDesc, dstTensor, convDesc, srcTensor,
                                                                 bwAlgo.algo, &ws_sizeInBytes));
@@ -74,13 +75,17 @@ void Conv2d::initCUDNN(bool back) {
         srcTensorDesc = dstTensor;
         dstTensorDesc = srcTensor;
     } else {
+        
         checkCUDNN( cudnnGetConvolutionForwardAlgorithm_v7(net->cudnnHandle,
                                                          srcTensor, filterDesc, convDesc, dstTensor,
-                                                        1, 0, &algo) );
+                                                        1, &algo_count, &algo) );
          checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
                                                             srcTensor, filterDesc, convDesc, dstTensor,
                                                            algo.algo, &ws_sizeInBytes));
     }
+
+    if(algo_count < 1)
+        FatalError("Cannot retrieve convolutional algo");
 }
 
 void Conv2d::inferCUDNN(dnnType* srcData, bool back) {

src/Yolo3Detection.cpp

@@ -52,28 +52,80 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
     return true;
 } 
 
-void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
+cv::Mat resize_image(cv::Mat im, int w, int h)
-#ifdef OPENCV_CUDACONTRIB
+{
-    cv::cuda::GpuMat orig_img, img_resized;
+    cv::Mat resized = cv::Mat(cv::Size(w,h), CV_32FC3, cv::Scalar(0) );
-    orig_img = cv::cuda::GpuMat(frame);
+    cv::Mat part = cv::Mat(cv::Size(w,im.rows), CV_32FC3, cv::Scalar(0) );
-    cv::cuda::resize(orig_img, img_resized, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+    int r, c, k;
-
+    float w_scale = (float)(im.cols - 1) / (w - 1);
-    img_resized.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
+    float h_scale = (float)(im.rows - 1) / (h - 1);
-
+    
-    //split channels
+    for(k = 0; k < im.channels(); ++k){
-    cv::cuda::split(imagePreproc,bgr);//split source
+        for(r = 0; r < im.rows; ++r){
-
+            for(c = 0; c < w; ++c){
-    //write channels
+                float val = 0;
-    for(int i=0; i<netRT->input_dim.c; i++) {
+                if(c == w-1 || im.cols == 1){
-        int size = imagePreproc.rows * imagePreproc.cols;
+                    val = im.at<cv::Vec3f>(r, im.cols-1)[k];
-        int ch = netRT->input_dim.c-1 -i;
+                } else {
-        bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
+                    float sx = c*w_scale;
-        checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
+                    int ix = (int) sx;
+                    float dx = sx - ix;
+                    val = (1 - dx) * im.at<cv::Vec3f>(r, ix)[k] + dx * im.at<cv::Vec3f>(r,ix+1)[k];
+                }
+                part.at<cv::Vec3f>(r,c)[k] = val;
+            }
+        }
     }
-#else
+    for(k = 0; k < im.channels(); ++k){
-    cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+        for(r = 0; r < h; ++r){
+            float sy = r*h_scale;
+            int iy = (int) sy;
+            float dy = sy - iy;
+            for(c = 0; c < w; ++c){
+                float val = (1-dy) * part.at<cv::Vec3f>(iy, c)[k];
+                resized.at<cv::Vec3f>(r, c)[k] = val;
+            }
+            if(r == h-1 || im.rows == 1) continue;
+            for(c = 0; c < w; ++c){
+                float val = dy * part.at<cv::Vec3f>(iy+1, c)[k];
+                resized.at<cv::Vec3f>(r,c)[k] += val;
+            }
+        }
+    }
+
+    return resized;
+}
+
+void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
     frame.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
 
+    if(letterbox){
+        int im_w = frame.cols;
+        int im_h = frame.rows;
+        int net_w = netRT->input_dim.w;
+        int net_h = netRT->input_dim.h;
+        if(net_w == net_h && letterbox){
+            float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
+
+            int new_h = im_h/ratio;
+            int new_w = im_w/ratio;
+
+            imagePreproc = resize_image(imagePreproc, new_w, new_h);
+
+            cv::Mat borders;
+            int top = (net_h - new_h)/2;
+            int bottom = (net_h - new_h) - top;
+            int left = (net_w - new_w)/2;
+            int right = (net_w - new_w) - left;
+
+            cv::copyMakeBorder(imagePreproc,imagePreproc, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0.5,0.5,0.5));
+        }
+        else
+            FatalError("letterbox not spported with h!=w");
+    }
+    else
+        imagePreproc = resize_image(imagePreproc, netRT->input_dim.w, netRT->input_dim.h);
+
     //split channels
     cv::split(imagePreproc,bgr);//split source
 
@@ -84,7 +136,6 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
         memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));     
     }
     checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
-#endif
 }
 
 void Yolo3Detection::postprocess(const int bi, const bool mAP){
@@ -105,41 +156,67 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
     }
     tk::dnn::Yolo::mergeDetections(dets, nDets, classes);
 
+    int im_w = originalSize[bi].width;
+    int im_h = originalSize[bi].height;
+    int net_w = netRT->input_dim.w;
+    int net_h = netRT->input_dim.h;
+    int new_h, new_w;
+
+    int top = 0, left = 0;
+
+    if(letterbox){
+        float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
+        x_ratio = ratio;
+        y_ratio = ratio;
+        std::cout<<ratio<<std::endl;
+
+        int new_h = im_h/ratio;
+        int new_w = im_w/ratio;
+
+        top = (net_h - new_h)/2;
+        left = (net_w - new_w)/2;
+    }
+    else{
+        new_h = net_h;
+        new_w = net_w;
+    }
+    
+    float deltaw = net_w - new_w;
+    float deltah = net_h - new_h;
+    float ratiow = (float)new_w / net_w;
+    float ratioh = (float)new_h / net_h;
+
     // fill detected
     detected.clear();
     for(int j=0; j<nDets; j++) {
         tk::dnn::Yolo::box b = dets[j].bbox;
-        int x0   = (b.x-b.w/2.);
+
-        int x1   = (b.x+b.w/2.);
+        float x0   = (b.x - left - b.w/2.);
-        int y0   = (b.y-b.h/2.);
+        float x1   = (b.x - left + b.w/2.);
-        int y1   = (b.y+b.h/2.);
+        float y0   = (b.y - top - b.h/2.);
-        int obj_class = -1;
+        float y1   = (b.y - top + b.h/2.);
-        float prob = 0;
+
+        // convert to image coords
+        x0 = x_ratio*x0;
+        x1 = x_ratio*x1;
+        y0 = y_ratio*y0;
+        y1 = y_ratio*y1;
+        
         for(int c=0; c<classes; c++) {
             if(dets[j].prob[c] >= confThreshold) {
-                obj_class = c;
+                int obj_class = c;
-                prob = dets[j].prob[c];
+                float prob = dets[j].prob[c];
-            }
-        }
 
-        if(obj_class >= 0) {
+                tk::dnn::box res;
-            // convert to image coords
+                res.cl = obj_class;
-            x0 = x_ratio*x0;
+                res.prob = prob;
-            x1 = x_ratio*x1;
+                res.x = x0;
-            y0 = y_ratio*y0;
+                res.y = y0;
-            y1 = y_ratio*y1;
+                res.w = x1 - x0;
-              
+                res.h = y1 - y0;
-            tk::dnn::box res;
+
-            res.cl = obj_class;
+                detected.push_back(res);
-            res.prob = prob;
+            }
-            res.x = x0;
-            res.y = y0;
-            res.w = x1 - x0;
-            res.h = y1 - y0;
-            if(mAP)
-                for(int c=0; c<classes; c++) 
-                    res.probs.push_back(dets[j].prob[c]);
-            detected.push_back(res);
         }
     }
     batchDetected.push_back(detected);

src/evaluation.cpp

@@ -342,14 +342,16 @@ void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path,
                 //min threshold confidence is set in DetectionNN.h
                 if (bbox[i].probs[j] > 0) {
 
-                    *out_file <<    "{\"image_id\":" << image_id <<
+                    *out_file << std::fixed  << std::setprecision(6) <<
+                                    "{\"image_id\":" << image_id <<
                                     ", \"category_id\":" << coco_ids[j] << 
                                     ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
                                     "], \"score\":" << bbox[i].probs[j] << "},\n";
                 }
             }
         else
-            *out_file <<    "{\"image_id\":" << image_id <<
+            *out_file <<     std::fixed  << std::setprecision(6) <<
+                            "{\"image_id\":" << image_id <<
                             ", \"category_id\":" << coco_ids[bbox[i].cl] << 
                             ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
                             "], \"score\":" << bbox[i].prob << "},\n";