From f44f377771bd6f628aac58f3d6515877ed7e1a03 Mon Sep 17 00:00:00 2001
From: Micaela Verucchi <micaelaverucchi@gmail.com>
Date: Thu, 19 Mar 2020 17:53:29 +0100
Subject: [PATCH] Add preprocess function, allow preprocess on GPU for
 mobilenetdetection

Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>
---
 include/tkDNN/MobilenetDetection.h            |  6 +-
 src/MobilenetDetection.cpp                    | 60 ++++++++++++++-----
 tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp |  3 +-
 3 files changed, 53 insertions(+), 16 deletions(-)

diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h
index f7b45a4..405f781 100644
--- a/include/tkDNN/MobilenetDetection.h
+++ b/include/tkDNN/MobilenetDetection.h
@@ -8,6 +8,9 @@
 #include <opencv2/videoio.hpp>
 #include <opencv2/imgproc/imgproc.hpp>
 
+#include "opencv2/opencv.hpp"
+
+
 #include "tkdnn.h"
 
 #define N_COORDS 4
@@ -65,7 +68,7 @@ private:
     int n_priors = 0;
 
     cv::Mat origImg;
-    cv::Mat bgr[3];
+    
 
     float *input, *input_d;
     float *locations_h, *confidences_h;
@@ -85,6 +88,7 @@ private:
     void generate_ssd_priors(const SSDSpec *specs, const int n_specs, bool clamp = true);
     void convert_locatios_to_boxes_and_center(float *priors, const int n_priors, float *locations, const float center_variance, const float size_variance);
     float iou(const tk::dnn::box &a, const tk::dnn::box &b);
+    void preprocess(const bool gpu = true);
     std::vector<tk::dnn::box> postprocess(float *locations, float *confidences, const int n_values, const float threshold, const int n_classes, const float iou_thresh, const int width, const int height);
     float get_color2(int c, int x, int max);
 
diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp
index d68f346..8b8ed0a 100644
--- a/src/MobilenetDetection.cpp
+++ b/src/MobilenetDetection.cpp
@@ -326,6 +326,50 @@ cv::Mat MobilenetDetection::draw()
     return origImg;
 }
 
+
+void MobilenetDetection::preprocess(const bool gpu)
+{
+    std::cout<<"preprocess"<<std::endl;
+    if(gpu){
+        cv::cuda::GpuMat im_Orig; 
+        cv::cuda::GpuMat frame_resize, frame_nomean, frame_scaled;
+
+        im_Orig = cv::cuda::GpuMat(origImg);
+        cv::cuda::resize (im_Orig, frame_resize, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); 
+
+        // resize(origImg, frame_resize, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+        frame_resize.convertTo(frame_nomean, CV_32FC3, 1, -127);
+        frame_nomean.convertTo(frame_scaled, CV_32FC3, 1 / 128.0, 0);
+
+        //copy image into tensor and copy it into GPU
+        cv::cuda::GpuMat bgr[3];
+        cv::cuda::split(frame_scaled, bgr);
+
+        for(int i=0; i < netRT->input_dim.c; i++){
+            int idx = i * frame_scaled.rows * frame_scaled.cols;
+            checkCuda( cudaMemcpy((void *)&input_d[idx], (void *)bgr[i].data, frame_scaled.rows * frame_scaled.cols* sizeof(float), cudaMemcpyDeviceToDevice) );
+        }
+    }
+    else{
+        //resize image, remove mean, divide by std
+        cv::Mat frame_resize, frame_nomean, frame_scaled;
+        resize(origImg, frame_resize, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+        frame_resize.convertTo(frame_nomean, CV_32FC3, 1, -127);
+        frame_nomean.convertTo(frame_scaled, CV_32FC3, 1 / 128.0, 0);
+
+        //copy image into tensor and copy it into GPU
+        cv::Mat bgr[3];
+        cv::split(frame_scaled, bgr);
+        for (int i = 0; i < netRT->input_dim.c; i++){
+            int idx = i * frame_scaled.rows * frame_scaled.cols;
+            memcpy((void *)&input[idx], (void *)bgr[i].data, frame_scaled.rows * frame_scaled.cols * sizeof(dnnType));
+        }
+        checkCuda(cudaMemcpyAsync(input_d, input, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
+    }
+    
+
+}
+
 void MobilenetDetection::update(cv::Mat &img)
 {
     TIMER_START
@@ -335,20 +379,8 @@ void MobilenetDetection::update(cv::Mat &img)
     origImg = img;
     cv::Size sz = origImg.size();
 
-    //resize image, remove mean, divide by std
-    cv::Mat frame_resize, frame_nomean, frame_scaled;
-    resize(origImg, frame_resize, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
-    frame_resize.convertTo(frame_nomean, CV_32FC3, 1, -127);
-    frame_nomean.convertTo(frame_scaled, CV_32FC3, 1 / 128.0, 0);
-
-    //copy image into tensor and copy it into GPU
-    cv::split(frame_scaled, bgr);
-    for (int i = 0; i < netRT->input_dim.c; i++)
-    {
-        int idx = i * frame_scaled.rows * frame_scaled.cols;
-        memcpy((void *)&input[idx], (void *)bgr[i].data, frame_scaled.rows * frame_scaled.cols * sizeof(dnnType));
-    }
-    checkCuda(cudaMemcpyAsync(input_d, input, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
+    //preprocess
+    preprocess();
 
     //do inference
     tk::dnn::dataDim_t dim2 = dim;
diff --git a/tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp b/tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp
index b207cec..81a9b33 100644
--- a/tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp
+++ b/tests/mobilenetv2ssd512/mobilenetv2ssd512.cpp
@@ -134,7 +134,8 @@ const char *regression_header5 = "../tests/mobilenetv2ssd512/layers/regression_h
 int main()
 {
 
-    // downloadWeightsifDoNotExist(input_bin, "./tests/mobilenetv2ssd512");
+    // downloadWeightsifDoNotExist(input_bin, "./tests/mobilenetv2ssd512", "https://cloud.hipert.unimore.it/s//download");
+    
 
     int classes = 81;