Add getMemoryUsage function, detection update moved in abstract lass, splitted execution time in pre-inf-post, other minors.

Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>

CMakeLists.txt

@@ -97,6 +97,9 @@ target_link_libraries(test_yolo3_tiny tkDNN)
 add_executable(test_yolo3_tiny512 tests/yolo3_tiny512/yolo3_tiny512.cpp)
 target_link_libraries(test_yolo3_tiny512 tkDNN)
 
+add_executable(test_yolo3_tinyNM512 tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp)
+target_link_libraries(test_yolo3_tinyNM512 tkDNN)
+
 add_executable(test_yolo3_tiny512tp tests/yolo3_tiny512tp/yolo3_tiny512tp.cpp)
 target_link_libraries(test_yolo3_tiny512tp tkDNN)
 

demo/demo/map.cpp

@@ -42,6 +42,9 @@ int main(int argc, char *argv[])
     int classes, map_points, map_levels;
     float map_step, IoU_thresh, conf_thresh;
 
+    double vm_total = 0, rss_total = 0;
+    double vm, rss;
+
     if(argc > 1)
         net = argv[1]; 
     if(argc > 2)
@@ -62,11 +65,12 @@ int main(int argc, char *argv[])
     readParams( config_filename, classes,  map_points, map_levels, map_step,
                 IoU_thresh, conf_thresh, verbose);
 
-    std::ofstream times;
+    std::ofstream times, memory;
     if(write_res_on_file)
     {
-        times.open ("times.csv", std::ios_base::app);
-        times<<net<<";";
+        times.open("times"+std::string(net)+".csv");
+        memory.open("memory.csv", std::ios_base::app);
+        memory<<net<<";";
     }
 
     tk::dnn::Yolo3Detection yolo;
@@ -77,6 +81,7 @@ int main(int argc, char *argv[])
 
     int n_classes = classes;   
     
+    
     switch(ntype)
     {
         case 'y':
@@ -95,6 +100,8 @@ int main(int argc, char *argv[])
 
     detNN->init(net, n_classes);
 
+    
+
     std::ifstream all_labels(labels_path);
     std::string l_filename;
     std::vector<Frame> images;
@@ -105,7 +112,8 @@ int main(int argc, char *argv[])
     if(show)
         cv::namedWindow("detection", cv::WINDOW_NORMAL);
 
-    for (int images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) 
+    int images_done;
+    for (images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) 
     {
         std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\n"<<COL_END;
 
@@ -129,18 +137,11 @@ int main(int argc, char *argv[])
 
         //inference 
         
-	    TIMER_START
-        
         detected_bbox.clear();
-        detNN->update(dnn_input);
+        detNN->update(dnn_input, write_res_on_file, &times);
         frame = detNN->draw(frame);
         detected_bbox = detNN->detected;
         
-        TIMER_STOP
-
-        if(write_res_on_file)
-	        times<<t_ns<<";";
-
         std::ofstream myfile;
         if(write_dets)
             myfile.open ("det/"+f.l_filename.substr(l_filename.find("000")));
@@ -191,6 +192,13 @@ int main(int argc, char *argv[])
             cv::imshow("detection", frame);
             cv::waitKey(0);
         }
+
+        
+        getMemUsage(vm, rss);
+        vm_total += vm;
+        rss_total += rss;
+
+        
     }
 
     std::cout<<"Done."<<std::endl;
@@ -204,10 +212,13 @@ int main(int argc, char *argv[])
     //compute average precision, recall and f1score
     computeTPFPFN(images,classes,IoU_thresh,conf_thresh, verbose, write_res_on_file, net);
 
+    std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl;
+
     if(write_res_on_file)
     {
-	times<<"\n";
+        memory<<vm_total/images_done/1024.0<<";"<<rss_total/images_done/1024.0<<"\n";
         times.close();
+        memory.close();
     }
 
 

include/tkDNN/CenternetDetection.h

@@ -75,8 +75,7 @@ public:
 
     bool init(const std::string& tensor_path, const int n_classes=80);
     void preprocess(cv::Mat &frame);
-    void update(cv::Mat &frame);
-    void postprocess(dnnType **rt_out, const int n_out);
+    void postprocess();
 };
 
 

include/tkDNN/DetectionNN.h

@@ -46,6 +46,20 @@ class DetectionNN {
         dnnType *input;
 #endif
 
+        /**
+         * This method preprocess the image, before feeding it to the NN.
+         *
+         * @param original frame to adapt for inference.
+         */
+        virtual void preprocess(cv::Mat &frame) = 0;
+
+        /**
+         * This method postprocess the output of the NN to obtain the correct 
+         * boundig boxes. 
+         * 
+         */
+        virtual void postprocess() = 0;
+
     public:
         int classes = 0;
         float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
@@ -66,28 +80,48 @@ class DetectionNN {
          */
         virtual bool init(const std::string& tensor_path, const int n_classes=80) = 0;
         
-        /**
-         * This method preprocess the image, before feeding it to the NN.
-         *
-         * @param original frame to adapt for inference.
-         */
-        virtual void preprocess(cv::Mat &frame) = 0;
-
         /**
          * This method performs the whole detection of the NN.
          * 
          * @param frame to run detection on.
+         * @param if set to true, preprocess, inference and postprocess times 
+         *        are saved on a csv file, otherwise not.
          */
-        virtual void update(cv::Mat &frame) = 0;
+        void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){
+            if(!frame.data)
+                FatalError("No image data feed to detection");
 
-        /**
-         * This method postprocess the output of the NN to obtain the correct 
-         * boundig boxes. 
-         * 
-         * @param outputs of the inference
-         * @param number of outputs of the inference
-         */
-        virtual void postprocess(dnnType **rt_out, const int n_out) = 0;
+            if(save_times && times==nullptr)
+                FatalError("save_times set to true, but no valid ofstream given");
+
+            originalSize = frame.size();
+            printCenteredTitle(" TENSORRT detection ", '=', 30); 
+            {
+                TIMER_START
+                preprocess(frame);
+                TIMER_STOP
+                if(save_times) *times<<t_ns<<";";
+            }
+
+            //do inference
+            tk::dnn::dataDim_t dim = netRT->input_dim;
+            {
+                dim.print();
+                TIMER_START
+                netRT->infer(dim, input_d);
+                TIMER_STOP
+                dim.print();
+                stats.push_back(t_ns);
+                if(save_times) *times<<t_ns<<";";
+            }
+
+            {
+                TIMER_START
+                postprocess();
+                TIMER_STOP
+                if(save_times) *times<<t_ns<<"\n";
+            }
+        }      
 
         /**
          * Method to draw boundixg boxes and labels on a frame.

include/tkDNN/MobilenetDetection.h

@@ -67,8 +67,7 @@ public:
 
     bool init(const std::string& tensor_path, const int n_classes);
     void preprocess(cv::Mat &frame);
-    void update(cv::Mat &frame);
-    void postprocess(dnnType **rt_out, const int n_out);
+    void postprocess();
 };
 
 

include/tkDNN/Yolo3Detection.h

@@ -26,8 +26,7 @@ public:
 
     bool init(const std::string& tensor_path, const int n_classes=80);
     void preprocess(cv::Mat &frame);
-    void update(cv::Mat &frame);
-    void postprocess(dnnType **rt_out, const int n_out);
+    void postprocess();
 };
 
 

include/tkDNN/utils.h

@@ -12,6 +12,10 @@
 #include <cublas_v2.h>
 #include <cudnn.h>
 
+#include <unistd.h>
+#include <ios>
+
+
 #define dnnType float
 
 
@@ -102,4 +106,6 @@ void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData,
 
 void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData, 
                     dnnType* add_vector, int dim, dnnType mul);
+
+void getMemUsage(double& vm_usage_kb, double& resident_set_kb);
 #endif //UTILS_H

src/CenternetDetection.cpp

@@ -264,40 +264,14 @@ void CenternetDetection::preprocess(cv::Mat &frame)
 #endif
 }
 
-void CenternetDetection::update(cv::Mat &frame) 
+void CenternetDetection::postprocess()
 {
-    originalSize = frame.size();
-    if(!frame.data) {
-        std::cout<<"CENTERNET: NO IMAGE DATA\n";
-        return;
-    }  
-    TIMER_START
-   
-    preprocess(frame);
-
-    printCenteredTitle(" TENSORRT inference ", '=', 30); {
-        dim2.print();
-        TIMER_START
-        netRT->infer(dim2, input_d);
-        TIMER_STOP
-        dim2.print();
-    }
-
     dnnType *rt_out[4];
     rt_out[0] = (dnnType *)netRT->buffersRT[1];
     rt_out[1] = (dnnType *)netRT->buffersRT[2];
     rt_out[2] = (dnnType *)netRT->buffersRT[3]; 
     rt_out[3] = (dnnType *)netRT->buffersRT[4]; 
 
-    postprocess(rt_out, 4);
-
-    // std::cout<<"TOTAL: \n";
-    TIMER_STOP
-    stats.push_back(t_ns);
-}
-
-void CenternetDetection::postprocess(dnnType **rt_out, const int n_out)
-{
     // auto start_t = std::chrono::steady_clock::now();
     // auto step_t = std::chrono::steady_clock::now();
     // auto end_t = std::chrono::steady_clock::now();

src/MobilenetDetection.cpp

@@ -243,45 +243,15 @@ void MobilenetDetection::preprocess(cv::Mat &frame)
 #endif
 }
 
-void MobilenetDetection::update(cv::Mat &frame)
+void MobilenetDetection::postprocess()
 {
-    TIMER_START
-    if(!frame.data) {
-        std::cout<<"MOBILENET: NO IMAGE DATA\n";
-        return;
-    } 
-    originalSize = frame.size();
-
-    //preprocess
-    preprocess(frame);
-
-    //do inference
-    tk::dnn::dataDim_t dim = tk::dnn::dataDim_t(1, 3, imageSize, imageSize, 1);;
-    printCenteredTitle(" TENSORRT inference ", '=', 30);
-    {
-        dim.print();
-        TIMER_START
-        netRT->infer(dim, input_d);
-        TIMER_STOP
-        dim.print();
-    }
-
     //get confidences and locations_h
     dnnType *rt_out[2];
     rt_out[0] = (dnnType *)netRT->buffersRT[3];
     rt_out[1] = (dnnType *)netRT->buffersRT[4];
 
     detected.clear();
-    //postprocess
-    postprocess(rt_out, 2);
 
-    TIMER_STOP
-    stats.push_back(t_ns);
-}
-
-
-void MobilenetDetection::postprocess(dnnType **rt_out, const int n_out)
-{
     checkCuda(cudaMemcpy(confidences_h, rt_out[0], nPriors * classes * sizeof(float), cudaMemcpyDeviceToHost));
     checkCuda(cudaMemcpy(locations_h, rt_out[1], N_COORDS * nPriors * sizeof(float), cudaMemcpyDeviceToHost));
     convert_locatios_to_boxes_and_center();

src/Yolo3Detection.cpp

@@ -84,52 +84,20 @@ void Yolo3Detection::preprocess(cv::Mat &frame)
 #endif
 }
 
-void Yolo3Detection::update(cv::Mat &frame)
+void Yolo3Detection::postprocess()
 {
-    TIMER_START
-    if(!frame.data) {
-        std::cout<<"YOLO: NO IMAGE DATA\n";
-        return;
-    }   
-
-    originalSize = frame.size();
-    preprocess(frame);
-
-    //do inference
-    tk::dnn::dataDim_t dim = netRT->input_dim;
-
-    // printDeviceVector(netRT->input_dim.tot()*sizeof(dnnType),input_d);
-    
-    
-    printCenteredTitle(" TENSORRT inference ", '=', 30); 
-    {
-        dim.print();
-        TIMER_START
-        netRT->infer(dim, input_d);
-        TIMER_STOP
-        dim.print();
-    }
-
     //get yolo outputs
     dnnType *rt_out[netRT->pluginFactory->n_yolos]; 
     for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
         rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
     }
 
-    postprocess(rt_out, netRT->pluginFactory->n_yolos);
-        
-    TIMER_STOP
-    stats.push_back(t_ns);
-}
-
-void Yolo3Detection::postprocess(dnnType **rt_out, const int n_out)
-{
     float x_ratio =  float(originalSize.width) / float(netRT->input_dim.w);
     float y_ratio =  float(originalSize.height) / float(netRT->input_dim.h);
 
     // compute dets
     nDets = 0;
-    for(int i=0; i<n_out; i++) {
+    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
         yolo[i]->dstData = rt_out[i];
         yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold);
     }

src/utils.cpp

@@ -168,3 +168,36 @@ void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData,
     checkERROR( cublasSaxpy(handle, dim, &alpha, srcData, 1, dstData, 1));
 
 }
+
+
+void getMemUsage(double& vm_usage_kb, double& resident_set_kb)
+{
+   using std::ios_base;
+   using std::ifstream;
+   using std::string;
+
+   vm_usage_kb     = 0.0;
+   resident_set_kb = 0.0;
+
+   ifstream stat_stream("/proc/self/stat",ios_base::in);
+
+   //all the stats
+   string pid, comm, state, ppid, pgrp, session, tty_nr;
+   string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+   string utime, stime, cutime, cstime, priority, nice;
+   string O, itrealvalue, starttime;
+
+   unsigned long vsize;
+   long rss;
+
+   stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
+               >> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
+               >> utime >> stime >> cutime >> cstime >> priority >> nice
+               >> O >> itrealvalue >> starttime >> vsize >> rss;
+
+   stat_stream.close();
+
+   long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
+   vm_usage_kb     = vsize / 1024.0;  
+   resident_set_kb = rss * page_size_kb;
+}

tests/yolo3_coco4/yolo3_coco4.cpp

@@ -15,6 +15,11 @@ int main() {
     tk::dnn::Yolo *yolo [3];
     #include "models/Yolo3.h"
 
+    // fill classes names
+    for(int i=0; i<3; i++) {
+        yolo[i]->classesNames = {"person" , "bicycle" , "car" , "motorbike" };
+    }
+
     // Load input
     dnnType *data;
     dnnType *input_h;

tests/yolo3_tiny512/yolo3_tiny512.cpp

@@ -22,7 +22,9 @@ const char *g23_bin    = "../tests/yolo3_tiny512/layers/g23.bin";
 const char *output_bin = "../tests/yolo3_tiny512/debug/layer23_out.bin";
 
 int main() {
-
+    
+    downloadWeightsifDoNotExist(input_bin, "../tests/yolo3_tiny512", "https://cloud.hipert.unimore.it/s/wRW9nmkibSe5HoS/download");
+    
     int classes = 80;
 
     // Network layout

tests/yolo3_tinyNM512/yolo3_tinyNM512.cpp

@@ -0,0 +1,124 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo3_tinyNM512/layers/input.bin";
+const char *c0_bin     = "../tests/yolo3_tinyNM512/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo3_tinyNM512/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo3_tinyNM512/layers/c4.bin";
+const char *c6_bin     = "../tests/yolo3_tinyNM512/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo3_tinyNM512/layers/c8.bin";
+const char *c10_bin    = "../tests/yolo3_tinyNM512/layers/c10.bin";
+const char *c11_bin    = "../tests/yolo3_tinyNM512/layers/c11.bin";
+const char *c12_bin    = "../tests/yolo3_tinyNM512/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo3_tinyNM512/layers/c13.bin";
+const char *c14_bin    = "../tests/yolo3_tinyNM512/layers/c14.bin";
+const char *c17_bin    = "../tests/yolo3_tinyNM512/layers/c17.bin";
+const char *c20_bin    = "../tests/yolo3_tinyNM512/layers/c20.bin";
+const char *c21_bin    = "../tests/yolo3_tinyNM512/layers/c21.bin";
+const char *g15_bin    = "../tests/yolo3_tinyNM512/layers/g15.bin";
+const char *g22_bin    = "../tests/yolo3_tinyNM512/layers/g22.bin";
+// const char *output_bin = "../tests/yolo3_tinyNM512/layers/output.bin";
+
+const char *output_bin = "../tests/yolo3_tinyNM512/debug/layer22_out.bin";
+
+int main() {
+    
+    // downloadWeightsifDoNotExist(input_bin, "../tests/yolo3_tinyNM512", "https://cloud.hipert.unimore.it/s/wRW9nmkibSe5HoS/download");
+    
+    int classes = 80;
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 512, 512, 1);
+    tk::dnn::Network net(dim);
+        
+
+    tk::dnn::Conv2d     c0 (&net, 16, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 32, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 64, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p5 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p7(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8(&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p9(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true);
+    tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Conv2d     c12(&net, 1024, 3, 3, 1, 1, 1, 1, c11_bin, true);
+    tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Conv2d     c13(&net, 256, 1, 1, 1, 1, 0, 0, c12_bin, true);
+    tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c14(&net, 512, 3, 3, 1, 1, 1, 1, c13_bin, true);
+    tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c15(&net, 255, 1, 1, 1, 1, 0, 0, c14_bin, false);
+
+    tk::dnn::Yolo     yolo0  (&net, classes, 2, g15_bin);
+
+    tk::dnn::Layer *m17_layers[1] = { &a13 };
+    tk::dnn::Route      m17  (&net, m17_layers, 1);
+    tk::dnn::Conv2d     c18(&net, 128, 1, 1, 1, 1, 0, 0, c17_bin, true);
+    tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Upsample   u19  (&net, 2);
+
+    tk::dnn::Layer *m20_layers[2] = { &u19, &a8 };
+    tk::dnn::Route      m20  (&net, m20_layers, 2);
+
+    tk::dnn::Conv2d     c21(&net, 256, 3, 3, 1, 1, 1, 1, c20_bin, true);
+    tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c22(&net, 255, 1, 1, 1, 1, 0, 0, c21_bin, false);
+
+    tk::dnn::Yolo     yolo1  (&net, classes, 2, g22_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+
+    //print network model
+    net.print();
+
+    // convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("yolo3_tinyNM512"));
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+ 
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+    
+    return 0;
+}