Merge branch 'ceccocats:master' into master

demo/demo/map.cpp

@@ -34,10 +34,12 @@ int main(int argc, char *argv[])
     const char *config_filename = "../demo/config.yaml";
     const char * net = "yolo3.rt";
     const char * labels_path = "../demo/COCO_val2017/all_labels.txt";
+    int n_batches = 1;
+    float confidence_thresh = 0.3;
     bool show = false;
     bool write_dets = false;
     bool write_res_on_file = true;
-    bool write_coco_json = true;
+    bool write_coco_json = false;
     int n_images = 5000;
 
     bool verbose;
@@ -56,6 +58,12 @@ int main(int argc, char *argv[])
         labels_path = argv[3]; 
     if(argc > 4)
         config_filename = argv[4]; 
+    if(argc > 5)
+        n_batches = atoi(argv[5]); 
+    if(argc > 6)
+        confidence_thresh = atof(argv[6]); 
+
+    std::cout<<"conf t: "<<confidence_thresh<<std::endl;
 
     //check if files needed exist
     if(!fileExist(config_filename))
@@ -83,9 +91,9 @@ int main(int argc, char *argv[])
     }
 
     if(write_res_on_file){
-        times.open("times_"+net_name+".csv");
+        times.open("times_"+net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh)+".csv");
         memory.open("memory.csv", std::ios_base::app);
-        memory<<net<<";";
+        memory<<net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh)<<";";
     }
 
     // instantiate detector
@@ -121,89 +129,108 @@ int main(int argc, char *argv[])
     if(show)
         cv::namedWindow("detection", cv::WINDOW_NORMAL);
 
+    bool file_ok = false;
+
     int images_done;
-    for (images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) {
+    for (images_done=0 ; images_done < n_images ;) {
-        std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\n"<<COL_END;
         
-        tk::dnn::Frame f;
-        f.lFilename = l_filename;
-        f.iFilename = l_filename;
-        convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
 
-        // read frame
+        int cur_batches = 0;
-        if(!fileExist(f.iFilename.c_str()))
-            FatalError("Wrong image file path.");
-
-        cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
         std::vector<cv::Mat> batch_frames;
-        batch_frames.push_back(frame);
-        int height = frame.rows;
-        int width = frame.cols;
-
-        if(!frame.data) 
-            break;
         std::vector<cv::Mat> batch_dnn_input;
-        batch_dnn_input.push_back(frame.clone());
+            
+        std::vector<tk::dnn::Frame> cur_frames;
+        for(;cur_batches<n_batches && images_done < n_images;cur_batches++, ++images_done){
+
+            std::getline(all_labels, l_filename);
+            file_ok = all_labels ? true : false ;
+            if (!file_ok)
+                break;
+
+            tk::dnn::Frame f;
+            f.lFilename = l_filename;
+            f.iFilename = l_filename;
+            convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
+
+            // read frame
+            if(!fileExist(f.iFilename.c_str()))
+                FatalError("Wrong image file path.");
+
+            cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
+            batch_frames.push_back(frame);
+            f.height = frame.rows;
+            f.width = frame.cols;
+
+            if(!frame.data) 
+                break;
+            batch_dnn_input.push_back(frame.clone());
+
+            // read and save groundtruth labels
+            if(fileExist(f.lFilename.c_str()))
+            {
+                std::ifstream labels(f.lFilename);
+                for(std::string line; std::getline(labels, line); ){
+                    std::istringstream in(line); 
+                    tk::dnn::BoundingBox b;
+                    in >> b.cl >> b.x >> b.y >> b.w >> b.h;  
+                    b.prob = 1;
+                    b.truthFlag = 1;
+                    f.gt.push_back(b);
+
+                    if(show)// draw rectangle for groundtruth
+                        cv::rectangle(batch_frames[cur_batches], cv::Point((b.x-b.w/2)*f.width, (b.y-b.h/2)*f.height), cv::Point((b.x+b.w/2)*f.width,(b.y+b.h/2)*f.height), cv::Scalar(0, 255, 0), 2);             
+                }
+            } 
+
+            cur_frames.push_back(f);
+        }
+        if (!file_ok)
+            break;
 
         //inference 
-        detected_bbox.clear();
+        detNN->update(batch_dnn_input,cur_batches,write_res_on_file, &times, write_coco_json);
-        detNN->update(batch_dnn_input,1,write_res_on_file, &times, write_coco_json);
         detNN->draw(batch_frames);
-        detected_bbox = detNN->detected;
 
-        if(write_coco_json)
+        for(int j=0;j<cur_frames.size(); ++j){
-            printJsonCOCOFormat(&coco_json, f.iFilename.c_str(), detected_bbox, classes,  width, height);        
+            if(write_coco_json)
+            printJsonCOCOFormat(&coco_json, cur_frames[j].iFilename.c_str(), detNN->batchDetected[j], classes,  cur_frames[j].width, cur_frames[j].height);        
 
-        std::ofstream myfile;
+            std::ofstream myfile;
-        if(write_dets)
+            if(write_dets)
-            myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("labels/") + 7));
+                myfile.open ("det/"+cur_frames[j].lFilename.substr(cur_frames[j].lFilename.find("labels/") + 7));
 
-        // save detections labels
+            // save detections labels
-        for(auto d:detected_bbox){
+            for(auto d:detNN->batchDetected[j]){
-            //convert detected bb in the same format as label
+                //convert detected bb in the same format as label
-            //<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
+                //<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
-            tk::dnn::BoundingBox b;
+                tk::dnn::BoundingBox b;
-            b.x = (d.x + d.w/2) / width;
+                b.x = (d.x + d.w/2) / cur_frames[j].width;
-            b.y = (d.y + d.h/2) / height;
+                b.y = (d.y + d.h/2) / cur_frames[j].height;
-            b.w = d.w / width;
+                b.w = d.w / cur_frames[j].width;
-            b.h = d.h / height;
+                b.h = d.h / cur_frames[j].height;
-            b.prob = d.prob;
+                b.prob = d.prob;
-            b.cl = d.cl;
+                b.cl = d.cl;
-            f.det.push_back(b);
+                cur_frames[j].det.push_back(b);
+
+                if(write_dets)
+                    myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n";
+
+                if(show)// draw rectangle for detection
+                    cv::rectangle(batch_frames[j], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);             
+                }
 
             if(write_dets)
-                myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n";
+                myfile.close();
       
-			if(show)// draw rectangle for detection
+            images.push_back(cur_frames[j]);
-                cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);             
-        }
         
-        if(write_dets)
+            if(show){
-            myfile.close();
+                cv::imshow("detection", batch_frames[j]);
-
+                cv::waitKey(0);
-        // read and save groundtruth labels
-        if(fileExist(f.lFilename.c_str()))
-        {
-            std::ifstream labels(l_filename);
-            for(std::string line; std::getline(labels, line); ){
-                std::istringstream in(line); 
-                tk::dnn::BoundingBox b;
-                in >> b.cl >> b.x >> b.y >> b.w >> b.h;  
-                b.prob = 1;
-                b.truthFlag = 1;
-                f.gt.push_back(b);
-
-                if(show)// draw rectangle for groundtruth
-                    cv::rectangle(batch_frames[0], cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2);             
             }
-        }    
             
-        images.push_back(f);
-        
-        if(show){
-            cv::imshow("detection", batch_frames[0]);
-            cv::waitKey(0);
         }
+        std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\tcur batch:\t"<<cur_batches<< "\n"<<COL_END;
         
         getMemUsage(vm, rss);
         vm_total += vm;
@@ -221,11 +248,11 @@ int main(int argc, char *argv[])
     std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl;
 
     //compute mAP
-    double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,conf_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name);
+    double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,confidence_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh));
     std::cout<<"mAP "<<IoU_thresh<<":"<<IoU_thresh+map_step*(map_levels-1)<<" = "<<AP<<std::endl;
 
     //compute average precision, recall and f1score
-    tk::dnn::computeTPFPFN(images,classes,IoU_thresh,conf_thresh, verbose, write_res_on_file, net_name);
+    tk::dnn::computeTPFPFN(images,classes,IoU_thresh,confidence_thresh, verbose, write_res_on_file, net_name +"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh));
 
     if(write_res_on_file){
         memory<<vm_total/images_done/1024.0<<";"<<rss_total/images_done/1024.0<<"\n";

include/tkDNN/Layer.h

@@ -56,6 +56,10 @@ public:
 
     int id = 0;
     bool final;        //if the layer is the final one
+    uint n_params = 0;
+    uint feature_map_size = 0;
+    long unsigned MACC = 0;
+
 
     std::string getLayerName() {
         layerType_t type = getLayerType();

include/tkDNN/Network.h

@@ -50,6 +50,7 @@ public:
     bool addLayer(Layer *l);
     void print();
     const char *getNetworkRTName(const char *network_name);
+    void adjustFeatureMapSizeWithShortcuts();
 
     cudnnDataType_t dataType;
     cudnnTensorFormat_t tensorFormat;

include/tkDNN/evaluation.h

@@ -18,6 +18,8 @@ struct Frame
     std::string iFilename;
     std::vector<BoundingBox> gt;
     std::vector<BoundingBox> det;
+    int width;
+    int height;
 
     void print() const;
 };

include/tkDNN/utils.h

@@ -40,7 +40,7 @@
 #define COL_PURPLEB "\033[1;35m"
 #define COL_CYANB "\033[1;36m"
 
-#define TKDNN_VERBOSE 1
+#define TKDNN_VERBOSE 0
 
 // Simple Timer 
 #ifdef __linux__
@@ -118,7 +118,7 @@ void printCenteredTitle(const char *title, char fill, int dim = 30);
 bool fileExist(const char *fname);
 void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url);
 void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
-int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10);
+int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10, bool verbose=true);
 void printDeviceVector(int size, dnnType* vec_d, bool device = true);
 float getColor(const int c, const int x, const int max);
 void resize(int size, dnnType **data);

scripts/test_all_tests.sh

@@ -75,10 +75,15 @@ do
     test_net shelfnet
     test_net shelfnet_berkeley
     test_net yolo4
+    test_net yolo4_320
+    test_net yolo4_320_coco2
+    test_net yolo4_512
+    test_net yolo4_608
     test_net yolo4-csp
     test_net yolo4x
     test_net yolo4_berkeley
     test_net yolo4tiny
+    test_net yolo4tiny_512
     test_net yolo3
     test_net yolo3_berkeley
     test_net yolo3_coco4

scripts/test_inference.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+function test_inference {
+    ./test_$1
+    ./test_rtinference $1_$2.rt 1 
+    ./test_rtinference $1_$2.rt 4
+}
+
+sudo jeston_clock
+
+# modes=( 1 ) # only FP32
+# modes=( 1 2 ) # FP32 and FP16
+modes=( 1 2 3 ) # FP32, FP16 and INT8
+
+rm times_rtinference.csv
+for i in "${modes[@]}"
+do
+    rm *rt
+    if [ $i -eq 1 ]
+    then
+        export TKDNN_MODE=FP32
+        mode=fp32
+        echo -e "${ORANGE}Test FP32${NC}"
+    fi
+    if [ $i -eq 2 ]
+    then
+        export TKDNN_MODE=FP16
+        mode=fp16
+        echo -e "${ORANGE}Test FP16${NC}"
+    fi
+    if [ $i -eq 3 ]
+    then
+        export TKDNN_MODE=INT8
+        export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
+        export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
+        mode=int8
+        echo -e "${ORANGE}Test INT8${NC}"
+        
+	fi
+
+    export TKDNN_BATCHSIZE=4
+    echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
+    
+    test_inference yolo4_320 $mode
+    test_inference yolo4 $mode
+    test_inference yolo4_512 $mode
+    test_inference yolo4_608 $mode
+    test_inference yolo4tiny $mode
+done
+
+
+

src/Conv2d.cpp

@@ -166,6 +166,11 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
     }
     initCUDNN(deConv);
 
+    if(this->groups != 1)
+        MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
+    else
+        MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
+
     // allocate warkspace
     if (ws_sizeInBytes!=0) {
         checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) );

src/DeformConv2d.cpp

@@ -73,6 +73,12 @@ DeformConv2d::DeformConv2d( Network *net, int out_ch, int deformable_group, int
     
     output_dim.c = out_ch;
     initCUDNN();
+
+    if(this->deformableGroup != 1)
+        MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
+    else
+        MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
+        
     //allocate data for infer result
     checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 }

src/Layer.cpp

@@ -18,6 +18,8 @@ Layer::Layer(Network *net) {
         if(!net->addLayer(this))
             FatalError("Net reached max number of layers");    
     }
+
+    feature_map_size = input_dim.tot() + output_dim.tot();
 }
 
 Layer::~Layer() {

src/LayerWgs.cpp

@@ -19,6 +19,8 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
     int seek = 0;
     readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
     seek += inputs*outputs*kh*kw*kl;
+    n_params = seek;
+    
     this->additional_bias = additional_bias;
     if(additional_bias) {
         readBinaryFile(weights_path.c_str(), outputs, &bias2_h, &bias2_d, seek);
@@ -36,6 +38,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
         readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek);
         seek += outputs;
         readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek);
+        seek += outputs;
 
         float eps = TKDNN_BN_MIN_EPSILON;
 

src/Network.cpp

@@ -96,6 +96,28 @@ dataDim_t Network::getOutputDim() {
             return layers[num_layers-1]->output_dim;
 }
 
+void Network::adjustFeatureMapSizeWithShortcuts(){
+    layerType_t layer_type;
+    int shortcutted_idx;
+
+    for(int i=0; i<num_layers; i++) {
+        layer_type = layers[i]->getLayerType();
+        if(layer_type == LAYER_SHORTCUT){
+            shortcutted_idx = -1;
+            for(int j=0; j<num_layers; j++) {
+                if(static_cast<tk::dnn::Shortcut*>(layers[i])->backLayer == layers[j]){
+                    shortcutted_idx = j;
+                    break;
+                }
+            }
+            if(shortcutted_idx == -1)
+                FatalError("Problem when computing featuer_map_size with shortcuts");
+            for(int j=shortcutted_idx+1; j<i; ++j)
+                layers[j]->feature_map_size += layers[shortcutted_idx]->output_dim.tot();
+        }
+    }
+}
+
 void Network::print() {
 
     printCenteredTitle(" NETWORK MODEL ", '=', 60);
@@ -106,10 +128,21 @@ void Network::print() {
     std::cout.width(16); std::cout<<std::left<<"output (H*W,CH)";
     std::cout<<"\n";
 
+    adjustFeatureMapSizeWithShortcuts();
+    
+    long long unsigned int tot_params = 0;
+    long long unsigned int max_feature_map_size = 0;
+    long long unsigned int tot_MACC = 0;
+    
     for(int i=0; i<num_layers; i++) {
         dataDim_t in = layers[i]->input_dim;
         dataDim_t out = layers[i]->output_dim;
 
+        tot_params += layers[i]->n_params;
+        tot_MACC += layers[i]->MACC;
+        if(layers[i]->feature_map_size> max_feature_map_size)
+            max_feature_map_size = layers[i]->feature_map_size;
+
         std::cout.width(3); std::cout<<std::right<<i;
         std::cout<<" ";
         std::cout.width(16); std::cout<<std::left<<layers[i]->getLayerName();
@@ -128,6 +161,9 @@ void Network::print() {
     }
     printCenteredTitle("", '=', 60);
     std::cout<<"\n";
+    std::cout<<"N params: "<<tot_params<<std::endl;
+    std::cout<<"Max feature map size: "<<max_feature_map_size<<std::endl;
+    std::cout<<"N MACC: "<<tot_MACC<<std::endl<<std::endl;
     printCudaMemUsage();
 }
 const char *Network::getNetworkRTName(const char *network_name){

src/Yolo.cpp

@@ -88,7 +88,6 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) {
     for (int b = 0; b < dim.n; ++b){
         for(int n = 0; n < n_masks; ++n){
             int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim);
-            std::cout<<"new_coords"<<new_coords<<std::endl;
             if (new_coords == 1){
                 if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
             }

src/utils.cpp

@@ -92,7 +92,7 @@ void printDeviceVector(int size, dnnType* vec_d, bool device){
         delete [] vec;
 }
 
-int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit) {
+int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit, bool verbose) {
 
     dnnType *data_h, *correct_h;
     const float eps = 0.02f;
@@ -127,13 +127,15 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int
         delete [] correct_h;
     }
 
-    std::cout<<" | ";
+    if(verbose){
-    if(diffs == 0)
+        std::cout<<" | ";
-        std::cout<<COL_GREENB<<"OK";
+        if(diffs == 0)
-    else
+            std::cout<<COL_GREENB<<"OK";
-        std::cout<<COL_REDB<<"Wrongs: "<<diffs;
+        else
+            std::cout<<COL_REDB<<"Wrongs: "<<diffs;
 
-    std::cout<<COL_END<<" ~"<<eps<<"\n";
+        std::cout<<COL_END<<" ~"<<eps<<"\n";
+    }
     return diffs;
 }
 

tests/centernet/dla34_cnet/dla34_cnet.cpp

@@ -479,6 +479,18 @@ int main()
     //print network model
     net.print();
 
+//     for(int i=0; i<net.num_layers; i++) {
+//         if(net.layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net.layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net.layers[i]->dstData != nullptr) {
+//             cudaFree(net.layers[i]->dstData);
+//             net.layers[i]->dstData = nullptr;
+//         }
+//    }
+
     //convert network to tensorRT
     tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet"));
 

tests/centernet/resnet101_cnet/resnet101_cnet.cpp

@@ -353,6 +353,18 @@ int main()
     //print network model
     net.print();
 
+//     for(int i=0; i<net.num_layers; i++) {
+//         if(net.layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net.layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net.layers[i]->dstData != nullptr) {
+//             cudaFree(net.layers[i]->dstData);
+//             net.layers[i]->dstData = nullptr;
+//         }
+//    }
+
     //convert network to tensorRT
     tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("resnet101_cnet"));
 

tests/darknet/cfg/yolo4tiny_512.cfg

@@ -0,0 +1,281 @@
+[net]
+# Testing
+#batch=1
+#subdivisions=1
+# Training
+batch=64
+subdivisions=1
+width=512
+height=512
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.00261
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers=-1
+groups=2
+group_id=1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers = -1,-2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers = -6,-1
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers=-1
+groups=2
+group_id=1
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers = -1,-2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers = -6,-1
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers=-1
+groups=2
+group_id=1
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers = -1,-2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[route]
+layers = -6,-1
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+##################################
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+
+[yolo]
+mask = 3,4,5
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
+jitter=.3
+scale_x_y = 1.05
+cls_normalizer=1.0
+iou_normalizer=0.07
+iou_loss=ciou
+ignore_thresh = .7
+truth_thresh = 1
+random=0
+resize=1.5
+nms_kind=greedynms
+beta_nms=0.6
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 23
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+[yolo]
+mask = 1,2,3
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
+jitter=.3
+scale_x_y = 1.05
+cls_normalizer=1.0
+iou_normalizer=0.07
+iou_loss=ciou
+ignore_thresh = .7
+truth_thresh = 1
+random=0
+resize=1.5
+nms_kind=greedynms
+beta_nms=0.6

tests/darknet/names/coco2.names

@@ -0,0 +1,2 @@
+person
+stop sign

tests/darknet/yolo3_512.cpp

@@ -23,6 +23,18 @@ int main() {
     tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
     net->print();
 
+//     for(int i=0; i<net->num_layers; i++) {
+//         if(net->layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net->layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net->layers[i]->dstData != nullptr) {
+//             cudaFree(net->layers[i]->dstData);
+//             net->layers[i]->dstData = nullptr;
+//         }
+//    }
+
     //convert network to tensorRT
     tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
     

tests/darknet/yolo3tiny_512.cpp

@@ -22,6 +22,19 @@ int main() {
     tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
     net->print();
 
+//     for(int i=0; i<net->num_layers; i++) {
+//         if(net->layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net->layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net->layers[i]->dstData != nullptr) {
+//             cudaFree(net->layers[i]->dstData);
+//             net->layers[i]->dstData = nullptr;
+//         }
+//    }
+
+
     //convert network to tensorRT
     tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
     

tests/darknet/yolo4.cpp

@@ -15,9 +15,9 @@ int main() {
         bin_path + "/debug/layer161_out.bin"
     };
     std::string wgs_path  = bin_path + "/layers";
-    std::string cfg_path  = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4.cfg";
+    std::string cfg_path  = "../tests/darknet/cfg/yolo4.cfg";
-    std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names";
+    std::string name_path = "../tests/darknet/names/coco.names";
-    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download");
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/982LxTQcNQfFQc4/download");
 
     // parse darknet network
     tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);

tests/darknet/yolo4_320.cpp

@@ -0,0 +1,34 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path  = "yolo4_320";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer139_out.bin",
+        bin_path + "/debug/layer150_out.bin",
+        bin_path + "/debug/layer161_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = "../tests/darknet/cfg/yolo4_320.cfg";
+    std::string name_path = "../tests/darknet/names/coco.names";
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/64PHAwrM6RCZbiR/download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/darknet/yolo4_320_coco2.cpp

@@ -0,0 +1,34 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path  = "yolo4_320_coco2";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer139_out.bin",
+        bin_path + "/debug/layer150_out.bin",
+        bin_path + "/debug/layer161_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = "../tests/darknet/cfg/yolo4_320_coco2.cfg";
+    std::string name_path = "../tests/darknet/names/coco2.names";
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/f3wk99iG5y7tEr8/download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/darknet/yolo4_512.cpp

@@ -0,0 +1,47 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path  = "yolo4_512";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer139_out.bin",
+        bin_path + "/debug/layer150_out.bin",
+        bin_path + "/debug/layer161_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = "../tests/darknet/cfg/yolo4_512.cfg";
+    std::string name_path = "../tests/darknet/names/coco.names";
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/fjFDqFmiSARKxFe/download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+//     for(int i=0; i<net->num_layers; i++) {
+//         if(net->layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net->layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net->layers[i]->dstData != nullptr) {
+//             cudaFree(net->layers[i]->dstData);
+//             net->layers[i]->dstData = nullptr;
+//         }
+//    }
+
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/darknet/yolo4_608.cpp

@@ -0,0 +1,34 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path  = "yolo4_608";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer139_out.bin",
+        bin_path + "/debug/layer150_out.bin",
+        bin_path + "/debug/layer161_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = "../tests/darknet/cfg/yolo4_608.cfg";
+    std::string name_path = "../tests/darknet/names/coco.names";
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/Bg9r7kqDFJiFB4c/download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/darknet/yolo4tiny_512.cpp

@@ -0,0 +1,45 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path  = "yolo4tiny_512";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer30_out.bin",
+        bin_path + "/debug/layer37_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4tiny_512.cfg";
+    std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/coco.names";
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/qa2ws4GXg7mS5nN/download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+//         for(int i=0; i<net->num_layers; i++) {
+//         if(net->layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net->layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net->layers[i]->dstData != nullptr) {
+//             cudaFree(net->layers[i]->dstData);
+//             net->layers[i]->dstData = nullptr;
+//         }
+//    }
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp

@@ -469,6 +469,19 @@ int main()
     //print network model
     net.print();
 
+//     for(int i=0; i<net.num_layers; i++) {
+//         if(net.layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
+//             tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net.layers[i];
+//             c->releaseDevice();
+//             c->releaseHost(true, false);
+//         }
+//         if(net.layers[i]->dstData != nullptr) {
+//             cudaFree(net.layers[i]->dstData);
+//             net.layers[i]->dstData = nullptr;
+//         }
+//    }
+
+
     // convert network to tensorRT
     tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("mobilenetv2ssd512"));
 

tests/test_rtinference/rtinference.cpp

@@ -1,4 +1,5 @@
 #include<iostream>
+#include<algorithm>
 #include "tkdnn.h"
 #include <stdlib.h>     /* srand, rand */
 
@@ -18,6 +19,8 @@ int main(int argc, char *argv[]) {
     //convert network to tensorRT
     tk::dnn::NetworkRT netRT(NULL, argv[1]);
 
+    
+    
     tk::dnn::dataDim_t idim = netRT.input_dim;
     tk::dnn::dataDim_t odim = netRT.output_dim;
     idim.n = BATCH_SIZE;
@@ -29,6 +32,7 @@ int main(int argc, char *argv[]) {
 
     int ret_tensorrt = 0; 
     std::cout<<"Testing with batchsize: "<<BATCH_SIZE<<"\n";
+    std::vector<double> stats;
     printCenteredTitle(" TENSORRT inference ", '=', 30); 
     float total_time = 0;
     for(int i=0; i<64; i++) {
@@ -46,18 +50,43 @@ int main(int argc, char *argv[]) {
         netRT.infer(dim, input_d);
         TKDNN_TSTOP
         total_time+= t_ns;
+        if(i> 1)
+        stats.push_back(t_ns);
 
         // control output
-        std::cout<<"Output Buffers: "<<netRT.getBuffersN()-1<<"\n";
+        // std::cout<<"Output Buffers: "<<netRT.getBuffersN()-1<<"\n";
+        std::cout<<"Img: "<<i<<"\n";
 	for(int o=1; o<netRT.getBuffersN(); o++) {
             for(int b=1; b<BATCH_SIZE; b++) {
                 dnnType *out_d = (dnnType*) netRT.buffersRT[o];
                 dnnType *out0_d = out_d;
                 dnnType *outI_d = out_d + netRT.buffersDIM[o].tot()*b;
-                ret_tensorrt |= checkResult(netRT.buffersDIM[o].tot(), outI_d, out0_d) == 0 ? 0 : ERROR_TENSORRT;
+                ret_tensorrt |= checkResult(netRT.buffersDIM[o].tot(), outI_d, out0_d,true, 10, false) == 0 ? 0 : ERROR_TENSORRT;
             }
         }
     }
-    std::cout<<"avg: "<<total_time/64.<<std::endl;
+
+    double min = *std::min_element(stats.begin(), stats.end())/BATCH_SIZE;
+    double max = *std::max_element(stats.begin(), stats.end())/BATCH_SIZE;
+    double mean =0;
+    for(int i=0; i<stats.size(); i++) mean += stats[i]; mean /= stats.size();
+    mean /=BATCH_SIZE;
+
+    std::cout<<"Min: "<<min<<" ms\n";    
+    std::cout<<"Max: "<<max<<" ms\n";    
+    std::cout<<"Avg: "<<mean<<" ms\t"<<1000/(mean)<<" FPS\n"<<COL_END;   
+
+
+    std::ofstream times;
+    times.open("times_rtinference.csv", std::ios_base::app);
+
+    std::string net_name;
+    removePathAndExtension(argv[1], net_name);
+
+    times << net_name<< "_" << BATCH_SIZE << ";" << mean << ";" << min << ";" << max << ";" << 1000./mean << "\n";
+
+    times.close();
+
     return ret_tensorrt;
 }
+