Add camera calibration file reading for demo3d and demoTracker.

Some minor fixes.

Signed-off-by: Davide Sapienza <sapienza.dav@gmail.com>

demo/demo/demo3D.cpp

@@ -31,27 +31,31 @@ int main(int argc, char *argv[]) {
 
     if(argc > 2)
         input = argv[2]; 
-    char ntype = 'c';
+    std::string calib_params = "";
     if(argc > 3)
-        ntype = argv[3][0]; 
+        calib_params = argv[3]; 
-    int n_classes = 3;
+    char ntype = 'c';
     if(argc > 4)
-        n_classes = atoi(argv[4]); 
+        ntype = argv[4][0]; 
-    int n_batch = 1;
+    int n_classes = 3;
     if(argc > 5)
-        n_batch = atoi(argv[5]); 
+        n_classes = atoi(argv[5]); 
-    bool show = true;
+    int n_batch = 1;
     if(argc > 6)
-        show = atoi(argv[6]); 
+        n_batch = atoi(argv[6]); 
-    float conf_thresh=0.3;
+        
+    bool show = true;
     if(argc > 7)
-        conf_thresh = atof(argv[7]);     
+        show = atoi(argv[7]); 
+    float conf_thresh=0.3;
+    if(argc > 8)
+        conf_thresh = atof(argv[8]);     
 
     if(n_batch < 1 || n_batch > 64)
         FatalError("Batch dim not supported");
 
     if(!show)
-	SAVE_RESULT = true;
+	    SAVE_RESULT = true;
 
     tk::dnn::CenternetDetection3D cnet;
 
@@ -66,15 +70,14 @@ int main(int argc, char *argv[]) {
         FatalError("Network type not allowed (3rd parameter)\n");
     }
     std::vector<cv::Mat> calibs;
-    // cv::Mat calib = cv::Mat::zeros(cv::Size(3,3), CV_32F);
+    if(!calib_params.empty() && calib_params!="NULL") {
-    // calib.at<float>(0,0) = 864.1243196486207;// * 512.0;//884.081444212;//864.1243196486207 * 512.0;// 633.0;
+        std::cout<<"calib_params: "<<calib_params<<std::endl;
-    // calib.at<float>(0,2) = 726.7271690557819;// * 512.0;//0.0;//726.7271690557819 * 512.0;// 0.0; //w/2
+        cv::Mat calib;
-    // calib.at<float>(1,1) = 883.6552349216504;// * 512.0;//884.081444212;//883.6552349216504 * 512.0;// 633.0;
+        // the calibration matrix must be a 3x3 matrix
-    // calib.at<float>(1,2) = 506.8548506986564;// * 512.0;//0.0;//506.8548506986564 * 512.0;// 0.0; //h/2
+        readCalibrationMatrix(calib_params, calib);
-    // calibs.push_back(calib);
+        for(int bi=0; bi< n_batch; ++bi)
-    // calibs.push_back(calib);
+            calibs.push_back(calib);
-    // calibs.push_back(calib);
+    }
-    // calibs.push_back(calib);
     detNN->init(net, n_classes, n_batch, conf_thresh, calibs);
 
     gRun = true;

demo/demo/demoTracker.cpp

@@ -31,29 +31,32 @@ int main(int argc, char *argv[]) {
 
     if(argc > 2)
         input = argv[2]; 
-    char ntype = 'c';
+    std::string calib_params = "";
     if(argc > 3)
-        ntype = argv[3][0]; 
+        calib_params = argv[3];
-    int n_classes = 3;
+    char ntype = 'c';
     if(argc > 4)
-        n_classes = atoi(argv[4]); 
+        ntype = argv[4][0]; 
-    int n_batch = 1;
+    int n_classes = 3;
     if(argc > 5)
-        n_batch = atoi(argv[5]); 
+        n_classes = atoi(argv[5]); 
-    bool show = true;
+    int n_batch = 1;
     if(argc > 6)
-        show = atoi(argv[6]); 
+        n_batch = atoi(argv[6]); 
-    float conf_thresh=0.3;
+    bool show = true;
     if(argc > 7)
-        conf_thresh = atof(argv[7]);   
+        show = atoi(argv[7]); 
-    bool t3d = true;
+    float conf_thresh=0.3;
     if(argc > 8)
-        t3d = atoi(argv[8]);
+        conf_thresh = atof(argv[8]);   
+    bool t3d = true;
+    if(argc > 9)
+        t3d = atoi(argv[9]);
     if(n_batch < 1 || n_batch > 64)
         FatalError("Batch dim not supported");
 
     if(!show)
-	SAVE_RESULT = true;
+	    SAVE_RESULT = true;
 
     tk::dnn::CenterTrack ctrack;
 
@@ -68,15 +71,14 @@ int main(int argc, char *argv[]) {
         FatalError("Network type not allowed (3rd parameter)\n");
     }
     std::vector<cv::Mat> calibs;
-    // cv::Mat calib = cv::Mat::zeros(cv::Size(3,3), CV_32F);
+    if(!calib_params.empty() && calib_params!="NULL") {
-    // calib.at<float>(0,0) = 864.1243196486207;// * 512.0;//884.081444212;//864.1243196486207 * 512.0;// 633.0;
+        std::cout<<"calib_params: "<<calib_params<<std::endl;
-    // calib.at<float>(0,2) = 726.7271690557819;// * 512.0;//0.0;//726.7271690557819 * 512.0;// 0.0; //w/2
+        cv::Mat calib;
-    // calib.at<float>(1,1) = 883.6552349216504;// * 512.0;//884.081444212;//883.6552349216504 * 512.0;// 633.0;
+        // the calibration matrix must be a 3x3 matrix
-    // calib.at<float>(1,2) = 506.8548506986564;// * 512.0;//0.0;//506.8548506986564 * 512.0;// 0.0; //h/2
+        readCalibrationMatrix(calib_params, calib);
-    // calibs.push_back(calib);
+        for(int bi=0; bi< n_batch; ++bi)
-    // calibs.push_back(calib);
+            calibs.push_back(calib);
-    // calibs.push_back(calib);
+    }
-    // calibs.push_back(calib);
     trackNN->init(net, n_classes, n_batch, conf_thresh, t3d, calibs);
 
     gRun = true;

docs/demo.md

@@ -77,12 +77,15 @@ To run the 3D object detection demo follow these steps (example with CenterNet b
 ```
 rm dla34_cnet3d_fp32.rt        # be sure to delete(or move) old tensorRT files
 ./test_dla34_cnet3d            # run the yolo test (is slow)
-./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 c
+./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 NULL c
 ```
 The demo3D program takes the same parameters of the demo program:
 ```
-./demo3D <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
+./demo3D <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
 ```
+where
+
+* ```calibration-file``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
 
 ### Object Detection and Tracking
 
@@ -90,16 +93,17 @@ To run the 3D object detection & tracking demo follow these steps (example with
 ```
 rm dla34_ctrack_fp32.rt  # be sure to delete(or move) old tensorRT files
 ./test_dla34_ctrack      # run the yolo test (is slow)
-./demoTracker dla34_ctrack_fp32.rt ../demo/yolo_test.mp4 c
+./demoTracker dla34_ctrack_fp32.rt ../demo/yolo_test.mp4 NULL c
 ```
 
 The demoTracker program takes the same parameters of the demo program:
 ```
-./demoTracker <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh> <2D/3D-flag>
+./demoTracker <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh> <2D/3D-flag>
 ```
 
 where
 
+* ```calibration-file``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
 *  ```<2D/3D-flag>``` if set to 0 the demo will be in the 2D mode, while if set to 1 the demo will be in the 3D mode (Default is 1 - 3D mode).
 
 ### FP16 inference

include/tkDNN/pluginsRT/DeformableConvRT.h

@@ -30,7 +30,7 @@ public:
 		height_ones = (i_h + 2 * ph - (1 * (kh - 1) + 1)) / sh + 1;
 		width_ones = (i_w + 2 * pw - (1 * (kw - 1) + 1)) / sw + 1;
 		dim_ones = i_c * kh * kw * 1 * height_ones * width_ones;
-		std::cout<<i_c * o_c * kh * kw * 1<<"\n";
+		
 		checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
 		checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
 		checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));

include/tkDNN/utils.h

@@ -12,8 +12,14 @@
 #include <cublas_v2.h>
 #include <cudnn.h>
 
+
 #ifdef __linux__
 #include <unistd.h>
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include <yaml-cpp/yaml.h>
 #endif
 
 #include <ios>
@@ -131,6 +137,7 @@ void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData,
 void getMemUsage(double& vm_usage_kb, double& resident_set_kb);
 void printCudaMemUsage();
 void removePathAndExtension(const std::string &full_string, std::string &name);
+void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat);
 static inline bool isCudaPointer(void *data) {
   cudaPointerAttributes attr;
   return cudaPointerGetAttributes(&attr, data) == 0;

scripts/test_all_tests.sh

@@ -91,10 +91,11 @@ do
     #test_net csresnext50-panet-spp_berkeley
     test_net resnet101_cnet
     test_net dla34_cnet
+    test_net dla34_cnet3d
     test_net mobilenetv2ssd
     test_net mobilenetv2ssd512
     test_net bdd-mobilenetv2ssd
-    test_net dla34_cnet3d_track
+    test_net dla34_ctrack
 done
 
 echo "If errors occured, check logfile $out_file" 

src/utils.cpp

@@ -240,4 +240,21 @@ void removePathAndExtension(const std::string &full_string, std::string &name){
         name = name.substr(previous, current);
     
     // std::cout<<"full string: "<<full_string<<" name: "<<name<<std::endl;
-}
+}
+
+void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat){
+    YAML::Node config   = YAML::LoadFile(path);
+
+    //read camera matrix
+    int rows = config["camera_matrix"]["rows"].as<int>();
+    int cols = config["camera_matrix"]["cols"].as<int>();
+
+    cv::Mat calib = cv::Mat(cv::Size(rows, cols), CV_32F);
+    float *vals = (float *)calib.data;
+    
+    for(int i=0; i < config["camera_matrix"]["data"].size(); ++i )
+        vals[i] = config["camera_matrix"]["data"][i].as<float>();
+
+    calib_mat = calib;
+
+}