Refactoring and modularization

Signed-off-by: Micaela Verucchi <micaela.verucchi@unimore.it>

CMakeLists.txt

@@ -38,12 +38,20 @@ cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
 file(GLOB tkdnn_SRC "src/*.cpp")
 set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} -lcudnn -lnvinfer ${OpenCV_LIBS} -lgdal)
 
+file(GLOB class_SRC "src/class_src/*.cpp")
+set(class_LIBS ${OpenCV_LIBS} -lgdal yaml-cpp python2.7)
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES} "~/repos/cereal/include" ${CMAKE_CURRENT_SOURCE_DIR}/tracker_CLASS/c++/src)
 include_directories( BEFORE ${MY_SOURCE_DIR}/src /usr/include/python2.7 )
+
 add_library(tkDNN SHARED ${tkdnn_SRC})
 target_link_libraries(tkDNN ${tkdnn_LIBS})
 
+add_library(CLASS SHARED ${class_SRC})
+target_link_libraries(CLASS ${class_LIBS})
+
+
 #static 
 #add_library(tkDNN_static STATIC ${tkdnn_SRC})
 #target_link_libraries(tkDNN_static ${tkdnn_LIBS})
@@ -104,8 +112,7 @@ add_executable(yolo3_demo demo/demo/demo.cpp
                         tracker_CLASS/c++/src/tracker.cpp )
 
                         
-target_link_libraries(yolo3_demo tkDNN)
+target_link_libraries(yolo3_demo tkDNN CLASS)
-target_link_libraries(yolo3_demo python2.7 yaml-cpp)
 
 
 

demo/demo/demo.cpp

@@ -1,89 +1,34 @@
-#include <iostream>
-#include <signal.h>
-#include <stdlib.h> /* srand, rand */
-#include <unistd.h>
-#include <mutex>
-#include <ctime>
-#include <pthread.h>
-
 #include <time.h>
-#include <chrono>
-#include <math.h>
-#include <typeinfo>
 
 #include "utils.h"
-#include "BoxDetection.h"
-
-#include <opencv2/core/core.hpp>
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-
-//saliency
-#include <opencv2/core/utility.hpp>
-#include <opencv2/saliency.hpp>
-#include <opencv2/highgui.hpp>
-
 #include "Yolo3Detection.h"
-#include "classutils.h"
+#include "message.h"
+#include "visualization.h"
+
+#include "tracker.h"
 #include "../masa_protocol/include/send.hpp"
 #include "../masa_protocol/include/serialize.hpp"
 
-#include "ekf.h"
+// #include <assert.h>
-#include "trackutils.h"
+// #include <unistd.h>
-#include "plot.h"
+// #include <mutex>
-#include "tracker.h"
+// #include <ctime>
-#include <assert.h> 
+// #include <pthread.h>
+// #include <signal.h>
+// #include <chrono>
+// #include <math.h>
+// #include <typeinfo>
+// #include <iostream>
 
 #define MAX_DETECT_SIZE 100
 
-std::chrono::steady_clock::time_point local_clock_start;
-
-std::mutex mutexgRun;
-
 bool gRun;
+std::chrono::steady_clock::time_point local_clock_start;
+std::mutex mutexgRun;
 std::string obj_class[10]{"person", "car", "truck", "bus", "motor", "bike", "rider", "traffic light", "traffic sign", "train"};
-
 //mutex for some opencv operations
 std::mutex mutex_cv;
-
+Show_t updates;
-struct ModFrame_t{
-    std::vector<Tracker> trackers;
-    geodetic_converter::GeodeticConverter gc;
-    double adfGeoTransform[6];
-    cv::Mat H;
-    cv::Mat original_frame;
-    tk::dnn::Yolo3Detection yolo;
-    cv::Mat mask;
-    // sem for mainthread, detectionthread and topviewthread
-    std::mutex sem;
-};
-
-struct Frame_t{
-    char *input;
-    cv::Mat frame;
-    int frame_nbr;
-    // sem_vc for mainthread, videocapturethread, originalthread and disparitythread
-    std::mutex sem_vc;
-};
-
-struct Camera_t{ 
-    int CAM_IDX;
-    char *input;
-    char *pmatrix;
-    char *maskfile;
-    char *cameraCalib;
-    char *maskFileOrient;
-    bool to_show;
-    tk::dnn::Yolo3Detection yolo;
-    double adfGeoTransform[6];
-};
-
-struct Show_t{
-    cv::Mat original, detection, topview, disparity;
-    bool update_o, update_de, update_t, update_di; 
-    // a single mutex for each operation - the show_updates function must get all mutex
-    std::mutex mutex_o, mutex_de, mutex_t, mutex_di;
-}updates;
 
 void sig_handler(int signo)
 {
@@ -95,9 +40,9 @@ void sig_handler(int signo)
 
 void *readVideoCapture(void *x_void_ptr)
 {
-    std::cout<<"readVideoCapture start...\n";
+    std::cout << "readVideoCapture start...\n";
 
-    Frame_t *info_f = (Frame_t *) x_void_ptr;
+    Frame_t *info_f = (Frame_t *)x_void_ptr;
     mutex_cv.lock();
     cv::VideoCapture cap(info_f->input, cv::CAP_FFMPEG);
     mutex_cv.unlock();
@@ -113,7 +58,6 @@ void *readVideoCapture(void *x_void_ptr)
     else
         std::cout << "camera started\n";
 
-
     // cap.set(cv::CAP_PROP_BUFFERSIZE,3);
     // std::cout<<"buf size: "<<cap.get(CV_CAP_PROP_BUFFERSIZE)<<std::endl;
     auto start_t = std::chrono::steady_clock::now();
@@ -124,31 +68,31 @@ void *readVideoCapture(void *x_void_ptr)
     // compute fps and find camera's clock
     double shift, mean_time = 0;
     std::cout << "Frames per second using video.get(CV_CAP_PROP_FPS) : " << cap.get(CV_CAP_PROP_FPS) << std::endl;
-    std::cout<<"readVideoCapture computes frame rate...\n";
+    std::cout << "readVideoCapture computes frame rate...\n";
     // //compute frame rate
-    int i=0;
+    int i = 0;
     int num_f = 120;
     // the first 20 frames are null
-    while (i<21)
+    while (i < 21)
     {
         cap >> frame_loc;
         i++;
     }
 
-    i=0;
+    i = 0;
     start_t = std::chrono::steady_clock::now();
-    while (i<num_f)
+    while (i < num_f)
     {
         step_t = std::chrono::steady_clock::now();
         cap >> frame_loc;
         mean_time = mean_time + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - step_t).count();
-        std::cout << " step "<<i<<" : "<<std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - step_t).count() << " ms"<<std::endl;
+        std::cout << " step " << i << " : " << std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - step_t).count() << " ms" << std::endl;
         i++;
     }
     end_t = std::chrono::steady_clock::now();
 
-    std::cout << "Capturing " << num_f << " frames" << std::endl ;
+    std::cout << "Capturing " << num_f << " frames" << std::endl;
-    std::cout << " Time taken : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms"<<std::endl;
+    std::cout << " Time taken : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms" << std::endl;
 
     /*
     mean_time indicates the milliseconds from a frame and the next. (frame rate)
@@ -158,27 +102,27 @@ void *readVideoCapture(void *x_void_ptr)
     otherwise it will be considered old.
     */
     auto local_clock_sync = std::chrono::steady_clock::now();
-    mean_time = mean_time/num_f;
+    mean_time = mean_time / num_f;
     shift = ((double)std::chrono::duration_cast<std::chrono::milliseconds>(local_clock_sync - local_clock_start).count()) / mean_time;
     shift = (shift - (int)shift) * mean_time;
-    std::cout<<".-------------------------------\n";
+    std::cout << ".-------------------------------\n";
-    std::cout<<" mean time: "<<mean_time<<std::endl;
+    std::cout << " mean time: " << mean_time << std::endl;
-    std::cout<<" shift: "<<shift<<std::endl;
+    std::cout << " shift: " << shift << std::endl;
-    std::cout<<" TIMEDIFFERENCE:  "<<std::chrono::duration_cast<std::chrono::milliseconds>(local_clock_sync - local_clock_start).count()<<std::endl;
+    std::cout << " TIMEDIFFERENCE:  " << std::chrono::duration_cast<std::chrono::milliseconds>(local_clock_sync - local_clock_start).count() << std::endl;
-    std::cout<<"\n\n\n\n";
+    std::cout << "\n\n\n\n";
-    std::cout<<"readVideoCapture start to capture...\n";
+    std::cout << "readVideoCapture start to capture...\n";
     while (gRun)
     {
         // mutex_cv.lock();
         cap >> frame_loc;
         // mutex_cv.unlock();
         current_timestamp = std::chrono::steady_clock::now();
-        shift = std::chrono::duration_cast<std::chrono::milliseconds>(current_timestamp - local_clock_sync).count() ;
+        shift = std::chrono::duration_cast<std::chrono::milliseconds>(current_timestamp - local_clock_sync).count();
-        std::cout << " RELATIVE TIMESTAMP FRAME : "<<shift << " ms"<<std::endl;
+        std::cout << " RELATIVE TIMESTAMP FRAME : " << shift << " ms" << std::endl;
         shift = shift / mean_time;
         shift = (shift - (int)shift) * mean_time;
-        shift = (shift - mean_time/2 >= 0)? -(mean_time - shift) : shift;
+        shift = (shift - mean_time / 2 >= 0) ? -(mean_time - shift) : shift;
-        std::cout<<"DELAY frame_"<<frame_nbr_loc<<" : "<< shift << " ms"<<std::endl;
+        std::cout << "DELAY frame_" << frame_nbr_loc << " : " << shift << " ms" << std::endl;
         // TODO: here introduce a tollerance to discard old frame
 
         // std::cout<< "CV_CAP_PROP_POS_MSEC:   "<< cap.get( cv::CAP_PROP_POS_MSEC) <<std::endl;
@@ -186,10 +130,9 @@ void *readVideoCapture(void *x_void_ptr)
         // std::cout<< "CV_CAP_PROP_FPS:  "<< cap.get( cv::CAP_PROP_FPS)<<std::endl;
         // std::cout << "Format: " << cap.get(CV_CAP_PROP_FORMAT) << "\n";
         // CAP_PROP_POS_MSEC Current position of the video file in milliseconds or video capture timestamp.
-        std::cout<<"id: "<<cap.get(cv::CAP_PROP_POS_MSEC)<<std::endl;
+        std::cout << "id: " << cap.get(cv::CAP_PROP_POS_MSEC) << std::endl;
         // CAP_PROP_FRAME_COUNT Number of frames in the video file.
-        std::cout<<"id: "<<cap.get(cv::CAP_PROP_FRAME_COUNT)<<std::endl;
+        std::cout << "id: " << cap.get(cv::CAP_PROP_FRAME_COUNT) << std::endl;
-
 
         if (!frame_loc.data)
         {
@@ -202,7 +145,7 @@ void *readVideoCapture(void *x_void_ptr)
         }
 
         end_t = std::chrono::steady_clock::now();
-        std::cout << " VC-TIME 1 : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms"<<std::endl;
+        std::cout << " VC-TIME 1 : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms" << std::endl;
         start_t = end_t;
 
         info_f->sem_vc.lock();
@@ -211,400 +154,22 @@ void *readVideoCapture(void *x_void_ptr)
         info_f->sem_vc.unlock();
         // usleep(50000);
         end_t = std::chrono::steady_clock::now();
-        std::cout << " VC-TIME 2 : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms"<<std::endl;
+        std::cout << " VC-TIME 2 : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms" << std::endl;
         start_t = end_t;
-        frame_nbr_loc ++;
+        frame_nbr_loc++;
-    }
-    return (void *)0;
-}
-
-/* Thread function to show the updated images 
-**/
-void *show_updates(void *x_void_ptr)
-{
-    cv::namedWindow("original", cv::WINDOW_NORMAL);
-    cv::namedWindow("detection", cv::WINDOW_NORMAL);
-    cv::namedWindow("topview", cv::WINDOW_NORMAL);
-    cv::namedWindow("disparity", cv::WINDOW_NORMAL);
-    cv::Mat original_loc, detection_loc, topview_loc, disparity_loc;
-    bool update_o_loc, update_de_loc, update_t_loc, update_di_loc;
-    
-    while (gRun)
-    {
-        TIMER_START
-        // critical section: copy the struct in local variable
-        // in this way we can unlock the sem for the main thread
-        if(updates.mutex_o.try_lock())
-            {
-            update_o_loc = updates.update_o;
-            updates.update_o = false;
-            if(update_o_loc)
-                original_loc = updates.original.clone();
-            updates.mutex_o.unlock();
-            }
-
-        if(updates.mutex_de.try_lock())
-            {    
-            update_de_loc = updates.update_de;
-            updates.update_de = false;
-            if(update_de_loc)
-                detection_loc = updates.detection.clone();
-            updates.mutex_de.unlock();
-            }
-
-        if(updates.mutex_t.try_lock())
-            {  
-            update_t_loc = updates.update_t;
-            updates.update_t = false;
-            if(update_t_loc)
-                topview_loc = updates.topview.clone();
-            updates.mutex_t.unlock();
-            }
-
-        if(updates.mutex_di.try_lock())
-            {  
-            update_di_loc = updates.update_di;
-            updates.update_di = false;
-            if(update_di_loc)
-                disparity_loc = updates.disparity.clone();
-            updates.mutex_di.unlock();
-            }
-
-        if(update_o_loc)
-            cv::imshow("original", original_loc);
-        if(update_de_loc)
-            cv::imshow("detection", detection_loc);
-        if(update_t_loc)
-            cv::imshow("topview", topview_loc);
-        if(update_di_loc)
-            cv::imshow("disparity", disparity_loc);
-        cv::waitKey(1);
-        // usleep(20000);  //sleep 20 msec
-        std::cout<<"show_updates: ";
-        TIMER_STOP
-    }
-    return (void *)0;
-}
-
-void *originalFrame(void *x_void_ptr)
-{
-    Frame_t *info_show_orig = (Frame_t *) x_void_ptr;
-    cv::Mat frame_loc;
-    int frame_nbr_loc = 0;
-    while (gRun)
-    {
-        TIMER_START
-        // critical section: copy the struct in local variable
-        // in this way we can unlock the sem for the main thread
-        info_show_orig->sem_vc.lock();
-        frame_loc = info_show_orig->frame.clone();
-        frame_nbr_loc = info_show_orig->frame_nbr;
-        info_show_orig->sem_vc.unlock();
-        if (frame_nbr_loc == 0)
-        {
-            usleep(1000000);
-            printf("no frame received\n");
-            continue;
-        }
-        updates.mutex_o.lock();
-        updates.original = frame_loc.clone();
-        updates.update_o = true;
-        updates.mutex_o.unlock();
-        usleep(10000);  //sleep 10 msec
-        std::cout<<"originalFrame: ";
-        TIMER_STOP
-    }
-    return (void *)0;
-}
-
-void *detectionFrame(void *x_void_ptr)
-{
-    ModFrame_t *info_show = (ModFrame_t *) x_void_ptr;
-    double lat, lon, alt;
-    int pix_x, pix_y;
-    cv::Mat original_frame_loc;
-    std::vector<Tracker> trackers;
-
-    geodetic_converter::GeodeticConverter gc;
-    double adfGeoTransform[6];
-    cv::Mat H;
-    tk::dnn::Yolo3Detection yolo;
-    int num_detected;
-    cv::Mat mask;
-
-    // box variable
-    tk::dnn::box b;
-    int x0, w, x1, y0, h, y1;
-    int objClass;
-    std::string det_class;;
-    // float prob; 
-    cv::Scalar intensity;
-    std::vector<cv::Point2f> map_p, camera_p;
-    int baseline = 0;
-    float fontScale = 0.5;
-    int thickness = 2;
-
-    while (gRun)
-    {
-        TIMER_START
-        // critical section: copy the struct in local variable
-        // in this way we can unlock the sem for the main thread
-        info_show->sem.lock();
-        original_frame_loc = info_show->original_frame.clone();
-        // std::vector<Tracker> trackers;
-        trackers = info_show->trackers;
-        // geodetic_converter::GeodeticConverter gc;
-        gc = info_show->gc;
-        for(int i = 0; i < 6; i++ )
-            adfGeoTransform[i] = info_show->adfGeoTransform[i];
-        // cv::Mat H;
-        H = info_show->H.clone();
-        yolo = info_show->yolo;
-        mask = info_show->mask.clone();
-        info_show->sem.unlock();
-        
-        if (trackers.empty())
-        {
-            usleep(1000000);
-            printf("no data available\n");
-            continue;
-        }
-
-        num_detected = yolo.detected.size();
-        for (int i = 0; i < num_detected; i++)
-        {
-            b = yolo.detected[i];
-            x0 = b.x;
-            w = b.w;
-            x1 = b.x + w;
-            y0 = b.y;
-            h = b.h;
-            y1 = b.y + h;
-            objClass = b.cl;
-            det_class = obj_class[b.cl];
-            // prob = b.prob;
-
-            intensity = mask.at<uchar>(cv::Point(int(x0 + b.w / 2), y1));
-            
-            if (intensity[0] && objClass < 6)
-            {
-                //std::cout<<objClass<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
-                cv::rectangle(original_frame_loc, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[objClass], 2);
-                // draw label
-                cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
-                cv::rectangle(original_frame_loc, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);
-                cv::putText(original_frame_loc, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness); 
-            }
-        }
-
-        for (auto t : trackers)
-        {
-            for (size_t p = 1; p < t.pred_list_.size(); p++)
-            {
-                gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt);
-                coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
-
-                map_p.clear();
-                camera_p.clear();
-                map_p.push_back(cv::Point2f(pix_x, pix_y));
-
-                //transform camera pixel to map pixel
-                cv::perspectiveTransform(map_p, camera_p, H.inv());
-                // std::cout<<"x,y: "<<pix_x<<", "<<pix_y<<std::endl;
-                // std::cout<<"map_p: "<<map_p<<std::endl;
-                // std::cout<<"camera_p: "<<camera_p<<std::endl;
-                // std::cout<<"size original_frame_loc: "<<original_frame_loc.cols<<", "<<original_frame_loc.rows<<std::endl;
-                // assert (camera_p[0].x < original_frame_loc.cols);
-                // assert (camera_p[0].y < original_frame_loc.rows);
-                if (camera_p[0].x < original_frame_loc.cols && camera_p[0].y < original_frame_loc.rows && camera_p[0].x >= 0 && camera_p[0].y >= 0)
-                    cv::circle(original_frame_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0);
-            }
-        }
-
-        updates.mutex_de.lock();
-        updates.detection = original_frame_loc.clone();
-        updates.update_de = true;
-        updates.mutex_de.unlock();
-
-        std::cout<<"detectionFrame: ";
-        TIMER_STOP
-
-    }
-    return (void *)0;
-}
-
-void *topviewFrame(void *x_void_ptr)
-{
-    ModFrame_t *info_show = (ModFrame_t *) x_void_ptr;
-    double lat, lon, alt;
-    int pix_x, pix_y;
-    cv::Mat frame_top;
-    cv::Mat original_frame_top;
-    // original_frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg");
-    original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png");
-    // original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670_V.png");
-    
-    std::vector<Tracker> trackers;
-    geodetic_converter::GeodeticConverter gc;
-    double adfGeoTransform[6];
-    cv::Mat H;
-    while (gRun)
-    {
-        TIMER_START
-        // critical section: copy the struct in local variable
-        // in this way we can unlock the sem for the main thread
-        info_show->sem.lock();
-        // std::vector<Tracker> trackers;
-        trackers = info_show->trackers;
-        // geodetic_converter::GeodeticConverter gc;
-        gc = info_show->gc;
-        for(int i = 0; i < 6; i++ )
-            adfGeoTransform[i] = info_show->adfGeoTransform[i];
-        // cv::Mat H;
-        H = info_show->H.clone();
-        info_show->sem.unlock();
-        if (trackers.empty())
-        {
-            usleep(1000000);
-            printf("no data available\n");
-            continue;
-        }
-        frame_top = original_frame_top.clone();
-        for (auto t : trackers)
-        {
-            for (size_t p = 1; p < t.pred_list_.size(); p++)
-            {
-                gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt);
-                coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
-                if (pix_x < frame_top.cols && pix_y < frame_top.rows && pix_x >= 0 && pix_y >= 0)
-                    cv::circle(frame_top, cv::Point(pix_x, pix_y), 7.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0);
-
-            }
-        }
-        //outputVideo<< frame_top;
-        // ------------------------------------------------
-        updates.mutex_t.lock();
-        updates.topview = frame_top.clone();
-        updates.update_t = true;
-        updates.mutex_t.unlock();
-
-        std::cout<<"topviewFrame: ";
-        TIMER_STOP
-
-    }    
-    return (void *)0;
-}
-
-void *disparityFrame(void *x_void_ptr)
-{
-    Frame_t *info_show_disparity = (Frame_t *) x_void_ptr;
-    bool first_iteration = true;
-    cv::Mat frame_loc;
-    int frame_nbr_loc = 0, pre_frame_nbr_loc = 0;
-    auto start_t = std::chrono::steady_clock::now();
-    auto step_t = std::chrono::steady_clock::now();
-    auto end_t = std::chrono::steady_clock::now();
-
-    // information for the disparity map
-    cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB;
-    cv::Mat canny_img;
-    cv::Mat disparity_frame;
-    while (gRun)
-    {
-        start_t = std::chrono::steady_clock::now();
-        step_t = start_t;
-        // critical section: copy the struct in local variable
-        // in this way we can unlock the sem for the main thread
-        info_show_disparity->sem_vc.lock();
-        frame_loc = info_show_disparity->frame.clone();
-        frame_nbr_loc = info_show_disparity->frame_nbr;
-        info_show_disparity->sem_vc.unlock();
-        if (frame_nbr_loc == 0)
-        {
-            usleep(1000000);
-            printf("no frame received\n");
-            continue;
-        }
-        // compute frame disparity only in there is a new frame
-        if(frame_nbr_loc - pre_frame_nbr_loc > 0)
-        {
-            pre_frame_nbr_loc = frame_nbr_loc;
-            //preprocessing frame
-            step_t = std::chrono::steady_clock::now();
-            // src_gray
-            canny_img = img_laplacian(frame_loc,0);
-            cv::Canny(canny_img, canny, 100, 100*2 );
-            // sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr_loc, 999);
-            // cv::imwrite(buf_frame_crop_name, canny);
-            end_t = std::chrono::steady_clock::now();
-            std::cout << " TIME END pre canny : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms"<<std::endl;
-            step_t = end_t;
-            // std::cout<<"o: "<<frame_loc.cols<<" - "<<frame_loc.rows<<std::endl;
-            // std::cout<<"canny: "<<canny.cols<<" - "<<canny.rows<<std::endl;
-            // std::cout<<"pre: "<<pre_canny.cols<<" - "<<pre_canny.rows<<std::endl;
-            if(!first_iteration)
-            {
-                // backtorgb = cv::cvtColor(pre_canny,cv::COLOR_GRAY2RGB) 
-                cv::cvtColor(pre_canny, pre_canny_RGB, CV_GRAY2RGB);
-                cv::cvtColor(canny, canny_RGB, CV_GRAY2RGB);
-                disparity_frame = frame_disparity(pre_canny_RGB, canny_RGB, frame_nbr_loc, 999, 0);
-                // std::cout<<"size: "<<disparity_frame.rows<<" - "<<disparity_frame.cols<<std::endl;
-                // if (disparity_frame.rows == 0 || disparity_frame.cols == 0)
-                //     return -1;
-                // if (disparity_frame.empty()) 
-                // { // only fools don't check...
-                //     std::cout << "image not loaded !" << std::endl;
-                //     return -1;
-                // }
-                end_t = std::chrono::steady_clock::now();
-                std::cout << " TIME canny : frame_disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms"<<std::endl;
-                step_t = end_t;
-
-                // //--------------------------------
-                // //frame box disparity on the original image
-                // step_t_segmentation = std::chrono::steady_clock::now();
-                // frame_box_disparity(pre_frame, frame, pre_rois, frame_nbr_loc);
-                // // reset pre_rois for the new roi of the current frame
-                // // pre_rois.erase(pre_rois.begin(), pre_rois.end());
-                // end_t_segmentation = std::chrono::steady_clock::now();
-                // std::cout << " TIME Frame disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
-                // step_t_segmentation = end_t_segmentation;
-
-                // //frame box disparity on the preprocessed image
-                // cv::cvtColor(pre_canny, pre_canny_RGB, CV_GRAY2RGB);
-                // cv::cvtColor(canny, canny_RGB, CV_GRAY2RGB);
-                // frame_box_disparity(pre_canny_RGB, canny_RGB, pre_rois, frame_nbr_loc);
-                // // reset pre_rois for the new roi of the current frame
-                // pre_rois.erase(pre_rois.begin(), pre_rois.end());
-                // end_t_segmentation = std::chrono::steady_clock::now();
-                // std::cout << " TIME Canny Frame disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
-                // step_t_segmentation = end_t_segmentation;
-                // //---------------------------------
-
-                updates.mutex_di.lock();
-                updates.disparity = disparity_frame.clone();
-                updates.update_di = true;
-                updates.mutex_di.unlock();
-            }
-            pre_canny = canny.clone();
-            if(first_iteration)
-                first_iteration = false;
-            end_t = std::chrono::steady_clock::now();
-            std::cout<<"disparityFrame : TIME END pre canny : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms"<<std::endl;
-        }
     }
     return (void *)0;
 }
 
 void *computationTask(void *x_void_ptr)
 {
-    Camera_t *camera = (Camera_t *) x_void_ptr;
+    Camera_t *camera = (Camera_t *)x_void_ptr;
     pthread_t visual, originalshow, detectionshow, topviewshow, disparityshow;
     pthread_t videocap;
     //create video capture thread
     Frame_t info_f;
     info_f.input = camera->input;
-    if (pthread_create(&videocap, NULL, readVideoCapture, (void*)&info_f))
+    if (pthread_create(&videocap, NULL, readVideoCapture, (void *)&info_f))
     {
         fprintf(stderr, "Error creating thread\n");
         return (void *)1;
@@ -612,7 +177,7 @@ void *computationTask(void *x_void_ptr)
 
     bool to_show = camera->to_show;
     double adfGeoTransform[6];
-    for(int i=0; i<6; i++)
+    for (int i = 0; i < 6; i++)
         adfGeoTransform[i] = camera->adfGeoTransform[i];
 
     ModFrame_t info_show;
@@ -623,39 +188,39 @@ void *computationTask(void *x_void_ptr)
         updates.update_de = false;
         updates.update_t = false;
         updates.update_di = false;
-        if (pthread_create(&visual, NULL, show_updates, (void*)NULL))
+        if (pthread_create(&visual, NULL, show_updates, (void *)NULL))
         {
             fprintf(stderr, "Error creating thread\n");
             return (void *)1;
         };
-        if (pthread_create(&originalshow, NULL, originalFrame, (void*)&info_f))
+        if (pthread_create(&originalshow, NULL, originalFrame, (void *)&info_f))
         {
             fprintf(stderr, "Error creating thread\n");
             return (void *)1;
         };
-        if (pthread_create(&disparityshow, NULL, disparityFrame, (void*)&info_f))
+        if (pthread_create(&disparityshow, NULL, disparityFrame, (void *)&info_f))
         {
             fprintf(stderr, "Error creating thread\n");
             return (void *)1;
         };
         info_show.H = cv::Mat(cv::Size(3, 3), CV_64FC1);
-        if (pthread_create(&detectionshow, NULL, detectionFrame, (void*)&info_show))
+        if (pthread_create(&detectionshow, NULL, detectionFrame, (void *)&info_show))
         {
             fprintf(stderr, "Error creating thread\n");
             return (void *)1;
         };
-        if (pthread_create(&topviewshow, NULL, topviewFrame, (void*)&info_show))
+        if (pthread_create(&topviewshow, NULL, topviewFrame, (void *)&info_show))
         {
             fprintf(stderr, "Error creating thread\n");
             return (void *)1;
         };
     }
 
-    char* pmatrix = camera->pmatrix;
+    char *pmatrix = camera->pmatrix;
     /*projection matrix from camera to map*/
     cv::Mat H(cv::Size(3, 3), CV_64FC1);
     read_projection_matrix(H, pmatrix);
-    assert (cv::countNonZero(H) > 0);
+    assert(cv::countNonZero(H) > 0);
     // std::cout<<H<<std::endl;
     // return (void*)0;
     /*Camera calibration*/
@@ -731,7 +296,8 @@ void *computationTask(void *x_void_ptr)
     tk::dnn::box b;
     int x0, h, y1; //w, x1, y0;
     int objClass;
-    std::string det_class;;
+    std::string det_class;
+    ;
     // float prob;
     cv::Scalar intensity;
 
@@ -747,11 +313,11 @@ void *computationTask(void *x_void_ptr)
 
         info_f.sem_vc.lock();
         frame = info_f.frame.clone();
-        if(info_f.frame_nbr - frame_nbr > 1)
+        if (info_f.frame_nbr - frame_nbr > 1)
-            std::cout<<"more than one - f_n (diff "<<info_f.frame_nbr - frame_nbr<<")\n";
+            std::cout << "more than one - f_n (diff " << info_f.frame_nbr - frame_nbr << ")\n";
         frame_nbr = info_f.frame_nbr;
         info_f.sem_vc.unlock();
-        std::cout<<"f_n: "<<frame_nbr<<std::endl;
+        std::cout << "f_n: " << frame_nbr << std::endl;
         // if (!frame.data)
         if (frame_nbr == 0)
         {
@@ -778,10 +344,10 @@ void *computationTask(void *x_void_ptr)
 
         coords.clear();
         end_t = std::chrono::steady_clock::now();
-        std::cout << " TIME 1 : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms"<<std::endl;
+        std::cout << " TIME 1 : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
         step_t = end_t;
         // draw dets
-        std::cout<<"camera: "<<camera->CAM_IDX <<" - num detected: "<<num_detected<<std::endl;
+        std::cout << "camera: " << camera->CAM_IDX << " - num detected: " << num_detected << std::endl;
 
         //TODO: move in a thread
         // //preprocessing frame
@@ -878,8 +444,7 @@ void *computationTask(void *x_void_ptr)
                     // segmentation(frame(roi), frame(roi), frame_nbr, i, 1);
 
                     /////
-                    convert_coords(coords, x0 + b.w / 2, y1, objClass, H, adfGeoTransform, frame_nbr);
+                    convert_coords(coords, x0 + b.w / 2, y1, objClass, H, adfGeoTransform);
-                    
 
                     // //std::cout<<objClass<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
                     // cv::rectangle(frame, cv::Point(x0, y0), cv::Point(x1, y1), camera->yolo.colors[objClass], 2);
@@ -895,7 +460,7 @@ void *computationTask(void *x_void_ptr)
         }
 
         end_t = std::chrono::steady_clock::now();
-        std::cout << " TIME 2 : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms"<<std::endl;
+        std::cout << " TIME 2 : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
         step_t = end_t;
         //convert from latitude and longitude to meters for ekf
         cur_frame.clear();
@@ -907,7 +472,7 @@ void *computationTask(void *x_void_ptr)
         if (first_iteration)
         {
             // if there aren't detections and it is the first iteration, we can't initialize the tracker, so continue
-            if(cur_frame.empty())
+            if (cur_frame.empty())
                 continue;
             for (auto f : cur_frame)
                 trackers.push_back(Tracker(f, initial_age, dt, n_states));
@@ -918,7 +483,7 @@ void *computationTask(void *x_void_ptr)
         }
         std::cout << "There are " << trackers.size() << " trackers" << std::endl;
         //prepare message with tracker info
-        if(trackers.size()==0)
+        if (trackers.size() == 0)
         {
             // mutex_cv.lock();
             addRoadUserfromTracker(trackers, m, gc, maskOrient, adfGeoTransform, H);
@@ -939,7 +504,7 @@ void *computationTask(void *x_void_ptr)
             info_show.trackers = trackers;
             // geodetic_converter::GeodeticConverter gc;
             info_show.gc = gc;
-            for(int i = 0; i < 6; i++ )
+            for (int i = 0; i < 6; i++)
                 info_show.adfGeoTransform[i] = adfGeoTransform[i];
             // cv::Mat H;
             info_show.H = H.clone();
@@ -952,11 +517,11 @@ void *computationTask(void *x_void_ptr)
         // update pre_frame for the disparity map
         // pre_frame = orig_frame.clone();
         // pre_canny = canny.clone();
-        if(first_iteration)
+        if (first_iteration)
             first_iteration = false;
 
         frame_nbr++;
-        std::cout<<camera->CAM_IDX<<" camera thread: ";
+        std::cout << camera->CAM_IDX << " camera thread: ";
         TIMER_STOP
     }
     return (void *)0;
@@ -980,27 +545,27 @@ int main(int argc, char *argv[])
     if (argc > 3)
     {
         n = argv[3];
-        if(strcmp(n, "-n"))
+        if (strcmp(n, "-n"))
             return -1;
     };
     int n_cameras = 0;
     if (argc > 4)
     {
         n_cameras = atoi(argv[4]);
-        if(argc < 5+ 7 * n_cameras)
+        if (argc < 5 + 7 * n_cameras)
         {
-            std::cout<<"too few parameters\n";
+            std::cout << "too few parameters\n";
             return -1;
         }
-        if(!n_cameras)
+        if (!n_cameras)
         {
             n_cameras = 1;
             no_params = true;
         }
     }
     Camera_t cameras[n_cameras];
-    bool *to_show = (bool *) malloc(n_cameras * sizeof(bool)); 
+    bool *to_show = (bool *)malloc(n_cameras * sizeof(bool));
-    if(no_params) 
+    if (no_params)
     {
         cameras[0].CAM_IDX = 20936;
         cameras[0].input = (char *)"../demo/demo/data/single_ped_2.mp4";
@@ -1013,34 +578,33 @@ int main(int argc, char *argv[])
     }
     else
     {
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
-            cameras[i].CAM_IDX = atoi(argv[5+i]);
+            cameras[i].CAM_IDX = atoi(argv[5 + i]);
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
-            cameras[i].input = argv[5+n_cameras+i];
+            cameras[i].input = argv[5 + n_cameras + i];
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
-            cameras[i].pmatrix = argv[5+2*n_cameras+i];
+            cameras[i].pmatrix = argv[5 + 2 * n_cameras + i];
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
-            cameras[i].maskfile = argv[5+3*n_cameras+i];
+            cameras[i].maskfile = argv[5 + 3 * n_cameras + i];
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
-            cameras[i].cameraCalib = argv[5+4*n_cameras+i];
+            cameras[i].cameraCalib = argv[5 + 4 * n_cameras + i];
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
-            cameras[i].maskFileOrient = argv[5+5*n_cameras+i];
+            cameras[i].maskFileOrient = argv[5 + 5 * n_cameras + i];
-        for(int i = 0; i<n_cameras; i++)
+        for (int i = 0; i < n_cameras; i++)
         {
-            cameras[i].to_show = atoi(argv[5+6*n_cameras+i]); //only one camera can be shown
+            cameras[i].to_show = atoi(argv[5 + 6 * n_cameras + i]); //only one camera can be shown
-            to_show[i] = atoi(argv[5+6*n_cameras+i]);
+            to_show[i] = atoi(argv[5 + 6 * n_cameras + i]);
         }
     }
     //TODO now only one camera can be visualized
-    int check_visualization=0;
+    int check_visualization = 0;
-    for(int i = 0; i<n_cameras; i++)
+    for (int i = 0; i < n_cameras; i++)
         check_visualization += to_show[i];
-    if(check_visualization > 1)
+    if (check_visualization > 1)
         return -1;
 
-    
     tk::dnn::Yolo3Detection yolo[n_cameras];
-    for(int i=0; i<n_cameras; i++)
+    for (int i = 0; i < n_cameras; i++)
     {
         yolo[i].init(net);
         yolo[i].thresh = 0.25;
@@ -1058,31 +622,30 @@ int main(int argc, char *argv[])
     double *adfGeoTransform = (double *)malloc(6 * sizeof(double));
     readTiff(tiffile, adfGeoTransform);
 
-    for(int i=0; i<n_cameras; i++)
+    for (int i = 0; i < n_cameras; i++)
     {
-        for(int j = 0; j < 6; j++ )
+        for (int j = 0; j < 6; j++)
             cameras[i].adfGeoTransform[j] = adfGeoTransform[j];
         cameras[i].yolo = yolo[i];
         // cameras[i].yolo = yolo;
-        
     }
-    std::cout<<"INIZIA:\n";
+    std::cout << "INIZIA:\n";
     pthread_t camera_task[n_cameras];
-    for(int i=0; i<n_cameras; i++)
+    for (int i = 0; i < n_cameras; i++)
     {
-        std::cout<<"creating thread\n";
+        std::cout << "creating thread\n";
-        if(pthread_create(&camera_task[i], NULL, computationTask, (void*)&cameras[i]))
+        if (pthread_create(&camera_task[i], NULL, computationTask, (void *)&cameras[i]))
         {
             fprintf(stderr, "error creating thread\n");
             return 1;
         }
-        std::cout<<"WHAT2\n";
+        std::cout << "WHAT2\n";
     }
-    for(int i=0; i<n_cameras; i++)
+    for (int i = 0; i < n_cameras; i++)
     {
         pthread_join(camera_task[i], NULL);
     }
-    std::cout <<" free adfGeoT \n";
+    std::cout << " free adfGeoT \n";
     free(adfGeoTransform);
     std::cout << "detection end\n";
     return 0;

include/Layer.h

@@ -1,13 +1,17 @@
 #ifndef LAYER_H
 #define LAYER_H
 
-#include<iostream>
+#include <iostream>
 #include "utils.h"
 #include "Network.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-enum layerType_t {
+enum layerType_t
+{
     LAYER_DENSE,
     LAYER_CONV2D,
     LAYER_ACTIVATION,
@@ -26,56 +30,73 @@ enum layerType_t {
 /** 
     Simple layer Father class
 */
-class Layer {
+class Layer
+{
 
 public:
     Layer(Network *net);
     virtual ~Layer();
     virtual layerType_t getLayerType() = 0;
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) {
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData)
-        std::cout<<"No infer action for this layer\n";
+    {
+        std::cout << "No infer action for this layer\n";
         return NULL;
     }
 
     dataDim_t input_dim, output_dim;
     dnnType *dstData; //where results will be putted
 
-    std::string getLayerName() {
+    std::string getLayerName()
+    {
         layerType_t type = getLayerType();
-        switch(type) {
+        switch (type)
-            case LAYER_DENSE:       return "Dense";
+        {
-            case LAYER_CONV2D:      return "Conv2d";
+        case LAYER_DENSE:
-            case LAYER_ACTIVATION:  return "Activation";
+            return "Dense";
-            case LAYER_FLATTEN:     return "Flatten";
+        case LAYER_CONV2D:
-            case LAYER_MULADD:      return "MulAdd";
+            return "Conv2d";
-            case LAYER_POOLING:     return "Pooling";
+        case LAYER_ACTIVATION:
-            case LAYER_SOFTMAX:     return "Softmax";
+            return "Activation";
-            case LAYER_ROUTE:       return "Route";            
+        case LAYER_FLATTEN:
-            case LAYER_REORG:       return "Reorg";
+            return "Flatten";
-            case LAYER_SHORTCUT:    return "Shortcut";
+        case LAYER_MULADD:
-            case LAYER_UPSAMPLE:    return "Upsample";
+            return "MulAdd";
-            case LAYER_REGION:      return "Region";
+        case LAYER_POOLING:
-            case LAYER_YOLO:        return "Yolo";
+            return "Pooling";
-            default:                return "unknown";
+        case LAYER_SOFTMAX:
+            return "Softmax";
+        case LAYER_ROUTE:
+            return "Route";
+        case LAYER_REORG:
+            return "Reorg";
+        case LAYER_SHORTCUT:
+            return "Shortcut";
+        case LAYER_UPSAMPLE:
+            return "Upsample";
+        case LAYER_REGION:
+            return "Region";
+        case LAYER_YOLO:
+            return "Yolo";
+        default:
+            return "unknown";
         }
     }
 
 protected:
     Network *net;
     cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc;
-
 };
 
-
 /**
     Father class of all layer that need to load trained weights
 */
-class LayerWgs : public Layer {
+class LayerWgs : public Layer
+{
 
 public:
     LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt,
-             const char* fname_weights, bool batchnorm = false); 
+             const char *fname_weights, bool batchnorm = false);
     virtual ~LayerWgs();
 
     int inputs, outputs;
@@ -101,25 +122,25 @@ public:
     __half *variance16_h, *variance16_d;
 };
 
-
 /**
     Dense (full interconnection) layer
 */
-class Dense : public LayerWgs {
+class Dense : public LayerWgs
+{
 
 public:
-    Dense(Network *net, int out_ch, const char* fname_weights); 
+    Dense(Network *net, int out_ch, const char *fname_weights);
     virtual ~Dense();
     virtual layerType_t getLayerType() { return LAYER_DENSE; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 };
 
-
 /**
     Avaible activation functions
 */
-typedef enum {
+typedef enum
+{
     ACTIVATION_ELU = 100,
     ACTIVATION_LEAKY = 101
 } tkdnnActivationMode_t;
@@ -127,7 +148,8 @@ typedef enum {
 /**
     Activation layer (it doesnt need weigths)
 */
-class Activation : public Layer {
+class Activation : public Layer
+{
 
 public:
     int act_mode;
@@ -136,26 +158,26 @@ public:
     virtual ~Activation();
     virtual layerType_t getLayerType() { return LAYER_ACTIVATION; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
 protected:
     cudnnActivationDescriptor_t activDesc;
 };
 
-
 /**
     Convolutional 2D layer
 */
-class Conv2d : public LayerWgs {
+class Conv2d : public LayerWgs
+{
 
 public:
-    Conv2d( Network *net, int out_ch, int kernelH, int kernelW, 
+    Conv2d(Network *net, int out_ch, int kernelH, int kernelW,
            int strideH, int strideW, int paddingH, int paddingW,
-                const char* fname_weights, bool batchnorm = false); 
+           const char *fname_weights, bool batchnorm = false);
     virtual ~Conv2d();
     virtual layerType_t getLayerType() { return LAYER_CONV2D; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
     int kernelH, kernelW, strideH, strideW, paddingH, paddingW;
 
@@ -165,50 +187,49 @@ protected:
     cudnnConvolutionFwdAlgo_t algo;
     cudnnTensorDescriptor_t biasTensorDesc;
 
-    void*  workSpace;
+    void *workSpace;
     size_t ws_sizeInBytes;
 };
 
-
 /**
     Flatten layer
     is actually a matrix transposition
 */
-class Flatten : public Layer {
+class Flatten : public Layer
+{
 
 public:
     Flatten(Network *net);
     virtual ~Flatten();
     virtual layerType_t getLayerType() { return LAYER_FLATTEN; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 };
 
-
 /**
     MulAdd layer
     apply a multiplication and then an addition for each data
 */
-class MulAdd : public Layer {
+class MulAdd : public Layer
+{
 
 public:
     MulAdd(Network *net, dnnType mul, dnnType add);
     virtual ~MulAdd();
     virtual layerType_t getLayerType() { return LAYER_MULADD; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
 protected:
     dnnType mul, add;
     dnnType *add_vector;
 };
 
-
-
 /**
     Avaible pooling functions (padding on tkDNN is not supported)
 */
-typedef enum {
+typedef enum
+{
     POOLING_MAX = 0,
     POOLING_AVERAGE = 1,                // count for average includes padded values
     POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values
@@ -218,7 +239,8 @@ typedef enum {
     Pooling layer
     currenty supported only 2d pooing (also on 3d input)
 */
-class Pooling : public Layer {
+class Pooling : public Layer
+{
 
 public:
     int winH, winW;
@@ -230,10 +252,9 @@ public:
     virtual ~Pooling();
     virtual layerType_t getLayerType() { return LAYER_POOLING; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
 protected:
-
     cudnnPoolingDescriptor_t poolingDesc;
     tkdnnPoolingMode_t pool_mode;
     dnnType *tmpInputData, *tmpOutputData;
@@ -243,47 +264,49 @@ protected:
 /**
     Softmax layer
 */
-class Softmax : public Layer {
+class Softmax : public Layer
+{
 
 public:
     Softmax(Network *net);
     virtual ~Softmax();
     virtual layerType_t getLayerType() { return LAYER_SOFTMAX; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 };
 
 /**
     Route layer
     Merge a list of layers
 */
-class Route : public Layer {
+class Route : public Layer
+{
 
 public:
     Route(Network *net, Layer **layers, int layers_n);
     virtual ~Route();
     virtual layerType_t getLayerType() { return LAYER_ROUTE; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
 public:
     Layer **layers; //ids of layers to be merged
     int layers_n;   //number of layers
 };
 
-
 /**
     Reorg layer
     Mantain same dimension but change C*H*W distribution
 */
-class Reorg : public Layer {
+class Reorg : public Layer
+{
 
 public:
     Reorg(Network *net, int stride);
     virtual ~Reorg();
     virtual layerType_t getLayerType() { return LAYER_REORG; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
     int stride;
 };
@@ -292,14 +315,15 @@ public:
     Shortcut layer
     sum with stride another layer
 */
-class Shortcut : public Layer {
+class Shortcut : public Layer
+{
 
 public:
     Shortcut(Network *net, Layer *backLayer);
     virtual ~Shortcut();
     virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
 public:
     Layer *backLayer;
@@ -309,25 +333,28 @@ public:
     Upsample layer
     Mantain same dimension but change C*H*W distribution
 */
-class Upsample : public Layer {
+class Upsample : public Layer
+{
 
 public:
     Upsample(Network *net, int stride);
     virtual ~Upsample();
     virtual layerType_t getLayerType() { return LAYER_UPSAMPLE; };
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 
     int stride;
     bool reverse;
 };
 
-struct box {
+struct box
+{
     int cl;
     float x, y, w, h;
     float prob;
 };
-struct sortable_bbox {
+struct sortable_bbox
+{
     int index;
     int cl;
     float **probs;
@@ -336,14 +363,17 @@ struct sortable_bbox {
 /**
     Yolo3 layer
 */
-class Yolo : public Layer {
+class Yolo : public Layer
+{
 
 public:
-    struct box {
+    struct box
+    {
         float x, y, w, h;
     };
 
-    struct detection{
+    struct detection
+    {
         Yolo::box bbox;
         int classes;
         float *prob;
@@ -352,7 +382,7 @@ public:
         int sort_class;
     };
 
-    Yolo(Network *net, int classes, int num, const char* fname_weights);
+    Yolo(Network *net, int classes, int num, const char *fname_weights);
     virtual ~Yolo();
     virtual layerType_t getLayerType() { return LAYER_YOLO; };
 
@@ -360,7 +390,7 @@ public:
     dnnType *mask_h, *mask_d; //anchors
     dnnType *bias_h, *bias_d; //anchors
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
     int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh);
 
     dnnType *predictions;
@@ -373,7 +403,8 @@ public:
 /**
     Region layer
 */
-class Region : public Layer {
+class Region : public Layer
+{
 
 public:
     Region(Network *net, int classes, int coords, int num);
@@ -382,14 +413,15 @@ public:
 
     int classes, coords, num;
 
-    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    virtual dnnType *infer(dataDim_t &dim, dnnType *srcData);
 };
 
-class RegionInterpret {
+class RegionInterpret
+{
 
 public:
     RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
-                    int classes, int coords, int num, float thresh, const char* fname_weights);
+                    int classes, int coords, int num, float thresh, const char *fname_weights);
     ~RegionInterpret();
 
     dataDim_t input_dim, output_dim;
@@ -397,7 +429,6 @@ public:
     int classes, coords, num;
     float thresh;
 
-    
     box *boxes;
     float **probs;
     sortable_bbox *s;
@@ -405,7 +436,7 @@ public:
     int res_boxes_n;
 
     box get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride);
-    void get_region_boxes(  float *input, int w, int h, int netw, int neth, float thresh, 
+    void get_region_boxes(float *input, int w, int h, int netw, int neth, float thresh,
                           float **probs, box *boxes, int only_objectness,
                           int *map, float tree_thresh, int relative);
     void correct_region_boxes(box *boxes, int n, int w, int h, int netw, int neth, int relative);
@@ -415,5 +446,6 @@ public:
     static float box_iou(box a, box b);
 };
 
-}}
+} // namespace dnn
+} // namespace tk
 #endif //LAYER_H

include/Network.h

@@ -3,7 +3,10 @@
 
 #include "utils.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
 /**
     Data rapresentation beetween layers
@@ -13,28 +16,31 @@ namespace tk { namespace dnn {
     w = width  (rows)
     l = lenght (3rd dimension)
 */
-struct dataDim_t {
+struct dataDim_t
+{
 
     int n, c, h, w, l;
 
-    dataDim_t() : n(1), c(1), h(1), w(1), l(1) {};
+    dataDim_t() : n(1), c(1), h(1), w(1), l(1){};
 
-    dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) :
+    dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) : n(_n), c(_c), h(_h), w(_w), l(_l){};
-        n(_n), c(_c), h(_h), w(_w), l(_l) {};
 
-    void print() {
+    void print()
-        std::cout<<"Data dim: "<<n<<" "<<c<<" "<<h<<" "<<w<<" "<<l<<"\n";
+    {
+        std::cout << "Data dim: " << n << " " << c << " " << h << " " << w << " " << l << "\n";
     }
 
-    int tot() {
+    int tot()
-        return n*c*h*w*l;
+    {
+        return n * c * h * w * l;
     }
 };
 
 class Layer;
 const int MAX_LAYERS = 256;
 
-class Network {
+class Network
+{
 
 public:
     Network(dataDim_t input_dim);
@@ -43,7 +49,7 @@ public:
     /**
         Do inferece for every added layer
     */
-    dnnType* infer(dataDim_t &dim, dnnType* data);
+    dnnType *infer(dataDim_t &dim, dnnType *data);
 
     bool addLayer(Layer *l);
     void print();
@@ -53,7 +59,7 @@ public:
     cudnnHandle_t cudnnHandle;
     cublasHandle_t cublasHandle;
 
-    Layer* layers[MAX_LAYERS];  //contains layers of the net
+    Layer *layers[MAX_LAYERS]; //contains layers of the net
     int num_layers;            //current number of layers
 
     dataDim_t input_dim;
@@ -62,5 +68,6 @@ public:
     bool fp16, dla;
 };
 
-}}
+} // namespace dnn
+} // namespace tk
 #endif //NETWORK_H

include/NetworkRT.h

@@ -7,17 +7,22 @@
 #include "Layer.h"
 #include "NvInfer.h"
 
-namespace tk { namespace dnn {
+namespace tk
-
-template<typename T> void writeBUF(char*& buffer, const T& val)
 {
-    *reinterpret_cast<T*>(buffer) = val;
+namespace dnn
+{
+
+template <typename T>
+void writeBUF(char *&buffer, const T &val)
+{
+    *reinterpret_cast<T *>(buffer) = val;
     buffer += sizeof(T);
 }
 
-template<typename T> T readBUF(const char*& buffer)
+template <typename T>
+T readBUF(const char *&buffer)
 {
-    T val = *reinterpret_cast<const T*>(buffer);
+    T val = *reinterpret_cast<const T *>(buffer);
     buffer += sizeof(T);
     return val;
 }
@@ -38,12 +43,11 @@ public:
     YoloRT *yolos[16];
     int n_yolos;
 
-	virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength);
+    virtual IPlugin *createPlugin(const char *layerName, const void *serialData, size_t serialLength);
 };
 
-
+class NetworkRT
-
+{
-class NetworkRT {
 
 public:
     nvinfer1::DataType dtRT;
@@ -55,7 +59,7 @@ public:
     nvinfer1::IExecutionContext *contextRT;
 
     const static int MAX_BUFFERS_RT = 10;
-    void* buffersRT[MAX_BUFFERS_RT];
+    void *buffersRT[MAX_BUFFERS_RT];
     int buf_input_idx, buf_output_idx;
 
     dataDim_t input_dim, output_dim;
@@ -70,25 +74,26 @@ public:
     /**
         Do inferece
     */
-    dnnType* infer(dataDim_t &dim, dnnType* data);
+    dnnType *infer(dataDim_t &dim, dnnType *data);
     void enqueue();
 
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Layer *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Conv2d *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Activation *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Dense *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Pooling *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Softmax *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Route *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Reorg *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Region *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Shortcut *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Yolo *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
+    nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Upsample *l);
 
     bool serialize(const char *filename);
     bool deserialize(const char *filename);
 };
 
-}}
+} // namespace dnn
+} // namespace tk
 #endif //NETWORKRT_H

include/Yolo3Detection.h

@@ -1,3 +1,6 @@
+#ifndef YOLO3DDETECTION_H
+#define YOLO3DDETECTION_H
+
 #include <iostream>
 #include <signal.h>
 #include <stdlib.h> /* srand, rand */
@@ -11,17 +14,21 @@
 
 #include "tkdnn.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
 /**
  * 
  * @author Francesco Gatti
  */
-class Yolo3Detection {
+class Yolo3Detection
+{
 
-    private:
+private:
     tk::dnn::NetworkRT *netRT = nullptr;
-        tk::dnn::Yolo* yolo[3];
+    tk::dnn::Yolo *yolo[3];
     dnnType *input, *input_d;
 
     int ndets = 0;
@@ -30,9 +37,7 @@ class Yolo3Detection {
     cv::Mat imageF;
     cv::Mat bgr[3];
 
-        
+public:
-
-    public:
     int classes = 0;
     int num = 0;
     float thresh = 0.3;
@@ -46,14 +51,16 @@ class Yolo3Detection {
     virtual ~Yolo3Detection() {}
 
     /**
-         * Method used for inizialize the class
+         * Method used to inizialize the class
          * 
          * @return Success of the initialization
          */
     bool init(std::string tensor_path);
     void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left);
     void update(cv::Mat &frame);
-
 };
 
-}}
+} // namespace dnn
+} // namespace tk
+
+#endif /*YOLO3DDETECTION_H*/

include/calibration.h

@@ -0,0 +1,32 @@
+#ifndef CALIBRATION_H
+#define CALIBRATION_H
+
+#include "gdal.h"
+#include <gdal_priv.h>
+#include <gdal/gdal.h>
+#include "gdal/gdal_priv.h"
+#include "gdal/cpl_conv.h"
+
+#include <yaml-cpp/yaml.h>
+#include <opencv2/calib3d.hpp>
+#include <opencv2/core.hpp>
+
+#include <iostream>
+#include <cstring>
+
+struct ObjCoords
+{
+    double lat_;
+    double long_;
+    int class_;
+};
+
+void readTiff(char *filename, double *adfGeoTransform);
+void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff);
+void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform);
+void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform);
+void fillMatrix(cv::Mat &H, double *matrix, bool show = false);
+void read_projection_matrix(cv::Mat &H, char *path);
+void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform);
+
+#endif /*CALIBRATION_H*/

include/cameraUtils.h

@@ -0,0 +1,45 @@
+#ifndef CAMERAUTILS_H
+#define CAMERAUTILS_H
+
+#include <vector>
+#include <mutex>
+#include <opencv2/core/core.hpp>
+#include "tracker.h"
+#include "Yolo3Detection.h"
+
+struct Camera_t
+{
+    int CAM_IDX;
+    char *input;
+    char *pmatrix;
+    char *maskfile;
+    char *cameraCalib;
+    char *maskFileOrient;
+    bool to_show;
+    tk::dnn::Yolo3Detection yolo;
+    double adfGeoTransform[6];
+};
+
+struct Frame_t
+{
+    char *input;
+    cv::Mat frame;
+    int frame_nbr;
+    // sem_vc for mainthread, videocapturethread, originalthread and disparitythread
+    std::mutex sem_vc;
+};
+
+struct ModFrame_t
+{
+    std::vector<Tracker> trackers;
+    geodetic_converter::GeodeticConverter gc;
+    double adfGeoTransform[6];
+    cv::Mat H;
+    cv::Mat original_frame;
+    tk::dnn::Yolo3Detection yolo;
+    cv::Mat mask;
+    // sem for mainthread, detectionthread and topviewthread
+    std::mutex sem;
+};
+
+#endif /*CAMERAUTILS_H*/

include/classutils.h

@@ -1,316 +0,0 @@
-#ifndef CLASSUTILS_H
-#define CLASSUTILS_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/socket.h> //socket
-#include <arpa/inet.h>  //inet_addr
-#include <unistd.h>     //write
-
-#include <opencv2/calib3d.hpp>
-#include <opencv2/core.hpp>
-#include "gdal.h"
-#include <gdal_priv.h>
-#include <gdal/gdal.h>
-#include "gdal/gdal_priv.h"
-#include "gdal/cpl_conv.h"
-
-#include "tracker.h"
-
-#include <yaml-cpp/yaml.h>
-
-#include "../masa_protocol/include/send.hpp"
-#include "../masa_protocol/include/serialize.hpp"
-
-struct ObjCoords
-{
-    double lat_;
-    double long_;
-    int class_;
-};
-
-void readTiff(char *filename, double *adfGeoTransform)
-{
-    GDALDataset *poDataset;
-    GDALAllRegister();
-    poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly);
-    if (poDataset != NULL)
-    {
-        poDataset->GetGeoTransform(adfGeoTransform);
-    }
-}
-
-void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff)
-{
-    YAML::Node config = YAML::LoadFile(cameraCalib);
-    const YAML::Node &node_test1 = config["camera_matrix"];
-
-    float data_cm[9];
-    for (std::size_t i = 0; i < node_test1["data"].size(); i++)
-        data_cm[i] = node_test1["data"][i].as<float>();
-    cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm);
-    cameraMat = cameraMat_.clone();
-    std::cout << cameraMat << std::endl;
-    const YAML::Node &node_test2 = config["distortion_coefficients"];
-
-    float data_dc[5];
-    for (std::size_t i = 0; i < node_test2["data"].size(); i++)
-        data_dc[i] = node_test2["data"][i].as<float>();
-    cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc);
-    distCoeff = distCoeff_.clone();
-    std::cout << distCoeff << std::endl;
-}
-
-void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform)
-{
-    //Returns global coordinates from pixel x, y coordinates
-    double xoff, a, b, yoff, d, e;
-    xoff = adfGeoTransform[0];
-    a = adfGeoTransform[1];
-    b = adfGeoTransform[2];
-    yoff = adfGeoTransform[3];
-    d = adfGeoTransform[4];
-    e = adfGeoTransform[5];
-
-    //printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e );
-
-    lon = a * x + b * y + xoff;
-    lat = d * x + e * y + yoff;
-}
-void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform)
-{
-    x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1]));
-    y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5]));
-}
-
-void fillMatrix(cv::Mat &H, double *matrix, bool show = false)
-{
-    double *vals = (double *)H.data;
-    for (int i = 0; i < 9; i++)
-    {
-        vals[i] = matrix[i];
-    }
-    if (show)
-        std::cout << H << "\n";
-}
-
-//FILE *out_file = fopen("prova_pixel.txt", "w");
-
-void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform, int frame_nbr)
-{
-    double latitude, longitude;
-    std::vector<cv::Point2f> x_y, ll;
-    x_y.push_back(cv::Point2f(x, y));
-    //transform camera pixel to map pixel
-    cv::perspectiveTransform(x_y, ll, H);
-    //tranform to map pixel to map gps
-    pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform);
-    //printf("lat: %f, long:%f \n", latitude, longitude);
-
-    ObjCoords coord;
-    coord.lat_ = latitude;
-    coord.long_ = longitude;
-    coord.class_ = detected_class;
-    coords.push_back(coord);
-
-    /*if (detected_class == 0)
-    {
-
-        struct timeval tv;
-        gettimeofday(&tv, NULL);
-        unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
-
-        //printf(out_file, "%d %lld %d %d\n",frame_nbr, t_stamp_ms, int(ll[0].x), int(ll[0].y));
-        fprintf(out_file, "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG);
-        //printf( "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG);
-    }*/
-}
-
-void read_projection_matrix(cv::Mat &H, char *path)
-{
-    FILE *fp;
-    char *line = NULL;
-    size_t len = 0;
-    ssize_t read;
-
-    // float *proj_matrix = (float *)malloc(9 * sizeof(float));
-    double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
-    int i = 0;
-    fp = fopen(path, "r");
-    if (fp == NULL)
-        exit(EXIT_FAILURE);
-        
-    while ((read = getline(&line, &len, fp)) != -1)
-    {
-        std::cout<<line<<std::endl;
-        std::stringstream ss(line);
-        while (ss >> proj_matrix[i])
-            i++;
-    }
-    fclose(fp);
-    fillMatrix(H, proj_matrix);
-    free(line);
-    // free(proj_matrix);
-}
-
-void draw_arrow(float angleRad, float vel, cv::Scalar color, cv::Point center, cv::Mat &frame)
-{
-    int angle = angleRad * 180.0 / CV_PI;
-    auto length = 10 * vel;
-    auto direction = cv::Point(length * cos(angleRad), length * sin(angleRad)); // calculate direction
-    double tipLength = .2 + 0.4 * (angle % 180) / 360;
-    int lineType = 8;
-    int thickness = 2;
-    cv::arrowedLine(frame, center, center + direction, color, thickness, lineType, 0, tipLength); // draw arrow!
-}
-
-unsigned long long time_in_ms()
-{
-    struct timeval tv;
-    gettimeofday(&tv, NULL);
-    unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
-    return t_stamp_ms;
-}
-
-void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat& maskOrient, double *adfGeoTransform, cv::Mat H)
-{
-    m->t_stamp_ms = time_in_ms();
-    m->objects.clear();
-    double lat, lon, alt;
-
-    for (auto t : trackers)
-    {
-        if (t.pred_list_.size() > 0)
-        {
-            Categories cat;
-            switch (t.class_)
-            {
-            case 0:
-                cat = Categories::C_person;
-                break;
-            case 1:
-                cat = Categories::C_car;
-                break;
-            case 2:
-                cat = Categories::C_car;
-                break;
-            case 3:
-                cat = Categories::C_bus;
-                break;
-            case 4:
-                cat = Categories::C_motorbike;
-                break;
-            case 5:
-                cat = Categories::C_bycicle;
-                break;
-            }
-            //std::cout << t.pred_list_.size() << std::endl;
-            gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt);
-
-            int pix_x, pix_y;
-            coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
-
-            // TODO: test correctness - added perspective transform call to converter pix_x and pix_y
-            // sometimes some values are wrong. float ok? 
-            // std::vector<cv::Point2f> map_p, camera_p;
-            // std::cout<<"--- pix_x, pix_y: "<<pix_x<<", "<<pix_y<<std::endl;
-            // map_p.push_back(cv::Point2f(pix_x, pix_y));
-            // std::cout<<"map_p: "<<map_p<<std::endl;
-            // //transform camera pixel to map pixel
-            // cv::perspectiveTransform(map_p, camera_p, H.inv());
-            // std::cout<<"size H: "<<H.cols<<", "<<H.rows<<std::endl;
-            // std::cout<<"camera_p: "<<camera_p<<std::endl;    
-            // // TODO: in some cases these lines causes seg fault!
-            // std::cout<<"y, x :"<<camera_p[0].y<<",  "<<camera_p[0].x<<std::endl;
-            // std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
-            // // std::cout<<"vec3b: "<<(cv::Vec3b)(pix_y,pix_x);
-            // assert (camera_p[0].x < maskOrient.cols);
-            // assert (camera_p[0].y < maskOrient.rows);
-            // uint8_t maskOrientPixel = maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)[0]; 
-            // std::cout<<"boo: "<<maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)<<std::endl;
-            // uint8_t orientation;
-            // if(maskOrientPixel != 0)
-            // {
-            //     orientation = maskOrientPixel;
-            //     // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
-            // }
-            // else
-            // {
-            //     orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
-            //     //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
-            // }
-
-            // TODO: to validate -> it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);")
-            // TODO: include perspective transform
-            // std::cout<<"y, x :"<<pix_y<<", "<<pix_x<<std::endl;
-            // std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
-            // std::cout<<"point: "<<(cv::Point)(pix_y,pix_x);
-            // uint8_t maskOrientPixel = maskOrient.at<uchar>(pix_y,pix_x);
-            // uint8_t orientation;
-            // if(maskOrientPixel != 0)
-            // {
-            //     orientation = maskOrientPixel;
-            //     // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
-            // }
-            // else
-            // {
-            //     orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
-            //     //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
-            // }
-
-            uint8_t orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
-            // std::cout<<"orient: "<<unsigned(orientation)<<std::endl;
-            //std::cout << "lat: " << lat << " lon: " << lon << std::endl;
-            uint8_t velocity = uint8_t(std::abs(t.pred_list_.back().vel_ * 3.6 / 2));
-            // std::cout<<"vel: "<<unsigned(velocity)<<std::endl;
-            RoadUser r{static_cast<float>(lat), static_cast<float>(lon), velocity, orientation, cat};
-            //std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl;
-            m->objects.push_back(r);
-        }
-    }
-    m->num_objects = m->objects.size();
-}
-
-void prepare_message(Message *m, const std::vector<ObjCoords> &coords, int idx)
-{
-    m->cam_idx = idx;
-    m->t_stamp_ms = time_in_ms();
-    m->num_objects = coords.size();
-
-    m->objects.clear();
-    for (unsigned int i = 0; i < coords.size(); i++)
-    {
-        Categories cat;
-
-        switch (coords[i].class_)
-        {
-        case 0:
-            cat = Categories::C_person;
-            break;
-        case 1:
-            cat = Categories::C_car;
-            break;
-        case 2:
-            cat = Categories::C_car;
-            break;
-        case 3:
-            cat = Categories::C_bus;
-            break;
-        case 4:
-            cat = Categories::C_motorbike;
-            break;
-        case 5:
-            cat = Categories::C_bycicle;
-            break;
-        }
-        RoadUser r{static_cast<float>(coords[i].lat_), static_cast<float>(coords[i].long_), 0, 1, cat};
-        std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << cat << std::endl;
-        m->objects.push_back(r);
-    }
-
-    m->lights.clear();
-}
-
-#endif /*CLASSUTILS_H*/

include/message.h

@@ -0,0 +1,22 @@
+#ifndef MESSAGE_H
+#define MESSAGE_H
+
+#include <iostream>
+#include <cstdlib>
+#include <ctime>
+#include <opencv2/calib3d.hpp>
+#include <opencv2/core.hpp>
+// #include <sys/socket.h> //socket
+// #include <arpa/inet.h>  //inet_addr
+// #include <unistd.h>     //write
+
+#include "tracker.h"
+
+#include "../masa_protocol/include/send.hpp"
+#include "../masa_protocol/include/serialize.hpp"
+
+unsigned long long time_in_ms();
+
+void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H);
+
+#endif /*MESSAGE_H*/

include/utils.h

@@ -32,13 +32,17 @@
 #define COL_CYANB "\033[1;36m"
 
 // Simple Timer
-#define TIMER_START timespec start, end;                               \
+#define TIMER_START    \
+  timespec start, end; \
   clock_gettime(CLOCK_MONOTONIC, &start);
 
-#define TIMER_STOP_C(col)  clock_gettime(CLOCK_MONOTONIC, &end);       \
+#define TIMER_STOP_C(col)                                         \
+  clock_gettime(CLOCK_MONOTONIC, &end);                           \
   double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 +    \
-                  (double)(end.tv_nsec - start.tv_nsec))/1.0e6;        \
+                 (double)(end.tv_nsec - start.tv_nsec)) /         \
-    std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END; 
+                1.0e6;                                            \
+  std::cout << col << "Time:" << std::setw(16) << t_ns << " ms\n" \
+            << COL_END;
 
 #define TIMER_STOP TIMER_STOP_C(COL_CYANB)
 
@@ -47,56 +51,66 @@
  * ******************************************************/
 #define EXIT_WAIVED 0
 
-#define FatalError(s) {                                                \
+#define FatalError(s)                                 \
+  {                                                   \
     std::stringstream _where, _message;               \
     _where << __FILE__ << ':' << __LINE__;            \
-    _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__;\
+    _message << std::string(s) + "\n"                 \
+             << __FILE__ << ':' << __LINE__;          \
     std::cerr << _message.str() << "\nAborting...\n"; \
     cudaDeviceReset();                                \
     exit(EXIT_FAILURE);                               \
-}
+  }
 
-#define checkCUDNN(status) {                                           \
+#define checkCUDNN(status)                                        \
+  {                                                               \
     std::stringstream _error;                                     \
-    if (status != CUDNN_STATUS_SUCCESS) {                              \
+    if (status != CUDNN_STATUS_SUCCESS)                           \
-      _error << "CUDNN failure: " <<cudnnGetErrorString(status);       \
+    {                                                             \
+      _error << "CUDNN failure: " << cudnnGetErrorString(status); \
       FatalError(_error.str());                                   \
     }                                                             \
-}
+  }
 
-#define checkCuda(status) {                                            \
+#define checkCuda(status)                                       \
+  {                                                             \
     std::stringstream _error;                                   \
-    if (status != 0) {                                                 \
+    if (status != 0)                                            \
-      _error << "Cuda failure: "<<cudaGetErrorString(status);          \
+    {                                                           \
+      _error << "Cuda failure: " << cudaGetErrorString(status); \
       FatalError(_error.str());                                 \
     }                                                           \
-}
+  }
 
-#define checkERROR(status) {                                           \
+#define checkERROR(status)                     \
+  {                                            \
     std::stringstream _error;                  \
-    if (status != 0) {                                                 \
+    if (status != 0)                           \
+    {                                          \
       _error << "Generic failure: " << status; \
       FatalError(_error.str());                \
     }                                          \
-}
+  }
 
-#define checkNULL(ptr) {                                               \
+#define checkNULL(ptr)          \
+  {                             \
     std::stringstream _error;   \
-    if (ptr == nullptr) {                                              \
+    if (ptr == nullptr)         \
+    {                           \
       _error << "Null pointer"; \
       FatalError(_error.str()); \
     }                           \
-}
+  }
 
 void printCenteredTitle(const char *title, char fill, int dim);
 bool fileExist(const char *fname);
-void readBinaryFile(const char* fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
+void readBinaryFile(const char *fname, int size, dnnType **data_h, dnnType **data_d, int seek = 0);
 int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true);
-void printDeviceVector(int size, dnnType* vec_d, bool device = true);
+void printDeviceVector(int size, dnnType *vec_d, bool device = true);
 void resize(int size, dnnType **data);
 
-void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols);
+void matrixTranspose(cublasHandle_t handle, dnnType *srcData, dnnType *dstData, int rows, int cols);
 
-void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData, 
+void matrixMulAdd(cublasHandle_t handle, dnnType *srcData, dnnType *dstData,
-                    dnnType* add_vector, int dim, dnnType mul);
+                  dnnType *add_vector, int dim, dnnType mul);
 #endif //UTILS_H

include/visualization.h

@@ -0,0 +1,42 @@
+#ifndef VIZUALIZATION_H
+#define VIZUALIZATION_H
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+//saliency
+#include <opencv2/core/utility.hpp>
+#include <opencv2/saliency.hpp>
+#include <opencv2/highgui.hpp>
+
+#include <chrono>
+#include <iostream>
+#include <cstring>
+
+#include "tracker.h"
+#include "cameraUtils.h"
+#include "calibration.h"
+#include "boxDetection.h"
+
+struct Show_t
+{
+    cv::Mat original, detection, topview, disparity;
+    bool update_o, update_de, update_t, update_di;
+    // a single mutex for each operation - the show_updates function must get all mutex
+    std::mutex mutex_o, mutex_de, mutex_t, mutex_di;
+};
+
+extern Show_t updates;
+extern bool gRun;
+extern std::string obj_class[10];
+
+/* Thread function to show the updated images 
+**/
+void *show_updates(void *x_void_ptr);
+void *originalFrame(void *x_void_ptr);
+void *detectionFrame(void *x_void_ptr);
+void *topviewFrame(void *x_void_ptr);
+void *disparityFrame(void *x_void_ptr);
+
+#endif /*VIZUALIZATION_H*/

src/Activation.cpp

@@ -3,64 +3,72 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-Activation::Activation(Network *net, int act_mode) : 
+Activation::Activation(Network *net, int act_mode) : Layer(net)
-    Layer(net) {
+{
 
     this->act_mode = act_mode;
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
+    checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType)));
 
-    if(int(act_mode) < 100) {
+    if (int(act_mode) < 100)
+    {
 
-        checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
+        checkCUDNN(cudnnSetTensor4dDescriptor(srcTensorDesc,
                                               net->tensorFormat,
                                               net->dataType,
-                                            input_dim.n*input_dim.l, 
+                                              input_dim.n * input_dim.l,
                                               input_dim.c,
-                                            input_dim.h, input_dim.w) );
+                                              input_dim.h, input_dim.w));
-        checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
+        checkCUDNN(cudnnSetTensor4dDescriptor(dstTensorDesc,
                                               net->tensorFormat,
                                               net->dataType,
-                                                input_dim.n*input_dim.l, 
+                                              input_dim.n * input_dim.l,
                                               input_dim.c,
-                                                input_dim.h, input_dim.w) );
+                                              input_dim.h, input_dim.w));
 
-
+        checkCUDNN(cudnnCreateActivationDescriptor(&activDesc));
-        checkCUDNN( cudnnCreateActivationDescriptor(&activDesc) );
+        checkCUDNN(cudnnSetActivationDescriptor(activDesc,
-        checkCUDNN( cudnnSetActivationDescriptor(activDesc,
+                                                (cudnnActivationMode_t)act_mode,
-                                                (cudnnActivationMode_t) act_mode,
                                                 CUDNN_PROPAGATE_NAN,
-                                                0.0) );
+                                                0.0));
     }
 }
 
-Activation::~Activation() {
+Activation::~Activation()
+{
 
-    checkCuda( cudaFree(dstData) );
+    checkCuda(cudaFree(dstData));
 
-    if(int(act_mode) < 100)
+    if (int(act_mode) < 100)
-        checkCUDNN( cudnnDestroyActivationDescriptor(activDesc) );
+        checkCUDNN(cudnnDestroyActivationDescriptor(activDesc));
 }
 
-dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
+dnnType *Activation::infer(dataDim_t &dim, dnnType *srcData)
+{
 
-    if(act_mode == ACTIVATION_LEAKY) {
+    if (act_mode == ACTIVATION_LEAKY)
+    {
         activationLEAKYForward(srcData, dstData, dim.tot());
-
+    }
-    } else {
+    else
+    {
         dnnType alpha = dnnType(1);
         dnnType beta = dnnType(0);
-        checkCUDNN( cudnnActivationForward(net->cudnnHandle,
+        checkCUDNN(cudnnActivationForward(net->cudnnHandle,
                                           activDesc,
                                           &alpha,
                                           srcTensorDesc,
                                           srcData,
                                           &beta,
                                           dstTensorDesc,
-                                            dstData) );   
+                                          dstData));
     }
     return dstData;
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/Conv2d.cpp

@@ -2,14 +2,18 @@
 
 #include "Layer.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, 
+Conv2d::Conv2d(Network *net, int out_ch, int kernelH, int kernelW,
                int strideH, int strideW, int paddingH, int paddingW,
-                const char* fname_weights, bool batchnorm) : 
+               const char *fname_weights, bool batchnorm) :
 
                 LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1,
-             fname_weights, batchnorm) {
+                            fname_weights, batchnorm)
+{
 
     this->kernelH = kernelH;
     this->kernelW = kernelW;
@@ -18,56 +22,55 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
     this->paddingH = paddingH;
     this->paddingW = paddingW;
 
-    checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) );
+    checkCUDNN(cudnnCreateFilterDescriptor(&filterDesc));
-    checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) );
+    checkCUDNN(cudnnCreateConvolutionDescriptor(&convDesc));
-    checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) );
+    checkCUDNN(cudnnCreateTensorDescriptor(&biasTensorDesc));
 
     int n = input_dim.n;
     int c = input_dim.c;
     int h = input_dim.h;
     int w = input_dim.w;
 
-    checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
+    checkCUDNN(cudnnSetTensor4dDescriptor(srcTensorDesc,
-                net->tensorFormat, net->dataType, n, c, h, w) );
+                                          net->tensorFormat, net->dataType, n, c, h, w));
 
-    checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc,
+    checkCUDNN(cudnnSetFilter4dDescriptor(filterDesc,
                                           net->dataType, net->tensorFormat, out_ch, input_dim.c,
-                kernelH, kernelW) );
+                                          kernelH, kernelW));
 
-    checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc,
+    checkCUDNN(cudnnSetConvolution2dDescriptor(convDesc,
                                                paddingH, paddingW, // padding
                                                strideH, strideW,   // stride
-                1,1, // upscale
+                                               1, 1,               // upscale
-                CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) );
+                                               CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT));
 
     // find dimension of convolution output
-    checkCUDNN( cudnnGetConvolution2dForwardOutputDim(
+    checkCUDNN(cudnnGetConvolution2dForwardOutputDim(
         convDesc, srcTensorDesc, filterDesc,
-                &n, &c, &h, &w) );
+        &n, &c, &h, &w));
 
-    checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
+    checkCUDNN(cudnnSetTensor4dDescriptor(dstTensorDesc,
-                net->tensorFormat, net->dataType, n, c, h, w) );
+                                          net->tensorFormat, net->dataType, n, c, h, w));
 
-    checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle,
+    checkCUDNN(cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle,
                                                    srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
-                CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) );
+                                                   CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo));
 
     workSpace = NULL;
     ws_sizeInBytes = 0;
 
-    checkCUDNN( cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
+    checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
                                                        srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
-                algo, &ws_sizeInBytes) );
+                                                       algo, &ws_sizeInBytes));
 
-    if (ws_sizeInBytes!=0) {
+    if (ws_sizeInBytes != 0)
-        checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) );
+    {
+        checkCuda(cudaMalloc(&workSpace, ws_sizeInBytes));
     }
 
-
+    checkCUDNN(cudnnSetTensor4dDescriptor(biasTensorDesc,
-    checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc,
                                           net->tensorFormat, net->dataType,
-                1, out_ch, 1, 1) );
+                                          1, out_ch, 1, 1));
-
 
     output_dim.n = n;
     output_dim.c = c;
@@ -76,40 +79,44 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
     output_dim.l = 1;
 
     //allocate data for infer result
-    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+    checkCuda(cudaMalloc(&dstData, output_dim.tot() * sizeof(dnnType)));
 }
 
-Conv2d::~Conv2d() {
+Conv2d::~Conv2d()
+{
 
-    checkCUDNN( cudnnDestroyFilterDescriptor(filterDesc) );
+    checkCUDNN(cudnnDestroyFilterDescriptor(filterDesc));
-    checkCUDNN( cudnnDestroyConvolutionDescriptor(convDesc) );
+    checkCUDNN(cudnnDestroyConvolutionDescriptor(convDesc));
-    checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) );
+    checkCUDNN(cudnnDestroyTensorDescriptor(biasTensorDesc));
 
-    if (ws_sizeInBytes!=0)
+    if (ws_sizeInBytes != 0)
-        checkCuda( cudaFree(workSpace) );
+        checkCuda(cudaFree(workSpace));
 
-    checkCuda( cudaFree(dstData) );
+    checkCuda(cudaFree(dstData));
 }
 
-dnnType* Conv2d::infer(dataDim_t &dim, dnnType* srcData) {
+dnnType *Conv2d::infer(dataDim_t &dim, dnnType *srcData)
-
+{
 
     // convolution
     dnnType alpha = dnnType(1);
     dnnType beta = dnnType(0);
-    checkCUDNN( cudnnConvolutionForward(net->cudnnHandle,
+    checkCUDNN(cudnnConvolutionForward(net->cudnnHandle,
                                        &alpha, srcTensorDesc, srcData, filterDesc,
                                        data_d, convDesc, algo, workSpace, ws_sizeInBytes,
-                &beta, dstTensorDesc, dstData) );
+                                       &beta, dstTensorDesc, dstData));
 
-    if(!batchnorm) {
+    if (!batchnorm)
+    {
         // bias
         alpha = dnnType(1);
         beta = dnnType(1);
-        checkCUDNN( cudnnAddTensor(net->cudnnHandle,
+        checkCUDNN(cudnnAddTensor(net->cudnnHandle,
                                   &alpha, biasTensorDesc, bias_d,
-                    &beta, dstTensorDesc, dstData) );
+                                  &beta, dstTensorDesc, dstData));
-    } else {
+    }
+    else
+    {
         float one = 1;
         float zero = 0;
         cudnnBatchNormalizationForwardInference(net->cudnnHandle,
@@ -125,4 +132,5 @@ dnnType* Conv2d::infer(dataDim_t &dim, dnnType* srcData) {
     return dstData;
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/Dense.cpp

@@ -2,10 +2,13 @@
 
 #include "Layer.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-Dense::Dense(Network *net, int out_ch, const char* fname_weights) : 
+Dense::Dense(Network *net, int out_ch, const char *fname_weights) : LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights)
-    LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights) {
+{
 
     output_dim.n = 1;
     output_dim.c = out_ch;
@@ -14,15 +17,17 @@ Dense::Dense(Network *net, int out_ch, const char* fname_weights) :
     output_dim.l = 1;
 
     //allocate data for infer result
-    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+    checkCuda(cudaMalloc(&dstData, output_dim.tot() * sizeof(dnnType)));
 }
 
-Dense::~Dense() {
+Dense::~Dense()
+{
 
-    checkCuda( cudaFree(dstData) );
+    checkCuda(cudaFree(dstData));
 }
 
-dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
+dnnType *Dense::infer(dataDim_t &dim, dnnType *srcData)
+{
 
     if (dim.n != 1)
         FatalError("Not Implemented");
@@ -35,16 +40,16 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
 
     dnnType alpha = dnnType(1), beta = dnnType(1);
     // place bias into dstData
-    checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
+    checkCuda(cudaMemcpy(dstData, bias_d, dim_y * sizeof(dnnType), cudaMemcpyDeviceToDevice));
 
     //do matrix moltiplication
-    checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
+    checkERROR(cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
                            dim_x, dim_y,
                            &alpha,
                            data_d, dim_x,
                            srcData, 1,
                            &beta,
-                            dstData, 1) );
+                           dstData, 1));
 
     //update data dimensions
     dim.h = 1;
@@ -55,4 +60,5 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
     return dstData;
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/Flatten.cpp

@@ -3,29 +3,34 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-Flatten::Flatten(Network *net) : Layer(net) {
+Flatten::Flatten(Network *net) : Layer(net)
+{
 
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
+    checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType)));
 
     output_dim.n = 1;
     output_dim.c = input_dim.tot();
     output_dim.h = 1;
     output_dim.w = 1;
     output_dim.l = 1;
-
 }
 
-Flatten::~Flatten() {
+Flatten::~Flatten()
+{
 
-    checkCuda( cudaFree(dstData) );
+    checkCuda(cudaFree(dstData));
 }
 
-dnnType* Flatten::infer(dataDim_t &dim, dnnType* srcData) {
+dnnType *Flatten::infer(dataDim_t &dim, dnnType *srcData)
+{
 
     //transpose per channel
-    matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h*dim.w*dim.l);
+    matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h * dim.w * dim.l);
 
     //update data dimensions
     dim = output_dim;
@@ -33,4 +38,5 @@ dnnType* Flatten::infer(dataDim_t &dim, dnnType* srcData) {
     return dstData;
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/Layer.cpp

@@ -2,28 +2,35 @@
 
 #include "Layer.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-Layer::Layer(Network *net) {
+Layer::Layer(Network *net)
+{
 
     this->net = net;
 
-    if(net != nullptr) {
+    if (net != nullptr)
+    {
         this->input_dim = net->getOutputDim();
         this->output_dim = input_dim;
 
-        checkCUDNN( cudnnCreateTensorDescriptor(&srcTensorDesc) );
+        checkCUDNN(cudnnCreateTensorDescriptor(&srcTensorDesc));
-        checkCUDNN( cudnnCreateTensorDescriptor(&dstTensorDesc) );
+        checkCUDNN(cudnnCreateTensorDescriptor(&dstTensorDesc));
 
-        if(!net->addLayer(this))
+        if (!net->addLayer(this))
             FatalError("Net reached max number of layers");
     }
 }
 
-Layer::~Layer() {
+Layer::~Layer()
+{
 
-    checkCUDNN( cudnnDestroyTensorDescriptor(srcTensorDesc) );
+    checkCUDNN(cudnnDestroyTensorDescriptor(srcTensorDesc));
-    checkCUDNN( cudnnDestroyTensorDescriptor(dstTensorDesc) );
+    checkCUDNN(cudnnDestroyTensorDescriptor(dstTensorDesc));
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/LayerWgs.cpp

@@ -4,24 +4,29 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
 LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
                    int kh, int kw, int kl,
-                   const char* fname_weights, bool batchnorm) : Layer(net) {
+                   const char *fname_weights, bool batchnorm) : Layer(net)
+{
 
     this->inputs = inputs;
     this->outputs = outputs;
     this->weights_path = std::string(fname_weights);
 
-    std::cout<<"Reading weights: I="<<inputs<<" O="<<outputs<<" KERNEL="<<kh<<"x"<<kw<<"x"<<kl<<"\n";
+    std::cout << "Reading weights: I=" << inputs << " O=" << outputs << " KERNEL=" << kh << "x" << kw << "x" << kl << "\n";
     int seek = 0;
-    readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
+    readBinaryFile(weights_path.c_str(), inputs * outputs * kh * kw * kl, &data_h, &data_d, seek);
-    seek += inputs*outputs*kh*kw*kl;
+    seek += inputs * outputs * kh * kw * kl;
     readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
 
     this->batchnorm = batchnorm;
-    if(batchnorm) {
+    if (batchnorm)
+    {
         seek += outputs;
         readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek);
         seek += outputs;
@@ -32,86 +37,90 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
         float eps = CUDNN_BN_MIN_EPSILON;
 
         power_h = new dnnType[outputs];
-        for(int i=0; i<outputs; i++) power_h[i] = 1.0f;
+        for (int i = 0; i < outputs; i++)
+            power_h[i] = 1.0f;
 
-        for(int i=0; i<outputs; i++)
+        for (int i = 0; i < outputs; i++)
             mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]);
 
-        for(int i=0; i<outputs; i++)
+        for (int i = 0; i < outputs; i++)
             variance_h[i] = 1.0f / sqrt(eps + variance_h[i]);
     }
 
-
+    if (!net->fp16)
-    if(!net->fp16)
         return;
 
     //convert to fp16
-    int w_size = inputs*outputs*kh*kw*kl;
+    int w_size = inputs * outputs * kh * kw * kl;
     data16_h = new __half[w_size];
-    cudaMalloc(&data16_d, w_size*sizeof(__half));
+    cudaMalloc(&data16_d, w_size * sizeof(__half));
     float2half(data_d, data16_d, w_size);
-    cudaMemcpy(data16_h, data16_d, w_size*sizeof(__half), cudaMemcpyDeviceToHost);
+    cudaMemcpy(data16_h, data16_d, w_size * sizeof(__half), cudaMemcpyDeviceToHost);
 
     int b_size = outputs;
     bias16_h = new __half[b_size];
-    cudaMalloc(&bias16_d, w_size*sizeof(__half));
+    cudaMalloc(&bias16_d, w_size * sizeof(__half));
     float2half(bias_d, bias16_d, b_size);
-    cudaMemcpy(bias16_h, bias16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+    cudaMemcpy(bias16_h, bias16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
 
-    if(batchnorm) {
+    if (batchnorm)
+    {
 
         power16_h = new __half[b_size];
         mean16_h = new __half[b_size];
         variance16_h = new __half[b_size];
         scales16_h = new __half[b_size];
 
-        cudaMalloc(&power16_d, b_size*sizeof(__half));
+        cudaMalloc(&power16_d, b_size * sizeof(__half));
-        cudaMalloc(&mean16_d, b_size*sizeof(__half));
+        cudaMalloc(&mean16_d, b_size * sizeof(__half));
-        cudaMalloc(&variance16_d, b_size*sizeof(__half));
+        cudaMalloc(&variance16_d, b_size * sizeof(__half));
-        cudaMalloc(&scales16_d, b_size*sizeof(__half));
+        cudaMalloc(&scales16_d, b_size * sizeof(__half));
 
         //temporary buffers
         float *tmp_d;
-        cudaMalloc(&tmp_d, b_size*sizeof(float));
+        cudaMalloc(&tmp_d, b_size * sizeof(float));
 
         //init power array of ones
-        cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
+        cudaMemcpy(tmp_d, power_h, b_size * sizeof(float), cudaMemcpyHostToDevice);
         float2half(tmp_d, power16_d, b_size);
-        cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+        cudaMemcpy(power16_h, power16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
 
         //mean array
 
-        cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
+        cudaMemcpy(tmp_d, mean_h, b_size * sizeof(float), cudaMemcpyHostToDevice);
         float2half(tmp_d, mean16_d, b_size);
-        cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+        cudaMemcpy(mean16_h, mean16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
 
         //convert variance
 
-        cudaMemcpy(tmp_d, variance_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
+        cudaMemcpy(tmp_d, variance_h, b_size * sizeof(float), cudaMemcpyHostToDevice);
         float2half(tmp_d, variance16_d, b_size);
-        cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+        cudaMemcpy(variance16_h, variance16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
 
         //conver scales
         float2half(scales_d, scales16_d, b_size);
-        cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+        cudaMemcpy(scales16_h, scales16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost);
     }
 }
 
-LayerWgs::~LayerWgs() {
+LayerWgs::~LayerWgs()
+{
 
-    delete [] data_h;
+    delete[] data_h;
-    delete [] bias_h;
+    delete[] bias_h;
-    checkCuda( cudaFree(data_d) );
+    checkCuda(cudaFree(data_d));
-    checkCuda( cudaFree(bias_d) );
+    checkCuda(cudaFree(bias_d));
 
-    if(batchnorm) {
+    if (batchnorm)
-        delete [] scales_h;
+    {
-        delete [] mean_h;
+        delete[] scales_h;
-        delete [] variance_h;
+        delete[] mean_h;
-        checkCuda( cudaFree(scales_d) );
+        delete[] variance_h;
-        checkCuda( cudaFree(mean_d) );
+        checkCuda(cudaFree(scales_d));
-        checkCuda( cudaFree(variance_d) );
+        checkCuda(cudaFree(mean_d));
+        checkCuda(cudaFree(variance_d));
     }
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/MulAdd.cpp

@@ -3,9 +3,13 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) {
+MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net)
+{
 
     this->mul = mul;
     this->add = add;
@@ -14,24 +18,25 @@ MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) {
 
     // create a vector with all value setted to add
     dnnType *add_vector_h = new dnnType[size];
-    for(int i=0; i<size; i++)
+    for (int i = 0; i < size; i++)
         add_vector_h[i] = add;
 
-    checkCuda( cudaMalloc(&add_vector, size*sizeof(dnnType)));
+    checkCuda(cudaMalloc(&add_vector, size * sizeof(dnnType)));
-    checkCuda( cudaMemcpy(add_vector, add_vector_h, size*sizeof(dnnType), cudaMemcpyHostToDevice));
+    checkCuda(cudaMemcpy(add_vector, add_vector_h, size * sizeof(dnnType), cudaMemcpyHostToDevice));
-    delete [] add_vector_h;
+    delete[] add_vector_h;
 
-
+    checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType)));
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 }
 
-MulAdd::~MulAdd() {
+MulAdd::~MulAdd()
+{
 
-    checkCuda( cudaFree(add_vector) );
+    checkCuda(cudaFree(add_vector));
-    checkCuda( cudaFree(dstData) );
+    checkCuda(cudaFree(dstData));
 }
 
-dnnType* MulAdd::infer(dataDim_t &dim, dnnType* srcData) {
+dnnType *MulAdd::infer(dataDim_t &dim, dnnType *srcData)
+{
 
     matrixMulAdd(net->cublasHandle, srcData, dstData, add_vector, input_dim.tot(), mul);
 
@@ -41,4 +46,5 @@ dnnType* MulAdd::infer(dataDim_t &dim, dnnType* srcData) {
     return dstData;
 }
 
-}}
+} // namespace dnn
+} // namespace tk

src/Network.cpp

@@ -5,106 +5,131 @@
 #include "Network.h"
 #include "Layer.h"
 
-namespace tk { namespace dnn {
+namespace tk
+{
+namespace dnn
+{
 
-Network::Network(dataDim_t input_dim) {
+Network::Network(dataDim_t input_dim)
+{
     this->input_dim = input_dim;
 
-    float tk_ver = float(TKDNN_VERSION)/1000;
+    float tk_ver = float(TKDNN_VERSION) / 1000;
-    float cu_ver = float(cudnnGetVersion())/1000;
+    float cu_ver = float(cudnnGetVersion()) / 1000;
 
-    std::cout<<"New NETWORK (tkDNN v"<<tk_ver
+    std::cout << "New NETWORK (tkDNN v" << tk_ver
-             <<", CUDNN v"<<cu_ver<<")\n";
+              << ", CUDNN v" << cu_ver << ")\n";
     dataType = CUDNN_DATA_FLOAT;
     tensorFormat = CUDNN_TENSOR_NCHW;
 
-    checkCUDNN( cudnnCreate(&cudnnHandle) );
+    checkCUDNN(cudnnCreate(&cudnnHandle));
-    checkERROR( cublasCreate(&cublasHandle) );
+    checkERROR(cublasCreate(&cublasHandle));
 
     num_layers = 0;
 
     fp16 = false;
     dla = false;
-    if(const char* env_p = std::getenv("TKDNN_MODE")) {
+    if (const char *env_p = std::getenv("TKDNN_MODE"))
-        if(strcmp(env_p, "FP16") == 0)
+    {
+        if (strcmp(env_p, "FP16") == 0)
             fp16 = true;
-	else if(strcmp(env_p, "DLA") == 0) {
+        else if (strcmp(env_p, "DLA") == 0)
+        {
             dla = true;
             fp16 = true;
         }
     }
 
-    if(fp16)
+    if (fp16)
-        std::cout<<COL_REDB<<"!! FP16 INERENCE ENABLED !!"<<COL_END<<"\n";
+        std::cout << COL_REDB << "!! FP16 INERENCE ENABLED !!" << COL_END << "\n";
-    if(dla)
+    if (dla)
-        std::cout<<COL_GREENB<<"!! DLA INERENCE ENABLED !!"<<COL_END<<"\n";
+        std::cout << COL_GREENB << "!! DLA INERENCE ENABLED !!" << COL_END << "\n";
 }
 
-Network::~Network() {
+Network::~Network()
+{
 
-    checkCUDNN( cudnnDestroy(cudnnHandle) );
+    checkCUDNN(cudnnDestroy(cudnnHandle));
-    checkERROR( cublasDestroy(cublasHandle) );
+    checkERROR(cublasDestroy(cublasHandle));
 }
 
-dnnType* Network::infer(dataDim_t &dim, dnnType* data) {
+dnnType *Network::infer(dataDim_t &dim, dnnType *data)
+{
 
     //do infer for every layer
-    for(int i=0; i<num_layers; i++) {
+    for (int i = 0; i < num_layers; i++)
+    {
         data = layers[i]->infer(dim, data);
     }
     checkCuda(cudaDeviceSynchronize());
     return data;
 }
 
-bool Network::addLayer(Layer *l) {
+bool Network::addLayer(Layer *l)
-    if(num_layers == MAX_LAYERS)
+{
+    if (num_layers == MAX_LAYERS)
         return false;
 
     layers[num_layers++] = l;
     return true;
 }
 
-dataDim_t Network::getOutputDim() {
+dataDim_t Network::getOutputDim()
+{
 
-        if(num_layers == 0)
+    if (num_layers == 0)
         return input_dim;
     else
-            return layers[num_layers-1]->output_dim;
+        return layers[num_layers - 1]->output_dim;
 }
 
-void Network::print() {
+void Network::print()
+{
 
     printCenteredTitle(" NETWORK MODEL ", '=', 60);
-    std::cout.width(3); std::cout<<std::left<<"N.";
+    std::cout.width(3);
-    std::cout<<" ";
+    std::cout << std::left << "N.";
-    std::cout.width(17); std::cout<<std::left<<"Layer type";
+    std::cout << " ";
-    std::cout.width(22); std::cout<<std::left<<"input (H*W,CH)";
+    std::cout.width(17);
-    std::cout.width(16); std::cout<<std::left<<"output (H*W,CH)";
+    std::cout << std::left << "Layer type";
-    std::cout<<"\n";
+    std::cout.width(22);
+    std::cout << std::left << "input (H*W,CH)";
+    std::cout.width(16);
+    std::cout << std::left << "output (H*W,CH)";
+    std::cout << "\n";
 
-    for(int i=0; i<num_layers; i++) {
+    for (int i = 0; i < num_layers; i++)
+    {
         dataDim_t in = layers[i]->input_dim;
         dataDim_t out = layers[i]->output_dim;
 
-        std::cout.width(3); std::cout<<std::right<<i;
+        std::cout.width(3);
-        std::cout<<" ";
+        std::cout << std::right << i;
-        std::cout.width(16); std::cout<<std::left<<layers[i]->getLayerName();
+        std::cout << " ";
-        std::cout.width(4);  std::cout<<std::right<<in.h;
+        std::cout.width(16);
-        std::cout<<" x ";
+        std::cout << std::left << layers[i]->getLayerName();
-        std::cout.width(4);  std::cout<<std::right<<in.w;
+        std::cout.width(4);
-        std::cout<<", ";
+        std::cout << std::right << in.h;
-        std::cout.width(4);  std::cout<<std::right<<in.c;
+        std::cout << " x ";
-        std::cout<<"  -> ";
+        std::cout.width(4);
-        std::cout.width(4);  std::cout<<std::right<<out.h;
+        std::cout << std::right << in.w;
-        std::cout<<" x ";
+        std::cout << ", ";
-        std::cout.width(4);  std::cout<<std::right<<out.w;
+        std::cout.width(4);
-        std::cout<<", ";
+        std::cout << std::right << in.c;
-        std::cout.width(4);  std::cout<<std::right<<out.c;
+        std::cout << "  -> ";
-        std::cout<<"\n";
+        std::cout.width(4);
+        std::cout << std::right << out.h;
+        std::cout << " x ";
+        std::cout.width(4);
+        std::cout << std::right << out.w;
+        std::cout << ", ";
+        std::cout.width(4);
+        std::cout << std::right << out.c;
+        std::cout << "\n";
     }
     printCenteredTitle("", '=', 60);
-    std::cout<<"\n";
+    std::cout << "\n";
 }
 
-
+} // namespace dnn
-}}
+} // namespace tk

src/class_src/boxDetection.cpp

@@ -1,6 +1,6 @@
-#include "BoxDetection.h"
+#include "boxDetection.h"
 #include <string.h>
-char buf_frame_crop_name [200];
+char buf_frame_crop_name[200];
 
 cv::Mat img_threshold(cv::Mat frame_crop)
 {
@@ -24,10 +24,10 @@ cv::Mat img_background(cv::Mat frame_crop)
     cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0);
     cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU);
     // get background
-    cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1));
+    cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1, 1, 1, 1));
     cv::erode(gray, opening, M);
     cv::dilate(gray, opening, M);
-    cv::Point p = cv::Point(-1,-1);
+    cv::Point p = cv::Point(-1, -1);
     cv::dilate(opening, coinsBg, M, p, 3);
     return coinsBg;
 }
@@ -43,10 +43,10 @@ cv::Mat img_dist_transform(cv::Mat frame_crop)
     cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU);
     // cv::Mat::ones M(3,3,cv::CV_8U);
     // get background
-    cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1));
+    cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1, 1, 1, 1));
     cv::erode(gray, opening, M);
     cv::dilate(gray, opening, M);
-    cv::Point p = cv::Point(-1,-1);
+    cv::Point p = cv::Point(-1, -1);
     cv::dilate(opening, coinsBg, M, p, 3);
     // distance transorm
     cv::distanceTransform(opening, distTrans, cv::DIST_L2, 5);
@@ -111,7 +111,7 @@ cv::Mat img_dist_transform(cv::Mat frame_crop)
 
 //////
 
-cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0)
+cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret = 0)
 {
     //ret = 0 --> dstx
     //ret = 1 --> dsty
@@ -122,15 +122,15 @@ cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0)
     // compute image gradient on two different directions
 
     cv::Mat f = frame_crop.clone();
-    int x,y;
+    int x, y;
-    (ret == 0 || ret == 2)?x=1, y=0 : NULL; 
+    (ret == 0 || ret == 2) ? x = 1, y = 0 : NULL;
-    (ret == 1 || ret == 3)?x=0, y=1 : NULL; 
+    (ret == 1 || ret == 3) ? x = 0, y = 1 : NULL;
     cv::Mat dst;
     cv::cvtColor(f, f, cv::COLOR_RGB2GRAY, 0);
     // You can try more different parameters
     cv::Sobel(f, dst, CV_8U, x, y, 3, 1, 0, cv::BORDER_DEFAULT);
     // for absSobel
-    if(ret == 2 || ret == 3)
+    if (ret == 2 || ret == 3)
         cv::convertScaleAbs(dst, dst, 1, 0);
     // next 3 rows to be checked
     //// ??cv::Mat f2 = frame_crop.clone();
@@ -139,7 +139,7 @@ cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0)
     return dst;
 }
 
-cv::Mat img_laplacian(cv::Mat frame_crop, int ret=1)
+cv::Mat img_laplacian(cv::Mat frame_crop, int ret = 1)
 {
     //ret = 0 --> src_gray
     //ret = 1 --> dst
@@ -151,30 +151,30 @@ cv::Mat img_laplacian(cv::Mat frame_crop, int ret=1)
     int scale = 1;
     int delta = 0;
     int ddepth = CV_16S;
-    cv::GaussianBlur( f, f, cv::Size(3,3), 0, 0, cv::BORDER_DEFAULT );
+    cv::GaussianBlur(f, f, cv::Size(3, 3), 0, 0, cv::BORDER_DEFAULT);
     /// Convert the image to grayscale
-    cv::cvtColor( f, src_gray, CV_RGB2GRAY );
+    cv::cvtColor(f, src_gray, CV_RGB2GRAY);
     if (ret == 0)
         return src_gray;
 
     // else: Apply Laplace function
     cv::Mat abs_dst;
-    cv::Laplacian( src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT );
+    cv::Laplacian(src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT);
     // //compute sharpness
     // float sharpnessValue = cv::mean(dst);
     return dst;
 }
 
-cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines=1)
+cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines = 1)
 {
     // n_line: number of line to plot on image
     cv::Mat img_line = frame_crop.clone();
     cv::Mat ret_thresh;
-    std::vector<std::vector<cv::Point> > contours;
+    std::vector<std::vector<cv::Point>> contours;
     double thresh = 127;
     double maxValue = 255;
-    cv::threshold(img, ret_thresh, thresh, maxValue, 0);//0); // = cv2.threshold(img,127,255,0)
+    cv::threshold(img, ret_thresh, thresh, maxValue, 0); //0); // = cv2.threshold(img,127,255,0)
-    cv::findContours(canny_output, contours, 1, 2);//cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2)
+    cv::findContours(canny_output, contours, 1, 2);      //cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2)
     // cv::threshold(img2, ret2, thresh, maxValue, 0);
     // cv::findContours(canny_output2, contours2, 1, 2);
     // cv::threshold(img3a, ret3a, thresh, maxValue, 0);
@@ -183,18 +183,18 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int
     // cv::findContours(canny_output3b, contours3b, 1, 2);
 
     cv::Vec4f line;
-    float vx,vy,x,y;
+    float vx, vy, x, y;
     int lefty, righty;
-    for(int i=0; i<n_lines; i++)
+    for (int i = 0; i < n_lines; i++)
     {
-        cv::fitLine(contours[i],line,CV_DIST_L2,0,0.01,0.01);
+        cv::fitLine(contours[i], line, CV_DIST_L2, 0, 0.01, 0.01);
         vx = line(0);
         vy = line(1);
         x = line(2);
         y = line(3);
-        lefty = int((-x*vy/vx) + y);
+        lefty = int((-x * vy / vx) + y);
-        righty = int(((img.cols-x)*vy/vx)+y);
+        righty = int(((img.cols - x) * vy / vx) + y);
-        cv::line(img_line,cv::Point(img.cols-1,righty),cv::Point(0,lefty),(255, 0 ,0),2);
+        cv::line(img_line, cv::Point(img.cols - 1, righty), cv::Point(0, lefty), (255, 0, 0), 2);
     }
 
     // cv::imshow("bla", img);
@@ -226,7 +226,7 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int
 //     return img_clone;
 // }
 
-cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr<cv::saliency::Saliency>  saliencyAlgorithm, int const_molt_mat, int ret=0)
+cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr<cv::saliency::Saliency> saliencyAlgorithm, int const_molt_mat, int ret = 0)
 {
     //ret=0 --> saliencyMap
     //ret=1 --> binaryMap
@@ -235,21 +235,21 @@ cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr<cv::saliency::Saliency>  sa
     cv::Mat saliencyMap;
     cv::Mat binaryMap;
 
-    if( saliencyAlgorithm->computeSaliency( f, saliencyMap ) )
+    if (saliencyAlgorithm->computeSaliency(f, saliencyMap))
     {
-        if(ret==0)
+        if (ret == 0)
-            return saliencyMap*const_molt_mat;
+            return saliencyMap * const_molt_mat;
 
         cv::saliency::StaticSaliencySpectralResidual spec;
-        spec.computeBinaryMap( saliencyMap, binaryMap );
+        spec.computeBinaryMap(saliencyMap, binaryMap);
 
         // imshow( "Saliency Map", saliencyMap );
         // imshow( "Original Image", image );
         // imshow( "Binary Map", binaryMap );
         // waitKey( 0 );
-        return binaryMap*const_molt_mat;
+        return binaryMap * const_molt_mat;
     }
-    return cv::Mat(0,0,CV_8U, cv::Scalar(0,0,0,0));
+    return cv::Mat(0, 0, CV_8U, cv::Scalar(0, 0, 0, 0));
 }
 
 //////
@@ -263,19 +263,22 @@ void image_segmentation(cv::Mat frame_crop, int frame_nbr, int i)
     auto end_t_segmentation = std::chrono::steady_clock::now();
     cv::Mat ret;
     // ret = img_threshold(frame_crop);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgthr.jpg", frame_nbr, i, img_threshold(frame_crop));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgthr.jpg", frame_nbr, i, img_threshold(frame_crop));
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME imgthr ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME imgthr (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // ret =img_background(frame_crop);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgback.jpg", frame_nbr, i, img_background(frame_crop));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgback.jpg", frame_nbr, i, img_background(frame_crop));
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME imgback ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME imgback (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // ret = img_dist_transform(frame_crop);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgtrans.jpg", frame_nbr, i, img_dist_transform(frame_crop));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgtrans.jpg", frame_nbr, i, img_dist_transform(frame_crop));
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME imgtrans ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME imgtrans (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // // ret = img_watershed(frame_crop);
     // if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgwatershed.jpg", frame_nbr, i, img_watershed(frame_crop));
@@ -291,45 +294,50 @@ void image_gradients(cv::Mat frame_crop, int frame_nbr, int i)
     cv::Mat ret;
     // sobel
     // ret = img_sobel_abssobel(frame_crop, 0);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_8U.jpgg", frame_nbr, i, img_sobel_abssobel(frame_crop, 0));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_8U.jpgg", frame_nbr, i, img_sobel_abssobel(frame_crop, 0));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME sobel0 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME sobel0 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // ret = img_sobel_abssobel(frame_crop, 1);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_8U.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 1));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_8U.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 1));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME sobel1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME sobel1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // ret = img_sobel_abssobel(frame_crop, 2);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 2));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 2));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME sobel2 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME sobel2 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // ret = img_sobel_abssobel(frame_crop, 3);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 3));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 3));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME sobel3 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME sobel3 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // laplacian
     // ret = img_laplacian(frame_crop, 0);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_gr.jpg", frame_nbr, i, img_laplacian(frame_crop, 0));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_gr.jpg", frame_nbr, i, img_laplacian(frame_crop, 0));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME laplacian0 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME laplacian0 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // ret = img_laplacian(frame_crop, 1);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_dst.jpg", frame_nbr, i, img_laplacian(frame_crop, 1));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_dst.jpg", frame_nbr, i, img_laplacian(frame_crop, 1));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME laplacian1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME laplacian1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     //////
-    
 }
 
 void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
@@ -337,7 +345,6 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
     // Finding contours in your image
     // https://docs.opencv.org/3.4/df/d0d/tutorial_find_contours.html
 
-    
     auto step_t_segmentation = std::chrono::steady_clock::now();
     auto end_t_segmentation = std::chrono::steady_clock::now();
     // plot lines on figure. 3 ways:
@@ -348,60 +355,67 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
     cv::Mat contours;
     // src_gray
     cv::Mat img1 = img_laplacian(frame_crop, 0);
-    cv::Canny(img1, canny_output1, 100, 100*2 );
+    cv::Canny(img1, canny_output1, 100, 100 * 2);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny1.jpg", frame_nbr, i, canny_output1);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny1.jpg", frame_nbr, i, canny_output1);
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME canny1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME canny1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // // dst
     // cv::Mat img2 = img_laplacian(frame_crop, 2);
     // cv::Canny(img2, canny_output2, 100, 100*2 );
-    cv::Canny(img1, canny_output2, 100, 100*2 );
+    cv::Canny(img1, canny_output2, 100, 100 * 2);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny2.jpg", frame_nbr, i, canny_output2);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny2.jpg", frame_nbr, i, canny_output2);
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME canny2 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME canny2 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // dstx
     cv::Mat img3a = img_sobel_abssobel(frame_crop, 0);
-    cv::Canny(img3a, canny_output3a, 100, 100*2 );
+    cv::Canny(img3a, canny_output3a, 100, 100 * 2);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3a.jpg", frame_nbr, i, canny_output3a);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3a.jpg", frame_nbr, i, canny_output3a);
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME canny3a ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME canny3a (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // dsty
     cv::Mat img3b = img_sobel_abssobel(frame_crop, 1);
-    cv::Canny(img3b, canny_output3b, 100, 100*2 );
+    cv::Canny(img3b, canny_output3b, 100, 100 * 2);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3b.jpg", frame_nbr, i, canny_output3b);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3b.jpg", frame_nbr, i, canny_output3b);
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME canny3b ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME canny3b (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // 1 line
     // contours = find_contours(frame_crop, img1, canny_output1, 1);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line1.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output1, 1));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_line1.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output1, 1));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME line1 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME line1 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // 3 line
     // contours = find_contours(frame_crop, img1, canny_output2, 1);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line2.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output2, 1));
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_line2.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output2, 1));
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME line2 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME line2 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
     // mix 1 line of image with 1 line of another
     cv::Mat img_line = frame_crop.clone();
     img_line = find_contours(img_line, img3a, canny_output3a, 1);
     img_line = find_contours(img_line, img3b, canny_output3b, 1);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line3.jpg", frame_nbr, i, img_line);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_line3.jpg", frame_nbr, i, img_line);
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME line3 ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME line3 (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // img_line = frame_crop.clone();
@@ -410,7 +424,6 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
     // sprintf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_line3bis.jpg",frame_nbr, i);
     // cv::imwrite(buf_frame_crop_name, img_line);
 
-    
     // cv::Mat canny_output4;
     // cv::Mat img4 = img_laplacian(frame_crop, 0);
     // cv::Canny(img4, canny_output4, 100, 100*2 );
@@ -418,7 +431,6 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i)
     // cv::imwrite(buf_frame_crop_name, fit_rectangular(frame_crop, img4, canny_output4));
 }
 
-
 void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
 {
     // https://github.com/opencv/opencv_contrib/blob/master/modules/saliency/samples/computeSaliency.cpp
@@ -432,47 +444,50 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
     const_molt_mat = 255;
     saliencyAlgorithm = cv::saliency::StaticSaliencySpectralResidual::create();
     cv::Mat spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0);
-    if(!spect_res.empty())
+    if (!spect_res.empty())
     {
-        if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg", frame_nbr, i, spect_res);
+        if (SAVE)
+            SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg", frame_nbr, i, spect_res);
     }
     else
     {
-        std::cout<<"something is wrond (image_saliency)"<<std::endl;
+        std::cout << "something is wrond (image_saliency)" << std::endl;
     }
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME SPECTRAL_RESIDUAL ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME SPECTRAL_RESIDUAL (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // BINARY SPECTRAL_RESIDUAL
     const_molt_mat = 255;
     spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 1);
-    if(!spect_res.empty())
+    if (!spect_res.empty())
     {
-        if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinarySpectralResidual.jpg", frame_nbr, i, spect_res);
+        if (SAVE)
+            SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinarySpectralResidual.jpg", frame_nbr, i, spect_res);
     }
     else
     {
-        std::cout<<"something is wrond (image_saliency)"<<std::endl;
+        std::cout << "something is wrond (image_saliency)" << std::endl;
     }
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME BINARY SPECTRAL_RESIDUAL ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME BINARY SPECTRAL_RESIDUAL (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // FINE_GRAINED
     const_molt_mat = 1;
     saliencyAlgorithm = cv::saliency::StaticSaliencyFineGrained::create();
     spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0);
-    if(!spect_res.empty())
+    if (!spect_res.empty())
     {
-        if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_FineGrained.jpg", frame_nbr, i, spect_res);
+        if (SAVE)
+            SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_FineGrained.jpg", frame_nbr, i, spect_res);
     }
     else
     {
-        std::cout<<"something is wrond (image_saliency)"<<std::endl;
+        std::cout << "something is wrond (image_saliency)" << std::endl;
     }
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME FINE_GRAINED ("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME FINE_GRAINED (" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // saliencyAlgorithm = cv::saliency::ObjectnessBING::create();
@@ -505,25 +520,26 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i)
     cv::Mat saliencyMap;
     cv::Mat frame_sal = frame_crop.clone();
     saliencyAlgorithm = cv::saliency::MotionSaliencyBinWangApr2014::create();
-    saliencyAlgorithm.dynamicCast<cv::saliency::MotionSaliencyBinWangApr2014>()->setImagesize( frame_sal.cols, frame_sal.rows );
+    saliencyAlgorithm.dynamicCast<cv::saliency::MotionSaliencyBinWangApr2014>()->setImagesize(frame_sal.cols, frame_sal.rows);
     saliencyAlgorithm.dynamicCast<cv::saliency::MotionSaliencyBinWangApr2014>()->init();
-    cvtColor( frame_sal, frame_sal, cv::COLOR_BGR2GRAY );
+    cvtColor(frame_sal, frame_sal, cv::COLOR_BGR2GRAY);
-    saliencyAlgorithm->computeSaliency( frame_sal, saliencyMap);
+    saliencyAlgorithm->computeSaliency(frame_sal, saliencyMap);
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg", frame_nbr, i, saliencyMap);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg", frame_nbr, i, saliencyMap);
 
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " - TIME BING WANG APR 2014("<<frame_nbr<<"-"<<i<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " - TIME BING WANG APR 2014(" << frame_nbr << "-" << i << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 }
 
-cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret=0)
+cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret = 0)
 {
     // https://stackoverflow.com/questions/27035672/cv-extract-differences-between-two-images
     cv::Mat backgroundImage = pre_frame.clone();
     cv::Mat currentImage = frame.clone();
     cv::Mat diffImage;
     // pass to HSV color
-    if(ret)
+    if (ret)
     {
         cv::cvtColor(backgroundImage, backgroundImage, CV_BGR2HSV);
         cv::cvtColor(currentImage, currentImage, CV_BGR2HSV);
@@ -537,27 +553,28 @@ cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i,
     float threshold = 30.0f;
     float dist;
 
-    for(int j=0; j<diffImage.rows; ++j)
+    for (int j = 0; j < diffImage.rows; ++j)
     {
-        for(int k=0; k<diffImage.cols; ++k)
+        for (int k = 0; k < diffImage.cols; ++k)
         {
-            cv::Vec3b pix = diffImage.at<cv::Vec3b>(j,k);
+            cv::Vec3b pix = diffImage.at<cv::Vec3b>(j, k);
 
-            dist = (pix[0]*pix[0] + pix[1]*pix[1] + pix[2]*pix[2]);
+            dist = (pix[0] * pix[0] + pix[1] * pix[1] + pix[2] * pix[2]);
             dist = sqrt(dist);
 
-            if(dist>threshold)
+            if (dist > threshold)
             {
-                foregroundMask.at<unsigned char>(j,k) = 255;
+                foregroundMask.at<unsigned char>(j, k) = 255;
             }
         }
     }
-    if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_dif.jpg", frame_nbr, i, foregroundMask);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_disparity/%d_%d_dif.jpg", frame_nbr, i, foregroundMask);
 
     return foregroundMask;
 }
 
-void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector <cv::Rect>  pre_rois, int frame_nbr)
+void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector<cv::Rect> pre_rois, int frame_nbr)
 {
 
     int roi_tollerance = 10;
@@ -567,18 +584,19 @@ void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector <cv::Rect
     auto step_t_segmentation = std::chrono::steady_clock::now();
     auto end_t_segmentation = std::chrono::steady_clock::now();
 
-    for(auto r : pre_rois)
+    for (auto r : pre_rois)
     {
-        if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_orig.jpg", frame_nbr, id, pre_frame(r));
+        if (SAVE)
+            SAVE_TO("../demo/demo/data/img_disparity/%d_%d_orig.jpg", frame_nbr, id, pre_frame(r));
 
         //resize last roi with a tollerance
         dx = r.width / roi_tollerance;
         dy = r.height / roi_tollerance;
-        r.x = (r.x - dx > 0)? (r.x - dx) : 0;
+        r.x = (r.x - dx > 0) ? (r.x - dx) : 0;
-        r.y = (r.y - dy > 0)? (r.y - dy) : 0;
+        r.y = (r.y - dy > 0) ? (r.y - dy) : 0;
         // std::cout<<"disp: x "<<r.x<<" - y "<<r.y<<std::endl;
-        r.width = ((r.x+r.width+dx+dx) >= frame.cols)? (frame.cols-1-r.x) : (r.width+dx+dx);
+        r.width = ((r.x + r.width + dx + dx) >= frame.cols) ? (frame.cols - 1 - r.x) : (r.width + dx + dx);
-        r.height = ((r.y+r.height+dy+dy) >= frame.rows)? (frame.rows-1-r.y) : (r.height+dy+dy);
+        r.height = ((r.y + r.height + dy + dy) >= frame.rows) ? (frame.rows - 1 - r.y) : (r.height + dy + dy);
         // std::cout<<"disp: w "<<r.width<<" - h "<<r.height<<std::endl;
         // std::cout<<"disp: wf "<<frame.cols<<" - hf "<<frame.rows<<std::endl;
         // std::cout<<"---"<<std::endl;
@@ -588,14 +606,16 @@ void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector <cv::Rect
         //crop pre_frame and current frame
         pre_frame_crop = pre_frame(r);
         frame_crop = frame(r);
-        if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_cur.jpg", frame_nbr, id, frame_crop);
+        if (SAVE)
-        if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_pre.jpg", frame_nbr, id, pre_frame_crop);
+            SAVE_TO("../demo/demo/data/img_disparity/%d_%d_cur.jpg", frame_nbr, id, frame_crop);
+        if (SAVE)
+            SAVE_TO("../demo/demo/data/img_disparity/%d_%d_pre.jpg", frame_nbr, id, pre_frame_crop);
 
         // difference from two consecutive frame
         step_t_segmentation = std::chrono::steady_clock::now();
         frame_disparity(pre_frame_crop, frame_crop, frame_nbr, id, 0);
         end_t_segmentation = std::chrono::steady_clock::now();
-        std::cout << " TIME frame_disparity ("<<frame_nbr<<"-"<<id<<") : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+        std::cout << " TIME frame_disparity (" << frame_nbr << "-" << id << ") : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
         step_t_segmentation = end_t_segmentation;
         id++;
     }
@@ -606,54 +626,53 @@ void segmentation(cv::Mat pre_frame, cv::Mat frame_crop, int frame_nbr, int i, i
     //mode=0 (for whole frame), it computes the frame disparity
     //mode=1 (for single box), it doesn't compute the frame disparity (it has already been done-see frame_box_disparity())
     // whole figure
-    char buf_str [15];
+    char buf_str[15];
-    if(!mode)
+    if (!mode)
-        sprintf(buf_str,"whole frame");
+        sprintf(buf_str, "whole frame");
     else
-        sprintf(buf_str,"a box frame");
+        sprintf(buf_str, "a box frame");
 
-    if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d.jpg", frame_nbr, i, frame_crop);
+    if (SAVE)
+        SAVE_TO("../demo/demo/data/img_crop/%d_%d.jpg", frame_nbr, i, frame_crop);
 
     auto step_t_segmentation = std::chrono::steady_clock::now();
     auto end_t_segmentation = std::chrono::steady_clock::now();
 
     // Watershed Algorithm
-    std::cout<<"image segmentation:"<<std::endl;
+    std::cout << "image segmentation:" << std::endl;
     image_segmentation(frame_crop, frame_nbr, i);
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " TIME "<<buf_str<<": image_segmentation : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " TIME " << buf_str << ": image_segmentation : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // Image Gradients
-    std::cout<<"image gradients:"<<std::endl;
+    std::cout << "image gradients:" << std::endl;
     image_gradients(frame_crop, frame_nbr, i);
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " TIME "<<buf_str<<": image_gradients : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " TIME " << buf_str << ": image_gradients : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     // Find contours
-    std::cout<<"image find contours:"<<std::endl;
+    std::cout << "image find contours:" << std::endl;
     image_find_contours(frame_crop, frame_nbr, i);
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " TIME "<<buf_str<<": image_find_contours : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " TIME " << buf_str << ": image_find_contours : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     //saliency map
-    std::cout<<"image saliency:"<<std::endl;
+    std::cout << "image saliency:" << std::endl;
     image_saliency(frame_crop, frame_nbr, i);
     end_t_segmentation = std::chrono::steady_clock::now();
-    std::cout << " TIME "<<buf_str<<": image_saliency : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+    std::cout << " TIME " << buf_str << ": image_saliency : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
     step_t_segmentation = end_t_segmentation;
 
     //frame disparity
-    if(!mode && frame_nbr!=0)
+    if (!mode && frame_nbr != 0)
     {
-        std::cout<<"frame disparity:"<<std::endl;
+        std::cout << "frame disparity:" << std::endl;
         frame_disparity(pre_frame, frame_crop, frame_nbr, i, 0);
         end_t_segmentation = std::chrono::steady_clock::now();
-        std::cout << " TIME "<<buf_str<<": frame_disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+        std::cout << " TIME " << buf_str << ": frame_disparity : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl;
         step_t_segmentation = end_t_segmentation;
     }
-    
-
 }

src/class_src/calibration.cpp

@@ -0,0 +1,108 @@
+#include "calibration.h"
+
+void readTiff(char *filename, double *adfGeoTransform)
+{
+    GDALDataset *poDataset;
+    GDALAllRegister();
+    poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly);
+    if (poDataset != NULL)
+    {
+        poDataset->GetGeoTransform(adfGeoTransform);
+    }
+}
+
+void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff)
+{
+    YAML::Node config = YAML::LoadFile(cameraCalib);
+    const YAML::Node &node_test1 = config["camera_matrix"];
+
+    float data_cm[9];
+    for (std::size_t i = 0; i < node_test1["data"].size(); i++)
+        data_cm[i] = node_test1["data"][i].as<float>();
+    cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm);
+    cameraMat = cameraMat_.clone();
+    std::cout << cameraMat << std::endl;
+    const YAML::Node &node_test2 = config["distortion_coefficients"];
+
+    float data_dc[5];
+    for (std::size_t i = 0; i < node_test2["data"].size(); i++)
+        data_dc[i] = node_test2["data"][i].as<float>();
+    cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc);
+    distCoeff = distCoeff_.clone();
+    std::cout << distCoeff << std::endl;
+}
+
+void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform)
+{
+    //Returns global coordinates from pixel x, y coordinates
+    double xoff, a, b, yoff, d, e;
+    xoff = adfGeoTransform[0];
+    a = adfGeoTransform[1];
+    b = adfGeoTransform[2];
+    yoff = adfGeoTransform[3];
+    d = adfGeoTransform[4];
+    e = adfGeoTransform[5];
+
+    //printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e );
+
+    lon = a * x + b * y + xoff;
+    lat = d * x + e * y + yoff;
+}
+void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform)
+{
+    x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1]));
+    y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5]));
+}
+
+void fillMatrix(cv::Mat &H, double *matrix, bool show)
+{
+    double *vals = (double *)H.data;
+    for (int i = 0; i < 9; i++)
+    {
+        vals[i] = matrix[i];
+    }
+    if (show)
+        std::cout << H << "\n";
+}
+
+void read_projection_matrix(cv::Mat &H, char *path)
+{
+    FILE *fp;
+    char *line = NULL;
+    size_t len = 0;
+    ssize_t read;
+
+    double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
+    int i = 0;
+    fp = fopen(path, "r");
+    if (fp == NULL)
+        exit(EXIT_FAILURE);
+
+    while ((read = getline(&line, &len, fp)) != -1)
+    {
+        std::cout << line << std::endl;
+        std::stringstream ss(line);
+        while (ss >> proj_matrix[i])
+            i++;
+    }
+    fclose(fp);
+    fillMatrix(H, proj_matrix);
+    free(line);
+}
+
+void convert_coords(std::vector<ObjCoords> &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform)
+{
+    double latitude, longitude;
+    std::vector<cv::Point2f> x_y, ll;
+    x_y.push_back(cv::Point2f(x, y));
+    //transform camera pixel to map pixel
+    cv::perspectiveTransform(x_y, ll, H);
+    //tranform to map pixel to map gps
+    pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform);
+
+    ObjCoords coord;
+    coord.lat_ = latitude;
+    coord.long_ = longitude;
+    coord.class_ = detected_class;
+    coords.push_back(coord);
+}

src/class_src/message.cpp

@@ -0,0 +1,109 @@
+#include "message.h"
+#include "calibration.h"
+
+unsigned long long time_in_ms()
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000;
+    return t_stamp_ms;
+}
+
+void addRoadUserfromTracker(const std::vector<Tracker> &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H)
+{
+    m->t_stamp_ms = time_in_ms();
+    m->objects.clear();
+    double lat, lon, alt;
+
+    for (auto t : trackers)
+    {
+        if (t.pred_list_.size() > 0)
+        {
+            Categories cat;
+            switch (t.class_)
+            {
+            case 0:
+                cat = Categories::C_person;
+                break;
+            case 1:
+                cat = Categories::C_car;
+                break;
+            case 2:
+                cat = Categories::C_car;
+                break;
+            case 3:
+                cat = Categories::C_bus;
+                break;
+            case 4:
+                cat = Categories::C_motorbike;
+                break;
+            case 5:
+                cat = Categories::C_bycicle;
+                break;
+            }
+            //std::cout << t.pred_list_.size() << std::endl;
+            gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt);
+
+            int pix_x, pix_y;
+            coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
+
+            // TODO: test correctness - added perspective transform call to converter pix_x and pix_y
+            // sometimes some values are wrong. float ok?
+            // std::vector<cv::Point2f> map_p, camera_p;
+            // std::cout<<"--- pix_x, pix_y: "<<pix_x<<", "<<pix_y<<std::endl;
+            // map_p.push_back(cv::Point2f(pix_x, pix_y));
+            // std::cout<<"map_p: "<<map_p<<std::endl;
+            // //transform camera pixel to map pixel
+            // cv::perspectiveTransform(map_p, camera_p, H.inv());
+            // std::cout<<"size H: "<<H.cols<<", "<<H.rows<<std::endl;
+            // std::cout<<"camera_p: "<<camera_p<<std::endl;
+            // // TODO: in some cases these lines causes seg fault!
+            // std::cout<<"y, x :"<<camera_p[0].y<<",  "<<camera_p[0].x<<std::endl;
+            // std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
+            // // std::cout<<"vec3b: "<<(cv::Vec3b)(pix_y,pix_x);
+            // assert (camera_p[0].x < maskOrient.cols);
+            // assert (camera_p[0].y < maskOrient.rows);
+            // uint8_t maskOrientPixel = maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)[0];
+            // std::cout<<"boo: "<<maskOrient.at<cv::Vec3b>(camera_p[0].y,camera_p[0].x)<<std::endl;
+            // uint8_t orientation;
+            // if(maskOrientPixel != 0)
+            // {
+            //     orientation = maskOrientPixel;
+            //     // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
+            // }
+            // else
+            // {
+            //     orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
+            //     //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
+            // }
+
+            // TODO: to validate -> it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);")
+            // TODO: include perspective transform
+            // std::cout<<"y, x :"<<pix_y<<", "<<pix_x<<std::endl;
+            // std::cout<<"size maskorient: "<<maskOrient.cols<<", "<<maskOrient.rows<<std::endl;
+            // std::cout<<"point: "<<(cv::Point)(pix_y,pix_x);
+            // uint8_t maskOrientPixel = maskOrient.at<uchar>(pix_y,pix_x);
+            // uint8_t orientation;
+            // if(maskOrientPixel != 0)
+            // {
+            //     orientation = maskOrientPixel;
+            //     // std::cout<<"orientation given by the mask "<< int(orientation)<<std::endl;
+            // }
+            // else
+            // {
+            //     orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
+            //     //std::cout<<"orientation given by the tracker "<< int(orientation)<<std::endl;
+            // }
+
+            uint8_t orientation = uint8_t((int((t.pred_list_.back().yaw_ * 57.29 + 360)) % 360) * 17 / 24);
+            // std::cout<<"orient: "<<unsigned(orientation)<<std::endl;
+            //std::cout << "lat: " << lat << " lon: " << lon << std::endl;
+            uint8_t velocity = uint8_t(std::abs(t.pred_list_.back().vel_ * 3.6 / 2));
+            // std::cout<<"vel: "<<unsigned(velocity)<<std::endl;
+            RoadUser r{static_cast<float>(lat), static_cast<float>(lon), velocity, orientation, cat};
+            //std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl;
+            m->objects.push_back(r);
+        }
+    }
+    m->num_objects = m->objects.size();
+}

src/class_src/visualization.cpp

@@ -0,0 +1,377 @@
+#include "visualization.h"
+
+/* Thread function to show the updated images 
+**/
+void *show_updates(void *x_void_ptr)
+{
+    cv::namedWindow("original", cv::WINDOW_NORMAL);
+    cv::namedWindow("detection", cv::WINDOW_NORMAL);
+    cv::namedWindow("topview", cv::WINDOW_NORMAL);
+    cv::namedWindow("disparity", cv::WINDOW_NORMAL);
+    cv::Mat original_loc, detection_loc, topview_loc, disparity_loc;
+    bool update_o_loc, update_de_loc, update_t_loc, update_di_loc;
+
+    while (gRun)
+    {
+        TIMER_START
+        // critical section: copy the struct in local variable
+        // in this way we can unlock the sem for the main thread
+        if (updates.mutex_o.try_lock())
+        {
+            update_o_loc = updates.update_o;
+            updates.update_o = false;
+            if (update_o_loc)
+                original_loc = updates.original.clone();
+            updates.mutex_o.unlock();
+        }
+
+        if (updates.mutex_de.try_lock())
+        {
+            update_de_loc = updates.update_de;
+            updates.update_de = false;
+            if (update_de_loc)
+                detection_loc = updates.detection.clone();
+            updates.mutex_de.unlock();
+        }
+
+        if (updates.mutex_t.try_lock())
+        {
+            update_t_loc = updates.update_t;
+            updates.update_t = false;
+            if (update_t_loc)
+                topview_loc = updates.topview.clone();
+            updates.mutex_t.unlock();
+        }
+
+        if (updates.mutex_di.try_lock())
+        {
+            update_di_loc = updates.update_di;
+            updates.update_di = false;
+            if (update_di_loc)
+                disparity_loc = updates.disparity.clone();
+            updates.mutex_di.unlock();
+        }
+
+        if (update_o_loc)
+            cv::imshow("original", original_loc);
+        if (update_de_loc)
+            cv::imshow("detection", detection_loc);
+        if (update_t_loc)
+            cv::imshow("topview", topview_loc);
+        if (update_di_loc)
+            cv::imshow("disparity", disparity_loc);
+        cv::waitKey(1);
+        // usleep(20000);  //sleep 20 msec
+        std::cout << "show_updates: ";
+        TIMER_STOP
+    }
+    return (void *)0;
+}
+
+void *originalFrame(void *x_void_ptr)
+{
+    Frame_t *info_show_orig = (Frame_t *)x_void_ptr;
+    cv::Mat frame_loc;
+    int frame_nbr_loc = 0;
+    while (gRun)
+    {
+        TIMER_START
+        // critical section: copy the struct in local variable
+        // in this way we can unlock the sem for the main thread
+        info_show_orig->sem_vc.lock();
+        frame_loc = info_show_orig->frame.clone();
+        frame_nbr_loc = info_show_orig->frame_nbr;
+        info_show_orig->sem_vc.unlock();
+        if (frame_nbr_loc == 0)
+        {
+            usleep(1000000);
+            printf("no frame received\n");
+            continue;
+        }
+        updates.mutex_o.lock();
+        updates.original = frame_loc.clone();
+        updates.update_o = true;
+        updates.mutex_o.unlock();
+        usleep(10000); //sleep 10 msec
+        std::cout << "originalFrame: ";
+        TIMER_STOP
+    }
+    return (void *)0;
+}
+
+void *detectionFrame(void *x_void_ptr)
+{
+    ModFrame_t *info_show = (ModFrame_t *)x_void_ptr;
+    double lat, lon, alt;
+    int pix_x, pix_y;
+    cv::Mat original_frame_loc;
+    std::vector<Tracker> trackers;
+
+    geodetic_converter::GeodeticConverter gc;
+    double adfGeoTransform[6];
+    cv::Mat H;
+    tk::dnn::Yolo3Detection yolo;
+    int num_detected;
+    cv::Mat mask;
+
+    // box variable
+    tk::dnn::box b;
+    int x0, w, x1, y0, h, y1;
+    int objClass;
+    std::string det_class;
+    ;
+    // float prob;
+    cv::Scalar intensity;
+    std::vector<cv::Point2f> map_p, camera_p;
+    int baseline = 0;
+    float fontScale = 0.5;
+    int thickness = 2;
+
+    while (gRun)
+    {
+        TIMER_START
+        // critical section: copy the struct in local variable
+        // in this way we can unlock the sem for the main thread
+        info_show->sem.lock();
+        original_frame_loc = info_show->original_frame.clone();
+        // std::vector<Tracker> trackers;
+        trackers = info_show->trackers;
+        // geodetic_converter::GeodeticConverter gc;
+        gc = info_show->gc;
+        for (int i = 0; i < 6; i++)
+            adfGeoTransform[i] = info_show->adfGeoTransform[i];
+        // cv::Mat H;
+        H = info_show->H.clone();
+        yolo = info_show->yolo;
+        mask = info_show->mask.clone();
+        info_show->sem.unlock();
+
+        if (trackers.empty())
+        {
+            usleep(1000000);
+            printf("no data available\n");
+            continue;
+        }
+
+        num_detected = yolo.detected.size();
+        for (int i = 0; i < num_detected; i++)
+        {
+            b = yolo.detected[i];
+            x0 = b.x;
+            w = b.w;
+            x1 = b.x + w;
+            y0 = b.y;
+            h = b.h;
+            y1 = b.y + h;
+            objClass = b.cl;
+            det_class = obj_class[b.cl];
+            // prob = b.prob;
+
+            intensity = mask.at<uchar>(cv::Point(int(x0 + b.w / 2), y1));
+
+            if (intensity[0] && objClass < 6)
+            {
+                //std::cout<<objClass<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
+                cv::rectangle(original_frame_loc, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[objClass], 2);
+                // draw label
+                cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
+                cv::rectangle(original_frame_loc, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);
+                cv::putText(original_frame_loc, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
+            }
+        }
+
+        for (auto t : trackers)
+        {
+            for (size_t p = 1; p < t.pred_list_.size(); p++)
+            {
+                gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt);
+                coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
+
+                map_p.clear();
+                camera_p.clear();
+                map_p.push_back(cv::Point2f(pix_x, pix_y));
+
+                //transform camera pixel to map pixel
+                cv::perspectiveTransform(map_p, camera_p, H.inv());
+                // std::cout<<"x,y: "<<pix_x<<", "<<pix_y<<std::endl;
+                // std::cout<<"map_p: "<<map_p<<std::endl;
+                // std::cout<<"camera_p: "<<camera_p<<std::endl;
+                // std::cout<<"size original_frame_loc: "<<original_frame_loc.cols<<", "<<original_frame_loc.rows<<std::endl;
+                // assert (camera_p[0].x < original_frame_loc.cols);
+                // assert (camera_p[0].y < original_frame_loc.rows);
+                if (camera_p[0].x < original_frame_loc.cols && camera_p[0].y < original_frame_loc.rows && camera_p[0].x >= 0 && camera_p[0].y >= 0)
+                    cv::circle(original_frame_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0);
+            }
+        }
+
+        updates.mutex_de.lock();
+        updates.detection = original_frame_loc.clone();
+        updates.update_de = true;
+        updates.mutex_de.unlock();
+
+        std::cout << "detectionFrame: ";
+        TIMER_STOP
+    }
+    return (void *)0;
+}
+
+void *topviewFrame(void *x_void_ptr)
+{
+    ModFrame_t *info_show = (ModFrame_t *)x_void_ptr;
+    double lat, lon, alt;
+    int pix_x, pix_y;
+    cv::Mat frame_top;
+    cv::Mat original_frame_top;
+    // original_frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg");
+    original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png");
+    // original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670_V.png");
+
+    std::vector<Tracker> trackers;
+    geodetic_converter::GeodeticConverter gc;
+    double adfGeoTransform[6];
+    cv::Mat H;
+    while (gRun)
+    {
+        TIMER_START
+        // critical section: copy the struct in local variable
+        // in this way we can unlock the sem for the main thread
+        info_show->sem.lock();
+        // std::vector<Tracker> trackers;
+        trackers = info_show->trackers;
+        // geodetic_converter::GeodeticConverter gc;
+        gc = info_show->gc;
+        for (int i = 0; i < 6; i++)
+            adfGeoTransform[i] = info_show->adfGeoTransform[i];
+        // cv::Mat H;
+        H = info_show->H.clone();
+        info_show->sem.unlock();
+        if (trackers.empty())
+        {
+            usleep(1000000);
+            printf("no data available\n");
+            continue;
+        }
+        frame_top = original_frame_top.clone();
+        for (auto t : trackers)
+        {
+            for (size_t p = 1; p < t.pred_list_.size(); p++)
+            {
+                gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt);
+                coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform);
+                if (pix_x < frame_top.cols && pix_y < frame_top.rows && pix_x >= 0 && pix_y >= 0)
+                    cv::circle(frame_top, cv::Point(pix_x, pix_y), 7.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0);
+            }
+        }
+        //outputVideo<< frame_top;
+        // ------------------------------------------------
+        updates.mutex_t.lock();
+        updates.topview = frame_top.clone();
+        updates.update_t = true;
+        updates.mutex_t.unlock();
+
+        std::cout << "topviewFrame: ";
+        TIMER_STOP
+    }
+    return (void *)0;
+}
+
+void *disparityFrame(void *x_void_ptr)
+{
+    Frame_t *info_show_disparity = (Frame_t *)x_void_ptr;
+    bool first_iteration = true;
+    cv::Mat frame_loc;
+    int frame_nbr_loc = 0, pre_frame_nbr_loc = 0;
+    auto start_t = std::chrono::steady_clock::now();
+    auto step_t = std::chrono::steady_clock::now();
+    auto end_t = std::chrono::steady_clock::now();
+
+    // information for the disparity map
+    cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB;
+    cv::Mat canny_img;
+    cv::Mat disparity_frame;
+    while (gRun)
+    {
+        start_t = std::chrono::steady_clock::now();
+        step_t = start_t;
+        // critical section: copy the struct in local variable
+        // in this way we can unlock the sem for the main thread
+        info_show_disparity->sem_vc.lock();
+        frame_loc = info_show_disparity->frame.clone();
+        frame_nbr_loc = info_show_disparity->frame_nbr;
+        info_show_disparity->sem_vc.unlock();
+        if (frame_nbr_loc == 0)
+        {
+            usleep(1000000);
+            printf("no frame received\n");
+            continue;
+        }
+        // compute frame disparity only in there is a new frame
+        if (frame_nbr_loc - pre_frame_nbr_loc > 0)
+        {
+            pre_frame_nbr_loc = frame_nbr_loc;
+            //preprocessing frame
+            step_t = std::chrono::steady_clock::now();
+            // src_gray
+            canny_img = img_laplacian(frame_loc, 0);
+            cv::Canny(canny_img, canny, 100, 100 * 2);
+            // sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr_loc, 999);
+            // cv::imwrite(buf_frame_crop_name, canny);
+            end_t = std::chrono::steady_clock::now();
+            std::cout << " TIME END pre canny : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
+            step_t = end_t;
+            // std::cout<<"o: "<<frame_loc.cols<<" - "<<frame_loc.rows<<std::endl;
+            // std::cout<<"canny: "<<canny.cols<<" - "<<canny.rows<<std::endl;
+            // std::cout<<"pre: "<<pre_canny.cols<<" - "<<pre_canny.rows<<std::endl;
+            if (!first_iteration)
+            {
+                // backtorgb = cv::cvtColor(pre_canny,cv::COLOR_GRAY2RGB)
+                cv::cvtColor(pre_canny, pre_canny_RGB, CV_GRAY2RGB);
+                cv::cvtColor(canny, canny_RGB, CV_GRAY2RGB);
+                disparity_frame = frame_disparity(pre_canny_RGB, canny_RGB, frame_nbr_loc, 999, 0);
+                // std::cout<<"size: "<<disparity_frame.rows<<" - "<<disparity_frame.cols<<std::endl;
+                // if (disparity_frame.rows == 0 || disparity_frame.cols == 0)
+                //     return -1;
+                // if (disparity_frame.empty())
+                // { // only fools don't check...
+                //     std::cout << "image not loaded !" << std::endl;
+                //     return -1;
+                // }
+                end_t = std::chrono::steady_clock::now();
+                std::cout << " TIME canny : frame_disparity : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - step_t).count() << " ms" << std::endl;
+                step_t = end_t;
+
+                // //--------------------------------
+                // //frame box disparity on the original image
+                // step_t_segmentation = std::chrono::steady_clock::now();
+                // frame_box_disparity(pre_frame, frame, pre_rois, frame_nbr_loc);
+                // // reset pre_rois for the new roi of the current frame
+                // // pre_rois.erase(pre_rois.begin(), pre_rois.end());
+                // end_t_segmentation = std::chrono::steady_clock::now();
+                // std::cout << " TIME Frame disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+                // step_t_segmentation = end_t_segmentation;
+
+                // //frame box disparity on the preprocessed image
+                // cv::cvtColor(pre_canny, pre_canny_RGB, CV_GRAY2RGB);
+                // cv::cvtColor(canny, canny_RGB, CV_GRAY2RGB);
+                // frame_box_disparity(pre_canny_RGB, canny_RGB, pre_rois, frame_nbr_loc);
+                // // reset pre_rois for the new roi of the current frame
+                // pre_rois.erase(pre_rois.begin(), pre_rois.end());
+                // end_t_segmentation = std::chrono::steady_clock::now();
+                // std::cout << " TIME Canny Frame disparity : "<<std::chrono::duration_cast<std::chrono::milliseconds>(end_t_segmentation - step_t_segmentation).count() << " ms"<<std::endl;
+                // step_t_segmentation = end_t_segmentation;
+                // //---------------------------------
+
+                updates.mutex_di.lock();
+                updates.disparity = disparity_frame.clone();
+                updates.update_di = true;
+                updates.mutex_di.unlock();
+            }
+            pre_canny = canny.clone();
+            if (first_iteration)
+                first_iteration = false;
+            end_t = std::chrono::steady_clock::now();
+            std::cout << "disparityFrame : TIME END pre canny : " << std::chrono::duration_cast<std::chrono::milliseconds>(end_t - start_t).count() << " ms" << std::endl;
+        }
+    }
+    return (void *)0;
+}