diff --git a/CMakeLists.txt b/CMakeLists.txt index 43cd091..bb87f2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,12 +38,20 @@ cuda_add_library(kernels SHARED ${tkdnn_CUSRC}) file(GLOB tkdnn_SRC "src/*.cpp") set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} -lcudnn -lnvinfer ${OpenCV_LIBS} -lgdal) +file(GLOB class_SRC "src/class_src/*.cpp") +set(class_LIBS ${OpenCV_LIBS} -lgdal yaml-cpp python2.7) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11") include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES} "~/repos/cereal/include" ${CMAKE_CURRENT_SOURCE_DIR}/tracker_CLASS/c++/src) include_directories( BEFORE ${MY_SOURCE_DIR}/src /usr/include/python2.7 ) + add_library(tkDNN SHARED ${tkdnn_SRC}) target_link_libraries(tkDNN ${tkdnn_LIBS}) +add_library(CLASS SHARED ${class_SRC}) +target_link_libraries(CLASS ${class_LIBS}) + + #static #add_library(tkDNN_static STATIC ${tkdnn_SRC}) #target_link_libraries(tkDNN_static ${tkdnn_LIBS}) @@ -104,8 +112,7 @@ add_executable(yolo3_demo demo/demo/demo.cpp tracker_CLASS/c++/src/tracker.cpp ) -target_link_libraries(yolo3_demo tkDNN) -target_link_libraries(yolo3_demo python2.7 yaml-cpp) +target_link_libraries(yolo3_demo tkDNN CLASS) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 2b7e870..3d311ca 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -1,89 +1,34 @@ -#include -#include -#include /* srand, rand */ -#include -#include -#include -#include - #include -#include -#include -#include #include "utils.h" -#include "BoxDetection.h" - -#include -#include -#include - -//saliency -#include -#include -#include - #include "Yolo3Detection.h" -#include "classutils.h" +#include "message.h" +#include "visualization.h" + +#include "tracker.h" #include "../masa_protocol/include/send.hpp" #include "../masa_protocol/include/serialize.hpp" -#include "ekf.h" -#include "trackutils.h" -#include "plot.h" -#include "tracker.h" -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include #define MAX_DETECT_SIZE 100 -std::chrono::steady_clock::time_point local_clock_start; - -std::mutex mutexgRun; - bool gRun; +std::chrono::steady_clock::time_point local_clock_start; +std::mutex mutexgRun; std::string obj_class[10]{"person", "car", "truck", "bus", "motor", "bike", "rider", "traffic light", "traffic sign", "train"}; - //mutex for some opencv operations std::mutex mutex_cv; - -struct ModFrame_t{ - std::vector trackers; - geodetic_converter::GeodeticConverter gc; - double adfGeoTransform[6]; - cv::Mat H; - cv::Mat original_frame; - tk::dnn::Yolo3Detection yolo; - cv::Mat mask; - // sem for mainthread, detectionthread and topviewthread - std::mutex sem; -}; - -struct Frame_t{ - char *input; - cv::Mat frame; - int frame_nbr; - // sem_vc for mainthread, videocapturethread, originalthread and disparitythread - std::mutex sem_vc; -}; - -struct Camera_t{ - int CAM_IDX; - char *input; - char *pmatrix; - char *maskfile; - char *cameraCalib; - char *maskFileOrient; - bool to_show; - tk::dnn::Yolo3Detection yolo; - double adfGeoTransform[6]; -}; - -struct Show_t{ - cv::Mat original, detection, topview, disparity; - bool update_o, update_de, update_t, update_di; - // a single mutex for each operation - the show_updates function must get all mutex - std::mutex mutex_o, mutex_de, mutex_t, mutex_di; -}updates; +Show_t updates; void sig_handler(int signo) { @@ -95,9 +40,9 @@ void sig_handler(int signo) void *readVideoCapture(void *x_void_ptr) { - std::cout<<"readVideoCapture start...\n"; - - Frame_t *info_f = (Frame_t *) x_void_ptr; + std::cout << "readVideoCapture start...\n"; + + Frame_t *info_f = (Frame_t *)x_void_ptr; mutex_cv.lock(); cv::VideoCapture cap(info_f->input, cv::CAP_FFMPEG); mutex_cv.unlock(); @@ -112,7 +57,6 @@ void *readVideoCapture(void *x_void_ptr) } else std::cout << "camera started\n"; - // cap.set(cv::CAP_PROP_BUFFERSIZE,3); // std::cout<<"buf size: "<> frame_loc; i++; } - i=0; + i = 0; start_t = std::chrono::steady_clock::now(); - while (i> frame_loc; mean_time = mean_time + std::chrono::duration_cast(std::chrono::steady_clock::now() - step_t).count(); - std::cout << " step "<(std::chrono::steady_clock::now() - step_t).count() << " ms"<(std::chrono::steady_clock::now() - step_t).count() << " ms" << std::endl; i++; } - end_t = std::chrono::steady_clock::now(); + end_t = std::chrono::steady_clock::now(); + + std::cout << "Capturing " << num_f << " frames" << std::endl; + std::cout << " Time taken : " << std::chrono::duration_cast(end_t - start_t).count() << " ms" << std::endl; - std::cout << "Capturing " << num_f << " frames" << std::endl ; - std::cout << " Time taken : "<(end_t - start_t).count() << " ms"<(local_clock_sync - local_clock_start).count()) / mean_time; shift = (shift - (int)shift) * mean_time; - std::cout<<".-------------------------------\n"; - std::cout<<" mean time: "<(local_clock_sync - local_clock_start).count()<(local_clock_sync - local_clock_start).count() << std::endl; + std::cout << "\n\n\n\n"; + std::cout << "readVideoCapture start to capture...\n"; while (gRun) - { - // mutex_cv.lock(); + { + // mutex_cv.lock(); cap >> frame_loc; // mutex_cv.unlock(); current_timestamp = std::chrono::steady_clock::now(); - shift = std::chrono::duration_cast(current_timestamp - local_clock_sync).count() ; - std::cout << " RELATIVE TIMESTAMP FRAME : "<(current_timestamp - local_clock_sync).count(); + std::cout << " RELATIVE TIMESTAMP FRAME : " << shift << " ms" << std::endl; shift = shift / mean_time; shift = (shift - (int)shift) * mean_time; - shift = (shift - mean_time/2 >= 0)? -(mean_time - shift) : shift; - std::cout<<"DELAY frame_"<= 0) ? -(mean_time - shift) : shift; + std::cout << "DELAY frame_" << frame_nbr_loc << " : " << shift << " ms" << std::endl; // TODO: here introduce a tollerance to discard old frame - // std::cout<< "CV_CAP_PROP_POS_MSEC: "<< cap.get( cv::CAP_PROP_POS_MSEC) <(end_t - start_t).count() << " ms"<(end_t - start_t).count() << " ms" << std::endl; start_t = end_t; info_f->sem_vc.lock(); info_f->frame = frame_loc.clone(); info_f->frame_nbr = frame_nbr_loc; info_f->sem_vc.unlock(); - // usleep(50000); + // usleep(50000); end_t = std::chrono::steady_clock::now(); - std::cout << " VC-TIME 2 : "<(end_t - start_t).count() << " ms"<(end_t - start_t).count() << " ms" << std::endl; start_t = end_t; - frame_nbr_loc ++; - } - return (void *)0; -} - -/* Thread function to show the updated images -**/ -void *show_updates(void *x_void_ptr) -{ - cv::namedWindow("original", cv::WINDOW_NORMAL); - cv::namedWindow("detection", cv::WINDOW_NORMAL); - cv::namedWindow("topview", cv::WINDOW_NORMAL); - cv::namedWindow("disparity", cv::WINDOW_NORMAL); - cv::Mat original_loc, detection_loc, topview_loc, disparity_loc; - bool update_o_loc, update_de_loc, update_t_loc, update_di_loc; - - while (gRun) - { - TIMER_START - // critical section: copy the struct in local variable - // in this way we can unlock the sem for the main thread - if(updates.mutex_o.try_lock()) - { - update_o_loc = updates.update_o; - updates.update_o = false; - if(update_o_loc) - original_loc = updates.original.clone(); - updates.mutex_o.unlock(); - } - - if(updates.mutex_de.try_lock()) - { - update_de_loc = updates.update_de; - updates.update_de = false; - if(update_de_loc) - detection_loc = updates.detection.clone(); - updates.mutex_de.unlock(); - } - - if(updates.mutex_t.try_lock()) - { - update_t_loc = updates.update_t; - updates.update_t = false; - if(update_t_loc) - topview_loc = updates.topview.clone(); - updates.mutex_t.unlock(); - } - - if(updates.mutex_di.try_lock()) - { - update_di_loc = updates.update_di; - updates.update_di = false; - if(update_di_loc) - disparity_loc = updates.disparity.clone(); - updates.mutex_di.unlock(); - } - - if(update_o_loc) - cv::imshow("original", original_loc); - if(update_de_loc) - cv::imshow("detection", detection_loc); - if(update_t_loc) - cv::imshow("topview", topview_loc); - if(update_di_loc) - cv::imshow("disparity", disparity_loc); - cv::waitKey(1); - // usleep(20000); //sleep 20 msec - std::cout<<"show_updates: "; - TIMER_STOP - } - return (void *)0; -} - -void *originalFrame(void *x_void_ptr) -{ - Frame_t *info_show_orig = (Frame_t *) x_void_ptr; - cv::Mat frame_loc; - int frame_nbr_loc = 0; - while (gRun) - { - TIMER_START - // critical section: copy the struct in local variable - // in this way we can unlock the sem for the main thread - info_show_orig->sem_vc.lock(); - frame_loc = info_show_orig->frame.clone(); - frame_nbr_loc = info_show_orig->frame_nbr; - info_show_orig->sem_vc.unlock(); - if (frame_nbr_loc == 0) - { - usleep(1000000); - printf("no frame received\n"); - continue; - } - updates.mutex_o.lock(); - updates.original = frame_loc.clone(); - updates.update_o = true; - updates.mutex_o.unlock(); - usleep(10000); //sleep 10 msec - std::cout<<"originalFrame: "; - TIMER_STOP - } - return (void *)0; -} - -void *detectionFrame(void *x_void_ptr) -{ - ModFrame_t *info_show = (ModFrame_t *) x_void_ptr; - double lat, lon, alt; - int pix_x, pix_y; - cv::Mat original_frame_loc; - std::vector trackers; - - geodetic_converter::GeodeticConverter gc; - double adfGeoTransform[6]; - cv::Mat H; - tk::dnn::Yolo3Detection yolo; - int num_detected; - cv::Mat mask; - - // box variable - tk::dnn::box b; - int x0, w, x1, y0, h, y1; - int objClass; - std::string det_class;; - // float prob; - cv::Scalar intensity; - std::vector map_p, camera_p; - int baseline = 0; - float fontScale = 0.5; - int thickness = 2; - - while (gRun) - { - TIMER_START - // critical section: copy the struct in local variable - // in this way we can unlock the sem for the main thread - info_show->sem.lock(); - original_frame_loc = info_show->original_frame.clone(); - // std::vector trackers; - trackers = info_show->trackers; - // geodetic_converter::GeodeticConverter gc; - gc = info_show->gc; - for(int i = 0; i < 6; i++ ) - adfGeoTransform[i] = info_show->adfGeoTransform[i]; - // cv::Mat H; - H = info_show->H.clone(); - yolo = info_show->yolo; - mask = info_show->mask.clone(); - info_show->sem.unlock(); - - if (trackers.empty()) - { - usleep(1000000); - printf("no data available\n"); - continue; - } - - num_detected = yolo.detected.size(); - for (int i = 0; i < num_detected; i++) - { - b = yolo.detected[i]; - x0 = b.x; - w = b.w; - x1 = b.x + w; - y0 = b.y; - h = b.h; - y1 = b.y + h; - objClass = b.cl; - det_class = obj_class[b.cl]; - // prob = b.prob; - - intensity = mask.at(cv::Point(int(x0 + b.w / 2), y1)); - - if (intensity[0] && objClass < 6) - { - //std::cout<= 0 && camera_p[0].y >= 0) - cv::circle(original_frame_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); - } - } - - updates.mutex_de.lock(); - updates.detection = original_frame_loc.clone(); - updates.update_de = true; - updates.mutex_de.unlock(); - - std::cout<<"detectionFrame: "; - TIMER_STOP - - } - return (void *)0; -} - -void *topviewFrame(void *x_void_ptr) -{ - ModFrame_t *info_show = (ModFrame_t *) x_void_ptr; - double lat, lon, alt; - int pix_x, pix_y; - cv::Mat frame_top; - cv::Mat original_frame_top; - // original_frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg"); - original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png"); - // original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670_V.png"); - - std::vector trackers; - geodetic_converter::GeodeticConverter gc; - double adfGeoTransform[6]; - cv::Mat H; - while (gRun) - { - TIMER_START - // critical section: copy the struct in local variable - // in this way we can unlock the sem for the main thread - info_show->sem.lock(); - // std::vector trackers; - trackers = info_show->trackers; - // geodetic_converter::GeodeticConverter gc; - gc = info_show->gc; - for(int i = 0; i < 6; i++ ) - adfGeoTransform[i] = info_show->adfGeoTransform[i]; - // cv::Mat H; - H = info_show->H.clone(); - info_show->sem.unlock(); - if (trackers.empty()) - { - usleep(1000000); - printf("no data available\n"); - continue; - } - frame_top = original_frame_top.clone(); - for (auto t : trackers) - { - for (size_t p = 1; p < t.pred_list_.size(); p++) - { - gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt); - coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform); - if (pix_x < frame_top.cols && pix_y < frame_top.rows && pix_x >= 0 && pix_y >= 0) - cv::circle(frame_top, cv::Point(pix_x, pix_y), 7.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); - - } - } - //outputVideo<< frame_top; - // ------------------------------------------------ - updates.mutex_t.lock(); - updates.topview = frame_top.clone(); - updates.update_t = true; - updates.mutex_t.unlock(); - - std::cout<<"topviewFrame: "; - TIMER_STOP - - } - return (void *)0; -} - -void *disparityFrame(void *x_void_ptr) -{ - Frame_t *info_show_disparity = (Frame_t *) x_void_ptr; - bool first_iteration = true; - cv::Mat frame_loc; - int frame_nbr_loc = 0, pre_frame_nbr_loc = 0; - auto start_t = std::chrono::steady_clock::now(); - auto step_t = std::chrono::steady_clock::now(); - auto end_t = std::chrono::steady_clock::now(); - - // information for the disparity map - cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB; - cv::Mat canny_img; - cv::Mat disparity_frame; - while (gRun) - { - start_t = std::chrono::steady_clock::now(); - step_t = start_t; - // critical section: copy the struct in local variable - // in this way we can unlock the sem for the main thread - info_show_disparity->sem_vc.lock(); - frame_loc = info_show_disparity->frame.clone(); - frame_nbr_loc = info_show_disparity->frame_nbr; - info_show_disparity->sem_vc.unlock(); - if (frame_nbr_loc == 0) - { - usleep(1000000); - printf("no frame received\n"); - continue; - } - // compute frame disparity only in there is a new frame - if(frame_nbr_loc - pre_frame_nbr_loc > 0) - { - pre_frame_nbr_loc = frame_nbr_loc; - //preprocessing frame - step_t = std::chrono::steady_clock::now(); - // src_gray - canny_img = img_laplacian(frame_loc,0); - cv::Canny(canny_img, canny, 100, 100*2 ); - // sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr_loc, 999); - // cv::imwrite(buf_frame_crop_name, canny); - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME END pre canny : "<(end_t - step_t).count() << " ms"<(end_t - step_t).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t - start_t).count() << " ms"<input; - if (pthread_create(&videocap, NULL, readVideoCapture, (void*)&info_f)) + if (pthread_create(&videocap, NULL, readVideoCapture, (void *)&info_f)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; @@ -612,9 +177,9 @@ void *computationTask(void *x_void_ptr) bool to_show = camera->to_show; double adfGeoTransform[6]; - for(int i=0; i<6; i++) + for (int i = 0; i < 6; i++) adfGeoTransform[i] = camera->adfGeoTransform[i]; - + ModFrame_t info_show; if (to_show) { @@ -623,39 +188,39 @@ void *computationTask(void *x_void_ptr) updates.update_de = false; updates.update_t = false; updates.update_di = false; - if (pthread_create(&visual, NULL, show_updates, (void*)NULL)) + if (pthread_create(&visual, NULL, show_updates, (void *)NULL)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; - if (pthread_create(&originalshow, NULL, originalFrame, (void*)&info_f)) + if (pthread_create(&originalshow, NULL, originalFrame, (void *)&info_f)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; - if (pthread_create(&disparityshow, NULL, disparityFrame, (void*)&info_f)) + if (pthread_create(&disparityshow, NULL, disparityFrame, (void *)&info_f)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; info_show.H = cv::Mat(cv::Size(3, 3), CV_64FC1); - if (pthread_create(&detectionshow, NULL, detectionFrame, (void*)&info_show)) + if (pthread_create(&detectionshow, NULL, detectionFrame, (void *)&info_show)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; - if (pthread_create(&topviewshow, NULL, topviewFrame, (void*)&info_show)) + if (pthread_create(&topviewshow, NULL, topviewFrame, (void *)&info_show)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; - } - - char* pmatrix = camera->pmatrix; + } + + char *pmatrix = camera->pmatrix; /*projection matrix from camera to map*/ cv::Mat H(cv::Size(3, 3), CV_64FC1); read_projection_matrix(H, pmatrix); - assert (cv::countNonZero(H) > 0); + assert(cv::countNonZero(H) > 0); // std::cout< coords; - + /*socket*/ Communicator Comm(SOCK_DGRAM); Comm.open_client_socket((char *)"127.0.0.1", 8888); - + Message *m = new Message; m->cam_idx = camera->CAM_IDX; m->lights.clear(); @@ -729,10 +294,11 @@ void *computationTask(void *x_void_ptr) // box variable tk::dnn::box b; - int x0, h, y1; //w, x1, y0; + int x0, h, y1; //w, x1, y0; int objClass; - std::string det_class;; - // float prob; + std::string det_class; + ; + // float prob; cv::Scalar intensity; cv::Mat frame; @@ -740,18 +306,18 @@ void *computationTask(void *x_void_ptr) cv::Mat dnn_input; bool first_iteration = true; while (gRun) - { + { TIMER_START start_t = std::chrono::steady_clock::now(); step_t = start_t; - + info_f.sem_vc.lock(); frame = info_f.frame.clone(); - if(info_f.frame_nbr - frame_nbr > 1) - std::cout<<"more than one - f_n (diff "< 1) + std::cout << "more than one - f_n (diff " << info_f.frame_nbr - frame_nbr << ")\n"; frame_nbr = info_f.frame_nbr; info_f.sem_vc.unlock(); - std::cout<<"f_n: "<(end_t - step_t).count() << " ms"<(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; // draw dets - std::cout<<"camera: "<CAM_IDX <<" - num detected: "<CAM_IDX << " - num detected: " << num_detected << std::endl; //TODO: move in a thread // //preprocessing frame @@ -799,14 +365,14 @@ void *computationTask(void *x_void_ptr) // // std::cout<<"pre: "<(cv::Point(int(x0 + b.w / 2), y1)); - + if (intensity[0]) { if (objClass < 6) { - // find the rectangular on the frame (sub-figure) + // find the rectangular on the frame (sub-figure) // roi.x = (x0 > 0)? x0 : 0; // roi.y = (y0 > 0)? y0 : 0; // // std::cout<<"x "<yolo.colors[objClass], 2); // // draw label @@ -889,13 +454,13 @@ void *computationTask(void *x_void_ptr) // int thickness = 2; // cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline); // cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), camera->yolo.colors[b.cl], -1); - // cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness); + // cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness); } } } end_t = std::chrono::steady_clock::now(); - std::cout << " TIME 2 : "<(end_t - step_t).count() << " ms"<(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; //convert from latitude and longitude to meters for ekf cur_frame.clear(); @@ -907,7 +472,7 @@ void *computationTask(void *x_void_ptr) if (first_iteration) { // if there aren't detections and it is the first iteration, we can't initialize the tracker, so continue - if(cur_frame.empty()) + if (cur_frame.empty()) continue; for (auto f : cur_frame) trackers.push_back(Tracker(f, initial_age, dt, n_states)); @@ -918,7 +483,7 @@ void *computationTask(void *x_void_ptr) } std::cout << "There are " << trackers.size() << " trackers" << std::endl; //prepare message with tracker info - if(trackers.size()==0) + if (trackers.size() == 0) { // mutex_cv.lock(); addRoadUserfromTracker(trackers, m, gc, maskOrient, adfGeoTransform, H); @@ -929,17 +494,17 @@ void *computationTask(void *x_void_ptr) if (!m->objects.empty()) Comm.send_message(m); } - + if (to_show) { - //populate the ModFrame_t + //populate the ModFrame_t info_show.sem.lock(); info_show.original_frame = frame.clone(); // std::vector trackers; info_show.trackers = trackers; // geodetic_converter::GeodeticConverter gc; info_show.gc = gc; - for(int i = 0; i < 6; i++ ) + for (int i = 0; i < 6; i++) info_show.adfGeoTransform[i] = adfGeoTransform[i]; // cv::Mat H; info_show.H = H.clone(); @@ -952,11 +517,11 @@ void *computationTask(void *x_void_ptr) // update pre_frame for the disparity map // pre_frame = orig_frame.clone(); // pre_canny = canny.clone(); - if(first_iteration) + if (first_iteration) first_iteration = false; - + frame_nbr++; - std::cout<CAM_IDX<<" camera thread: "; + std::cout << camera->CAM_IDX << " camera thread: "; TIMER_STOP } return (void *)0; @@ -968,7 +533,7 @@ int main(int argc, char *argv[]) std::cout << "detection\n"; signal(SIGINT, sig_handler); srand(time(NULL)); - bool no_params = false; //flag to indicate if there are camera parameters or we use mp4 test video + bool no_params = false; //flag to indicate if there are camera parameters or we use mp4 test video char *net = (char *)"yolo3_coco4.rt"; if (argc > 1) @@ -980,27 +545,27 @@ int main(int argc, char *argv[]) if (argc > 3) { n = argv[3]; - if(strcmp(n, "-n")) - return -1; + if (strcmp(n, "-n")) + return -1; }; int n_cameras = 0; if (argc > 4) { n_cameras = atoi(argv[4]); - if(argc < 5+ 7 * n_cameras) + if (argc < 5 + 7 * n_cameras) { - std::cout<<"too few parameters\n"; + std::cout << "too few parameters\n"; return -1; } - if(!n_cameras) + if (!n_cameras) { n_cameras = 1; no_params = true; - } + } } - Camera_t cameras[n_cameras]; - bool *to_show = (bool *) malloc(n_cameras * sizeof(bool)); - if(no_params) + Camera_t cameras[n_cameras]; + bool *to_show = (bool *)malloc(n_cameras * sizeof(bool)); + if (no_params) { cameras[0].CAM_IDX = 20936; cameras[0].input = (char *)"../demo/demo/data/single_ped_2.mp4"; @@ -1013,76 +578,74 @@ int main(int argc, char *argv[]) } else { - for(int i = 0; i 1) + if (check_visualization > 1) return -1; - tk::dnn::Yolo3Detection yolo[n_cameras]; - for(int i=0; i +#include #include "utils.h" #include "Network.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -enum layerType_t { +enum layerType_t +{ LAYER_DENSE, LAYER_CONV2D, LAYER_ACTIVATION, @@ -26,56 +30,73 @@ enum layerType_t { /** Simple layer Father class */ -class Layer { +class Layer +{ public: Layer(Network *net); virtual ~Layer(); virtual layerType_t getLayerType() = 0; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) { - std::cout<<"No infer action for this layer\n"; + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData) + { + std::cout << "No infer action for this layer\n"; return NULL; } dataDim_t input_dim, output_dim; - dnnType *dstData; //where results will be putted + dnnType *dstData; //where results will be putted - std::string getLayerName() { + std::string getLayerName() + { layerType_t type = getLayerType(); - switch(type) { - case LAYER_DENSE: return "Dense"; - case LAYER_CONV2D: return "Conv2d"; - case LAYER_ACTIVATION: return "Activation"; - case LAYER_FLATTEN: return "Flatten"; - case LAYER_MULADD: return "MulAdd"; - case LAYER_POOLING: return "Pooling"; - case LAYER_SOFTMAX: return "Softmax"; - case LAYER_ROUTE: return "Route"; - case LAYER_REORG: return "Reorg"; - case LAYER_SHORTCUT: return "Shortcut"; - case LAYER_UPSAMPLE: return "Upsample"; - case LAYER_REGION: return "Region"; - case LAYER_YOLO: return "Yolo"; - default: return "unknown"; + switch (type) + { + case LAYER_DENSE: + return "Dense"; + case LAYER_CONV2D: + return "Conv2d"; + case LAYER_ACTIVATION: + return "Activation"; + case LAYER_FLATTEN: + return "Flatten"; + case LAYER_MULADD: + return "MulAdd"; + case LAYER_POOLING: + return "Pooling"; + case LAYER_SOFTMAX: + return "Softmax"; + case LAYER_ROUTE: + return "Route"; + case LAYER_REORG: + return "Reorg"; + case LAYER_SHORTCUT: + return "Shortcut"; + case LAYER_UPSAMPLE: + return "Upsample"; + case LAYER_REGION: + return "Region"; + case LAYER_YOLO: + return "Yolo"; + default: + return "unknown"; } } protected: Network *net; cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc; - }; - /** Father class of all layer that need to load trained weights */ -class LayerWgs : public Layer { +class LayerWgs : public Layer +{ public: - LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt, - const char* fname_weights, bool batchnorm = false); + LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt, + const char *fname_weights, bool batchnorm = false); virtual ~LayerWgs(); int inputs, outputs; @@ -87,75 +108,76 @@ public: //batchnorm bool batchnorm; dnnType *power_h; - dnnType *scales_h, *scales_d; - dnnType *mean_h, *mean_d; + dnnType *scales_h, *scales_d; + dnnType *mean_h, *mean_d; dnnType *variance_h, *variance_d; //fp16 __half *data16_h, *bias16_h; __half *data16_d, *bias16_d; - __half *power16_h, *power16_d; - __half *scales16_h, *scales16_d; - __half *mean16_h, *mean16_d; + __half *power16_h, *power16_d; + __half *scales16_h, *scales16_d; + __half *mean16_h, *mean16_d; __half *variance16_h, *variance16_d; }; - /** Dense (full interconnection) layer */ -class Dense : public LayerWgs { +class Dense : public LayerWgs +{ public: - Dense(Network *net, int out_ch, const char* fname_weights); + Dense(Network *net, int out_ch, const char *fname_weights); virtual ~Dense(); virtual layerType_t getLayerType() { return LAYER_DENSE; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); }; - /** Avaible activation functions */ -typedef enum { - ACTIVATION_ELU = 100, - ACTIVATION_LEAKY = 101 +typedef enum +{ + ACTIVATION_ELU = 100, + ACTIVATION_LEAKY = 101 } tkdnnActivationMode_t; /** Activation layer (it doesnt need weigths) */ -class Activation : public Layer { +class Activation : public Layer +{ public: int act_mode; - Activation(Network *net, int act_mode); + Activation(Network *net, int act_mode); virtual ~Activation(); virtual layerType_t getLayerType() { return LAYER_ACTIVATION; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); protected: cudnnActivationDescriptor_t activDesc; }; - /** Convolutional 2D layer */ -class Conv2d : public LayerWgs { +class Conv2d : public LayerWgs +{ public: - Conv2d( Network *net, int out_ch, int kernelH, int kernelW, - int strideH, int strideW, int paddingH, int paddingW, - const char* fname_weights, bool batchnorm = false); + Conv2d(Network *net, int out_ch, int kernelH, int kernelW, + int strideH, int strideW, int paddingH, int paddingW, + const char *fname_weights, bool batchnorm = false); virtual ~Conv2d(); virtual layerType_t getLayerType() { return LAYER_CONV2D; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); int kernelH, kernelW, strideH, strideW, paddingH, paddingW; @@ -165,75 +187,74 @@ protected: cudnnConvolutionFwdAlgo_t algo; cudnnTensorDescriptor_t biasTensorDesc; - void* workSpace; + void *workSpace; size_t ws_sizeInBytes; }; - /** Flatten layer is actually a matrix transposition */ -class Flatten : public Layer { +class Flatten : public Layer +{ public: - Flatten(Network *net); + Flatten(Network *net); virtual ~Flatten(); virtual layerType_t getLayerType() { return LAYER_FLATTEN; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); }; - /** MulAdd layer apply a multiplication and then an addition for each data */ -class MulAdd : public Layer { +class MulAdd : public Layer +{ public: - MulAdd(Network *net, dnnType mul, dnnType add); + MulAdd(Network *net, dnnType mul, dnnType add); virtual ~MulAdd(); virtual layerType_t getLayerType() { return LAYER_MULADD; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); protected: dnnType mul, add; dnnType *add_vector; }; - - /** Avaible pooling functions (padding on tkDNN is not supported) */ -typedef enum { - POOLING_MAX = 0, - POOLING_AVERAGE = 1, // count for average includes padded values - POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values +typedef enum +{ + POOLING_MAX = 0, + POOLING_AVERAGE = 1, // count for average includes padded values + POOLING_AVERAGE_EXCLUDE_PADDING = 2 // count for average does not include padded values } tkdnnPoolingMode_t; /** Pooling layer currenty supported only 2d pooing (also on 3d input) */ -class Pooling : public Layer { +class Pooling : public Layer +{ public: int winH, winW; int strideH, strideW; int paddingH, paddingW; - Pooling(Network *net, int winH, int winW, - int strideH, int strideW, tkdnnPoolingMode_t pool_mode); + Pooling(Network *net, int winH, int winW, + int strideH, int strideW, tkdnnPoolingMode_t pool_mode); virtual ~Pooling(); virtual layerType_t getLayerType() { return LAYER_POOLING; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); protected: - cudnnPoolingDescriptor_t poolingDesc; tkdnnPoolingMode_t pool_mode; dnnType *tmpInputData, *tmpOutputData; @@ -243,47 +264,49 @@ protected: /** Softmax layer */ -class Softmax : public Layer { +class Softmax : public Layer +{ public: - Softmax(Network *net); + Softmax(Network *net); virtual ~Softmax(); virtual layerType_t getLayerType() { return LAYER_SOFTMAX; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); }; /** Route layer Merge a list of layers */ -class Route : public Layer { +class Route : public Layer +{ public: - Route(Network *net, Layer **layers, int layers_n); + Route(Network *net, Layer **layers, int layers_n); virtual ~Route(); virtual layerType_t getLayerType() { return LAYER_ROUTE; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); public: - Layer **layers; //ids of layers to be merged - int layers_n; //number of layers + Layer **layers; //ids of layers to be merged + int layers_n; //number of layers }; - /** Reorg layer Mantain same dimension but change C*H*W distribution */ -class Reorg : public Layer { +class Reorg : public Layer +{ public: Reorg(Network *net, int stride); virtual ~Reorg(); virtual layerType_t getLayerType() { return LAYER_REORG; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); int stride; }; @@ -292,14 +315,15 @@ public: Shortcut layer sum with stride another layer */ -class Shortcut : public Layer { +class Shortcut : public Layer +{ public: - Shortcut(Network *net, Layer *backLayer); + Shortcut(Network *net, Layer *backLayer); virtual ~Shortcut(); virtual layerType_t getLayerType() { return LAYER_SHORTCUT; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); public: Layer *backLayer; @@ -309,25 +333,28 @@ public: Upsample layer Mantain same dimension but change C*H*W distribution */ -class Upsample : public Layer { +class Upsample : public Layer +{ public: Upsample(Network *net, int stride); virtual ~Upsample(); virtual layerType_t getLayerType() { return LAYER_UPSAMPLE; }; - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); int stride; bool reverse; }; -struct box { +struct box +{ int cl; float x, y, w, h; float prob; }; -struct sortable_bbox { +struct sortable_bbox +{ int index; int cl; float **probs; @@ -336,14 +363,17 @@ struct sortable_bbox { /** Yolo3 layer */ -class Yolo : public Layer { +class Yolo : public Layer +{ public: - struct box { + struct box + { float x, y, w, h; }; - struct detection{ + struct detection + { Yolo::box bbox; int classes; float *prob; @@ -352,7 +382,7 @@ public: int sort_class; }; - Yolo(Network *net, int classes, int num, const char* fname_weights); + Yolo(Network *net, int classes, int num, const char *fname_weights); virtual ~Yolo(); virtual layerType_t getLayerType() { return LAYER_YOLO; }; @@ -360,20 +390,21 @@ public: dnnType *mask_h, *mask_d; //anchors dnnType *bias_h, *bias_d; //anchors - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh); dnnType *predictions; static const int MAX_DETECTIONS = 256; static Yolo::detection *allocateDetections(int nboxes, int classes); - static void mergeDetections(Yolo::detection *dets, int ndets, int classes); + static void mergeDetections(Yolo::detection *dets, int ndets, int classes); }; /** Region layer */ -class Region : public Layer { +class Region : public Layer +{ public: Region(Network *net, int classes, int coords, int num); @@ -381,15 +412,16 @@ public: virtual layerType_t getLayerType() { return LAYER_REGION; }; int classes, coords, num; - - virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + + virtual dnnType *infer(dataDim_t &dim, dnnType *srcData); }; -class RegionInterpret { +class RegionInterpret +{ public: - RegionInterpret(dataDim_t input_dim, dataDim_t output_dim, - int classes, int coords, int num, float thresh, const char* fname_weights); + RegionInterpret(dataDim_t input_dim, dataDim_t output_dim, + int classes, int coords, int num, float thresh, const char *fname_weights); ~RegionInterpret(); dataDim_t input_dim, output_dim; @@ -397,7 +429,6 @@ public: int classes, coords, num; float thresh; - box *boxes; float **probs; sortable_bbox *s; @@ -405,9 +436,9 @@ public: int res_boxes_n; box get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride); - void get_region_boxes( float *input, int w, int h, int netw, int neth, float thresh, - float **probs, box *boxes, int only_objectness, - int *map, float tree_thresh, int relative); + void get_region_boxes(float *input, int w, int h, int netw, int neth, float thresh, + float **probs, box *boxes, int only_objectness, + int *map, float tree_thresh, int relative); void correct_region_boxes(box *boxes, int n, int w, int h, int netw, int neth, int relative); void interpretData(dnnType *data_h, int imageW = 0, int imageH = 0); void showImageResult(dnnType *input_h); @@ -415,5 +446,6 @@ public: static float box_iou(box a, box b); }; -}} +} // namespace dnn +} // namespace tk #endif //LAYER_H diff --git a/include/Network.h b/include/Network.h index 9da5806..6b054a6 100644 --- a/include/Network.h +++ b/include/Network.h @@ -3,7 +3,10 @@ #include "utils.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ /** Data rapresentation beetween layers @@ -13,28 +16,31 @@ namespace tk { namespace dnn { w = width (rows) l = lenght (3rd dimension) */ -struct dataDim_t { +struct dataDim_t +{ int n, c, h, w, l; - dataDim_t() : n(1), c(1), h(1), w(1), l(1) {}; + dataDim_t() : n(1), c(1), h(1), w(1), l(1){}; - dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) : - n(_n), c(_c), h(_h), w(_w), l(_l) {}; + dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) : n(_n), c(_c), h(_h), w(_w), l(_l){}; - void print() { - std::cout<<"Data dim: "< void writeBUF(char*& buffer, const T& val) +namespace tk { - *reinterpret_cast(buffer) = val; +namespace dnn +{ + +template +void writeBUF(char *&buffer, const T &val) +{ + *reinterpret_cast(buffer) = val; buffer += sizeof(T); } -template T readBUF(const char*& buffer) +template +T readBUF(const char *&buffer) { - T val = *reinterpret_cast(buffer); + T val = *reinterpret_cast(buffer); buffer += sizeof(T); return val; } @@ -38,24 +43,23 @@ public: YoloRT *yolos[16]; int n_yolos; - virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength); + virtual IPlugin *createPlugin(const char *layerName, const void *serialData, size_t serialLength); }; - - -class NetworkRT { +class NetworkRT +{ public: nvinfer1::DataType dtRT; nvinfer1::IBuilder *builderRT; nvinfer1::IRuntime *runtimeRT; - nvinfer1::INetworkDefinition *networkRT; - + nvinfer1::INetworkDefinition *networkRT; + nvinfer1::ICudaEngine *engineRT; nvinfer1::IExecutionContext *contextRT; const static int MAX_BUFFERS_RT = 10; - void* buffersRT[MAX_BUFFERS_RT]; + void *buffersRT[MAX_BUFFERS_RT]; int buf_input_idx, buf_output_idx; dataDim_t input_dim, output_dim; @@ -70,25 +74,26 @@ public: /** Do inferece */ - dnnType* infer(dataDim_t &dim, dnnType* data); - void enqueue(); + dnnType *infer(dataDim_t &dim, dnnType *data); + void enqueue(); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l); - nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Layer *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Conv2d *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Activation *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Dense *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Pooling *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Softmax *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Route *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Reorg *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Region *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Shortcut *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Yolo *l); + nvinfer1::ILayer *convert_layer(nvinfer1::ITensor *input, Upsample *l); bool serialize(const char *filename); bool deserialize(const char *filename); }; -}} +} // namespace dnn +} // namespace tk #endif //NETWORKRT_H diff --git a/include/Yolo3Detection.h b/include/Yolo3Detection.h index 573b663..57f275c 100644 --- a/include/Yolo3Detection.h +++ b/include/Yolo3Detection.h @@ -1,6 +1,9 @@ +#ifndef YOLO3DDETECTION_H +#define YOLO3DDETECTION_H + #include #include -#include /* srand, rand */ +#include /* srand, rand */ #include #include #include "utils.h" @@ -11,49 +14,53 @@ #include "tkdnn.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ /** * * @author Francesco Gatti */ -class Yolo3Detection { +class Yolo3Detection +{ - private: - tk::dnn::NetworkRT *netRT = nullptr; - tk::dnn::Yolo* yolo[3]; - dnnType *input, *input_d; +private: + tk::dnn::NetworkRT *netRT = nullptr; + tk::dnn::Yolo *yolo[3]; + dnnType *input, *input_d; - int ndets = 0; - tk::dnn::Yolo::detection *dets = nullptr; + int ndets = 0; + tk::dnn::Yolo::detection *dets = nullptr; - cv::Mat imageF; - cv::Mat bgr[3]; + cv::Mat imageF; + cv::Mat bgr[3]; - +public: + int classes = 0; + int num = 0; + float thresh = 0.3; + cv::Scalar colors[256]; - public: - int classes = 0; - int num = 0; - float thresh = 0.3; - cv::Scalar colors[256]; + // this is filled with results + std::vector detected; - // this is filled with results - std::vector detected; + Yolo3Detection() {} - Yolo3Detection() {} + virtual ~Yolo3Detection() {} - virtual ~Yolo3Detection() {} - - /** - * Method used for inizialize the class + /** + * Method used to inizialize the class * * @return Success of the initialization */ - bool init(std::string tensor_path); - void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left); - void update(cv::Mat &frame); - + bool init(std::string tensor_path); + void addBorders(cv::Mat &imageORIG, cv::Mat &imageWBorders, int &top, int &left); + void update(cv::Mat &frame); }; -}} +} // namespace dnn +} // namespace tk + +#endif /*YOLO3DDETECTION_H*/ \ No newline at end of file diff --git a/include/BoxDetection.h b/include/boxDetection.h similarity index 100% rename from include/BoxDetection.h rename to include/boxDetection.h diff --git a/include/calibration.h b/include/calibration.h new file mode 100644 index 0000000..39b46e9 --- /dev/null +++ b/include/calibration.h @@ -0,0 +1,32 @@ +#ifndef CALIBRATION_H +#define CALIBRATION_H + +#include "gdal.h" +#include +#include +#include "gdal/gdal_priv.h" +#include "gdal/cpl_conv.h" + +#include +#include +#include + +#include +#include + +struct ObjCoords +{ + double lat_; + double long_; + int class_; +}; + +void readTiff(char *filename, double *adfGeoTransform); +void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff); +void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform); +void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform); +void fillMatrix(cv::Mat &H, double *matrix, bool show = false); +void read_projection_matrix(cv::Mat &H, char *path); +void convert_coords(std::vector &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform); + +#endif /*CALIBRATION_H*/ \ No newline at end of file diff --git a/include/cameraUtils.h b/include/cameraUtils.h new file mode 100644 index 0000000..ef9eb7d --- /dev/null +++ b/include/cameraUtils.h @@ -0,0 +1,45 @@ +#ifndef CAMERAUTILS_H +#define CAMERAUTILS_H + +#include +#include +#include +#include "tracker.h" +#include "Yolo3Detection.h" + +struct Camera_t +{ + int CAM_IDX; + char *input; + char *pmatrix; + char *maskfile; + char *cameraCalib; + char *maskFileOrient; + bool to_show; + tk::dnn::Yolo3Detection yolo; + double adfGeoTransform[6]; +}; + +struct Frame_t +{ + char *input; + cv::Mat frame; + int frame_nbr; + // sem_vc for mainthread, videocapturethread, originalthread and disparitythread + std::mutex sem_vc; +}; + +struct ModFrame_t +{ + std::vector trackers; + geodetic_converter::GeodeticConverter gc; + double adfGeoTransform[6]; + cv::Mat H; + cv::Mat original_frame; + tk::dnn::Yolo3Detection yolo; + cv::Mat mask; + // sem for mainthread, detectionthread and topviewthread + std::mutex sem; +}; + +#endif /*CAMERAUTILS_H*/ \ No newline at end of file diff --git a/include/classutils.h b/include/classutils.h deleted file mode 100644 index 8bf1e46..0000000 --- a/include/classutils.h +++ /dev/null @@ -1,316 +0,0 @@ -#ifndef CLASSUTILS_H -#define CLASSUTILS_H - -#include -#include -#include -#include -#include //socket -#include //inet_addr -#include //write - -#include -#include -#include "gdal.h" -#include -#include -#include "gdal/gdal_priv.h" -#include "gdal/cpl_conv.h" - -#include "tracker.h" - -#include - -#include "../masa_protocol/include/send.hpp" -#include "../masa_protocol/include/serialize.hpp" - -struct ObjCoords -{ - double lat_; - double long_; - int class_; -}; - -void readTiff(char *filename, double *adfGeoTransform) -{ - GDALDataset *poDataset; - GDALAllRegister(); - poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly); - if (poDataset != NULL) - { - poDataset->GetGeoTransform(adfGeoTransform); - } -} - -void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff) -{ - YAML::Node config = YAML::LoadFile(cameraCalib); - const YAML::Node &node_test1 = config["camera_matrix"]; - - float data_cm[9]; - for (std::size_t i = 0; i < node_test1["data"].size(); i++) - data_cm[i] = node_test1["data"][i].as(); - cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm); - cameraMat = cameraMat_.clone(); - std::cout << cameraMat << std::endl; - const YAML::Node &node_test2 = config["distortion_coefficients"]; - - float data_dc[5]; - for (std::size_t i = 0; i < node_test2["data"].size(); i++) - data_dc[i] = node_test2["data"][i].as(); - cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc); - distCoeff = distCoeff_.clone(); - std::cout << distCoeff << std::endl; -} - -void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform) -{ - //Returns global coordinates from pixel x, y coordinates - double xoff, a, b, yoff, d, e; - xoff = adfGeoTransform[0]; - a = adfGeoTransform[1]; - b = adfGeoTransform[2]; - yoff = adfGeoTransform[3]; - d = adfGeoTransform[4]; - e = adfGeoTransform[5]; - - //printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e ); - - lon = a * x + b * y + xoff; - lat = d * x + e * y + yoff; -} -void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform) -{ - x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1])); - y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5])); -} - -void fillMatrix(cv::Mat &H, double *matrix, bool show = false) -{ - double *vals = (double *)H.data; - for (int i = 0; i < 9; i++) - { - vals[i] = matrix[i]; - } - if (show) - std::cout << H << "\n"; -} - -//FILE *out_file = fopen("prova_pixel.txt", "w"); - -void convert_coords(std::vector &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform, int frame_nbr) -{ - double latitude, longitude; - std::vector x_y, ll; - x_y.push_back(cv::Point2f(x, y)); - //transform camera pixel to map pixel - cv::perspectiveTransform(x_y, ll, H); - //tranform to map pixel to map gps - pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform); - //printf("lat: %f, long:%f \n", latitude, longitude); - - ObjCoords coord; - coord.lat_ = latitude; - coord.long_ = longitude; - coord.class_ = detected_class; - coords.push_back(coord); - - /*if (detected_class == 0) - { - - struct timeval tv; - gettimeofday(&tv, NULL); - unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000; - - //printf(out_file, "%d %lld %d %d\n",frame_nbr, t_stamp_ms, int(ll[0].x), int(ll[0].y)); - fprintf(out_file, "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG); - //printf( "%d %lld %f %f\n", frame_nbr, t_stamp_ms, coord.LAT, coord.LONG); - }*/ -} - -void read_projection_matrix(cv::Mat &H, char *path) -{ - FILE *fp; - char *line = NULL; - size_t len = 0; - ssize_t read; - - // float *proj_matrix = (float *)malloc(9 * sizeof(float)); - double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - int i = 0; - fp = fopen(path, "r"); - if (fp == NULL) - exit(EXIT_FAILURE); - - while ((read = getline(&line, &len, fp)) != -1) - { - std::cout<> proj_matrix[i]) - i++; - } - fclose(fp); - fillMatrix(H, proj_matrix); - free(line); - // free(proj_matrix); -} - -void draw_arrow(float angleRad, float vel, cv::Scalar color, cv::Point center, cv::Mat &frame) -{ - int angle = angleRad * 180.0 / CV_PI; - auto length = 10 * vel; - auto direction = cv::Point(length * cos(angleRad), length * sin(angleRad)); // calculate direction - double tipLength = .2 + 0.4 * (angle % 180) / 360; - int lineType = 8; - int thickness = 2; - cv::arrowedLine(frame, center, center + direction, color, thickness, lineType, 0, tipLength); // draw arrow! -} - -unsigned long long time_in_ms() -{ - struct timeval tv; - gettimeofday(&tv, NULL); - unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000; - return t_stamp_ms; -} - -void addRoadUserfromTracker(const std::vector &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat& maskOrient, double *adfGeoTransform, cv::Mat H) -{ - m->t_stamp_ms = time_in_ms(); - m->objects.clear(); - double lat, lon, alt; - - for (auto t : trackers) - { - if (t.pred_list_.size() > 0) - { - Categories cat; - switch (t.class_) - { - case 0: - cat = Categories::C_person; - break; - case 1: - cat = Categories::C_car; - break; - case 2: - cat = Categories::C_car; - break; - case 3: - cat = Categories::C_bus; - break; - case 4: - cat = Categories::C_motorbike; - break; - case 5: - cat = Categories::C_bycicle; - break; - } - //std::cout << t.pred_list_.size() << std::endl; - gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt); - - int pix_x, pix_y; - coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform); - - // TODO: test correctness - added perspective transform call to converter pix_x and pix_y - // sometimes some values are wrong. float ok? - // std::vector map_p, camera_p; - // std::cout<<"--- pix_x, pix_y: "<(camera_p[0].y,camera_p[0].x)[0]; - // std::cout<<"boo: "<(camera_p[0].y,camera_p[0].x)< it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);") - // TODO: include perspective transform - // std::cout<<"y, x :"<(pix_y,pix_x); - // uint8_t orientation; - // if(maskOrientPixel != 0) - // { - // orientation = maskOrientPixel; - // // std::cout<<"orientation given by the mask "<< int(orientation)<(lat), static_cast(lon), velocity, orientation, cat}; - //std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl; - m->objects.push_back(r); - } - } - m->num_objects = m->objects.size(); -} - -void prepare_message(Message *m, const std::vector &coords, int idx) -{ - m->cam_idx = idx; - m->t_stamp_ms = time_in_ms(); - m->num_objects = coords.size(); - - m->objects.clear(); - for (unsigned int i = 0; i < coords.size(); i++) - { - Categories cat; - - switch (coords[i].class_) - { - case 0: - cat = Categories::C_person; - break; - case 1: - cat = Categories::C_car; - break; - case 2: - cat = Categories::C_car; - break; - case 3: - cat = Categories::C_bus; - break; - case 4: - cat = Categories::C_motorbike; - break; - case 5: - cat = Categories::C_bycicle; - break; - } - RoadUser r{static_cast(coords[i].lat_), static_cast(coords[i].long_), 0, 1, cat}; - std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << cat << std::endl; - m->objects.push_back(r); - } - - m->lights.clear(); -} - -#endif /*CLASSUTILS_H*/ \ No newline at end of file diff --git a/include/message.h b/include/message.h new file mode 100644 index 0000000..c67d5fe --- /dev/null +++ b/include/message.h @@ -0,0 +1,22 @@ +#ifndef MESSAGE_H +#define MESSAGE_H + +#include +#include +#include +#include +#include +// #include //socket +// #include //inet_addr +// #include //write + +#include "tracker.h" + +#include "../masa_protocol/include/send.hpp" +#include "../masa_protocol/include/serialize.hpp" + +unsigned long long time_in_ms(); + +void addRoadUserfromTracker(const std::vector &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H); + +#endif /*MESSAGE_H*/ \ No newline at end of file diff --git a/include/utils.h b/include/utils.h index a46b2c4..c6937e3 100644 --- a/include/utils.h +++ b/include/utils.h @@ -31,14 +31,18 @@ #define COL_PURPLEB "\033[1;35m" #define COL_CYANB "\033[1;36m" -// Simple Timer -#define TIMER_START timespec start, end; \ - clock_gettime(CLOCK_MONOTONIC, &start); +// Simple Timer +#define TIMER_START \ + timespec start, end; \ + clock_gettime(CLOCK_MONOTONIC, &start); -#define TIMER_STOP_C(col) clock_gettime(CLOCK_MONOTONIC, &end); \ - double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 + \ - (double)(end.tv_nsec - start.tv_nsec))/1.0e6; \ - std::cout< +#include +#include + +//saliency +#include +#include +#include + +#include +#include +#include + +#include "tracker.h" +#include "cameraUtils.h" +#include "calibration.h" +#include "boxDetection.h" + +struct Show_t +{ + cv::Mat original, detection, topview, disparity; + bool update_o, update_de, update_t, update_di; + // a single mutex for each operation - the show_updates function must get all mutex + std::mutex mutex_o, mutex_de, mutex_t, mutex_di; +}; + +extern Show_t updates; +extern bool gRun; +extern std::string obj_class[10]; + +/* Thread function to show the updated images +**/ +void *show_updates(void *x_void_ptr); +void *originalFrame(void *x_void_ptr); +void *detectionFrame(void *x_void_ptr); +void *topviewFrame(void *x_void_ptr); +void *disparityFrame(void *x_void_ptr); + +#endif /*VIZUALIZATION_H*/ \ No newline at end of file diff --git a/src/Activation.cpp b/src/Activation.cpp index 3c43e68..f6261cf 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -3,64 +3,72 @@ #include "Layer.h" #include "kernels.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -Activation::Activation(Network *net, int act_mode) : - Layer(net) { +Activation::Activation(Network *net, int act_mode) : Layer(net) +{ this->act_mode = act_mode; - checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) ); + checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType))); - if(int(act_mode) < 100) { + if (int(act_mode) < 100) + { - checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, - net->tensorFormat, - net->dataType, - input_dim.n*input_dim.l, - input_dim.c, - input_dim.h, input_dim.w) ); - checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc, - net->tensorFormat, - net->dataType, - input_dim.n*input_dim.l, - input_dim.c, - input_dim.h, input_dim.w) ); + checkCUDNN(cudnnSetTensor4dDescriptor(srcTensorDesc, + net->tensorFormat, + net->dataType, + input_dim.n * input_dim.l, + input_dim.c, + input_dim.h, input_dim.w)); + checkCUDNN(cudnnSetTensor4dDescriptor(dstTensorDesc, + net->tensorFormat, + net->dataType, + input_dim.n * input_dim.l, + input_dim.c, + input_dim.h, input_dim.w)); - - checkCUDNN( cudnnCreateActivationDescriptor(&activDesc) ); - checkCUDNN( cudnnSetActivationDescriptor(activDesc, - (cudnnActivationMode_t) act_mode, + checkCUDNN(cudnnCreateActivationDescriptor(&activDesc)); + checkCUDNN(cudnnSetActivationDescriptor(activDesc, + (cudnnActivationMode_t)act_mode, CUDNN_PROPAGATE_NAN, - 0.0) ); + 0.0)); } } -Activation::~Activation() { +Activation::~Activation() +{ - checkCuda( cudaFree(dstData) ); + checkCuda(cudaFree(dstData)); - if(int(act_mode) < 100) - checkCUDNN( cudnnDestroyActivationDescriptor(activDesc) ); + if (int(act_mode) < 100) + checkCUDNN(cudnnDestroyActivationDescriptor(activDesc)); } -dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { +dnnType *Activation::infer(dataDim_t &dim, dnnType *srcData) +{ - if(act_mode == ACTIVATION_LEAKY) { + if (act_mode == ACTIVATION_LEAKY) + { activationLEAKYForward(srcData, dstData, dim.tot()); - - } else { + } + else + { dnnType alpha = dnnType(1); - dnnType beta = dnnType(0); - checkCUDNN( cudnnActivationForward(net->cudnnHandle, - activDesc, - &alpha, - srcTensorDesc, - srcData, - &beta, - dstTensorDesc, - dstData) ); - } + dnnType beta = dnnType(0); + checkCUDNN(cudnnActivationForward(net->cudnnHandle, + activDesc, + &alpha, + srcTensorDesc, + srcData, + &beta, + dstTensorDesc, + dstData)); + } return dstData; } -}} +} // namespace dnn +} // namespace tk diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index bfb8316..bf13a33 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -2,14 +2,18 @@ #include "Layer.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, - int strideH, int strideW, int paddingH, int paddingW, - const char* fname_weights, bool batchnorm) : - - LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, - fname_weights, batchnorm) { +Conv2d::Conv2d(Network *net, int out_ch, int kernelH, int kernelW, + int strideH, int strideW, int paddingH, int paddingW, + const char *fname_weights, bool batchnorm) : + + LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, + fname_weights, batchnorm) +{ this->kernelH = kernelH; this->kernelW = kernelW; @@ -18,56 +22,55 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, this->paddingH = paddingH; this->paddingW = paddingW; - checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) ); - checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) ); - checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) ); + checkCUDNN(cudnnCreateFilterDescriptor(&filterDesc)); + checkCUDNN(cudnnCreateConvolutionDescriptor(&convDesc)); + checkCUDNN(cudnnCreateTensorDescriptor(&biasTensorDesc)); int n = input_dim.n; int c = input_dim.c; int h = input_dim.h; int w = input_dim.w; - checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, - net->tensorFormat, net->dataType, n, c, h, w) ); + checkCUDNN(cudnnSetTensor4dDescriptor(srcTensorDesc, + net->tensorFormat, net->dataType, n, c, h, w)); - checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc, - net->dataType, net->tensorFormat, out_ch, input_dim.c, - kernelH, kernelW) ); + checkCUDNN(cudnnSetFilter4dDescriptor(filterDesc, + net->dataType, net->tensorFormat, out_ch, input_dim.c, + kernelH, kernelW)); - checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc, - paddingH, paddingW, // padding - strideH, strideW, // stride - 1,1, // upscale - CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) ); + checkCUDNN(cudnnSetConvolution2dDescriptor(convDesc, + paddingH, paddingW, // padding + strideH, strideW, // stride + 1, 1, // upscale + CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT)); // find dimension of convolution output - checkCUDNN( cudnnGetConvolution2dForwardOutputDim( - convDesc, srcTensorDesc, filterDesc, - &n, &c, &h, &w) ); + checkCUDNN(cudnnGetConvolution2dForwardOutputDim( + convDesc, srcTensorDesc, filterDesc, + &n, &c, &h, &w)); + + checkCUDNN(cudnnSetTensor4dDescriptor(dstTensorDesc, + net->tensorFormat, net->dataType, n, c, h, w)); + + checkCUDNN(cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle, + srcTensorDesc, filterDesc, convDesc, dstTensorDesc, + CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo)); - checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc, - net->tensorFormat, net->dataType, n, c, h, w) ); - - checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle, - srcTensorDesc, filterDesc, convDesc, dstTensorDesc, - CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) ); - workSpace = NULL; ws_sizeInBytes = 0; - checkCUDNN( cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, - srcTensorDesc, filterDesc, convDesc, dstTensorDesc, - algo, &ws_sizeInBytes) ); + checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, + srcTensorDesc, filterDesc, convDesc, dstTensorDesc, + algo, &ws_sizeInBytes)); - if (ws_sizeInBytes!=0) { - checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) ); + if (ws_sizeInBytes != 0) + { + checkCuda(cudaMalloc(&workSpace, ws_sizeInBytes)); } - - checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc, - net->tensorFormat, net->dataType, - 1, out_ch, 1, 1) ); - + checkCUDNN(cudnnSetTensor4dDescriptor(biasTensorDesc, + net->tensorFormat, net->dataType, + 1, out_ch, 1, 1)); output_dim.n = n; output_dim.c = c; @@ -76,53 +79,58 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, output_dim.l = 1; //allocate data for infer result - checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); + checkCuda(cudaMalloc(&dstData, output_dim.tot() * sizeof(dnnType))); } -Conv2d::~Conv2d() { - - checkCUDNN( cudnnDestroyFilterDescriptor(filterDesc) ); - checkCUDNN( cudnnDestroyConvolutionDescriptor(convDesc) ); - checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) ); +Conv2d::~Conv2d() +{ - if (ws_sizeInBytes!=0) - checkCuda( cudaFree(workSpace) ); + checkCUDNN(cudnnDestroyFilterDescriptor(filterDesc)); + checkCUDNN(cudnnDestroyConvolutionDescriptor(convDesc)); + checkCUDNN(cudnnDestroyTensorDescriptor(biasTensorDesc)); - checkCuda( cudaFree(dstData) ); + if (ws_sizeInBytes != 0) + checkCuda(cudaFree(workSpace)); + + checkCuda(cudaFree(dstData)); } -dnnType* Conv2d::infer(dataDim_t &dim, dnnType* srcData) { - +dnnType *Conv2d::infer(dataDim_t &dim, dnnType *srcData) +{ // convolution dnnType alpha = dnnType(1); - dnnType beta = dnnType(0); - checkCUDNN( cudnnConvolutionForward(net->cudnnHandle, - &alpha, srcTensorDesc, srcData, filterDesc, - data_d, convDesc, algo, workSpace, ws_sizeInBytes, - &beta, dstTensorDesc, dstData) ); + dnnType beta = dnnType(0); + checkCUDNN(cudnnConvolutionForward(net->cudnnHandle, + &alpha, srcTensorDesc, srcData, filterDesc, + data_d, convDesc, algo, workSpace, ws_sizeInBytes, + &beta, dstTensorDesc, dstData)); - if(!batchnorm) { + if (!batchnorm) + { // bias alpha = dnnType(1); - beta = dnnType(1); - checkCUDNN( cudnnAddTensor(net->cudnnHandle, - &alpha, biasTensorDesc, bias_d, - &beta, dstTensorDesc, dstData) ); - } else { + beta = dnnType(1); + checkCUDNN(cudnnAddTensor(net->cudnnHandle, + &alpha, biasTensorDesc, bias_d, + &beta, dstTensorDesc, dstData)); + } + else + { float one = 1; float zero = 0; cudnnBatchNormalizationForwardInference(net->cudnnHandle, - CUDNN_BATCHNORM_SPATIAL, &one, &zero, - dstTensorDesc, dstData, dstTensorDesc, - dstData, biasTensorDesc, //same tensor descriptor as bias - scales_d, bias_d, mean_d, variance_d, - CUDNN_BN_MIN_EPSILON); + CUDNN_BATCHNORM_SPATIAL, &one, &zero, + dstTensorDesc, dstData, dstTensorDesc, + dstData, biasTensorDesc, //same tensor descriptor as bias + scales_d, bias_d, mean_d, variance_d, + CUDNN_BN_MIN_EPSILON); } - //update data dimensions + //update data dimensions dim = output_dim; return dstData; } -}} +} // namespace dnn +} // namespace tk diff --git a/src/Dense.cpp b/src/Dense.cpp index f86aa4a..f1a4da5 100644 --- a/src/Dense.cpp +++ b/src/Dense.cpp @@ -2,10 +2,13 @@ #include "Layer.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -Dense::Dense(Network *net, int out_ch, const char* fname_weights) : - LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights) { +Dense::Dense(Network *net, int out_ch, const char *fname_weights) : LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights) +{ output_dim.n = 1; output_dim.c = out_ch; @@ -14,19 +17,21 @@ Dense::Dense(Network *net, int out_ch, const char* fname_weights) : output_dim.l = 1; //allocate data for infer result - checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); + checkCuda(cudaMalloc(&dstData, output_dim.tot() * sizeof(dnnType))); } -Dense::~Dense() { +Dense::~Dense() +{ - checkCuda( cudaFree(dstData) ); + checkCuda(cudaFree(dstData)); } -dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) { +dnnType *Dense::infer(dataDim_t &dim, dnnType *srcData) +{ if (dim.n != 1) - FatalError("Not Implemented"); - + FatalError("Not Implemented"); + int dim_x = dim.tot(); int dim_y = output_dim.tot(); @@ -35,18 +40,18 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) { dnnType alpha = dnnType(1), beta = dnnType(1); // place bias into dstData - checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); - - //do matrix moltiplication - checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T, - dim_x, dim_y, - &alpha, - data_d, dim_x, - srcData, 1, - &beta, - dstData, 1) ); + checkCuda(cudaMemcpy(dstData, bias_d, dim_y * sizeof(dnnType), cudaMemcpyDeviceToDevice)); - //update data dimensions + //do matrix moltiplication + checkERROR(cublasSgemv(net->cublasHandle, CUBLAS_OP_T, + dim_x, dim_y, + &alpha, + data_d, dim_x, + srcData, 1, + &beta, + dstData, 1)); + + //update data dimensions dim.h = 1; dim.w = 1; dim.l = 1; @@ -55,4 +60,5 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) { return dstData; } -}} +} // namespace dnn +} // namespace tk diff --git a/src/Flatten.cpp b/src/Flatten.cpp index 8906c88..46237be 100644 --- a/src/Flatten.cpp +++ b/src/Flatten.cpp @@ -3,34 +3,40 @@ #include "Layer.h" #include "kernels.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -Flatten::Flatten(Network *net) : Layer(net) { +Flatten::Flatten(Network *net) : Layer(net) +{ - checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) ); + checkCuda(cudaMalloc(&dstData, input_dim.tot() * sizeof(dnnType))); output_dim.n = 1; output_dim.c = input_dim.tot(); output_dim.h = 1; output_dim.w = 1; output_dim.l = 1; - } -Flatten::~Flatten() { +Flatten::~Flatten() +{ - checkCuda( cudaFree(dstData) ); + checkCuda(cudaFree(dstData)); } -dnnType* Flatten::infer(dataDim_t &dim, dnnType* srcData) { +dnnType *Flatten::infer(dataDim_t &dim, dnnType *srcData) +{ //transpose per channel - matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h*dim.w*dim.l); + matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h * dim.w * dim.l); - //update data dimensions + //update data dimensions dim = output_dim; return dstData; } -}} \ No newline at end of file +} // namespace dnn +} // namespace tk \ No newline at end of file diff --git a/src/Layer.cpp b/src/Layer.cpp index dc1d494..fd04d9f 100644 --- a/src/Layer.cpp +++ b/src/Layer.cpp @@ -2,28 +2,35 @@ #include "Layer.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -Layer::Layer(Network *net) { +Layer::Layer(Network *net) +{ this->net = net; - if(net != nullptr) { + if (net != nullptr) + { this->input_dim = net->getOutputDim(); this->output_dim = input_dim; - - checkCUDNN( cudnnCreateTensorDescriptor(&srcTensorDesc) ); - checkCUDNN( cudnnCreateTensorDescriptor(&dstTensorDesc) ); - if(!net->addLayer(this)) - FatalError("Net reached max number of layers"); + checkCUDNN(cudnnCreateTensorDescriptor(&srcTensorDesc)); + checkCUDNN(cudnnCreateTensorDescriptor(&dstTensorDesc)); + + if (!net->addLayer(this)) + FatalError("Net reached max number of layers"); } } -Layer::~Layer() { +Layer::~Layer() +{ - checkCUDNN( cudnnDestroyTensorDescriptor(srcTensorDesc) ); - checkCUDNN( cudnnDestroyTensorDescriptor(dstTensorDesc) ); + checkCUDNN(cudnnDestroyTensorDescriptor(srcTensorDesc)); + checkCUDNN(cudnnDestroyTensorDescriptor(dstTensorDesc)); } -}} \ No newline at end of file +} // namespace dnn +} // namespace tk \ No newline at end of file diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index d8bb881..ec6cab8 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -4,24 +4,29 @@ #include "Layer.h" #include "kernels.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -LayerWgs::LayerWgs(Network *net, int inputs, int outputs, - int kh, int kw, int kl, - const char* fname_weights, bool batchnorm) : Layer(net) { +LayerWgs::LayerWgs(Network *net, int inputs, int outputs, + int kh, int kw, int kl, + const char *fname_weights, bool batchnorm) : Layer(net) +{ - this->inputs = inputs; - this->outputs = outputs; - this->weights_path = std::string(fname_weights); - - std::cout<<"Reading weights: I="<inputs = inputs; + this->outputs = outputs; + this->weights_path = std::string(fname_weights); + + std::cout << "Reading weights: I=" << inputs << " O=" << outputs << " KERNEL=" << kh << "x" << kw << "x" << kl << "\n"; int seek = 0; - readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek); - seek += inputs*outputs*kh*kw*kl; + readBinaryFile(weights_path.c_str(), inputs * outputs * kh * kw * kl, &data_h, &data_d, seek); + seek += inputs * outputs * kh * kw * kl; readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek); - + this->batchnorm = batchnorm; - if(batchnorm) { + if (batchnorm) + { seek += outputs; readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek); seek += outputs; @@ -32,86 +37,90 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, float eps = CUDNN_BN_MIN_EPSILON; power_h = new dnnType[outputs]; - for(int i=0; ifp16) + if (!net->fp16) return; //convert to fp16 - int w_size = inputs*outputs*kh*kw*kl; + int w_size = inputs * outputs * kh * kw * kl; data16_h = new __half[w_size]; - cudaMalloc(&data16_d, w_size*sizeof(__half)); + cudaMalloc(&data16_d, w_size * sizeof(__half)); float2half(data_d, data16_d, w_size); - cudaMemcpy(data16_h, data16_d, w_size*sizeof(__half), cudaMemcpyDeviceToHost); + cudaMemcpy(data16_h, data16_d, w_size * sizeof(__half), cudaMemcpyDeviceToHost); int b_size = outputs; bias16_h = new __half[b_size]; - cudaMalloc(&bias16_d, w_size*sizeof(__half)); + cudaMalloc(&bias16_d, w_size * sizeof(__half)); float2half(bias_d, bias16_d, b_size); - cudaMemcpy(bias16_h, bias16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + cudaMemcpy(bias16_h, bias16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); - if(batchnorm) { + if (batchnorm) + { - power16_h = new __half[b_size]; - mean16_h = new __half[b_size]; + power16_h = new __half[b_size]; + mean16_h = new __half[b_size]; variance16_h = new __half[b_size]; - scales16_h = new __half[b_size]; + scales16_h = new __half[b_size]; - cudaMalloc(&power16_d, b_size*sizeof(__half)); - cudaMalloc(&mean16_d, b_size*sizeof(__half)); - cudaMalloc(&variance16_d, b_size*sizeof(__half)); - cudaMalloc(&scales16_d, b_size*sizeof(__half)); + cudaMalloc(&power16_d, b_size * sizeof(__half)); + cudaMalloc(&mean16_d, b_size * sizeof(__half)); + cudaMalloc(&variance16_d, b_size * sizeof(__half)); + cudaMalloc(&scales16_d, b_size * sizeof(__half)); //temporary buffers float *tmp_d; - cudaMalloc(&tmp_d, b_size*sizeof(float)); + cudaMalloc(&tmp_d, b_size * sizeof(float)); //init power array of ones - cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy(tmp_d, power_h, b_size * sizeof(float), cudaMemcpyHostToDevice); float2half(tmp_d, power16_d, b_size); - cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + cudaMemcpy(power16_h, power16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); //mean array - cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy(tmp_d, mean_h, b_size * sizeof(float), cudaMemcpyHostToDevice); float2half(tmp_d, mean16_d, b_size); - cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + cudaMemcpy(mean16_h, mean16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); //convert variance - - cudaMemcpy(tmp_d, variance_h, b_size*sizeof(float), cudaMemcpyHostToDevice); + + cudaMemcpy(tmp_d, variance_h, b_size * sizeof(float), cudaMemcpyHostToDevice); float2half(tmp_d, variance16_d, b_size); - cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + cudaMemcpy(variance16_h, variance16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); //conver scales float2half(scales_d, scales16_d, b_size); - cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); + cudaMemcpy(scales16_h, scales16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); } } -LayerWgs::~LayerWgs() { +LayerWgs::~LayerWgs() +{ - delete [] data_h; - delete [] bias_h; - checkCuda( cudaFree(data_d) ); - checkCuda( cudaFree(bias_d) ); + delete[] data_h; + delete[] bias_h; + checkCuda(cudaFree(data_d)); + checkCuda(cudaFree(bias_d)); - if(batchnorm) { - delete [] scales_h; - delete [] mean_h; - delete [] variance_h; - checkCuda( cudaFree(scales_d) ); - checkCuda( cudaFree(mean_d) ); - checkCuda( cudaFree(variance_d) ); + if (batchnorm) + { + delete[] scales_h; + delete[] mean_h; + delete[] variance_h; + checkCuda(cudaFree(scales_d)); + checkCuda(cudaFree(mean_d)); + checkCuda(cudaFree(variance_d)); } } -}} +} // namespace dnn +} // namespace tk diff --git a/src/MulAdd.cpp b/src/MulAdd.cpp index 0c2a962..ccc32bb 100644 --- a/src/MulAdd.cpp +++ b/src/MulAdd.cpp @@ -3,42 +3,48 @@ #include "Layer.h" #include "kernels.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) { +MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) +{ this->mul = mul; this->add = add; int size = input_dim.tot(); - // create a vector with all value setted to add + // create a vector with all value setted to add dnnType *add_vector_h = new dnnType[size]; - for(int i=0; icublasHandle, srcData, dstData, add_vector, input_dim.tot(), mul); - - //update data dimensions + + //update data dimensions dim = output_dim; return dstData; } -}} \ No newline at end of file +} // namespace dnn +} // namespace tk \ No newline at end of file diff --git a/src/Network.cpp b/src/Network.cpp index 491e32b..7ceeca5 100644 --- a/src/Network.cpp +++ b/src/Network.cpp @@ -5,106 +5,131 @@ #include "Network.h" #include "Layer.h" -namespace tk { namespace dnn { +namespace tk +{ +namespace dnn +{ -Network::Network(dataDim_t input_dim) { +Network::Network(dataDim_t input_dim) +{ this->input_dim = input_dim; - float tk_ver = float(TKDNN_VERSION)/1000; - float cu_ver = float(cudnnGetVersion())/1000; + float tk_ver = float(TKDNN_VERSION) / 1000; + float cu_ver = float(cudnnGetVersion()) / 1000; - std::cout<<"New NETWORK (tkDNN v"<infer(dim, data); } checkCuda(cudaDeviceSynchronize()); return data; } -bool Network::addLayer(Layer *l) { - if(num_layers == MAX_LAYERS) +bool Network::addLayer(Layer *l) +{ + if (num_layers == MAX_LAYERS) return false; - + layers[num_layers++] = l; return true; } -dataDim_t Network::getOutputDim() { +dataDim_t Network::getOutputDim() +{ - if(num_layers == 0) - return input_dim; - else - return layers[num_layers-1]->output_dim; + if (num_layers == 0) + return input_dim; + else + return layers[num_layers - 1]->output_dim; } -void Network::print() { +void Network::print() +{ printCenteredTitle(" NETWORK MODEL ", '=', 60); - std::cout.width(3); std::cout<input_dim; dataDim_t out = layers[i]->output_dim; - std::cout.width(3); std::cout<getLayerName(); - std::cout.width(4); std::cout< "; - std::cout.width(4); std::cout<getLayerName(); + std::cout.width(4); + std::cout << std::right << in.h; + std::cout << " x "; + std::cout.width(4); + std::cout << std::right << in.w; + std::cout << ", "; + std::cout.width(4); + std::cout << std::right << in.c; + std::cout << " -> "; + std::cout.width(4); + std::cout << std::right << out.h; + std::cout << " x "; + std::cout.width(4); + std::cout << std::right << out.w; + std::cout << ", "; + std::cout.width(4); + std::cout << std::right << out.c; + std::cout << "\n"; } printCenteredTitle("", '=', 60); - std::cout<<"\n"; + std::cout << "\n"; } - -}} +} // namespace dnn +} // namespace tk diff --git a/src/BoxDetection.cpp b/src/class_src/boxDetection.cpp similarity index 60% rename from src/BoxDetection.cpp rename to src/class_src/boxDetection.cpp index 0ce26bb..3e5ed3b 100644 --- a/src/BoxDetection.cpp +++ b/src/class_src/boxDetection.cpp @@ -1,6 +1,6 @@ -#include "BoxDetection.h" +#include "boxDetection.h" #include -char buf_frame_crop_name [200]; +char buf_frame_crop_name[200]; cv::Mat img_threshold(cv::Mat frame_crop) { @@ -24,10 +24,10 @@ cv::Mat img_background(cv::Mat frame_crop) cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0); cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU); // get background - cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1)); + cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1, 1, 1, 1)); cv::erode(gray, opening, M); cv::dilate(gray, opening, M); - cv::Point p = cv::Point(-1,-1); + cv::Point p = cv::Point(-1, -1); cv::dilate(opening, coinsBg, M, p, 3); return coinsBg; } @@ -43,10 +43,10 @@ cv::Mat img_dist_transform(cv::Mat frame_crop) cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU); // cv::Mat::ones M(3,3,cv::CV_8U); // get background - cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1)); + cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1, 1, 1, 1)); cv::erode(gray, opening, M); cv::dilate(gray, opening, M); - cv::Point p = cv::Point(-1,-1); + cv::Point p = cv::Point(-1, -1); cv::dilate(opening, coinsBg, M, p, 3); // distance transorm cv::distanceTransform(opening, distTrans, cv::DIST_L2, 5); @@ -75,11 +75,11 @@ cv::Mat img_dist_transform(cv::Mat frame_crop) // // get foreground // cv::threshold(distTrans, coinsFg, 0.7 * 1, 255, cv::THRESH_BINARY); -// coinsFg.convertTo(coinsFg, CV_8U, 1, 0); +// coinsFg.convertTo(coinsFg, CV_8U, 1, 0); // cv::subtract(coinsBg, coinsFg, unknown); // // get connected components networks // cv::connectedComponents(coinsFg, markers); -// // intptr_t n = NULL; +// // intptr_t n = NULL; // for(int i = 0; i< markers.rows; i++) // { // for (int j = 0; j< markers.cols; j++) @@ -111,7 +111,7 @@ cv::Mat img_dist_transform(cv::Mat frame_crop) ////// -cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0) +cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret = 0) { //ret = 0 --> dstx //ret = 1 --> dsty @@ -120,61 +120,61 @@ cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0) // Image Sobel and Image AbsSobel // https://docs.opencv.org/trunk/da/d85/tutorial_js_gradients.html // compute image gradient on two different directions - + cv::Mat f = frame_crop.clone(); - int x,y; - (ret == 0 || ret == 2)?x=1, y=0 : NULL; - (ret == 1 || ret == 3)?x=0, y=1 : NULL; - cv::Mat dst; + int x, y; + (ret == 0 || ret == 2) ? x = 1, y = 0 : NULL; + (ret == 1 || ret == 3) ? x = 0, y = 1 : NULL; + cv::Mat dst; cv::cvtColor(f, f, cv::COLOR_RGB2GRAY, 0); // You can try more different parameters cv::Sobel(f, dst, CV_8U, x, y, 3, 1, 0, cv::BORDER_DEFAULT); // for absSobel - if(ret == 2 || ret == 3) + if (ret == 2 || ret == 3) cv::convertScaleAbs(dst, dst, 1, 0); // next 3 rows to be checked //// ??cv::Mat f2 = frame_crop.clone(); //// cv.Scharr(?(f,f2), dstx, cv.CV_8U, 1, 0, 1, 0, cv.BORDER_DEFAULT); - //// cv.Scharr(?(f,f2), dsty, cv.CV_8U, 0, 1, 1, 0, cv.BORDER_DEFAULT); + //// cv.Scharr(?(f,f2), dsty, cv.CV_8U, 0, 1, 1, 0, cv.BORDER_DEFAULT); return dst; } -cv::Mat img_laplacian(cv::Mat frame_crop, int ret=1) +cv::Mat img_laplacian(cv::Mat frame_crop, int ret = 1) { //ret = 0 --> src_gray //ret = 1 --> dst // Image Laplacian - // compute image gradient with laplacian + // compute image gradient with laplacian cv::Mat f = frame_crop.clone(); cv::Mat src_gray, dst; int kernel_size = 3; int scale = 1; int delta = 0; int ddepth = CV_16S; - cv::GaussianBlur( f, f, cv::Size(3,3), 0, 0, cv::BORDER_DEFAULT ); + cv::GaussianBlur(f, f, cv::Size(3, 3), 0, 0, cv::BORDER_DEFAULT); /// Convert the image to grayscale - cv::cvtColor( f, src_gray, CV_RGB2GRAY ); + cv::cvtColor(f, src_gray, CV_RGB2GRAY); if (ret == 0) return src_gray; // else: Apply Laplace function cv::Mat abs_dst; - cv::Laplacian( src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT ); + cv::Laplacian(src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT); // //compute sharpness // float sharpnessValue = cv::mean(dst); - return dst; + return dst; } -cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines=1) +cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines = 1) { // n_line: number of line to plot on image cv::Mat img_line = frame_crop.clone(); cv::Mat ret_thresh; - std::vector > contours; + std::vector> contours; double thresh = 127; double maxValue = 255; - cv::threshold(img, ret_thresh, thresh, maxValue, 0);//0); // = cv2.threshold(img,127,255,0) - cv::findContours(canny_output, contours, 1, 2);//cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2) + cv::threshold(img, ret_thresh, thresh, maxValue, 0); //0); // = cv2.threshold(img,127,255,0) + cv::findContours(canny_output, contours, 1, 2); //cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2) // cv::threshold(img2, ret2, thresh, maxValue, 0); // cv::findContours(canny_output2, contours2, 1, 2); // cv::threshold(img3a, ret3a, thresh, maxValue, 0); @@ -183,23 +183,23 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int // cv::findContours(canny_output3b, contours3b, 1, 2); cv::Vec4f line; - float vx,vy,x,y; + float vx, vy, x, y; int lefty, righty; - for(int i=0; i > boxPts2; -// cv::boxPoints(rect, boxPts1); +// cv::boxPoints(rect, boxPts1); // // boxPts = np.int0(boxPts); // for (int x = 0; x < img.cols; x++) // for (int y = 0; y < img.rows; y++) @@ -226,7 +226,7 @@ cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int // return img_clone; // } -cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr saliencyAlgorithm, int const_molt_mat, int ret=0) +cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr saliencyAlgorithm, int const_molt_mat, int ret = 0) { //ret=0 --> saliencyMap //ret=1 --> binaryMap @@ -234,110 +234,117 @@ cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr sa cv::Mat f = frame_crop.clone(); cv::Mat saliencyMap; cv::Mat binaryMap; - - if( saliencyAlgorithm->computeSaliency( f, saliencyMap ) ) + + if (saliencyAlgorithm->computeSaliency(f, saliencyMap)) { - if(ret==0) - return saliencyMap*const_molt_mat; + if (ret == 0) + return saliencyMap * const_molt_mat; cv::saliency::StaticSaliencySpectralResidual spec; - spec.computeBinaryMap( saliencyMap, binaryMap ); + spec.computeBinaryMap(saliencyMap, binaryMap); // imshow( "Saliency Map", saliencyMap ); // imshow( "Original Image", image ); // imshow( "Binary Map", binaryMap ); // waitKey( 0 ); - return binaryMap*const_molt_mat; + return binaryMap * const_molt_mat; } - return cv::Mat(0,0,CV_8U, cv::Scalar(0,0,0,0)); + return cv::Mat(0, 0, CV_8U, cv::Scalar(0, 0, 0, 0)); } ////// void image_segmentation(cv::Mat frame_crop, int frame_nbr, int i) -{ +{ // Watershed Algorithm // https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html - + auto step_t_segmentation = std::chrono::steady_clock::now(); auto end_t_segmentation = std::chrono::steady_clock::now(); cv::Mat ret; // ret = img_threshold(frame_crop); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgthr.jpg", frame_nbr, i, img_threshold(frame_crop)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgthr.jpg", frame_nbr, i, img_threshold(frame_crop)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME imgthr ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // ret =img_background(frame_crop); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgback.jpg", frame_nbr, i, img_background(frame_crop)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgback.jpg", frame_nbr, i, img_background(frame_crop)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME imgback ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // ret = img_dist_transform(frame_crop); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgtrans.jpg", frame_nbr, i, img_dist_transform(frame_crop)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgtrans.jpg", frame_nbr, i, img_dist_transform(frame_crop)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME imgtrans ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // // ret = img_watershed(frame_crop); - // if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgwatershed.jpg", frame_nbr, i, img_watershed(frame_crop)); + // if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgwatershed.jpg", frame_nbr, i, img_watershed(frame_crop)); } void image_gradients(cv::Mat frame_crop, int frame_nbr, int i) { // Image Gradients // https://docs.opencv.org/trunk/da/d85/tutorial_js_gradients.html - + auto step_t_segmentation = std::chrono::steady_clock::now(); auto end_t_segmentation = std::chrono::steady_clock::now(); cv::Mat ret; // sobel // ret = img_sobel_abssobel(frame_crop, 0); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_8U.jpgg", frame_nbr, i, img_sobel_abssobel(frame_crop, 0)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_8U.jpgg", frame_nbr, i, img_sobel_abssobel(frame_crop, 0)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME sobel0 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // ret = img_sobel_abssobel(frame_crop, 1); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_8U.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 1)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_8U.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 1)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME sobel1 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // ret = img_sobel_abssobel(frame_crop, 2); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 2)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_x_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 2)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME sobel2 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // ret = img_sobel_abssobel(frame_crop, 3); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 3)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imgsobel_y_64F.jpg", frame_nbr, i, img_sobel_abssobel(frame_crop, 3)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME sobel3 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; - + // laplacian // ret = img_laplacian(frame_crop, 0); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_gr.jpg", frame_nbr, i, img_laplacian(frame_crop, 0)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_gr.jpg", frame_nbr, i, img_laplacian(frame_crop, 0)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME laplacian0 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; + step_t_segmentation = end_t_segmentation; // ret = img_laplacian(frame_crop, 1); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_dst.jpg", frame_nbr, i, img_laplacian(frame_crop, 1)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_imglaplacian_dst.jpg", frame_nbr, i, img_laplacian(frame_crop, 1)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME laplacian1 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; ////// - -} +} void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i) { // Finding contours in your image // https://docs.opencv.org/3.4/df/d0d/tutorial_find_contours.html - auto step_t_segmentation = std::chrono::steady_clock::now(); auto end_t_segmentation = std::chrono::steady_clock::now(); // plot lines on figure. 3 ways: @@ -348,60 +355,67 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i) cv::Mat contours; // src_gray cv::Mat img1 = img_laplacian(frame_crop, 0); - cv::Canny(img1, canny_output1, 100, 100*2 ); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny1.jpg", frame_nbr, i, canny_output1); + cv::Canny(img1, canny_output1, 100, 100 * 2); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny1.jpg", frame_nbr, i, canny_output1); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME canny1 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // // dst // cv::Mat img2 = img_laplacian(frame_crop, 2); // cv::Canny(img2, canny_output2, 100, 100*2 ); - cv::Canny(img1, canny_output2, 100, 100*2 ); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny2.jpg", frame_nbr, i, canny_output2); + cv::Canny(img1, canny_output2, 100, 100 * 2); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny2.jpg", frame_nbr, i, canny_output2); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME canny2 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // dstx cv::Mat img3a = img_sobel_abssobel(frame_crop, 0); - cv::Canny(img3a, canny_output3a, 100, 100*2 ); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3a.jpg", frame_nbr, i, canny_output3a); + cv::Canny(img3a, canny_output3a, 100, 100 * 2); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3a.jpg", frame_nbr, i, canny_output3a); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME canny3a ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // dsty cv::Mat img3b = img_sobel_abssobel(frame_crop, 1); - cv::Canny(img3b, canny_output3b, 100, 100*2 ); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3b.jpg", frame_nbr, i, canny_output3b); + cv::Canny(img3b, canny_output3b, 100, 100 * 2); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_canny3b.jpg", frame_nbr, i, canny_output3b); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME canny3b ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; - + // 1 line // contours = find_contours(frame_crop, img1, canny_output1, 1); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line1.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output1, 1)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_line1.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output1, 1)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME line1 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // 3 line // contours = find_contours(frame_crop, img1, canny_output2, 1); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line2.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output2, 1)); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_line2.jpg", frame_nbr, i, find_contours(frame_crop, img1, canny_output2, 1)); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME line2 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // mix 1 line of image with 1 line of another cv::Mat img_line = frame_crop.clone(); img_line = find_contours(img_line, img3a, canny_output3a, 1); img_line = find_contours(img_line, img3b, canny_output3b, 1); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_line3.jpg", frame_nbr, i, img_line); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_line3.jpg", frame_nbr, i, img_line); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME line3 ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // img_line = frame_crop.clone(); @@ -409,8 +423,7 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i) // img_line = find_contours(img_line, img3b, canny_output3b, 2); // sprintf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_line3bis.jpg",frame_nbr, i); // cv::imwrite(buf_frame_crop_name, img_line); - - + // cv::Mat canny_output4; // cv::Mat img4 = img_laplacian(frame_crop, 0); // cv::Canny(img4, canny_output4, 100, 100*2 ); @@ -418,12 +431,11 @@ void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i) // cv::imwrite(buf_frame_crop_name, fit_rectangular(frame_crop, img4, canny_output4)); } - void image_saliency(cv::Mat frame_crop, int frame_nbr, int i) { // https://github.com/opencv/opencv_contrib/blob/master/modules/saliency/samples/computeSaliency.cpp cv::Ptr saliencyAlgorithm; - + int const_molt_mat = 0; auto step_t_segmentation = std::chrono::steady_clock::now(); auto end_t_segmentation = std::chrono::steady_clock::now(); @@ -432,47 +444,50 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i) const_molt_mat = 255; saliencyAlgorithm = cv::saliency::StaticSaliencySpectralResidual::create(); cv::Mat spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0); - if(!spect_res.empty()) + if (!spect_res.empty()) { - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg", frame_nbr, i, spect_res); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg", frame_nbr, i, spect_res); } else { - std::cout<<"something is wrond (image_saliency)"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // BINARY SPECTRAL_RESIDUAL const_molt_mat = 255; spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 1); - if(!spect_res.empty()) + if (!spect_res.empty()) { - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinarySpectralResidual.jpg", frame_nbr, i, spect_res); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinarySpectralResidual.jpg", frame_nbr, i, spect_res); } else { - std::cout<<"something is wrond (image_saliency)"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // FINE_GRAINED const_molt_mat = 1; - saliencyAlgorithm = cv::saliency::StaticSaliencyFineGrained::create(); + saliencyAlgorithm = cv::saliency::StaticSaliencyFineGrained::create(); spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0); - if(!spect_res.empty()) + if (!spect_res.empty()) { - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_FineGrained.jpg", frame_nbr, i, spect_res); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_FineGrained.jpg", frame_nbr, i, spect_res); } else { - std::cout<<"something is wrond (image_saliency)"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // saliencyAlgorithm = cv::saliency::ObjectnessBING::create(); @@ -486,7 +501,7 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i) // // // The result are sorted by objectness. We only use the first maxd boxes here. // // int maxd = 7, step = 255 / maxd, jitter=9; // jitter to seperate single rects // // cv::Mat draw = frame_crop.clone(); - // // for (int i = 0; i < std::min(maxd, ndet); i++) + // // for (int i = 0; i < std::min(maxd, ndet); i++) // // { // // cv::Vec4i bb = saliencyMap1[i]; // // cv::Scalar col = cv::Scalar(((i*step)%255), 100, 255-((i*step)%255)); @@ -499,34 +514,35 @@ void image_saliency(cv::Mat frame_crop, int frame_nbr, int i) // printf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_saliency_BING.jpg",frame_nbr, i); // cv::imwrite(buf_frame_crop_name, saliencyMap1); - //// + //// // BING WANG APR 2014 cv::Mat saliencyMap; cv::Mat frame_sal = frame_crop.clone(); saliencyAlgorithm = cv::saliency::MotionSaliencyBinWangApr2014::create(); - saliencyAlgorithm.dynamicCast()->setImagesize( frame_sal.cols, frame_sal.rows ); + saliencyAlgorithm.dynamicCast()->setImagesize(frame_sal.cols, frame_sal.rows); saliencyAlgorithm.dynamicCast()->init(); - cvtColor( frame_sal, frame_sal, cv::COLOR_BGR2GRAY ); - saliencyAlgorithm->computeSaliency( frame_sal, saliencyMap); - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg", frame_nbr, i, saliencyMap); + cvtColor(frame_sal, frame_sal, cv::COLOR_BGR2GRAY); + saliencyAlgorithm->computeSaliency(frame_sal, saliencyMap); + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg", frame_nbr, i, saliencyMap); end_t_segmentation = std::chrono::steady_clock::now(); - std::cout << " - TIME BING WANG APR 2014("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; } -cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret=0) +cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret = 0) { // https://stackoverflow.com/questions/27035672/cv-extract-differences-between-two-images cv::Mat backgroundImage = pre_frame.clone(); cv::Mat currentImage = frame.clone(); cv::Mat diffImage; // pass to HSV color - if(ret) + if (ret) { - cv::cvtColor(backgroundImage, backgroundImage, CV_BGR2HSV); - cv::cvtColor(currentImage, currentImage, CV_BGR2HSV); + cv::cvtColor(backgroundImage, backgroundImage, CV_BGR2HSV); + cv::cvtColor(currentImage, currentImage, CV_BGR2HSV); } cv::absdiff(backgroundImage, currentImage, diffImage); @@ -537,27 +553,28 @@ cv::Mat frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, float threshold = 30.0f; float dist; - for(int j=0; j(j,k); + cv::Vec3b pix = diffImage.at(j, k); - dist = (pix[0]*pix[0] + pix[1]*pix[1] + pix[2]*pix[2]); + dist = (pix[0] * pix[0] + pix[1] * pix[1] + pix[2] * pix[2]); dist = sqrt(dist); - if(dist>threshold) + if (dist > threshold) { - foregroundMask.at(j,k) = 255; + foregroundMask.at(j, k) = 255; } } } - if(SAVE) SAVE_TO("../demo/demo/data/img_disparity/%d_%d_dif.jpg", frame_nbr, i, foregroundMask); + if (SAVE) + SAVE_TO("../demo/demo/data/img_disparity/%d_%d_dif.jpg", frame_nbr, i, foregroundMask); return foregroundMask; } -void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector pre_rois, int frame_nbr) +void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector pre_rois, int frame_nbr) { int roi_tollerance = 10; @@ -567,35 +584,38 @@ void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector 0)? (r.x - dx) : 0; - r.y = (r.y - dy > 0)? (r.y - dy) : 0; + r.x = (r.x - dx > 0) ? (r.x - dx) : 0; + r.y = (r.y - dy > 0) ? (r.y - dy) : 0; // std::cout<<"disp: x "<= frame.cols)? (frame.cols-1-r.x) : (r.width+dx+dx); - r.height = ((r.y+r.height+dy+dy) >= frame.rows)? (frame.rows-1-r.y) : (r.height+dy+dy); + r.width = ((r.x + r.width + dx + dx) >= frame.cols) ? (frame.cols - 1 - r.x) : (r.width + dx + dx); + r.height = ((r.y + r.height + dy + dy) >= frame.rows) ? (frame.rows - 1 - r.y) : (r.height + dy + dy); // std::cout<<"disp: w "<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; id++; } @@ -606,54 +626,53 @@ void segmentation(cv::Mat pre_frame, cv::Mat frame_crop, int frame_nbr, int i, i //mode=0 (for whole frame), it computes the frame disparity //mode=1 (for single box), it doesn't compute the frame disparity (it has already been done-see frame_box_disparity()) // whole figure - char buf_str [15]; - if(!mode) - sprintf(buf_str,"whole frame"); + char buf_str[15]; + if (!mode) + sprintf(buf_str, "whole frame"); else - sprintf(buf_str,"a box frame"); - - if(SAVE) SAVE_TO("../demo/demo/data/img_crop/%d_%d.jpg", frame_nbr, i, frame_crop); + sprintf(buf_str, "a box frame"); + + if (SAVE) + SAVE_TO("../demo/demo/data/img_crop/%d_%d.jpg", frame_nbr, i, frame_crop); auto step_t_segmentation = std::chrono::steady_clock::now(); auto end_t_segmentation = std::chrono::steady_clock::now(); // Watershed Algorithm - std::cout<<"image segmentation:"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // Image Gradients - std::cout<<"image gradients:"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; // Find contours - std::cout<<"image find contours:"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; - + //saliency map - std::cout<<"image saliency:"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; //frame disparity - if(!mode && frame_nbr!=0) + if (!mode && frame_nbr != 0) { - std::cout<<"frame disparity:"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms" << std::endl; step_t_segmentation = end_t_segmentation; } - - } \ No newline at end of file diff --git a/src/class_src/calibration.cpp b/src/class_src/calibration.cpp new file mode 100644 index 0000000..1ccf0f7 --- /dev/null +++ b/src/class_src/calibration.cpp @@ -0,0 +1,108 @@ +#include "calibration.h" + +void readTiff(char *filename, double *adfGeoTransform) +{ + GDALDataset *poDataset; + GDALAllRegister(); + poDataset = (GDALDataset *)GDALOpen(filename, GA_ReadOnly); + if (poDataset != NULL) + { + poDataset->GetGeoTransform(adfGeoTransform); + } +} + +void readCameraCalibrationYaml(const std::string &cameraCalib, cv::Mat &cameraMat, cv::Mat &distCoeff) +{ + YAML::Node config = YAML::LoadFile(cameraCalib); + const YAML::Node &node_test1 = config["camera_matrix"]; + + float data_cm[9]; + for (std::size_t i = 0; i < node_test1["data"].size(); i++) + data_cm[i] = node_test1["data"][i].as(); + cv::Mat cameraMat_ = cv::Mat(3, 3, CV_32F, data_cm); + cameraMat = cameraMat_.clone(); + std::cout << cameraMat << std::endl; + const YAML::Node &node_test2 = config["distortion_coefficients"]; + + float data_dc[5]; + for (std::size_t i = 0; i < node_test2["data"].size(); i++) + data_dc[i] = node_test2["data"][i].as(); + cv::Mat distCoeff_ = cv::Mat(5, 1, CV_32F, data_dc); + distCoeff = distCoeff_.clone(); + std::cout << distCoeff << std::endl; +} + +void pixel2coord(int x, int y, double &lat, double &lon, double *adfGeoTransform) +{ + //Returns global coordinates from pixel x, y coordinates + double xoff, a, b, yoff, d, e; + xoff = adfGeoTransform[0]; + a = adfGeoTransform[1]; + b = adfGeoTransform[2]; + yoff = adfGeoTransform[3]; + d = adfGeoTransform[4]; + e = adfGeoTransform[5]; + + //printf("%f %f %f %f %f %f\n",xoff, a, b, yoff, d, e ); + + lon = a * x + b * y + xoff; + lat = d * x + e * y + yoff; +} +void coord2pixel(double lat, double lon, int &x, int &y, double *adfGeoTransform) +{ + x = int(round((lon - adfGeoTransform[0]) / adfGeoTransform[1])); + y = int(round((lat - adfGeoTransform[3]) / adfGeoTransform[5])); +} + +void fillMatrix(cv::Mat &H, double *matrix, bool show) +{ + double *vals = (double *)H.data; + for (int i = 0; i < 9; i++) + { + vals[i] = matrix[i]; + } + if (show) + std::cout << H << "\n"; +} + +void read_projection_matrix(cv::Mat &H, char *path) +{ + FILE *fp; + char *line = NULL; + size_t len = 0; + ssize_t read; + + double proj_matrix[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + int i = 0; + fp = fopen(path, "r"); + if (fp == NULL) + exit(EXIT_FAILURE); + + while ((read = getline(&line, &len, fp)) != -1) + { + std::cout << line << std::endl; + std::stringstream ss(line); + while (ss >> proj_matrix[i]) + i++; + } + fclose(fp); + fillMatrix(H, proj_matrix); + free(line); +} + +void convert_coords(std::vector &coords, int x, int y, int detected_class, cv::Mat H, double *adfGeoTransform) +{ + double latitude, longitude; + std::vector x_y, ll; + x_y.push_back(cv::Point2f(x, y)); + //transform camera pixel to map pixel + cv::perspectiveTransform(x_y, ll, H); + //tranform to map pixel to map gps + pixel2coord(ll[0].x, ll[0].y, latitude, longitude, adfGeoTransform); + + ObjCoords coord; + coord.lat_ = latitude; + coord.long_ = longitude; + coord.class_ = detected_class; + coords.push_back(coord); +} diff --git a/src/class_src/message.cpp b/src/class_src/message.cpp new file mode 100644 index 0000000..b9e2f08 --- /dev/null +++ b/src/class_src/message.cpp @@ -0,0 +1,109 @@ +#include "message.h" +#include "calibration.h" + +unsigned long long time_in_ms() +{ + struct timeval tv; + gettimeofday(&tv, NULL); + unsigned long long t_stamp_ms = (unsigned long long)(tv.tv_sec) * 1000 + (unsigned long long)(tv.tv_usec) / 1000; + return t_stamp_ms; +} + +void addRoadUserfromTracker(const std::vector &trackers, Message *m, geodetic_converter::GeodeticConverter &gc, const cv::Mat &maskOrient, double *adfGeoTransform, cv::Mat H) +{ + m->t_stamp_ms = time_in_ms(); + m->objects.clear(); + double lat, lon, alt; + + for (auto t : trackers) + { + if (t.pred_list_.size() > 0) + { + Categories cat; + switch (t.class_) + { + case 0: + cat = Categories::C_person; + break; + case 1: + cat = Categories::C_car; + break; + case 2: + cat = Categories::C_car; + break; + case 3: + cat = Categories::C_bus; + break; + case 4: + cat = Categories::C_motorbike; + break; + case 5: + cat = Categories::C_bycicle; + break; + } + //std::cout << t.pred_list_.size() << std::endl; + gc.enu2Geodetic(t.pred_list_.back().x_, t.pred_list_.back().y_, 0, &lat, &lon, &alt); + + int pix_x, pix_y; + coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform); + + // TODO: test correctness - added perspective transform call to converter pix_x and pix_y + // sometimes some values are wrong. float ok? + // std::vector map_p, camera_p; + // std::cout<<"--- pix_x, pix_y: "<(camera_p[0].y,camera_p[0].x)[0]; + // std::cout<<"boo: "<(camera_p[0].y,camera_p[0].x)< it works for grayscale image (see demo.cpp, row: "cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0);") + // TODO: include perspective transform + // std::cout<<"y, x :"<(pix_y,pix_x); + // uint8_t orientation; + // if(maskOrientPixel != 0) + // { + // orientation = maskOrientPixel; + // // std::cout<<"orientation given by the mask "<< int(orientation)<(lat), static_cast(lon), velocity, orientation, cat}; + //std::cout << std::setprecision(10) << r.latitude << " , " << r.longitude << " " << int(r.speed) << " " << int(r.orientation) << " " << r.category << std::endl; + m->objects.push_back(r); + } + } + m->num_objects = m->objects.size(); +} diff --git a/src/class_src/visualization.cpp b/src/class_src/visualization.cpp new file mode 100644 index 0000000..87ad0fe --- /dev/null +++ b/src/class_src/visualization.cpp @@ -0,0 +1,377 @@ +#include "visualization.h" + +/* Thread function to show the updated images +**/ +void *show_updates(void *x_void_ptr) +{ + cv::namedWindow("original", cv::WINDOW_NORMAL); + cv::namedWindow("detection", cv::WINDOW_NORMAL); + cv::namedWindow("topview", cv::WINDOW_NORMAL); + cv::namedWindow("disparity", cv::WINDOW_NORMAL); + cv::Mat original_loc, detection_loc, topview_loc, disparity_loc; + bool update_o_loc, update_de_loc, update_t_loc, update_di_loc; + + while (gRun) + { + TIMER_START + // critical section: copy the struct in local variable + // in this way we can unlock the sem for the main thread + if (updates.mutex_o.try_lock()) + { + update_o_loc = updates.update_o; + updates.update_o = false; + if (update_o_loc) + original_loc = updates.original.clone(); + updates.mutex_o.unlock(); + } + + if (updates.mutex_de.try_lock()) + { + update_de_loc = updates.update_de; + updates.update_de = false; + if (update_de_loc) + detection_loc = updates.detection.clone(); + updates.mutex_de.unlock(); + } + + if (updates.mutex_t.try_lock()) + { + update_t_loc = updates.update_t; + updates.update_t = false; + if (update_t_loc) + topview_loc = updates.topview.clone(); + updates.mutex_t.unlock(); + } + + if (updates.mutex_di.try_lock()) + { + update_di_loc = updates.update_di; + updates.update_di = false; + if (update_di_loc) + disparity_loc = updates.disparity.clone(); + updates.mutex_di.unlock(); + } + + if (update_o_loc) + cv::imshow("original", original_loc); + if (update_de_loc) + cv::imshow("detection", detection_loc); + if (update_t_loc) + cv::imshow("topview", topview_loc); + if (update_di_loc) + cv::imshow("disparity", disparity_loc); + cv::waitKey(1); + // usleep(20000); //sleep 20 msec + std::cout << "show_updates: "; + TIMER_STOP + } + return (void *)0; +} + +void *originalFrame(void *x_void_ptr) +{ + Frame_t *info_show_orig = (Frame_t *)x_void_ptr; + cv::Mat frame_loc; + int frame_nbr_loc = 0; + while (gRun) + { + TIMER_START + // critical section: copy the struct in local variable + // in this way we can unlock the sem for the main thread + info_show_orig->sem_vc.lock(); + frame_loc = info_show_orig->frame.clone(); + frame_nbr_loc = info_show_orig->frame_nbr; + info_show_orig->sem_vc.unlock(); + if (frame_nbr_loc == 0) + { + usleep(1000000); + printf("no frame received\n"); + continue; + } + updates.mutex_o.lock(); + updates.original = frame_loc.clone(); + updates.update_o = true; + updates.mutex_o.unlock(); + usleep(10000); //sleep 10 msec + std::cout << "originalFrame: "; + TIMER_STOP + } + return (void *)0; +} + +void *detectionFrame(void *x_void_ptr) +{ + ModFrame_t *info_show = (ModFrame_t *)x_void_ptr; + double lat, lon, alt; + int pix_x, pix_y; + cv::Mat original_frame_loc; + std::vector trackers; + + geodetic_converter::GeodeticConverter gc; + double adfGeoTransform[6]; + cv::Mat H; + tk::dnn::Yolo3Detection yolo; + int num_detected; + cv::Mat mask; + + // box variable + tk::dnn::box b; + int x0, w, x1, y0, h, y1; + int objClass; + std::string det_class; + ; + // float prob; + cv::Scalar intensity; + std::vector map_p, camera_p; + int baseline = 0; + float fontScale = 0.5; + int thickness = 2; + + while (gRun) + { + TIMER_START + // critical section: copy the struct in local variable + // in this way we can unlock the sem for the main thread + info_show->sem.lock(); + original_frame_loc = info_show->original_frame.clone(); + // std::vector trackers; + trackers = info_show->trackers; + // geodetic_converter::GeodeticConverter gc; + gc = info_show->gc; + for (int i = 0; i < 6; i++) + adfGeoTransform[i] = info_show->adfGeoTransform[i]; + // cv::Mat H; + H = info_show->H.clone(); + yolo = info_show->yolo; + mask = info_show->mask.clone(); + info_show->sem.unlock(); + + if (trackers.empty()) + { + usleep(1000000); + printf("no data available\n"); + continue; + } + + num_detected = yolo.detected.size(); + for (int i = 0; i < num_detected; i++) + { + b = yolo.detected[i]; + x0 = b.x; + w = b.w; + x1 = b.x + w; + y0 = b.y; + h = b.h; + y1 = b.y + h; + objClass = b.cl; + det_class = obj_class[b.cl]; + // prob = b.prob; + + intensity = mask.at(cv::Point(int(x0 + b.w / 2), y1)); + + if (intensity[0] && objClass < 6) + { + //std::cout<= 0 && camera_p[0].y >= 0) + cv::circle(original_frame_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); + } + } + + updates.mutex_de.lock(); + updates.detection = original_frame_loc.clone(); + updates.update_de = true; + updates.mutex_de.unlock(); + + std::cout << "detectionFrame: "; + TIMER_STOP + } + return (void *)0; +} + +void *topviewFrame(void *x_void_ptr) +{ + ModFrame_t *info_show = (ModFrame_t *)x_void_ptr; + double lat, lon, alt; + int pix_x, pix_y; + cv::Mat frame_top; + cv::Mat original_frame_top; + // original_frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg"); + original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png"); + // original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670_V.png"); + + std::vector trackers; + geodetic_converter::GeodeticConverter gc; + double adfGeoTransform[6]; + cv::Mat H; + while (gRun) + { + TIMER_START + // critical section: copy the struct in local variable + // in this way we can unlock the sem for the main thread + info_show->sem.lock(); + // std::vector trackers; + trackers = info_show->trackers; + // geodetic_converter::GeodeticConverter gc; + gc = info_show->gc; + for (int i = 0; i < 6; i++) + adfGeoTransform[i] = info_show->adfGeoTransform[i]; + // cv::Mat H; + H = info_show->H.clone(); + info_show->sem.unlock(); + if (trackers.empty()) + { + usleep(1000000); + printf("no data available\n"); + continue; + } + frame_top = original_frame_top.clone(); + for (auto t : trackers) + { + for (size_t p = 1; p < t.pred_list_.size(); p++) + { + gc.enu2Geodetic(t.pred_list_[p].x_, t.pred_list_[p].y_, 0, &lat, &lon, &alt); + coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform); + if (pix_x < frame_top.cols && pix_y < frame_top.rows && pix_x >= 0 && pix_y >= 0) + cv::circle(frame_top, cv::Point(pix_x, pix_y), 7.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); + } + } + //outputVideo<< frame_top; + // ------------------------------------------------ + updates.mutex_t.lock(); + updates.topview = frame_top.clone(); + updates.update_t = true; + updates.mutex_t.unlock(); + + std::cout << "topviewFrame: "; + TIMER_STOP + } + return (void *)0; +} + +void *disparityFrame(void *x_void_ptr) +{ + Frame_t *info_show_disparity = (Frame_t *)x_void_ptr; + bool first_iteration = true; + cv::Mat frame_loc; + int frame_nbr_loc = 0, pre_frame_nbr_loc = 0; + auto start_t = std::chrono::steady_clock::now(); + auto step_t = std::chrono::steady_clock::now(); + auto end_t = std::chrono::steady_clock::now(); + + // information for the disparity map + cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB; + cv::Mat canny_img; + cv::Mat disparity_frame; + while (gRun) + { + start_t = std::chrono::steady_clock::now(); + step_t = start_t; + // critical section: copy the struct in local variable + // in this way we can unlock the sem for the main thread + info_show_disparity->sem_vc.lock(); + frame_loc = info_show_disparity->frame.clone(); + frame_nbr_loc = info_show_disparity->frame_nbr; + info_show_disparity->sem_vc.unlock(); + if (frame_nbr_loc == 0) + { + usleep(1000000); + printf("no frame received\n"); + continue; + } + // compute frame disparity only in there is a new frame + if (frame_nbr_loc - pre_frame_nbr_loc > 0) + { + pre_frame_nbr_loc = frame_nbr_loc; + //preprocessing frame + step_t = std::chrono::steady_clock::now(); + // src_gray + canny_img = img_laplacian(frame_loc, 0); + cv::Canny(canny_img, canny, 100, 100 * 2); + // sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr_loc, 999); + // cv::imwrite(buf_frame_crop_name, canny); + end_t = std::chrono::steady_clock::now(); + std::cout << " TIME END pre canny : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + step_t = end_t; + // std::cout<<"o: "<(end_t - step_t).count() << " ms" << std::endl; + step_t = end_t; + + // //-------------------------------- + // //frame box disparity on the original image + // step_t_segmentation = std::chrono::steady_clock::now(); + // frame_box_disparity(pre_frame, frame, pre_rois, frame_nbr_loc); + // // reset pre_rois for the new roi of the current frame + // // pre_rois.erase(pre_rois.begin(), pre_rois.end()); + // end_t_segmentation = std::chrono::steady_clock::now(); + // std::cout << " TIME Frame disparity : "<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t - start_t).count() << " ms" << std::endl; + } + } + return (void *)0; +}