diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index f9772f2..732c151 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -38,17 +38,32 @@ bool gRun; std::string obj_class[10]{"person", "car", "truck", "bus", "motor", "bike", "rider", "traffic light", "traffic sign", "train"}; -std::mutex sem; +// sem for mainthread, detectionthread and topviewthread +// sem_vc for mainthread, videocapturethread, originalthread and disparitythread +std::mutex sem, sem_vc; +// a single mutex for each operation - the show_updates function must get all mutex +std::mutex mutex_o, mutex_de, mutex_t, mutex_di; -struct InfoShow{ +struct ModFrame_t{ std::vector trackers; geodetic_converter::GeodeticConverter gc; double adfGeoTransform[6]; cv::Mat H; - cv::Mat frame_v; - cv::Mat frame_disp; + cv::Mat original_frame; + tk::dnn::Yolo3Detection yolo; + cv::Mat mask; }; +struct Frame_t{ + char *input; + cv::Mat frame; + int frame_nbr; +}; + +struct Show_t{ + cv::Mat original, detection, topview, disparity; + bool update_o, update_de, update_t, update_di; +}updates; void sig_handler(int signo) { @@ -56,25 +71,208 @@ void sig_handler(int signo) gRun = false; } -void *showImages(void *x_void_ptr) +void *readVideoCapture(void *x_void_ptr) { - InfoShow *info_show = (InfoShow *) x_void_ptr; - double lat, lon, alt; - int pix_x, pix_y; - cv::Mat frame_top; - cv::Mat original_frame_top; + Frame_t *info_f = (Frame_t *) x_void_ptr; + cv::VideoCapture cap(info_f->input); + cv::Mat frame_loc; + int frame_nbr_loc = 0; + // bool to_show = false; + if (!cap.isOpened()) + gRun = false; + else + std::cout << "camera started\n"; + + + cap.set(cv::CAP_PROP_BUFFERSIZE,3); + // cap.set(CV_CAP_PROP_FPS, 10); + // // CV_CAP_PROP_BUFFERSIZE + std::cout<<"buf size: "<> frame; + // i++; + // } + + // i=0; + // start_t = std::chrono::steady_clock::now(); + // while (i> frame; + // i++; + // std::cout << " step "<(std::chrono::steady_clock::now() - step_t).count() << " ms"<(end_t - start_t).count() << " ms"<> frame_loc; + // CAP_PROP_POS_MSEC Current position of the video file in milliseconds or video capture timestamp. + std::cout<<"id: "<input); + printf("cap reinitialize\n"); + continue; + } + + end_t = std::chrono::steady_clock::now(); + std::cout << " VC-TIME 1 : "<(end_t - start_t).count() << " ms"<frame = frame_loc.clone(); + info_f->frame_nbr = frame_nbr_loc; + sem_vc.unlock(); + end_t = std::chrono::steady_clock::now(); + std::cout << " VC-TIME 2 : "<(end_t - start_t).count() << " ms"<frame.clone(); + frame_nbr_loc = info_show_orig->frame_nbr; + sem_vc.unlock(); + if (frame_nbr_loc == 0) + { + usleep(1000000); + printf("no frame received\n"); + continue; + } + mutex_o.lock(); + updates.original = frame_loc.clone(); + updates.update_o = true; + mutex_o.unlock(); + usleep(10000); //sleep 10 msec + std::cout<<"originalFrame: "; + TIMER_STOP + } +} + +void *detectionFrame(void *x_void_ptr) +{ + ModFrame_t *info_show = (ModFrame_t *) x_void_ptr; + double lat, lon, alt; + int pix_x, pix_y; + cv::Mat original_frame_loc; std::vector trackers; geodetic_converter::GeodeticConverter gc; double adfGeoTransform[6]; cv::Mat H; - cv::Mat frame_disp_loc; + tk::dnn::Yolo3Detection yolo; + int num_detected; + cv::Mat mask; + + // box variable + tk::dnn::box b; + int x0, w, x1, y0, h, y1; + int objClass; + std::string det_class;; + float prob; + cv::Scalar intensity; while (gRun) { @@ -82,7 +280,7 @@ void *showImages(void *x_void_ptr) // critical section: copy the struct in local variable // in this way we can unlock the sem for the main thread sem.lock(); - frame_v_loc = info_show->frame_v.clone(); + original_frame_loc = info_show->original_frame.clone(); // std::vector trackers; trackers = info_show->trackers; // geodetic_converter::GeodeticConverter gc; @@ -91,10 +289,111 @@ void *showImages(void *x_void_ptr) adfGeoTransform[i] = info_show->adfGeoTransform[i]; // cv::Mat H; H = info_show->H.clone(); - frame_disp_loc = info_show->frame_disp.clone(); - + yolo = info_show->yolo; + mask = info_show->mask.clone(); sem.unlock(); + if (trackers.empty()) + { + usleep(1000000); + printf("no data available\n"); + continue; + } + + num_detected = yolo.detected.size(); + for (int i = 0; i < num_detected; i++) + { + b = yolo.detected[i]; + x0 = b.x; + w = b.w; + x1 = b.x + w; + y0 = b.y; + h = b.h; + y1 = b.y + h; + objClass = b.cl; + det_class = obj_class[b.cl]; + prob = b.prob; + + intensity = mask.at(cv::Point(int(x0 + b.w / 2), y1)); + + if (intensity[0] && objClass < 6) + { + //std::cout< map_p, camera_p; + map_p.push_back(cv::Point2f(pix_x, pix_y)); + + //transform camera pixel to map pixel + cv::perspectiveTransform(map_p, camera_p, H.inv()); + cv::circle(original_frame_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); + } + } + + mutex_de.lock(); + updates.detection = original_frame_loc.clone(); + updates.update_de = true; + mutex_de.unlock(); + + std::cout<<"detectionFrame: "; + TIMER_STOP + + } +} + +void *topviewFrame(void *x_void_ptr) +{ + ModFrame_t *info_show = (ModFrame_t *) x_void_ptr; + double lat, lon, alt; + int pix_x, pix_y; + cv::Mat frame_top; + cv::Mat original_frame_top; + // original_frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg"); + // original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png"); + original_frame_top = cv::imread("../demo/demo/data/map/MASA_4670_V.png"); + + std::vector trackers; + geodetic_converter::GeodeticConverter gc; + double adfGeoTransform[6]; + cv::Mat H; + while (gRun) + { + TIMER_START + // critical section: copy the struct in local variable + // in this way we can unlock the sem for the main thread + sem.lock(); + // std::vector trackers; + trackers = info_show->trackers; + // geodetic_converter::GeodeticConverter gc; + gc = info_show->gc; + for(int i = 0; i < 6; i++ ) + adfGeoTransform[i] = info_show->adfGeoTransform[i]; + // cv::Mat H; + H = info_show->H.clone(); + sem.unlock(); + if (trackers.empty()) + { + usleep(1000000); + printf("no data available\n"); + continue; + } frame_top = original_frame_top.clone(); for (auto t : trackers) { @@ -110,40 +409,118 @@ void *showImages(void *x_void_ptr) if (pix_x < frame_top.cols && pix_y < frame_top.rows && pix_x >= 0 && pix_y >= 0) cv::circle(frame_top, cv::Point(pix_x, pix_y), 7.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); - std::vector map_p, camera_p; - map_p.push_back(cv::Point2f(pix_x, pix_y)); - - //transform camera pixel to map pixel - cv::perspectiveTransform(map_p, camera_p, H.inv()); - //std::cout << "pix_x: " << camera_p[0].x << " pix_y: " << camera_p[0].y << std::endl; - - cv::circle(frame_v_loc, cv::Point(camera_p[0].x, camera_p[0].y), 3.0, cv::Scalar(t.r_, t.g_, t.b_), CV_FILLED, 8, 0); - - /*if(p == t.pred_list_.size()-1) - { - auto center = cv::Point(camera_p[0].x, camera_p[0].y); - auto color = cv::Scalar(t.r_, t.g_, t.b_); - draw_arrow(t.pred_list_[p].yaw_, t.pred_list_[p].vel_,color, center, frame); - - center = cv::Point(pix_x,pix_y); - draw_arrow(t.pred_list_[p].yaw_, t.pred_list_[p].vel_,color, center, frame_top); - }*/ } } - - //outputVideo<< frame_top; + //outputVideo<< frame_top; // ------------------------------------------------ - std::cout<<"size: "<frame.clone(); + frame_nbr_loc = info_show_disparity->frame_nbr; + sem_vc.unlock(); + if (frame_nbr_loc == 0) + { + usleep(1000000); + printf("no frame received\n"); + continue; + } + // compute frame disparity only in there is a new frame + if(frame_nbr_loc - pre_frame_nbr_loc > 0) + { + pre_frame_nbr_loc = frame_nbr_loc; + //preprocessing frame + step_t = std::chrono::steady_clock::now(); + // src_gray + canny_img = img_laplacian(frame_loc,0); + cv::Canny(canny_img, canny, 100, 100*2 ); + // sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr_loc, 999); + // cv::imwrite(buf_frame_crop_name, canny); + end_t = std::chrono::steady_clock::now(); + std::cout << " TIME END pre canny : "<(end_t - step_t).count() << " ms"<(end_t - step_t).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t - start_t).count() << " ms"< pre_rois; - cv::Mat pre_frame; + // cv::Mat pre_frame; cv::Mat orig_frame; - cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB; - cv::Mat canny_img; + // cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB; + // cv::Mat canny_img; // box variable tk::dnn::box b; @@ -282,44 +664,79 @@ int main(int argc, char *argv[]) cv::Scalar intensity; cv::Rect roi; - InfoShow info_show; + ModFrame_t info_show; if (to_show) { - info_show.H = cv::Mat(cv::Size(3, 3), CV_64FC1); - if (pthread_create(&visual, NULL, showImages, (void *)&info_show)) + // initialize updates struct + updates.update_o = false; + updates.update_de = false; + updates.update_t = false; + updates.update_di = false; + if (pthread_create(&visual, NULL, show_updates, (void*)NULL)) { fprintf(stderr, "Error creating thread\n"); return 1; - } - } + }; + if (pthread_create(&originalshow, NULL, originalFrame, (void*)&info_f)) + { + fprintf(stderr, "Error creating thread\n"); + return 1; + }; + if (pthread_create(&disparityshow, NULL, disparityFrame, (void*)&info_f)) + { + fprintf(stderr, "Error creating thread\n"); + return 1; + }; + info_show.H = cv::Mat(cv::Size(3, 3), CV_64FC1); + if (pthread_create(&detectionshow, NULL, detectionFrame, (void*)&info_show)) + { + fprintf(stderr, "Error creating thread\n"); + return 1; + }; + if (pthread_create(&topviewshow, NULL, topviewFrame, (void*)&info_show)) + { + fprintf(stderr, "Error creating thread\n"); + return 1; + }; + // if (pthread_create(&detectionshow, NULL, showImages, (void *)&info_show)) + // { + // fprintf(stderr, "Error creating thread\n"); + // return 1; + // } + } + + bool first_iteration = true; while (gRun) { TIMER_START start_t = std::chrono::steady_clock::now(); step_t = start_t; - cap >> frame; - - orig_frame = frame.clone(); + sem_vc.lock(); + frame = info_f.frame.clone(); + if(info_f.frame_nbr - frame_nbr > 1) + std::cout<<"more than one - f_n\n"; + frame_nbr = info_f.frame_nbr; + sem_vc.unlock(); + std::cout<<"f_n: "<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<= frame.rows)? frame.rows-1-roi.y : h; // std::cout<<"w "<(end_t - step_t).count() << " ms"<(end_t - step_t).count() << " ms"< trackers; info_show.trackers = trackers; // geodetic_converter::GeodeticConverter gc; @@ -488,15 +906,19 @@ int main(int argc, char *argv[]) info_show.adfGeoTransform[i] = adfGeoTransform[i]; // cv::Mat H; info_show.H = H.clone(); - info_show.frame_disp = frame_disp.clone(); + info_show.yolo = yolo; + info_show.mask = mask.clone(); sem.unlock(); } // update pre_frame for the disparity map - pre_frame = orig_frame.clone(); - pre_canny = canny.clone(); - + // pre_frame = orig_frame.clone(); + // pre_canny = canny.clone(); + if(first_iteration) + first_iteration = false; + frame_nbr++; + std::cout<<"MAIN thread: "; TIMER_STOP } diff --git a/include/classutils.h b/include/classutils.h index ac91658..33b2be0 100644 --- a/include/classutils.h +++ b/include/classutils.h @@ -215,14 +215,14 @@ void addRoadUserfromTracker(const std::vector &trackers, Message *m, ge int pix_x, pix_y; coord2pixel(lat, lon, pix_x, pix_y, adfGeoTransform); - std::cout<(pix_y,pix_x)<(pix_y,pix_x)<(pix_y,pix_x)[0]; uint8_t orientation; if(maskOrientPixel != 0) { orientation = maskOrientPixel; - std::cout<<"orientation given by the mask "<< int(orientation)<