#include #include "utils.h" #include "Yolo3Detection.h" #include "message.h" #include "visualization.h" #include "configuration.h" #include "tracker.h" #include "../masa_protocol/include/send.hpp" #include "../masa_protocol/include/serialize.hpp" // #include // #include // #include // #include // #include // #include // #include // #include // #include // #include #define MAX_DETECT_SIZE 100 bool gRun; std::chrono::steady_clock::time_point local_clock_start; std::mutex mutexgRun; std::string obj_class[10]{"person", "car", "truck", "bus", "motor", "bike", "rider", "traffic light", "traffic sign", "train"}; //mutex for some opencv operations std::mutex mutex_cv; Show_t updates; bool SAVE_RESULT = false; void sig_handler(int signo) { std::cout << "request gateway stop\n"; mutexgRun.lock(); gRun = false; mutexgRun.unlock(); } void *readVideoCapture(void *x_void_ptr) { std::cout << "readVideoCapture start...\n"; Frame_t *info_f = (Frame_t *)x_void_ptr; mutex_cv.lock(); cv::VideoCapture cap(info_f->input, cv::CAP_FFMPEG); mutex_cv.unlock(); cv::Mat frame_loc, frame0; int frame_nbr_loc = 0; // bool to_show = false; if (!cap.isOpened()) { mutexgRun.lock(); gRun = false; mutexgRun.unlock(); } else std::cout << "camera started\n"; // cap.set(cv::CAP_PROP_BUFFERSIZE,3); // std::cout<<"buf size: "<> frame_loc; i++; } i = 0; start_t = std::chrono::steady_clock::now(); while (i < num_f) { step_t = std::chrono::steady_clock::now(); cap >> frame_loc; mean_time = mean_time + std::chrono::duration_cast(std::chrono::steady_clock::now() - step_t).count(); std::cout << " step " << i << " : " << std::chrono::duration_cast(std::chrono::steady_clock::now() - step_t).count() << " ms" << std::endl; i++; } end_t = std::chrono::steady_clock::now(); std::cout << "Capturing " << num_f << " frames" << std::endl; std::cout << " Time taken : " << std::chrono::duration_cast(end_t - start_t).count() << " ms" << std::endl; /* mean_time indicates the milliseconds from a frame and the next. (frame rate) local_clock_sync is the camera clock. shift is the difference from local camera clock and local process clock. a frame is allowed if its local timestamp minus its local clock is less then a tollerance, otherwise it will be considered old. */ auto local_clock_sync = std::chrono::steady_clock::now(); mean_time = mean_time / num_f; shift = ((double)std::chrono::duration_cast(local_clock_sync - local_clock_start).count()) / mean_time; shift = (shift - (int)shift) * mean_time; std::cout << ".-------------------------------\n"; std::cout << " mean time: " << mean_time << std::endl; std::cout << " shift: " << shift << std::endl; std::cout << " TIMEDIFFERENCE: " << std::chrono::duration_cast(local_clock_sync - local_clock_start).count() << std::endl; std::cout << "\n\n\n\n"; std::cout << "readVideoCapture start to capture...\n"; while (gRun) { // mutex_cv.lock(); cap >> frame_loc; // mutex_cv.unlock(); current_timestamp = std::chrono::steady_clock::now(); shift = std::chrono::duration_cast(current_timestamp - local_clock_sync).count(); std::cout << " RELATIVE TIMESTAMP FRAME : " << shift << " ms" << std::endl; shift = shift / mean_time; shift = (shift - (int)shift) * mean_time; shift = (shift - mean_time / 2 >= 0) ? -(mean_time - shift) : shift; std::cout << "DELAY frame_" << frame_nbr_loc << " : " << shift << " ms" << std::endl; // TODO: here introduce a tollerance to discard old frame // std::cout<< "CV_CAP_PROP_POS_MSEC: "<< cap.get( cv::CAP_PROP_POS_MSEC) <input); printf("cap reinitialize\n"); mutex_cv.unlock(); continue; } end_t = std::chrono::steady_clock::now(); std::cout << " VC-TIME 1 : " << std::chrono::duration_cast(end_t - start_t).count() << " ms" << std::endl; start_t = end_t; info_f->sem_vc.lock(); info_f->frame = frame_loc.clone(); info_f->frame_nbr = frame_nbr_loc; info_f->sem_vc.unlock(); // usleep(50000); end_t = std::chrono::steady_clock::now(); std::cout << " VC-TIME 2 : " << std::chrono::duration_cast(end_t - start_t).count() << " ms" << std::endl; start_t = end_t; frame_nbr_loc++; } return (void *)0; } void *computationTask(void *x_void_ptr) { Camera_t *camera = (Camera_t *)x_void_ptr; pthread_t visual, originalshow, detectionshow, topviewshow, disparityshow; pthread_t videocap; tk::dnn::Yolo3Detection yolo = *(camera->yolo); //create video capture thread Frame_t info_f; info_f.input = camera->input; if (pthread_create(&videocap, NULL, readVideoCapture, (void *)&info_f)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; bool to_show = camera->to_show; double adfGeoTransform[6]; for (int i = 0; i < 6; i++) adfGeoTransform[i] = camera->adfGeoTransform[i]; ModFrame_t info_show; if (to_show) { // initialize updates struct updates.update_o = false; updates.update_de = false; updates.update_t = false; updates.update_di = false; if (pthread_create(&visual, NULL, show_updates, (void *)NULL)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; if (pthread_create(&originalshow, NULL, originalFrame, (void *)&info_f)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; if (pthread_create(&disparityshow, NULL, disparityFrame, (void *)&info_f)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; info_show.H = cv::Mat(cv::Size(3, 3), CV_64FC1); if (pthread_create(&detectionshow, NULL, detectionFrame, (void *)&info_show)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; if (pthread_create(&topviewshow, NULL, topviewFrame, (void *)&info_show)) { fprintf(stderr, "Error creating thread\n"); return (void *)1; }; } char *pmatrix = camera->pmatrix; /*projection matrix from camera to map*/ cv::Mat H(cv::Size(3, 3), CV_64FC1); read_projection_matrix(H, pmatrix); assert(cv::countNonZero(H) > 0); // std::cout<cameraCalib, cameraMat, distCoeff); std::cout << cameraMat << std::endl; std::cout << distCoeff << std::endl; /*GPS information*/ std::vector coords; /*socket*/ Communicator Comm(SOCK_DGRAM); Comm.open_client_socket((char *)"127.0.0.1", 8888); Message *m = new Message; m->cam_idx = camera->CAM_IDX; m->lights.clear(); /*Conversion for tracker, from gps to meters and viceversa*/ // mutex_cv.lock(); geodetic_converter::GeodeticConverter gc; gc.initialiseReference(44.655540, 10.934315, 0); // mutex_cv.unlock(); double east, north, up; // double lat, lon, alt; /*Mask info*/ cv::Mat mask = cv::imread(camera->maskfile, cv::IMREAD_GRAYSCALE); cv::Mat maskOrient = cv::imread(camera->maskFileOrient); // cv::Mat maskOrient = cv::imread(camera->maskFileOrient, 0); /*for(int i=0; i< mask.cols; i++) { for(int j=0; j< mask.rows; j++) { std::cout<(i,j) < trackers; std::vector cur_frame; int initial_age = -5; int age_threshold = -8; int n_states = 5; float dt = 0.03; int frame_nbr = 0; //save video /*cv::VideoWriter outputVideo; cv::Size S = cv::Size((int)cap.get(cv::CAP_PROP_FRAME_WIDTH), //Acquire input size (int)cap.get(cv::CAP_PROP_FRAME_HEIGHT)); outputVideo.open("test.avi", static_cast(cap.get(cv::CAP_PROP_FOURCC)), cap.get(cv::CAP_PROP_FPS), S, true);*/ cv::Mat map1, map2; auto start_t = std::chrono::steady_clock::now(); auto step_t = std::chrono::steady_clock::now(); auto end_t = std::chrono::steady_clock::now(); // auto step_t_segmentation = std::chrono::steady_clock::now(); // auto end_t_segmentation = std::chrono::steady_clock::now(); //TODO: move in a thread // // information for the disparity map // std::vector pre_rois; // cv::Mat pre_frame; cv::Mat orig_frame; // cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB; // cv::Mat canny_img; // box variable tk::dnn::box b; int x0, h, y1; //w, x1, y0; int objClass; std::string det_class; ; // float prob; cv::Scalar intensity; // cv::VideoWriter resultVideo; // if(SAVE_RESULT) { // int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); // int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); // resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); // } cv::Mat frame; cv::Mat frame_crop; cv::Mat dnn_input; bool first_iteration = true; while (gRun) { TIMER_START start_t = std::chrono::steady_clock::now(); step_t = start_t; info_f.sem_vc.lock(); frame = info_f.frame.clone(); if (info_f.frame_nbr - frame_nbr > 1) std::cout << "more than one - f_n (diff " << info_f.frame_nbr - frame_nbr << ")\n"; frame_nbr = info_f.frame_nbr; info_f.sem_vc.unlock(); std::cout << "f_n: " << frame_nbr << std::endl; // if (!frame.data) if (frame_nbr == 0) { usleep(1000000); printf("no frame received\n"); continue; } orig_frame = frame.clone(); // mutex_cv.lock(); if (first_iteration) cv::initUndistortRectifyMap(cameraMat, distCoeff, cv::Mat(), cameraMat, frame.size(), CV_16SC2, map1, map2); cv::Mat temp = frame.clone(); cv::remap(temp, frame, map1, map2, 1); //undistort(temp, frame, cameraMat, distCoeff); // mutex_cv.unlock(); // this will be resized to the net format dnn_input = frame.clone(); // TODO: async infer yolo.update(dnn_input); int num_detected = yolo.detected.size(); if (num_detected > MAX_DETECT_SIZE) num_detected = MAX_DETECT_SIZE; coords.clear(); end_t = std::chrono::steady_clock::now(); std::cout << " TIME 1 : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; // draw dets std::cout << "camera: " << camera->CAM_IDX << " - num detected: " << num_detected << std::endl; //TODO: move in a thread // //preprocessing frame // step_t_segmentation = std::chrono::steady_clock::now(); // // src_gray // canny_img = img_laplacian(orig_frame,0); // cv::Canny(canny_img, canny, 100, 100*2 ); // // sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_canny.jpg",frame_nbr, 999); // // cv::imwrite(buf_frame_crop_name, canny); // end_t_segmentation = std::chrono::steady_clock::now(); // std::cout << " - TIME END pre canny : "<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(cv::Point(int(x0 + b.w / 2), y1)); if (intensity[0]) { if (objClass < 6) { // find the rectangular on the frame (sub-figure) // roi.x = (x0 > 0)? x0 : 0; // roi.y = (y0 > 0)? y0 : 0; // // std::cout<<"x "<= frame.cols)? frame.cols-1-roi.x : w; // roi.height = (roi.y+h >= frame.rows)? frame.rows-1-roi.y : h; // std::cout<<"w "<yolo.colors[objClass], 2); // // draw label // int baseline = 0; // float fontScale = 0.5; // int thickness = 2; // cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline); // cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), camera->yolo.colors[b.cl], -1); // cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness); } } } end_t = std::chrono::steady_clock::now(); std::cout << " TIME 2 : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; //convert from latitude and longitude to meters for ekf cur_frame.clear(); for (size_t i = 0; i < coords.size(); i++) { gc.geodetic2Enu(coords[i].lat_, coords[i].long_, 0, &east, &north, &up); cur_frame.push_back(Data(east, north, frame_nbr, coords[i].class_)); } if (first_iteration) { // if there aren't detections and it is the first iteration, we can't initialize the tracker, so continue if (cur_frame.empty()) continue; for (auto f : cur_frame) trackers.push_back(Tracker(f, initial_age, dt, n_states)); } else { Track(cur_frame, dt, n_states, initial_age, age_threshold, trackers); } std::cout << "There are " << trackers.size() << " trackers" << std::endl; //prepare message with tracker info if (trackers.size() != 0) { // mutex_cv.lock(); addRoadUserfromTracker(trackers, m, gc, maskOrient, adfGeoTransform, H); // mutex_cv.unlock(); //prepare the message with detection info //prepare_message(m, coords, CAM_IDX); //send message if (!m->objects.empty()) Comm.send_message(m); } if (to_show) { //populate the ModFrame_t info_show.sem.lock(); info_show.original_frame = frame.clone(); // std::vector trackers; info_show.trackers = trackers; // geodetic_converter::GeodeticConverter gc; info_show.gc = gc; for (int i = 0; i < 6; i++) info_show.adfGeoTransform[i] = adfGeoTransform[i]; // cv::Mat H; info_show.H = H.clone(); info_show.yolo = yolo; // std::copy(camera->yolo.begin(), camera->yolo.end(), info_show.yolo.begin()); info_show.mask = mask.clone(); info_show.sem.unlock(); } // update pre_frame for the disparity map // pre_frame = orig_frame.clone(); // pre_canny = canny.clone(); if (first_iteration) first_iteration = false; frame_nbr++; std::cout << camera->CAM_IDX << " camera thread: "; TIMER_STOP } return (void *)0; } int main(int argc, char *argv[]) { std::cout << "detection\n"; signal(SIGINT, sig_handler); srand(time(NULL)); Parameters_t par; if(!read_parameters(argc, argv, &par)) return -1; tk::dnn::Yolo3Detection yolo[par.n_cameras]; for(int i=0; i