diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 46aeef1..f6431f9 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -6,12 +6,23 @@ #include #include +#include +#include +#include +#include + #include "utils.h" +#include "BoxDetection.h" #include #include #include +//saliency +#include +#include +#include + #include "Yolo3Detection.h" #include "classutils.h" #include "../masa_protocol/include/send.hpp" @@ -96,6 +107,8 @@ int main(int argc, char *argv[]) std::cout << "camera started\n"; cv::Mat frame; + cv::Mat frame_crop; + char buf_frame_crop_name [200]; cv::Mat frame_top; cv::Mat dnn_input; cv::Mat original_frame_top; @@ -106,7 +119,8 @@ int main(int argc, char *argv[]) { cv::namedWindow("detection", cv::WINDOW_NORMAL); cv::namedWindow("topview", cv::WINDOW_NORMAL); - frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg"); + // frame_top = cv::imread("../demo/demo/data/map/map_geo.jpg"); + frame_top = cv::imread("../demo/demo/data/map/MASA_4670.png"); original_frame_top = frame_top.clone(); } @@ -171,12 +185,36 @@ int main(int argc, char *argv[]) outputVideo.open("test.avi", static_cast(cap.get(cv::CAP_PROP_FOURCC)), cap.get(cv::CAP_PROP_FPS), S, true);*/ cv::Mat map1, map2; + auto start_t = std::chrono::steady_clock::now(); + auto step_t = std::chrono::steady_clock::now(); + auto end_t = std::chrono::steady_clock::now(); + auto step_t_segmentation = std::chrono::steady_clock::now(); + auto end_t_segmentation = std::chrono::steady_clock::now(); + + // information for the disparity map + std::vector pre_rois; + cv::Mat pre_frame; + cv::Mat orig_frame; + cv::Mat canny, pre_canny, canny_RGB, pre_canny_RGB; + cv::Mat canny_img; + + // box variable + tk::dnn::box b; + int x0, w, x1, y0, h, y1; + int objClass; + std::string det_class;; + float prob; + cv::Scalar intensity; + cv::Rect roi; while (gRun) { TIMER_START - cap >> frame; + start_t = std::chrono::steady_clock::now(); + step_t = start_t; + cap >> frame; + orig_frame = frame.clone(); if (frame_nbr == 0) cv::initUndistortRectifyMap(cameraMat, distCoeff, cv::Mat(), cameraMat, frame.size(), CV_16SC2, map1, map2); cv::Mat temp = frame.clone(); @@ -205,26 +243,100 @@ int main(int argc, char *argv[]) coords.clear(); + end_t = std::chrono::steady_clock::now(); + std::cout << " TIME 1 : "<(end_t - step_t).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(cv::Point(int(x0 + b.w / 2), y1)); + b = yolo.detected[i]; + x0 = b.x; + w = b.w; + x1 = b.x + w; + y0 = b.y; + h = b.h; + y1 = b.y + h; + objClass = b.cl; + det_class = obj_class[b.cl]; + prob = b.prob; + intensity = mask.at(cv::Point(int(x0 + b.w / 2), y1)); + if (intensity[0]) { if (objClass < 6) { + + // find the rectangular on the frame (sub-figure) + roi.x = x0; + roi.y = y0; + // std::cout<<"x "<= frame.cols)? frame.cols-1-x0 : w; + roi.height = (y0+h >= frame.rows)? frame.rows-1-y0 : h; + // std::cout<<"w "<(end_t - step_t).count() << " ms"< +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cuda.h" +#include "cuda_runtime_api.h" +#include +#include + +#include +#include +#include + +//saliency +#include +#include +#include + +// cv::Mat img_threshold(cv::Mat frame_crop); +// cv::Mat img_background(cv::Mat frame_crop); +// cv::Mat img_dist_transform(cv::Mat frame_crop); +// cv::Mat img_watershed(cv::Mat frame_crop); +void image_segmentation(cv::Mat frame_crop, int frame_nbr, int i); +void image_gradients(cv::Mat frame_crop, int frame_nbr, int i); +void image_find_contours(cv::Mat frame_crop, int frame_nbr, int i); +void image_saliency(cv::Mat frame_crop, int frame_nbr, int i); +void frame_box_disparity(cv::Mat pre_frame, cv::Mat frame, std::vector pre_rois, int frame_nbr); +void segmentation(cv::Mat pre_frame, cv::Mat frame_crop, int frame_nbr, int i, int mode); + +//canny +cv::Mat img_laplacian(cv::Mat frame_crop, int ret); +void frame_disparity(cv::Mat pre_frame, cv::Mat frame, int frame_nbr, int i, int ret); \ No newline at end of file diff --git a/src/BoxDetection.cpp b/src/BoxDetection.cpp new file mode 100644 index 0000000..b8310d3 --- /dev/null +++ b/src/BoxDetection.cpp @@ -0,0 +1,663 @@ +#include "BoxDetection.h" +#include + + +cv::Mat img_threshold(cv::Mat frame_crop) +{ + // Image Threshold Example + // https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html + cv::Mat f = frame_crop.clone(); + cv::Mat dst, gray; + // gray and threshold image + cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0); + cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU); + return gray; +} + +cv::Mat img_background(cv::Mat frame_crop) +{ + // Image Background Example + // https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html + cv::Mat f = frame_crop.clone(); + cv::Mat dst, gray, opening, coinsBg; + // gray and threshold image + cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0); + cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU); + // get background + cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1)); + cv::erode(gray, opening, M); + cv::dilate(gray, opening, M); + cv::Point p = cv::Point(-1,-1); + cv::dilate(opening, coinsBg, M, p, 3); + return coinsBg; +} + +cv::Mat img_dist_transform(cv::Mat frame_crop) +{ + // Distance Transform Example + // https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html + cv::Mat f = frame_crop.clone(); + cv::Mat dst, gray, opening, coinsBg, coinsFg, distTrans; + // gray and threshold image + cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0); + cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU); + // cv::Mat::ones M(3,3,cv::CV_8U); + // get background + cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1)); + cv::erode(gray, opening, M); + cv::dilate(gray, opening, M); + cv::Point p = cv::Point(-1,-1); + cv::dilate(opening, coinsBg, M, p, 3); + // distance transorm + cv::distanceTransform(opening, distTrans, cv::DIST_L2, 5); + cv::normalize(distTrans, distTrans, 1, 0, cv::NORM_INF); + return distTrans; +} + +// cv::Mat img_watershed(cv::Mat frame_crop) +// { +// // Image Watershed Example +// // https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html +// cv::Mat f = frame_crop.clone(); +// cv::Mat dst, gray, opening, coinsBg, coinsFg, distTrans, unknown, markers; +// // gray and threshold image +// cv::cvtColor(f, gray, cv::COLOR_RGBA2GRAY, 0); +// cv::threshold(gray, gray, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU); +// // get background +// cv::Mat M = cv::Mat(3, 3, CV_8U, cv::Scalar(1,1,1,1)); +// cv::erode(gray, opening, M); +// cv::dilate(gray, opening, M); +// cv::Point p = cv::Point(-1,-1); +// cv::dilate(opening, coinsBg, M, p, 3); +// // distance transorm +// cv::distanceTransform(opening, distTrans, cv::DIST_L2, 5); +// cv::normalize(distTrans, distTrans, 1, 0, cv::NORM_INF); + +// // get foreground +// cv::threshold(distTrans, coinsFg, 0.7 * 1, 255, cv::THRESH_BINARY); +// coinsFg.convertTo(coinsFg, CV_8U, 1, 0); +// cv::subtract(coinsBg, coinsFg, unknown); +// // get connected components networks +// cv::connectedComponents(coinsFg, markers); +// // intptr_t n = NULL; +// for(int i = 0; i< markers.rows; i++) +// { +// for (int j = 0; j< markers.cols; j++) +// { +// M.at(0, 0); +// markers.intPtr(i,j)[0] = markers.ucharPtr(i,j)[0] +1; +// if(unknown.ucharPtr(i,j)[0] == 255) +// { +// markers.intPtr(i,j)[0] = 0; +// } +// } +// } +// cv::cvtColor(f, f, cv::COLOR_RGBA2RGB, 0); +// cv::watershed(f, markers); +// //draw barriers +// for(int i = 0; i< markers.rows; i++) +// { +// for (int j = 0; j< markers.cols; j++) +// { +// if(markers.IntPtr(i,j)[0] == -1) +// { +// f.ucharPtr(i,j)[0] = 255; // R +// f.ucharPtr(i,j)[1] = 0; // G +// f.ucharPtr(i,j)[2] = 0; // B +// } +// } +// } +// } + +////// + +cv::Mat img_sobel_abssobel(cv::Mat frame_crop, int ret=0) +{ + //ret = 0 --> dstx + //ret = 1 --> dsty + //ret = 2 --> absDstx + //ret = 3 --> absDsty + // Image Sobel and Image AbsSobel + // https://docs.opencv.org/trunk/da/d85/tutorial_js_gradients.html + // compute image gradient on two different directions + + cv::Mat f = frame_crop.clone(); + int x,y; + (ret == 0 || ret == 2)?x=1, y=0 : NULL; + (ret == 1 || ret == 3)?x=0, y=1 : NULL; + cv::Mat dst; + cv::cvtColor(f, f, cv::COLOR_RGB2GRAY, 0); + // You can try more different parameters + cv::Sobel(f, dst, CV_8U, x, y, 3, 1, 0, cv::BORDER_DEFAULT); + // for absSobel + if(ret == 2 || ret == 3) + cv::convertScaleAbs(dst, dst, 1, 0); + // next 3 rows to be checked + //// ??cv::Mat f2 = frame_crop.clone(); + //// cv.Scharr(?(f,f2), dstx, cv.CV_8U, 1, 0, 1, 0, cv.BORDER_DEFAULT); + //// cv.Scharr(?(f,f2), dsty, cv.CV_8U, 0, 1, 1, 0, cv.BORDER_DEFAULT); + return dst; +} + +cv::Mat img_laplacian(cv::Mat frame_crop, int ret=1) +{ + //ret = 0 --> src_gray + //ret = 1 --> dst + // Image Laplacian + // compute image gradient with laplacian + cv::Mat f = frame_crop.clone(); + cv::Mat src_gray, dst; + int kernel_size = 3; + int scale = 1; + int delta = 0; + int ddepth = CV_16S; + cv::GaussianBlur( f, f, cv::Size(3,3), 0, 0, cv::BORDER_DEFAULT ); + /// Convert the image to grayscale + cv::cvtColor( f, src_gray, CV_RGB2GRAY ); + if (ret == 0) + return src_gray; + + // else: Apply Laplace function + cv::Mat abs_dst; + cv::Laplacian( src_gray, dst, ddepth, kernel_size, scale, delta, cv::BORDER_DEFAULT ); + // //compute sharpness + // float sharpnessValue = cv::mean(dst); + return dst; +} + +cv::Mat find_contours(cv::Mat frame_crop, cv::Mat img, cv::Mat canny_output, int n_lines=1) +{ + // n_line: number of line to plot on image + cv::Mat img_line = frame_crop.clone(); + cv::Mat ret_thresh; + std::vector > contours; + double thresh = 127; + double maxValue = 255; + cv::threshold(img, ret_thresh, thresh, maxValue, 0);//0); // = cv2.threshold(img,127,255,0) + cv::findContours(canny_output, contours, 1, 2);//cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2) + // cv::threshold(img2, ret2, thresh, maxValue, 0); + // cv::findContours(canny_output2, contours2, 1, 2); + // cv::threshold(img3a, ret3a, thresh, maxValue, 0); + // cv::findContours(canny_output3a, contours3a, 1, 2); + // cv::threshold(img3b, ret3b, thresh, maxValue, 0); + // cv::findContours(canny_output3b, contours3b, 1, 2); + + cv::Vec4f line; + float vx,vy,x,y; + int lefty, righty; + for(int i=0; i > contours; +// double thresh = 127; +// double maxValue = 255; +// cv::threshold(img, ret_thresh, thresh, maxValue, 0);//0); // = cv2.threshold(img,127,255,0) +// cv::findContours(canny_output, contours, 1, 2);//cv::CHAIN_APPROX_SIMPLE );//1, 2); //contours,hierarchy = cv2.findContours(thresh, 1, 2) + +// cv::RotatedRect rect = cv::minAreaRect(contours[0]); +// cv::Mat boxPts1; +// std::vector > boxPts2; +// cv::boxPoints(rect, boxPts1); +// // boxPts = np.int0(boxPts); +// for (int x = 0; x < img.cols; x++) +// for (int y = 0; y < img.rows; y++) +// boxPts2.at(x).push_back(cv::Point(boxPts1.at(x, y))); + +// cv::drawContours(img_clone, boxPts2,0,(0,0,255),2); +// // drawContours( drawing, contours, i, color, 2, 8, hierarchy, 0, Point() ); +// return img_clone; +// } + +cv::Mat compute_saliency(cv::Mat frame_crop, cv::Ptr saliencyAlgorithm, int const_molt_mat, int ret=0) +{ + //ret=0 --> saliencyMap + //ret=1 --> binaryMap + // SPECTRAL_RESIDUAL algorithm + cv::Mat f = frame_crop.clone(); + cv::Mat saliencyMap; + cv::Mat binaryMap; + + if( saliencyAlgorithm->computeSaliency( f, saliencyMap ) ) + { + if(ret==0) + return saliencyMap*const_molt_mat; + + cv::saliency::StaticSaliencySpectralResidual spec; + spec.computeBinaryMap( saliencyMap, binaryMap ); + + // imshow( "Saliency Map", saliencyMap ); + // imshow( "Original Image", image ); + // imshow( "Binary Map", binaryMap ); + // waitKey( 0 ); + return binaryMap*const_molt_mat; + } + return cv::Mat(0,0,CV_8U, cv::Scalar(0,0,0,0)); +} + +////// + +void image_segmentation(cv::Mat frame_crop, int frame_nbr, int i) +{ + // Watershed Algorithm + // https://docs.opencv.org/3.4/d7/d1c/tutorial_js_watershed.html + char buf_frame_crop_name [200]; + auto step_t_segmentation = std::chrono::steady_clock::now(); + auto end_t_segmentation = std::chrono::steady_clock::now(); + + sprintf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_imgthr.jpg",frame_nbr, i); + cv::imwrite(buf_frame_crop_name, img_threshold(frame_crop)); + end_t_segmentation = std::chrono::steady_clock::now(); + std::cout << " - TIME imgthr ("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"< saliencyAlgorithm; + char buf_frame_crop_name [200]; + int const_molt_mat = 0; + auto step_t_segmentation = std::chrono::steady_clock::now(); + auto end_t_segmentation = std::chrono::steady_clock::now(); + + // SPECTRAL_RESIDUAL + const_molt_mat = 255; + saliencyAlgorithm = cv::saliency::StaticSaliencySpectralResidual::create(); + cv::Mat spect_res = compute_saliency(frame_crop, saliencyAlgorithm, const_molt_mat, 0); + if(!spect_res.empty()) + { + sprintf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_saliency_SpectralResidual.jpg",frame_nbr, i); + cv::imwrite(buf_frame_crop_name, spect_res); + + } + else + { + std::cout<<"something is wrond (image_saliency)"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"< saliencyMap1; + // saliencyAlgorithm.dynamicCast()->setTrainingPath( "" ); + // saliencyAlgorithm.dynamicCast()->setBBResDir( "Results" ); + // std::cout<<"mmm"<computeSaliency( frame_crop, saliencyMap1 ); + // int ndet = int(saliencyMap1.size()); + // std::cout << "Objectness done " << ndet << std::endl; + // // // The result are sorted by objectness. We only use the first maxd boxes here. + // // int maxd = 7, step = 255 / maxd, jitter=9; // jitter to seperate single rects + // // cv::Mat draw = frame_crop.clone(); + // // for (int i = 0; i < std::min(maxd, ndet); i++) + // // { + // // cv::Vec4i bb = saliencyMap1[i]; + // // cv::Scalar col = cv::Scalar(((i*step)%255), 100, 255-((i*step)%255)); + // // cv::Point off(cv::theRNG().uniform(-jitter,jitter), cv::theRNG().uniform(-jitter,jitter)); + // // cv::rectangle(draw, cv::Point(bb[0]+off.x, bb[1]+off.y), cv::Point(bb[2]+off.x, bb[3]+off.y), col, 2); + // // cv::rectangle(draw, cv::Rect(20, 20+i*10, 10,10), col, -1); // mini temperature scale + // // } + // // imshow("BING", draw); + // // waitKey(); + // printf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_saliency_BING.jpg",frame_nbr, i); + // cv::imwrite(buf_frame_crop_name, saliencyMap1); + + //// + + // BING WANG APR 2014 + cv::Mat saliencyMap; + cv::Mat frame_sal = frame_crop.clone(); + saliencyAlgorithm = cv::saliency::MotionSaliencyBinWangApr2014::create(); + saliencyAlgorithm.dynamicCast()->setImagesize( frame_sal.cols, frame_sal.rows ); + saliencyAlgorithm.dynamicCast()->init(); + cvtColor( frame_sal, frame_sal, cv::COLOR_BGR2GRAY ); + saliencyAlgorithm->computeSaliency( frame_sal, saliencyMap); + sprintf(buf_frame_crop_name,"../demo/demo/data/img_crop/%d_%d_saliency_BinWangApr.jpg",frame_nbr, i); + cv::imwrite(buf_frame_crop_name, saliencyMap); + end_t_segmentation = std::chrono::steady_clock::now(); + std::cout << " - TIME BING WANG APR 2014("<(end_t_segmentation - step_t_segmentation).count() << " ms"<(j,k); + + dist = (pix[0]*pix[0] + pix[1]*pix[1] + pix[2]*pix[2]); + dist = sqrt(dist); + + if(dist>threshold) + { + foregroundMask.at(j,k) = 255; + } + } + } + sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_dif.jpg",frame_nbr,i); + cv::imwrite(buf_frame_crop_name, foregroundMask); + std::cout<<"foregroundMask: "< pre_rois, int frame_nbr) +{ + + int roi_tollerance = 10; + cv::Mat pre_frame_crop, frame_crop; + char buf_frame_crop_name [200]; + int dx, dy; + int id = 1; + auto step_t_segmentation = std::chrono::steady_clock::now(); + auto end_t_segmentation = std::chrono::steady_clock::now(); + + for(auto r : pre_rois) + { + sprintf(buf_frame_crop_name,"../demo/demo/data/img_disparity/%d_%d_orig.jpg",frame_nbr, id); + cv::imwrite(buf_frame_crop_name, pre_frame(r)); + //resize last roi with a tollerance + dx = r.width / roi_tollerance; + dy = r.height / roi_tollerance; + r.x = (r.x - dx > 0)? (r.x - dx) : 0; + r.y = (r.y - dy > 0)? (r.y - dy) : 0; + // std::cout<<"disp: x "<= frame.cols)? (frame.cols-1-r.x) : (r.width+dx+dx); + r.height = ((r.y+r.height+dy+dy) >= frame.rows)? (frame.rows-1-r.y) : (r.height+dy+dy); + // std::cout<<"disp: w "<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<(end_t_segmentation - step_t_segmentation).count() << " ms"<