diff --git a/demo/demo/demo3D.cpp b/demo/demo/demo3D.cpp index 4286faa..c35ac51 100644 --- a/demo/demo/demo3D.cpp +++ b/demo/demo/demo3D.cpp @@ -1,7 +1,7 @@ #include #include #include /* srand, rand */ -#include +//#include #include #include "CenternetDetection3D.h" @@ -24,7 +24,12 @@ int main(int argc, char *argv[]) { std::string net = "dla34_cnet3d_fp32.rt"; if(argc > 1) net = argv[1]; - std::string input = "../demo/yolo_test.mp4"; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) input = argv[2]; char ntype = 'c'; @@ -33,9 +38,18 @@ int main(int argc, char *argv[]) { int n_classes = 3; if(argc > 4) n_classes = atoi(argv[4]); - bool show = false; + int n_batch = 1; if(argc > 5) - show = atoi(argv[5]); + n_batch = atoi(argv[5]); + bool show = true; + if(argc > 6) + show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); + + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); if(!show) SAVE_RESULT = true; @@ -57,7 +71,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes); + detNN->init(net, n_classes, n_batch, conf_thresh); gRun = true; @@ -75,30 +89,40 @@ int main(int argc, char *argv[]) { } cv::Mat frame; - cv::Mat dnn_input; if(show) - cv::namedWindow("detection", cv::WINDOW_NORMAL); + cv::namedWindow("detection", cv::WINDOW_NORMAL); - std::vector detected_bbox; + std::vector batch_frame; + std::vector batch_dnn_input; while(gRun) { - cap >> frame; - if(!frame.data) { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); + batch_dnn_input.clear(); + batch_frame.clear(); + for(int bi=0; bi< n_batch; ++bi){ + cap >> frame; + if(!frame.data) + break; + + batch_frame.push_back(frame); + + // this will be resized to the net format + batch_dnn_input.push_back(frame.clone()); + } + if(!frame.data) + break; + //inference - detNN->update(dnn_input); - frame = detNN->draw(frame); - - if(show) { - cv::imshow("detection", frame); - cv::waitKey(1); - } - if(SAVE_RESULT) + detNN->update(batch_dnn_input, n_batch); + detNN->draw(batch_frame); + + if(show){ + for(int bi=0; bi< n_batch; ++bi){ + cv::imshow("detection", batch_frame[bi]); + cv::waitKey(1); + } + } + if(n_batch == 1 && SAVE_RESULT) resultVideo << frame; } @@ -124,7 +148,6 @@ int main(int argc, char *argv[]) { std::cout<<"Avg: "< detected3D; - std::vectorcls3D; std::vector> face_id; public: CenternetDetection3D() {}; ~CenternetDetection3D() {}; - bool init(const std::string& tensor_path, const int n_classes=3); - void preprocess(cv::Mat &frame); - void postprocess(); - cv::Mat draw(cv::Mat &frame); + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); + void draw(std::vector& frames); }; diff --git a/include/tkDNN/CenternetDetection3DTrack.h b/include/tkDNN/CenternetDetection3DTrack.h index 5149d70..809d0c9 100644 --- a/include/tkDNN/CenternetDetection3DTrack.h +++ b/include/tkDNN/CenternetDetection3DTrack.h @@ -145,6 +145,7 @@ private: int count_det; //tracks std::vector tr_res; + std::vector> batchTracked; int count_tr; int track_id=0; @@ -153,7 +154,7 @@ private: bool init_pre_inf(); bool init_postprocessing(); bool init_visualization(const int n_classes); - void pre_inf(); + void pre_inf(const int bi); void _get_additional_inputs(); cv::Mat transform_preds_with_trans(float x1, float x2); void tracking(); @@ -161,11 +162,11 @@ private: public: tk::dnn::Network *pre_phase_net = nullptr; CenternetDetection3DTrack() {}; - ~CenternetDetection3DTrack() {}; - bool init(const std::string& tensor_path, const int n_classes=3); - void preprocess(cv::Mat &frame); - void postprocess(); - cv::Mat draw(cv::Mat &frame); + ~CenternetDetection3DTrack() {}; + bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3); + void preprocess(cv::Mat &frame, const int bi=0); + void postprocess(const int bi=0,const bool mAP=false); + void draw(std::vector& frames); }; diff --git a/include/tkDNN/DetectionNN3D.h b/include/tkDNN/DetectionNN3D.h index 2870aa0..65cd728 100644 --- a/include/tkDNN/DetectionNN3D.h +++ b/include/tkDNN/DetectionNN3D.h @@ -3,8 +3,11 @@ #include #include -#include +#include +#ifdef __linux__ #include +#endif + #include #include "utils.h" @@ -30,10 +33,12 @@ class DetectionNN3D { tk::dnn::NetworkRT *netRT = nullptr; dnnType *input_d; - cv::Size originalSize; + std::vector originalSize; cv::Scalar colors[256]; + int nBatches = 1; + #ifdef OPENCV_CUDACONTRIB cv::cuda::GpuMat bgr[3]; cv::cuda::GpuMat imagePreproc; @@ -47,21 +52,26 @@ class DetectionNN3D { * This method preprocess the image, before feeding it to the NN. * * @param frame original frame to adapt for inference. + * @param bi batch index */ - virtual void preprocess(cv::Mat &frame) = 0; + virtual void preprocess(cv::Mat &frame, const int bi=0) = 0; /** * This method postprocess the output of the NN to obtain the correct * boundig boxes. * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation */ - virtual void postprocess() = 0; + virtual void postprocess(const int bi=0,const bool mAP=false) = 0; public: int classes = 0; float confThreshold = 0.3; /*threshold on the confidence of the boxes*/ - - std::vector detected; /*bounding boxes in output*/ + + std::vector detected3D; /*bounding boxes in output*/ + std::vector> batchDetected; /*bounding boxes in output*/ std::vector pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/ std::vector classesNames; @@ -69,68 +79,79 @@ class DetectionNN3D { ~DetectionNN3D(){}; /** - * Method used to inialize the class, allocate memory and compute + * Method used to initialize the class, allocate memory and compute * needed data. * - * @param tensor_path path to the rt file og the NN. + * @param tensor_path path to the rt file of the NN. * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference. * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=3) = 0; - - /** - * Method to draw boundixg boxes and labels on a frame. - * - * @param frame orginal frame to draw bounding box on. - * @return frame with boundig boxes. - */ - virtual cv::Mat draw(cv::Mat &frame){}; + virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. * - * @param frame frame to run detection on. + * @param frames frames to run detection on. + * @param cur_batches number of batches to use in inference. * @param save_times if set to true, preprocess, inference and postprocess times * are saved on a csv file, otherwise not. - * @param times pointer to the output stream where to write times + * @param times pointer to the output stream where to write times. + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation. */ - void update(cv::Mat &frame, bool save_times=false, std::ofstream *times=nullptr){ - if(!frame.data) - FatalError("No image data feed to detection"); - + void update(std::vector& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){ if(save_times && times==nullptr) FatalError("save_times set to true, but no valid ofstream given"); + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); - originalSize = frame.size(); - printCenteredTitle(" TENSORRT detection ", '=', 30); + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); { TKDNN_TSTART - preprocess(frame); + for(int bi=0; biinput_dim; + dim.n = cur_batches; { - dim.print(); + if(TKDNN_VERBOSE) dim.print(); TKDNN_TSTART netRT->infer(dim, input_d); TKDNN_TSTOP - dim.print(); + if(TKDNN_VERBOSE) dim.print(); stats.push_back(t_ns); if(save_times) *times<& frames){}; + }; }} diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 1803381..53b3cf7 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -3,10 +3,12 @@ namespace tk { namespace dnn { -bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes){ +bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; + nBatches = n_batches; + confThreshold = conf_thresh; dim = netRT->input_dim; @@ -28,7 +30,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas trans = cv::Mat(cv::Size(3,2), CV_32F); trans2 = cv::Mat(cv::Size(3,2), CV_32F); - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1); dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); @@ -91,7 +93,7 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); mean << 0.485, 0.456, 0.406; stddev << 0.229, 0.224, 0.225; #endif @@ -154,13 +156,13 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); } -void CenternetDetection3D::preprocess(cv::Mat &frame){ +void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ // -----------------------------------pre-process ------------------------------------------ // auto start_t = std::chrono::steady_clock::now(); // auto step_t = std::chrono::steady_clock::now(); // auto end_t = std::chrono::steady_clock::now(); - cv::Size sz = originalSize; + cv::Size sz = originalSize[bi]; // std::cout<<"image: "<(end_t - step_t).count() << " us" << std::endl; // step_t = end_t; - checkCuda(cudaMemcpy(input_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); // end_t = std::chrono::steady_clock::now(); // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; @@ -280,21 +282,21 @@ void CenternetDetection3D::preprocess(cv::Mat &frame){ int idx = i*imageF.rows*imageF.cols; int ch = dim2.c-3 +i; // std::cout<<"i: "<input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType)); } - checkCuda(cudaMemcpyAsync(input_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice)); #endif } -void CenternetDetection3D::postprocess(){ +void CenternetDetection3D::postprocess(const int bi, const bool mAP) { dnnType *rt_out[7]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; - rt_out[4] = (dnnType *)netRT->buffersRT[5]; - rt_out[5] = (dnnType *)netRT->buffersRT[6]; - rt_out[6] = (dnnType *)netRT->buffersRT[7]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; + rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi; + rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi; + rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi; // ------------------------------------ process -------------------------------------------- activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); @@ -404,8 +406,7 @@ void CenternetDetection3D::postprocess(){ if(rot_y peakThreshold) { - if(scores[j] > centerThreshold) { + if(scores[j] > confThreshold) { if(z>0) { // compute_box_3d r.at(0,0) = std::cos(rot_y); @@ -457,16 +458,17 @@ void CenternetDetection3D::postprocess(){ } res.cl = i; res.prob = scores[j]; - res.print(); + //res.print(); detected3D.push_back(res); } } } } } + batchDetected.push_back(detected3D); } -cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { +void CenternetDetection3D::draw(std::vector& frames) { tk::dnn::box3D b; int x0, w, x1, y0, h, y1; int objClass; @@ -476,40 +478,41 @@ cv::Mat CenternetDetection3D::draw(cv::Mat &frame) { float font_scale = 0.5; int thickness = 2; - // draw dets - for(int i=0; i=0; ind_f--) { - for(int j=0; j<4; j++) { - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), - b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - colors[b.cl], 2); - if(ind_f == 0) { - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), - b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), - b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); - cv::line(frame, cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), - b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), - b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(j) * 2), + b.corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + b.corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + colors[b.cl], 2); + if(ind_f == 0) { + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(0) * 2), + b.corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(2) * 2), + b.corners.at(face_id.at(ind_f).at(2) * 2 + 1)), colors[b.cl], 2); + cv::line(frames[bi], cv::Point(b.corners.at(face_id.at(ind_f).at(1) * 2), + b.corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point(b.corners.at(face_id.at(ind_f).at(3) * 2), + b.corners.at(face_id.at(ind_f).at(3) * 2 + 1)), colors[b.cl], 2); + } } } + // draw label + cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); + cv::rectangle(frames[bi], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1)), + cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), + (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); + cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), + b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } - // draw label - cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline); - cv::rectangle(frame, cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1)), - cv::Point((b.corners.at(face_id.at(0).at(0) * 2) + text_size.width - 2), - (b.corners.at(face_id.at(0).at(0) * 2 + 1)) - text_size.height - 2), colors[b.cl], -1); - cv::putText(frame, classesNames[b.cl], cv::Point(b.corners.at(face_id.at(0).at(0) * 2), - b.corners.at(face_id.at(0).at(0) * 2 + 1) - (baseline / 2)), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness); } - return frame; } }} diff --git a/src/CenternetDetection3DTrack.cpp b/src/CenternetDetection3DTrack.cpp index ef3e161..dfc38f9 100644 --- a/src/CenternetDetection3DTrack.cpp +++ b/src/CenternetDetection3DTrack.cpp @@ -3,12 +3,14 @@ namespace tk { namespace dnn { -bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes){ - std::cout<<(tensor_path).c_str()<<"\n"; + +bool CenternetDetection3DTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); dim = netRT->input_dim; dim.c = 3; + nBatches = n_batches; + confThreshold = conf_thresh; init_preprocessing(); init_pre_inf(); @@ -46,13 +48,13 @@ bool CenternetDetection3DTrack::init_preprocessing(){ checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); #else - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot())); + checkCuda(cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches)); mean << 0.40789655, 0.44719303, 0.47026116; stddev << 0.2886383, 0.27408165, 0.27809834; #endif - checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); checkCuda(cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); } @@ -276,20 +278,20 @@ void CenternetDetection3DTrack::_get_additional_inputs(){ //None no additional input } -void CenternetDetection3DTrack::pre_inf(){ +void CenternetDetection3DTrack::pre_inf(const int bi){ TKDNN_TSTART tk::dnn::dataDim_t dim_aus; pre_phase_net->infer(dim_aus, nullptr); TKDNN_TSTOP checkCuda( cudaDeviceSynchronize() ); - checkCuda( cudaMemcpy(input_d, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); + checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); checkCuda( cudaDeviceSynchronize() ); } -void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ - // -----------------------------------pre-process ------------------------------------------ - - cv::Size sz = originalSize; +void CenternetDetection3DTrack::preprocess(cv::Mat &frame, const int bi){ + // -----------------------------------pre-process ------------------------------------------ + batchTracked.clear(); + cv::Size sz = originalSize[bi]; cv::Size sz_old; float scale = 1.0; float new_height = sz.height * scale; @@ -302,7 +304,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ // float s = new_width >= new_height ? new_width : new_height; // ----------- get_affine_transform // rot_rad = pi * 0 / 100 --> 0 - dim.print(); + //dim.print(); src.at(0,0)=c[0]; src.at(0,1)=c[1]; src.at(1,0)=c[0]; @@ -389,7 +391,7 @@ void CenternetDetection3DTrack::preprocess(cv::Mat &frame){ checkCuda( cudaDeviceSynchronize() ); iter0=false; } - pre_inf(); + pre_inf(bi); checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); checkCuda( cudaDeviceSynchronize() ); @@ -587,17 +589,17 @@ void CenternetDetection3DTrack::tracking(){ } -void CenternetDetection3DTrack::postprocess(){ +void CenternetDetection3DTrack::postprocess(const int bi, const bool mAP) { dnnType *rt_out[9]; - rt_out[0] = (dnnType *)netRT->buffersRT[1]; - rt_out[1] = (dnnType *)netRT->buffersRT[2]; - rt_out[2] = (dnnType *)netRT->buffersRT[3]; - rt_out[3] = (dnnType *)netRT->buffersRT[4]; - rt_out[4] = (dnnType *)netRT->buffersRT[5]; - rt_out[5] = (dnnType *)netRT->buffersRT[6]; - rt_out[6] = (dnnType *)netRT->buffersRT[7]; - rt_out[7] = (dnnType *)netRT->buffersRT[8]; - rt_out[8] = (dnnType *)netRT->buffersRT[9]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi; + rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; + rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; + rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi; + rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi; + rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi; + rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi; + rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi; // ------------------------------------ process -------------------------------------------- @@ -719,143 +721,144 @@ void CenternetDetection3DTrack::postprocess(){ } // track step tracking(); + batchTracked.push_back(tr_res); } -cv::Mat CenternetDetection3DTrack::draw(cv::Mat &frame) { - +void CenternetDetection3DTrack::draw(std::vector& frames) { + struct trackingRes t; float sc; int id; std::string txt; int baseline = 0; float font_scale = 0.8; - int thickness = 2; - for(int i=0; i vis_thresh){// && tr_res[i].active!=0) { - if(view2d) { - - - cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), tr_res[i].det_res.bb0.at(0,1)), - cv::Point(tr_res[i].det_res.bb1.at(0,0), tr_res[i].det_res.bb1.at(0,1)), tr_colors[tr_res[i].color], thickness); - cv::rectangle(frame, cv::Point(tr_res[i].det_res.bb0.at(0,0), - tr_res[i].det_res.bb0.at(0,1) - text_size.height - thickness), - cv::Point(tr_res[i].det_res.bb0.at(0,0) + text_size.width, - tr_res[i].det_res.bb0.at(0,1)), tr_colors[tr_res[i].color], -1); - - cv::putText(frame, txt, cv::Point(tr_res[i].det_res.bb0.at(0,0), - tr_res[i].det_res.bb0.at(0,1) - thickness -1), - cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); + int thickness = 2; + for(int bi=0; bi vis_thresh){// && t.active!=0) { + if(view2d) { + cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), t.det_res.bb0.at(0,1)), + cv::Point(t.det_res.bb1.at(0,0), t.det_res.bb1.at(0,1)), tr_colors[t.color], thickness); + cv::rectangle(frames[bi], cv::Point(t.det_res.bb0.at(0,0), + t.det_res.bb0.at(0,1) - text_size.height - thickness), + cv::Point(t.det_res.bb0.at(0,0) + text_size.width, + t.det_res.bb0.at(0,1)), tr_colors[t.color], -1); + + cv::putText(frames[bi], txt, cv::Point(t.det_res.bb0.at(0,0), + t.det_res.bb0.at(0,1) - thickness -1), + cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frame, cv::Point((int)tr_res[i].det_res.ct.at(0,0), - (int)tr_res[i].det_res.ct.at(0,1)), - cv::Point((int)(tr_res[i].det_res.ct.at(0,0) + tr_res[i].det_res.tr.at(0,0)), - (int)(tr_res[i].det_res.ct.at(0,1) + tr_res[i].det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); - } - //3d - if(!view2d && tr_res[i].det_res.z > 1){ - r.at(0,0) = std::cos(tr_res[i].det_res.rot_y); - r.at(0,2) = std::sin(tr_res[i].det_res.rot_y); - r.at(2,0) = -std::sin(tr_res[i].det_res.rot_y); - r.at(2,2) = std::cos(tr_res[i].det_res.rot_y); - - corners.at(0,0) = tr_res[i].det_res.dim[2]/2; - corners.at(0,1) = tr_res[i].det_res.dim[2]/2; - corners.at(0,2) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,3) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,4) = tr_res[i].det_res.dim[2]/2; - corners.at(0,5) = tr_res[i].det_res.dim[2]/2; - corners.at(0,6) = -tr_res[i].det_res.dim[2]/2; - corners.at(0,7) = -tr_res[i].det_res.dim[2]/2; - - corners.at(1,4) = -tr_res[i].det_res.dim[0]; - corners.at(1,5) = -tr_res[i].det_res.dim[0]; - corners.at(1,6) = -tr_res[i].det_res.dim[0]; - corners.at(1,7) = -tr_res[i].det_res.dim[0]; - - corners.at(2,0) = tr_res[i].det_res.dim[1]/2; - corners.at(2,1) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,2) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,3) = tr_res[i].det_res.dim[1]/2; - corners.at(2,4) = tr_res[i].det_res.dim[1]/2; - corners.at(2,5) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,6) = -tr_res[i].det_res.dim[1]/2; - corners.at(2,7) = tr_res[i].det_res.dim[1]/2; - - cv::Mat aus = r * corners; - - for(int k=0; k<8; k++) { - aus.at(0,k) += tr_res[i].det_res.x; - aus.at(1,k) += tr_res[i].det_res.y; - aus.at(2,k) += tr_res[i].det_res.z; + cv::arrowedLine(frames[bi], cv::Point((int)t.det_res.ct.at(0,0), + (int)t.det_res.ct.at(0,1)), + cv::Point((int)(t.det_res.ct.at(0,0) + t.det_res.tr.at(0,0)), + (int)(t.det_res.ct.at(0,1) + t.det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); } - - // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); - for(int k1=0; k1<3; k1++) { - for(int k2=0; k2<8; k2++) - pts3DHomo.at(k1,k2) = aus.at(k1,k2); - } - - aus.release(); - aus = calibs * pts3DHomo; - std::vector res_corners; - for(int k=0; k<8; k++) { - res_corners.push_back(aus.at(0,k) / aus.at(2,k)); - res_corners.push_back(aus.at(1,k) / aus.at(2,k)); - } - aus.release(); - for(int ind_f = 3; ind_f>=0; ind_f--) { - for(int j=0; j<4; j++) { - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), - (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), - (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), - tr_colors[tr_res[i].color], 2); - if(ind_f == 0 && j==3) { - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), - (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), - (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[tr_res[i].color], 2); - cv::line(frame, cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), - (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), - cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), - (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[tr_res[i].color], 2); + //3d + if(!view2d && t.det_res.z > 1){ + r.at(0,0) = std::cos(t.det_res.rot_y); + r.at(0,2) = std::sin(t.det_res.rot_y); + r.at(2,0) = -std::sin(t.det_res.rot_y); + r.at(2,2) = std::cos(t.det_res.rot_y); + + corners.at(0,0) = t.det_res.dim[2]/2; + corners.at(0,1) = t.det_res.dim[2]/2; + corners.at(0,2) = -t.det_res.dim[2]/2; + corners.at(0,3) = -t.det_res.dim[2]/2; + corners.at(0,4) = t.det_res.dim[2]/2; + corners.at(0,5) = t.det_res.dim[2]/2; + corners.at(0,6) = -t.det_res.dim[2]/2; + corners.at(0,7) = -t.det_res.dim[2]/2; + + corners.at(1,4) = -t.det_res.dim[0]; + corners.at(1,5) = -t.det_res.dim[0]; + corners.at(1,6) = -t.det_res.dim[0]; + corners.at(1,7) = -t.det_res.dim[0]; + + corners.at(2,0) = t.det_res.dim[1]/2; + corners.at(2,1) = -t.det_res.dim[1]/2; + corners.at(2,2) = -t.det_res.dim[1]/2; + corners.at(2,3) = t.det_res.dim[1]/2; + corners.at(2,4) = t.det_res.dim[1]/2; + corners.at(2,5) = -t.det_res.dim[1]/2; + corners.at(2,6) = -t.det_res.dim[1]/2; + corners.at(2,7) = t.det_res.dim[1]/2; + + cv::Mat aus = r * corners; + + for(int k=0; k<8; k++) { + aus.at(0,k) += t.det_res.x; + aus.at(1,k) += t.det_res.y; + aus.at(2,k) += t.det_res.z; + } + + // corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3))); + for(int k1=0; k1<3; k1++) { + for(int k2=0; k2<8; k2++) + pts3DHomo.at(k1,k2) = aus.at(k1,k2); + } + + aus.release(); + aus = calibs * pts3DHomo; + std::vector res_corners; + for(int k=0; k<8; k++) { + res_corners.push_back(aus.at(0,k) / aus.at(2,k)); + res_corners.push_back(aus.at(1,k) / aus.at(2,k)); + } + aus.release(); + for(int ind_f = 3; ind_f>=0; ind_f--) { + for(int j=0; j<4; j++) { + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(j) * 2), + (int)res_corners.at(face_id.at(ind_f).at(j) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2), + (int)res_corners.at(face_id.at(ind_f).at((j+1)%4) * 2 + 1)), + tr_colors[t.color], 2); + if(ind_f == 0 && j==3) { + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(0) * 2), + (int)res_corners.at(face_id.at(ind_f).at(0) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(2) * 2), + (int)res_corners.at(face_id.at(ind_f).at(2) * 2 + 1)), tr_colors[t.color], 2); + cv::line(frames[bi], cv::Point((int)res_corners.at(face_id.at(ind_f).at(1) * 2), + (int)res_corners.at(face_id.at(ind_f).at(1) * 2 + 1)), + cv::Point((int)res_corners.at(face_id.at(ind_f).at(3) * 2), + (int)res_corners.at(face_id.at(ind_f).at(3) * 2 + 1)), tr_colors[t.color], 2); + } } } - } - float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; - for(int k=0; k<8; k++) { - if(res_corners[2*k]bb1) - bb1=res_corners[2*k]; - if(res_corners[2*k+1]bb3) - bb3=res_corners[2*k+1]; - - } - // if(not no_bbox): - // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), - // tr_colors[tr_res[i].color], thickness); - cv::rectangle(frame, cv::Point(bb0, bb2 - text_size.height - thickness), - cv::Point(bb0 + text_size.width, bb2), tr_colors[tr_res[i].color], -1); - - cv::putText(frame, txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, - font_scale, cv::Scalar(255, 255, 255), 1); + float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0; + for(int k=0; k<8; k++) { + if(res_corners[2*k]bb1) + bb1=res_corners[2*k]; + if(res_corners[2*k+1]bb3) + bb3=res_corners[2*k+1]; + + } + // if(not no_bbox): + // cv::rectangle(frame, cv::Point(bb0, bb2), cv::Point(bb1, bb3), + // tr_colors[t.color], thickness); + cv::rectangle(frames[bi], cv::Point(bb0, bb2 - text_size.height - thickness), + cv::Point(bb0 + text_size.width, bb2), tr_colors[t.color], -1); + + cv::putText(frames[bi], txt, cv::Point(bb0, bb2 - thickness -1), cv::FONT_HERSHEY_SIMPLEX, + font_scale, cv::Scalar(255, 255, 255), 1); - cv::arrowedLine(frame, cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), - cv::Point((int)((bb0 + bb1)/2 + tr_res[i].det_res.tr.at(0,0)), - (int)((bb2 + bb3)/2 + tr_res[i].det_res.tr.at(0,1))), - cv::Scalar(255, 0, 255), 2); + cv::arrowedLine(frames[bi], cv::Point((int)((bb0 + bb1)/2), (int)((bb2 + bb3)/2)), + cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at(0,0)), + (int)((bb2 + bb3)/2 + t.det_res.tr.at(0,1))), + cv::Scalar(255, 0, 255), 2); + } } } - } - return frame; } }}