diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e6829e..a24a749 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,18 +118,11 @@ target_link_libraries(test_dla34_cnet tkDNN) add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) -add_executable(yolo3_demo demo/demo/demo_yolo3.cpp) -target_link_libraries(yolo3_demo tkDNN) - -add_executable(centernet_demo demo/demo/demo_centernet.cpp) -target_link_libraries(centernet_demo tkDNN) - -add_executable(mobilenet_demo demo/demo/demo_mobilenet.cpp) -target_link_libraries(mobilenet_demo tkDNN) - add_executable(map_demo demo/demo/map.cpp) target_link_libraries(map_demo tkDNN) +add_executable(demo demo/demo/demo.cpp) +target_link_libraries(demo tkDNN) #------------------------------------------------------------------------------- # Install diff --git a/README.md b/README.md index f8a3262..93af488 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ rm yolo3_berkeley.rt # be sure to delete(or move) old tensorRT files ``` this will genereate a yolo3_berkeley.rt file that can be used for live detection: ``` -./yolo3_demo # launch detection on a demo video -./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 +./demo # launch detection on a demo video +./demo yolo3_berkeley.rt /dev/video0 y # launch detection on device 0 ``` ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) @@ -110,9 +110,9 @@ rm dla34_cnet.rt # be sure to delete(or move) old tensorRT files this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used for live detection: ``` -./centernet_demo # launch detection on a demo video -./centernet_demo resnet101_cnet.rt /dev/video0 # launch detection on device 0 -./centernet_demo dla34_cnet.rt /dev/video0 # launch detection on device 0 +./demo dla34_cnet.rt ../demo/yolo_test.mp4 c # launch detection on a demo video +./demo resnet101_cnet.rt /dev/video0 c # launch detection on device 0 +./demo dla34_cnet.rt /dev/video0 c # launch detection on device 0 ``` ## mAP demo @@ -139,3 +139,36 @@ Example: cd build ./map_demo dla34_cnet.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` + +## Supported networks + +| Test Name | Network | Dataset | N Classes | Input size | Weights | +| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ | +| yolo | YOLO v21 | [COCO 2014](http://cocodataset.org/) | 80 | 608x608 | weights | +| yolo_224 | YOLO v21 | [COCO 2014](http://cocodataset.org/) | 80 | 224x224 | weights | +| yolo_berkeley | YOLO v21 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 416x736 | weights | +| yolo_relu | YOLO v2 (with ReLU, not Leaky)1 | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | weights | +| yolo_tiny | YOLO v2 tiny1 | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | weights | +| yolo_voc | YOLO v21 | [VOC ](http://host.robots.ox.ac.uk/pascal/VOC/) | 21 | 416x416 | weights | +| yolo3 | YOLO v32 | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download) | +| yolo3_berkeley | YOLO v32 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 320x544 | weights | +| yolo3_coco4 | YOLO v32 | [COCO 2014](http://cocodataset.org/) | 4 | 416x416 | weights | +| yolo3_flir | YOLO v32 | [FREE FLIR](https://www.flir.com/oem/adas/adas-dataset-form/) | 3 | 320x544 | weights | +| yolo3_tiny | YOLO v3 tiny2 | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/LMcSHtWaLeps8yN/download) | +| yolo3_tiny512 | YOLO v3 tiny2 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/njnYACnQfWQFKrn/download) | +| dla34 | Deep Leayer Aggreagtion (DLA) 343 | [COCO 2014](http://cocodataset.org/) | 80 | 224x224 | weights | +| dla34_cnet | Centernet (DLA34 backend)4 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/8AjXdgCeRzCa5AF/download) | +| mobilenetv2ssd | Mobilnet v2 SSD Lite5 | [VOC ](http://host.robots.ox.ac.uk/pascal/VOC/) | 21 | 300x300 | [weights](https://cloud.hipert.unimore.it/s/x4ZfxBKN23zAJQp/download) | +| resnet101 | Resnet 1016 | [COCO 2014](http://cocodataset.org/) | 80 | 224x224 | weights | +| resnet101_cnet | Centernet (Resnet101 backend)4 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/B6mj33k7beECXsY/download) | + + + +## References + +1. Redmon, Joseph, and Ali Farhadi. "YOLO9000: better, faster, stronger." Proceedings of the IEEE conference on computer vision and pattern recognition. 2017. +2. Redmon, Joseph, and Ali Farhadi. "Yolov3: An incremental improvement." arXiv preprint arXiv:1804.02767 (2018). +3. Yu, Fisher, et al. "Deep layer aggregation." Proceedings of the IEEE conference on computer vision and pattern recognition. 2018. +4. Zhou, Xingyi, Dequan Wang, and Philipp Krähenbühl. "Objects as points." arXiv preprint arXiv:1904.07850 (2019). +5. Sandler, Mark, et al. "Mobilenetv2: Inverted residuals and linear bottlenecks." Proceedings of the IEEE conference on computer vision and pattern recognition. 2018. +6. He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016. diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp new file mode 100644 index 0000000..a5a9d1a --- /dev/null +++ b/demo/demo/demo.cpp @@ -0,0 +1,138 @@ +#include +#include +#include /* srand, rand */ +#include +#include + +#include "CenternetDetection.h" +#include "MobilenetDetection.h" +#include "Yolo3Detection.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + char *net = "yolo3_berkeley.rt"; + if(argc > 1) + net = argv[1]; + char *input = "../demo/yolo_test.mp4"; + if(argc > 2) + input = argv[2]; + char ntype = 'y'; + if(argc > 3) + ntype = argv[3][0]; + + tk::dnn::Yolo3Detection yolo; + tk::dnn::CenternetDetection cnet; + tk::dnn::MobilenetDetection mbnet; + switch(ntype) + { + case 'y': + yolo.init(net); + break; + case 'c': + cnet.init(net); + break; + case 'm': + mbnet.init(net); + break; + default: + FatalError("Network type not allowed (3rd parameter)\n"); + } + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + cv::Mat dnn_input; + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + std::vector detected_bbox; + + while(gRun) { + cap >> frame; + if(!frame.data) { + break; + } + + // this will be resized to the net format + dnn_input = frame.clone(); + // TODO: async infer + switch(ntype) + { + case 'y': + yolo.update(dnn_input); + frame = yolo.draw(frame); + break; + case 'c': + cnet.update(dnn_input); + frame = cnet.draw(dnn_input); + break; + case 'm': + mbnet.update(dnn_input); + frame = mbnet.draw(); + break; + default: + FatalError("Network type not allowed!\n"); + } + + cv::imshow("detection", frame); + cv::waitKey(1); + if(SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + double mean = 0; + switch(ntype) + { + case 'y': + std::cout< -#include -#include /* srand, rand */ -#include -#include -#include "utils.h" - -#include -#include -#include -#include - -#include "CenternetDetection.h" - -bool gRun; -bool SAVE_RESULT = false; - -void sig_handler(int signo) { - std::cout<<"request gateway stop\n"; - gRun = false; -} - -int main(int argc, char *argv[]) { - - std::cout<<"detection\n"; - signal(SIGINT, sig_handler); - - - char *net = "resnet101_cnet.rt"; - if(argc > 1) - net = argv[1]; - char *input = "../demo/yolo_test.mp4"; - if(argc > 2) - input = argv[2]; - - tk::dnn::CenternetDetection cnet; - cnet.init(net); - - gRun = true; - - cv::VideoCapture cap(input); - if(!cap.isOpened()) - gRun = false; - else - std::cout<<"camera started\n"; - - - cv::VideoWriter resultVideo; - if(SAVE_RESULT) { - int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); - int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); - resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); - } - - cv::Mat frame; - cv::Mat dnn_input; - cv::namedWindow("detection", cv::WINDOW_NORMAL); - - while(gRun) { - cap >> frame; - if(!frame.data) { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); - // TODO: async infer - cnet.update(dnn_input); - // draw dets - frame = cnet.draw(dnn_input); - - cv::imshow("detection", frame); - cv::waitKey(1); - if(SAVE_RESULT) - resultVideo << frame; - } - - std::cout<<"detection end\n"; - - - std::cout< -#include -#include /* srand, rand */ -#include -#include -#include "utils.h" - -#include -#include -#include -#include - -#include "MobilenetDetection.h" - -bool gRun; -bool SAVE_RESULT = false; - -void sig_handler(int signo) -{ - std::cout << "request gateway stop\n"; - gRun = false; -} - -int main(int argc, char *argv[]) -{ - - std::cout << "detection\n"; - signal(SIGINT, sig_handler); - - char *net = "mobilenetv2ssd.rt"; - if (argc > 1) - net = argv[1]; - char *input = "../demo/yolo_test.mp4"; - if (argc > 2) - input = argv[2]; - - tk::dnn::MobilenetDetection mbnet; - mbnet.init(net); - - gRun = true; - - cv::VideoCapture cap(input); - if (!cap.isOpened()) - gRun = false; - else - std::cout << "camera started\n"; - - cv::VideoWriter resultVideo; - if (SAVE_RESULT) - { - int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); - int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); - resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M', 'P', '4', 'V'), 30, cv::Size(w, h)); - } - - cv::Mat frame; - cv::Mat dnn_input; - cv::namedWindow("detection", cv::WINDOW_NORMAL); - - while (gRun) - { - cap >> frame; - if (!frame.data) - { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); - // TODO: async infer - mbnet.update(dnn_input); - // draw dets - frame = mbnet.draw(); - - cv::imshow("detection", frame); - cv::waitKey(1); - if (SAVE_RESULT) - resultVideo << frame; - } - - std::cout << "detection end\n"; - - std::cout << COL_GREENB << "\n\nTime stats:\n"; - std::cout << "Min: " << *std::min_element(mbnet.stats.begin(), mbnet.stats.end()) << " ms\n"; - std::cout << "Max: " << *std::max_element(mbnet.stats.begin(), mbnet.stats.end()) << " ms\n"; - double mean = 0; - for (int i = 0; i < mbnet.stats.size(); i++) - mean += mbnet.stats[i]; - mean /= mbnet.stats.size(); - std::cout << "Avg: " << mean << " ms\n" - << COL_END; - return 0; -} diff --git a/demo/demo/demo_yolo3.cpp b/demo/demo/demo_yolo3.cpp deleted file mode 100644 index 2b34386..0000000 --- a/demo/demo/demo_yolo3.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include -#include -#include /* srand, rand */ -#include -#include -#include "utils.h" - -#include -#include -#include -#include - -#include "Yolo3Detection.h" - -bool gRun; -bool SAVE_RESULT = false; - -void sig_handler(int signo) { - std::cout<<"request gateway stop\n"; - gRun = false; -} - -int main(int argc, char *argv[]) { - - std::cout<<"detection\n"; - signal(SIGINT, sig_handler); - - - char *net = "yolo3_berkeley.rt"; - if(argc > 1) - net = argv[1]; - char *input = "../demo/yolo_test.mp4"; - if(argc > 2) - input = argv[2]; - - tk::dnn::Yolo3Detection yolo; - yolo.init(net); - - gRun = true; - - cv::VideoCapture cap(input); - if(!cap.isOpened()) - gRun = false; - else - std::cout<<"camera started\n"; - - - cv::VideoWriter resultVideo; - if(SAVE_RESULT) { - int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); - int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); - resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); - } - - cv::Mat frame; - cv::Mat dnn_input; - cv::namedWindow("detection", cv::WINDOW_NORMAL); - - while(gRun) { - cap >> frame; - if(!frame.data) { - break; - } - - // this will be resized to the net format - dnn_input = frame.clone(); - // TODO: async infer - yolo.update(dnn_input); - - // draw dets - for(int i=0; iclassesNames[b.cl]; - float prob = b.prob; - - // std::cout< 1) net = argv[1]; @@ -91,6 +99,7 @@ int main(int argc, char *argv[]) //inference detected_bbox.clear(); + TIMER_START switch(ntype) { case 'y': @@ -104,6 +113,9 @@ int main(int argc, char *argv[]) default: FatalError("Network type not allowed!\n"); } + TIMER_STOP + if(write_res_on_file) + times< &images,const int classes,const float IoU_thresh, const float conf_thresh=0.3, const int map_points=101, const bool verbose=false); -double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false, const bool write_on_file = false, std::string net = ""); -void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh=0.5, const float conf_thresh=0.3, bool verbose=false); +void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh=0.5, const float conf_thresh=0.3, bool verbose=false, const bool write_on_file=false, std::string net=""); -#endif /*EVALUATION_H*/ \ No newline at end of file +#endif /*EVALUATION_H*/ diff --git a/include/sorting.h b/include/sorting.h index 7e90f11..5153f3c 100644 --- a/include/sorting.h +++ b/include/sorting.h @@ -1,3 +1,6 @@ +#ifndef SORTING_H +#define SORTING_H + #include #include #include @@ -32,3 +35,5 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_be float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out); void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out); + +#endif /*SORTING_H*/ \ No newline at end of file diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h index 6bfd1b2..840a2cc 100644 --- a/include/tkDNN/MobilenetDetection.h +++ b/include/tkDNN/MobilenetDetection.h @@ -2,13 +2,14 @@ #define MOBILENETDETECTION_H #include -#include "tkdnn.h" #include #include #include #include +#include "tkdnn.h" + #define N_COORDS 4 diff --git a/include/tkDNN/Yolo3Detection.h b/include/tkDNN/Yolo3Detection.h index ac12a4f..51994e1 100644 --- a/include/tkDNN/Yolo3Detection.h +++ b/include/tkDNN/Yolo3Detection.h @@ -1,3 +1,6 @@ +#ifndef YOLODETECTION_H +#define YOLODETECTION_H + #include #include #include /* srand, rand */ @@ -34,7 +37,7 @@ class Yolo3Detection { int classes = 0; int num = 0; int n_masks = 0; - float thresh = 0.3; + float thresh = 0.05; cv::Scalar colors[256]; // this is filled with results @@ -53,7 +56,7 @@ class Yolo3Detection { * @return Success of the initialization */ bool init(std::string tensor_path); - + cv::Mat draw(cv::Mat &frame); void update(cv::Mat &frame); tk::dnn::Yolo* getYoloLayer(int n=0) { @@ -66,3 +69,5 @@ class Yolo3Detection { }; }} + +#endif /* YOLODETECTION_H*/ diff --git a/include/tkDNN/pluginsRT/SoftmaxRT.h b/include/tkDNN/pluginsRT/SoftmaxRT.h deleted file mode 100644 index 226f6f6..0000000 --- a/include/tkDNN/pluginsRT/SoftmaxRT.h +++ /dev/null @@ -1,64 +0,0 @@ -#include - -class SoftmaxRT : public IPlugin { - -public: - SoftmaxRT(const tk::dnn::dataDim_t* dim) { - assert(dim != nullptr); - this->dim.n = dim->n; - this->dim.c = dim->c; - this->dim.h = dim->h; - this->dim.w = dim->w; - this->dim.l = dim->l; - } - - ~SoftmaxRT(){ - - } - - int getNbOutputs() const override { - return 1; - } - - Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { - return DimsNCHW{this->dim.n,this->dim.c,this->dim.h,this->dim.w }; - } - - void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { - } - - int initialize() override { - return 0; - } - - virtual void terminate() override { - } - - virtual size_t getWorkspaceSize(int maxBatchSize) const override { - return 0; - } - - virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { - dnnType *srcData = (dnnType*)reinterpret_cast(inputs[0]); - dnnType *dstData = reinterpret_cast(outputs[0]); - - - return 0; - } - - - virtual size_t getSerializationSize() override { - return 5*sizeof(int); - } - - virtual void serialize(void* buffer) override { - char *buf = reinterpret_cast(buffer); - tk::dnn::writeBUF(buf, this->dim.n); - tk::dnn::writeBUF(buf, this->dim.c); - tk::dnn::writeBUF(buf, this->dim.h); - tk::dnn::writeBUF(buf, this->dim.w); - tk::dnn::writeBUF(buf, this->dim.l); - } - - dataDim_t dim; -}; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 5508b88..03edf22 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -1,3 +1,6 @@ +#ifndef CENTERNETDETECTION_H +#define CENTERNETDETECTION_H + #include "CenternetDetection.h" #include "opencv2/imgproc/imgproc.hpp" #include @@ -419,3 +422,5 @@ void CenternetDetection::update(cv::Mat &imageORIG) { stats.push_back(t_ns); } }} + +#endif /*CENTERNETDETECTION_H*/ diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index f21efa7..08742c6 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -60,6 +60,37 @@ bool Yolo3Detection::init(std::string tensor_path) { return true; } +cv::Mat Yolo3Detection::draw(cv::Mat &imageORIG) { + + tk::dnn::box b; + int x0, w, x1, y0, h, y1; + int objClass; + std::string det_class; + float prob; + int baseline = 0; + float fontScale = 0.5; + int thickness = 2; + // draw dets + for(int i=0; iclassesNames[b.cl]; + prob = b.prob; + + // std::cout< void BoundingBox::clear() @@ -283,27 +284,51 @@ double computeMap(std::vector &images,const int classes,const float IoU_t return mean_average_precision; } -double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh, const float conf_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh, const float conf_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose, const bool write_on_file, std::string net) { - double AP = 0; + std::ofstream out_file; + if(write_on_file) + { + out_file.open("map.csv", std::ios_base::app); + out_file< &images,const int classes,const float IoU_thresh, const float conf_thresh, bool verbose) +void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, bool verbose, const bool write_on_file, std::string net) { + + std::ofstream out_file; + if(write_on_file) + { + out_file.open("pr.csv", std::ios_base::app); + out_file< truth_classes_count(classes,0); std::vector dets_classes_count(classes,0); - std::vector pr( classes); + std::vector pr(classes); for(auto &img:images) { @@ -355,6 +380,8 @@ void computeTPFPFN(std::vector &images,const int classes,const float IoU_ double avg_precision = 0, avg_recall = 0, f1_score = 0; + + int TP = 0, FP = 0, FN = 0; for(size_t i=0; i 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fp) : 0; @@ -364,13 +391,23 @@ void computeTPFPFN(std::vector &images,const int classes,const float IoU_ // std::cout< 0 ? 2 * ( avg_precision * avg_recall ) / ( avg_precision + avg_recall ) : 0; + if(write_on_file) + { + out_file< #include "tkdnn.h" -#include -#include -#include -#include const char *output_bin1 = "../tests/mobilenetv2ssd/debug/classification_headers-5.bin"; const char *output_bin2 = "../tests/mobilenetv2ssd/debug/regression_headers-5.bin";