From eca10ac0a85339c97877de30416ac806280229f4 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Sun, 19 Sep 2021 00:16:31 +0200 Subject: [PATCH 01/15] Update weights Signed-off-by: Micaela Verucchi --- tests/darknet/yolo4_berkeley_f1.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/darknet/yolo4_berkeley_f1.cpp b/tests/darknet/yolo4_berkeley_f1.cpp index 6dfbc63..b2f1139 100644 --- a/tests/darknet/yolo4_berkeley_f1.cpp +++ b/tests/darknet/yolo4_berkeley_f1.cpp @@ -17,7 +17,7 @@ int main() { std::string wgs_path = bin_path + "/layers"; std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4_berkeley.cfg"; std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/berkeley.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/M7WJdGoGDaDACnN/download"); + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/q9dwoqQ5YQqEi7s/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); From d6fb6c6af44498c19fd49c0870c75fbf648f28d2 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 4 Nov 2021 19:28:55 +0100 Subject: [PATCH 02/15] Fix max elem (remove thrust) for segmentation Signed-off-by: Micaela Verucchi --- src/kernels/postprocessing.cu | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/kernels/postprocessing.cu b/src/kernels/postprocessing.cu index 63643eb..c175e1f 100644 --- a/src/kernels/postprocessing.cu +++ b/src/kernels/postprocessing.cu @@ -46,11 +46,16 @@ void maxElem_kernel(float *src_begin, float *dst_begin, const int n_classes, con if (i > size) return; - thrust::device_ptr dPbeg ( &src_begin[i*n_classes] ) ; - thrust::device_ptr dPend = dPbeg + n_classes; - thrust::device_ptr result = thrust::max_element(thrust::device,dPbeg, dPend); + float max = 0; + int max_idx = 0; + for( int j = i*n_classes; j < i*n_classes + n_classes; ++j ){ + if( src_begin[j] > max ){ + max = src_begin[j]; + max_idx = j; + } + } - dst_begin[i] = result - dPbeg; + dst_begin[i] = max_idx - i*n_classes; } void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w){ From 55df97afe1625c834fe931a52373b6f4f2db51ad Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 23 Nov 2021 13:01:07 +0100 Subject: [PATCH 03/15] Fix warnings, upgrade to C++14 Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 4 ++-- include/tkDNN/SegmentationNN.h | 1 + src/CenterTrack.cpp | 10 +++++++++- src/CenternetDetection.cpp | 1 + src/CenternetDetection3D.cpp | 2 ++ src/MobilenetDetection.cpp | 4 ++-- 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b366416..20ede6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,10 +3,10 @@ cmake_minimum_required(VERSION 3.15) project (tkDNN) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) if(UNIX) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable ") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations") endif() if(WIN32) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_FLAGS "/O2 /FS /EHsc") set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif(WIN32) diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index b691cbc..b73ffa2 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -182,6 +182,7 @@ class SegmentationNN { checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); + return true; } /** diff --git a/src/CenterTrack.cpp b/src/CenterTrack.cpp index dc15823..991ff69 100644 --- a/src/CenterTrack.cpp +++ b/src/CenterTrack.cpp @@ -17,6 +17,8 @@ bool CenterTrack::init(const std::string& tensor_path, const int n_classes, cons init_pre_inf(); init_postprocessing(); init_visualization(n_classes); + + return true; } bool CenterTrack::init_preprocessing(){ @@ -59,6 +61,8 @@ bool CenterTrack::init_preprocessing(){ checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches)); checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot())); checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) ); + + return true; } bool CenterTrack::init_pre_inf(){ @@ -202,6 +206,8 @@ bool CenterTrack::init_postprocessing(){ trRes.resize(nBatches); countTr.resize(nBatches, 0); trackId.resize(nBatches, 0); + + return true; } bool CenterTrack::init_visualization(const int n_classes){ @@ -274,6 +280,8 @@ bool CenterTrack::init_visualization(const int n_classes){ faceId.push_back({3,0,4,7}); faceId.push_back({2,3,7,6}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); + + return true; } void CenterTrack::_get_additional_inputs(){ @@ -308,7 +316,7 @@ void CenterTrack::preprocess(cv::Mat &frame, const int bi){ } float c[] = {new_width / 2.0f, new_height /2.0f}; - float s[] = {dim.w, dim.h}; + float s[] = {float(dim.w), float(dim.h)}; // float s = new_width >= new_height ? new_width : new_height; // ----------- get_affine_transform // rot_rad = pi * 0 / 100 --> 0 diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 46757f4..dce1455 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -119,6 +119,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe dst2.at(2,0)=dst2.at(1,0) + (-dst2.at(0,1)+dst2.at(1,1) ); dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); + return true; } diff --git a/src/CenternetDetection3D.cpp b/src/CenternetDetection3D.cpp index 653624b..14f674d 100644 --- a/src/CenternetDetection3D.cpp +++ b/src/CenternetDetection3D.cpp @@ -167,6 +167,8 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas faceId.push_back({2,3,7,6}); faceId.push_back({3,0,4,7}); // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]); + + return true; } void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){ diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index 3c54e28..13b836f 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -198,7 +198,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe "bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" , "apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" , "donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" , - "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , + "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , "cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" , "book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"}; classesNames = std::vector(classes_names_, std::end(classes_names_)); @@ -207,7 +207,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe else{ FatalError("Number of classes not supported for mobilenet"); } - return 1; + return true; } void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){ From 75c3cb003818b59512b3d42acceaaf560c641496 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 23 Nov 2021 13:01:58 +0100 Subject: [PATCH 04/15] Add script to compare times_rtinference.csv files Signed-off-by: Micaela Verucchi --- scripts/checkExecTimes.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 scripts/checkExecTimes.py diff --git a/scripts/checkExecTimes.py b/scripts/checkExecTimes.py new file mode 100644 index 0000000..c3e1b0b --- /dev/null +++ b/scripts/checkExecTimes.py @@ -0,0 +1,37 @@ +import sys +import pandas as pd + +if len(sys.argv) < 3: + print("Error: two csv files are needed, old first new second") + exit(1) + +old_perf_file = str(sys.argv[1]) +new_perf_file = str(sys.argv[2]) + +verbose = False +if len(sys.argv) == 4: + verbose = bool(sys.argv[3]) + +print("Comparing {} vs {}".format(old_perf_file, new_perf_file)) + +df_old = pd.read_csv (old_perf_file, sep=';', header=None, index_col=0) +df_new = pd.read_csv (new_perf_file, sep=';', header=None, index_col=0) + +for index, row in df_new.iterrows(): + if index in df_old.index: + if verbose: + print("New: ",row[1], row[2], row[3]) + print("Old: ",df_old.loc[index][1], df_old.loc[index][2], df_old.loc[index][3]) + + print(index, end=': ') + if abs(row[1] - df_old.loc[index][1]) < df_old.loc[index][1]*0.1: + print("similar performance") + elif (row[1] < df_old.loc[index][1]): + print('\x1b[3;30;42m' + 'faster' + '\x1b[0m') + elif (row[1] > df_old.loc[index][1]): + if row[1] > df_old.loc[index][1] + df_old.loc[index][1] * 0.5 : + print('\x1b[3;30;41m' + 'WAY SLOWER' + '\x1b[0m') + else: + print('\x1b[3;30;41m' + 'slower' + '\x1b[0m') + + From be5864748af0cbd28c2603b62986a442ef744bca Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 23 Nov 2021 16:29:40 +0100 Subject: [PATCH 05/15] Use yaml config file for the demo instead of param list Signed-off-by: Micaela Verucchi --- demo/demo/demo.cpp | 81 ++++++++++++++++++++++++------------------- demo/demoConfig.yaml | 14 ++++++++ docs/demo.md | 31 +++++++++-------- include/tkDNN/utils.h | 16 +++++++++ 4 files changed, 91 insertions(+), 51 deletions(-) create mode 100644 demo/demoConfig.yaml diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 317a574..069c272 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -9,7 +9,6 @@ #include "Yolo3Detection.h" bool gRun; -bool SAVE_RESULT = false; void sig_handler(int signo) { std::cout<<"request gateway stop\n"; @@ -18,43 +17,53 @@ void sig_handler(int signo) { int main(int argc, char *argv[]) { - std::cout<<"detection\n"; signal(SIGINT, sig_handler); - - std::string net = "yolo4tiny_fp32.rt"; - if(argc > 1) - net = argv[1]; + // get config file path and read it #ifdef __linux__ - std::string input = "../demo/yolo_test.mp4"; + std::string config_file = "../demo/demoConfig.yaml"; #elif _WIN32 - std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + std::string config_file = "..\\..\\..\\demo\\demoConfig.yaml"; #endif + if(argc > 1) + config_file = config_file[1]; + + YAML::Node conf = YAMLloadConf(config_file); + if(!conf) + FatalError("Problem with config file"); - if(argc > 2) - input = argv[2]; - char ntype = 'y'; - if(argc > 3) - ntype = argv[3][0]; - int n_classes = 80; - if(argc > 4) - n_classes = atoi(argv[4]); - int n_batch = 1; - if(argc > 5) - n_batch = atoi(argv[5]); - bool show = true; - if(argc > 6) - show = atoi(argv[6]); - float conf_thresh=0.3; - if(argc > 7) - conf_thresh = atof(argv[7]); + // read settings from config file + std::string net = YAMLgetConf(conf, "net", "yolo4tiny_fp32.rt"); + if(!fileExist(net.c_str())) + FatalError("The given network does not exist. Create the rt first."); + #ifdef __linux__ + std::string input = YAMLgetConf(conf, "input", "../demo/yolo_test.mp4"); + #elif _WIN32 + std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); + #endif + if(!fileExist(input.c_str())) + FatalError("The given input video does not exist."); + + char ntype = YAMLgetConf(conf, "ntype", 'y'); + int n_classes = YAMLgetConf(conf, "n_classes", 80); + int n_batch = YAMLgetConf(conf, "n_batch", 1); if(n_batch < 1 || n_batch > 64) FatalError("Batch dim not supported"); + float conf_thresh = YAMLgetConf(conf, "conf_thresh", 0.3); + bool show = YAMLgetConf(conf, "show", true); + bool save = YAMLgetConf(conf, "save", false); - if(!show) - SAVE_RESULT = true; - + std::cout <<"Net settings - net: "<< net + <<", ntype: "<< ntype + <<", n_classes: "<< n_classes + <<", n_batch: "<< n_batch + <<", conf_thresh: "<< conf_thresh<<"\n"; + std::cout <<"Demo settings - input: "<< input + <<", show: "<< show + <<", save: "<< save<<"\n\n"; + + // create detection network tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; tk::dnn::MobilenetDetection mbnet; @@ -79,8 +88,7 @@ int main(int argc, char *argv[]) { detNN->init(net, n_classes, n_batch, conf_thresh); - gRun = true; - + // open video stream cv::VideoCapture cap(input); if(!cap.isOpened()) gRun = false; @@ -88,19 +96,21 @@ int main(int argc, char *argv[]) { std::cout<<"camera started\n"; cv::VideoWriter resultVideo; - if(SAVE_RESULT) { + if(save) { int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } - cv::Mat frame; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); + cv::Mat frame; std::vector batch_frame; std::vector batch_dnn_input; + // start detection loop + gRun = true; while(gRun) { batch_dnn_input.clear(); batch_frame.clear(); @@ -128,19 +138,18 @@ int main(int argc, char *argv[]) { cv::waitKey(1); } } - if(n_batch == 1 && SAVE_RESULT) + if(n_batch == 1 && save) resultVideo << frame; } std::cout<<"detection end\n"; - double mean = 0; + double mean = 0; std::cout<stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; + std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); std::cout<<"Avg: "< ``` -In general the demo program takes 7 parameters: -``` -./demo -``` -where +In general the demo program takes 1 parameter, the `````` that is the path to che configuration file. The parameter is optional and its default value is ```"../demo/demoConfig.yaml"```. -* `````` is the rt file generated by a test -* ```<``` is the path to a video file or a camera input -* `````` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family) -* ``````is the number of classes the network is trained on -* `````` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). -* `````` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1) -* `````` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed. +The config file is a yaml file with the following attributes: +* ```net``` is the rt file generated by a test +* ```input``` is the path to a video file or a camera input (on Linux) +* ```win_input``` is the path to a video file or a camera input (on Windows) +* ```ntype``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family) +* ```n_classes``` is the number of classes the network is trained on +* ```n_batch``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). +* ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed. +* ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1) +* ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1) N.B. By default it is used FP32 inference @@ -61,7 +60,8 @@ To run the demo with FP16 inference follow these steps (example with yolov3): export TKDNN_MODE=FP16 # set the half floating point optimization rm yolo3_fp16.rt # be sure to delete(or move) old tensorRT files ./test_yolo3 # run the yolo test (is slow) -./demo yolo3_fp16.rt ../demo/yolo_test.mp4 y +# set net: yolo3_fp16.rt in the config-file +./demo ``` N.B. Using FP16 inference will lead to some errors in the results (first or second decimal). @@ -86,7 +86,8 @@ export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files ./test_yolo3 # run the yolo test (is slow) -./demo yolo3_int8.rt ../demo/yolo_test.mp4 y +# set net: yolo3_int8.rt in the config-file +./demo ``` N.B. diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index 65375ba..4965a7a 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -21,6 +21,7 @@ #include #include +#include #define dnnType float @@ -137,4 +138,19 @@ static inline bool isCudaPointer(void *data) { cudaPointerAttributes attr; return cudaPointerGetAttributes(&attr, data) == 0; } + +inline YAML::Node YAMLloadConf(const std::string& conf_file) { + std::cerr<<"Loading YAML: "< +inline T YAMLgetConf(YAML::Node conf, std::string key, T defaultVal) { + T val = defaultVal; + if(conf && conf[key]) { + val = conf[key].as(); + } + return val; +} + #endif //UTILS_H From 24cdb4c4a7abe18ff764bb7d9f32fe681523d1c8 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 24 Nov 2021 18:20:26 +0100 Subject: [PATCH 06/15] Update Readme Signed-off-by: Micaela Verucchi --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b630fec..a71e1cd 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,15 @@ If you use tkDNN in your research, please cite the [following paper](https://iee } ``` -### What's new (20 July 2021) +### What's new +#### 20 July 2021 - [x] Support to sematic segmentation [README](docs/README_seg.md) - [x] Support 2D/3D Object Detection and Tracking [README](docs/README_2d3dtracking.md) -- [ ] Support to TensorRT8 (WIP) +#### 24 November 2021 +- [x] Support to sematic segmentation on cuda 11 +- [x] Support to TensorRT8 (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg)). + +TensorRT8 (and therefore Jetpack 4.6) is currently supported only on the branch tensorrt8 due to [performance issue with TensorRT8](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/tensorrt-8.html)). We will merge it to the master as soon as those issues are fixed (probably in future minor releases). ## FPS Results Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on From 9cbac460bc7b1920a4a1195d547e0d2e6a8bc57b Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 25 Nov 2021 11:32:38 +0100 Subject: [PATCH 07/15] Update README.md --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a71e1cd..ddfd4ef 100644 --- a/README.md +++ b/README.md @@ -173,11 +173,10 @@ For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows | yolo4_320 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 320x320 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | | yolo4_512 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | | yolo4_608 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 608x608 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | -| yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | +| yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 544x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | | yolo4tiny | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | -| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | +| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) | | yolo4tiny_512 | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | -80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) | | yolo4x-cps | Scaled Yolov4 10 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download) | | shelfnet | ShelfNet18_realtime11 | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) | | shelfnet_berkeley | ShelfNet18_realtime11 | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) | From 04de9908a68588c32dcf94d84b59fdf71cf8d9a8 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Mon, 29 Nov 2021 17:33:57 +0100 Subject: [PATCH 08/15] Add yolo4-csp for crowdhuman dataset, add shelfnet for coco-stuff dataset, fix minor in demo.cpp Signed-off-by: Micaela Verucchi --- CMakeLists.txt | 3 + demo/demo/demo.cpp | 2 +- tests/darknet/cfg/yolo4-csp_crowd.cfg | 1279 +++++++++++++++++++++++++ tests/darknet/names/crowdhuman.names | 2 + tests/darknet/yolo4-csp_crowd.cpp | 34 + tests/shelfnet/shelfnet_coco.cpp | 295 ++++++ 6 files changed, 1614 insertions(+), 1 deletion(-) create mode 100644 tests/darknet/cfg/yolo4-csp_crowd.cfg create mode 100644 tests/darknet/names/crowdhuman.names create mode 100644 tests/darknet/yolo4-csp_crowd.cpp create mode 100644 tests/shelfnet/shelfnet_coco.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 20ede6c..877b506 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,6 +140,9 @@ target_link_libraries(test_shelfnet_berkeley tkDNN) add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp) target_link_libraries(test_shelfnet_mapillary tkDNN) +add_executable(test_shelfnet_coco tests/shelfnet/shelfnet_coco.cpp) +target_link_libraries(test_shelfnet_coco tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 069c272..5445d86 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -26,7 +26,7 @@ int main(int argc, char *argv[]) { std::string config_file = "..\\..\\..\\demo\\demoConfig.yaml"; #endif if(argc > 1) - config_file = config_file[1]; + config_file = argv[1]; YAML::Node conf = YAMLloadConf(config_file); if(!conf) diff --git a/tests/darknet/cfg/yolo4-csp_crowd.cfg b/tests/darknet/cfg/yolo4-csp_crowd.cfg new file mode 100644 index 0000000..4c50f4d --- /dev/null +++ b/tests/darknet/cfg/yolo4-csp_crowd.cfg @@ -0,0 +1,1279 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=16 +width=512 +height=512 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 8000 +policy=steps +steps=6400,7200 +scales=.1,.1 + +mosaic=1 + +letter_box=1 + +ema_alpha=0.9998 + +#optimized_memory=1 + +#23:104x104 54:52x52 85:26x26 104:13x13 for 416 + + + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -2 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -1,-7 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -13 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 79 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 48 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=logistic + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=0 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=4.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -20 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=logistic + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=1.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -49 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=logistic + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=0.4 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=2 diff --git a/tests/darknet/names/crowdhuman.names b/tests/darknet/names/crowdhuman.names new file mode 100644 index 0000000..5ba1275 --- /dev/null +++ b/tests/darknet/names/crowdhuman.names @@ -0,0 +1,2 @@ +person +head \ No newline at end of file diff --git a/tests/darknet/yolo4-csp_crowd.cpp b/tests/darknet/yolo4-csp_crowd.cpp new file mode 100644 index 0000000..b8f0e6b --- /dev/null +++ b/tests/darknet/yolo4-csp_crowd.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4-csp_crowd"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer144_out.bin", + bin_path + "/debug/layer159_out.bin", + bin_path + "/debug/layer174_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4-csp_crowd.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/crowdhuman.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/RKWfWNmWXfJigsK/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} \ No newline at end of file diff --git a/tests/shelfnet/shelfnet_coco.cpp b/tests/shelfnet/shelfnet_coco.cpp new file mode 100644 index 0000000..276a09d --- /dev/null +++ b/tests/shelfnet/shelfnet_coco.cpp @@ -0,0 +1,295 @@ +#include +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" + + +const char *input_bin = "shelfnet_coco/debug/input.bin"; + +const char *backbone[] = { + "shelfnet_coco/layers/backbone-conv1.bin", + "shelfnet_coco/layers/backbone-layer1-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer1-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer1-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer1-1-conv2.bin", + "shelfnet_coco/layers/backbone-layer2-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer2-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet_coco/layers/backbone-layer2-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer2-1-conv2.bin", + "shelfnet_coco/layers/backbone-layer3-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer3-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet_coco/layers/backbone-layer3-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer3-1-conv2.bin", + "shelfnet_coco/layers/backbone-layer4-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer4-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet_coco/layers/backbone-layer4-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet_coco/layers/conv_out-conv-conv.bin", + "shelfnet_coco/layers/conv_out-conv_out.bin", + "shelfnet_coco/layers/conv_out16-conv-conv.bin", + "shelfnet_coco/layers/conv_out16-conv_out.bin", + "shelfnet_coco/layers/conv_out32-conv-conv.bin", + "shelfnet_coco/layers/conv_out32-conv_out.bin" + }; + +const char *decoder[] = { + "shelfnet_coco/layers/decoder-bottom-conv1.bin", + "shelfnet_coco/layers/decoder-bottom-conv12.bin", + "shelfnet_coco/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet_coco/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet_coco/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet_coco/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet_coco/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet_coco/layers/decoder-up_dense_list-1-conv.bin" + }; + + +const char *ladder[] = { + "shelfnet_coco/layers/ladder-inconv-conv1.bin", + "shelfnet_coco/layers/ladder-inconv-conv12.bin", + "shelfnet_coco/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet_coco/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet_coco/layers/ladder-down_conv_list-0.bin", + + "shelfnet_coco/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet_coco/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet_coco/layers/ladder-down_conv_list-1.bin", + + "shelfnet_coco/layers/ladder-bottom-conv1.bin", + "shelfnet_coco/layers/ladder-bottom-conv12.bin", + + + + "shelfnet_coco/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet_coco/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet_coco/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet_coco/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet_coco/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet_coco/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet_coco/layers/trans1-conv.bin", + "shelfnet_coco/layers/trans2-conv.bin", + "shelfnet_coco/layers/trans3-conv.bin"}; +int main() +{ + + downloadWeightsifDoNotExist(input_bin, "shelfnet_coco", "https://cloud.hipert.unimore.it/s/KfQ9fGJQsgzNbiW/download"); + + int classes = 183; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0, di = 0, li = 0, ci = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); + // // } + + new tk::dnn::Softmax(&net); + + const char *output_bin = "shelfnet_coco/debug/softmax.bin"; + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + std::cout<<"Input:"< +``` +where +* `````` is the rt file generated by a test +* ```<``` is the path to a video file or a camera input +* `````` if set to 0 the demo will not show the visualization, it will otherwise (default=1) +* `````` if set to 1 the demo will save the video into result.mp4, it won't otherwise (default=1) + +NB) By default it is used FP32 inference + + +![demo](https://user-images.githubusercontent.com/11939259/160845358-0d6ab15d-c5f4-46ae-b9da-bfaf3903389d.gif "Results on yolo_test.mp4") + + + + diff --git a/docs/exporting_weights.md b/docs/exporting_weights.md index 811431d..b4cb366 100644 --- a/docs/exporting_weights.md +++ b/docs/exporting_weights.md @@ -86,6 +86,18 @@ mkdir layer debug python export.py ``` +### 6)Export weights for monodepth2 +To get the weights needed to run Shelfnet tests use [this](https://github.com/perseusdg/monodepth2) fork of a Pytorch implementation of monodepth2 network. + +``` +git clone https://github.com/perseusdg/monodepth2 +cd monodepth2 +mkdir models # Download the official weights and put depth.pth and encorder.pth inside this new folder +conda env create --file monodepth.yaml +conda activate monodepth2 +python exporter.py # you will find the weights inside the tkDNN_bin folder +``` + ## Darknet Parser tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*: ``` From 7c0620e3915d2053f23232e947d75ed8df686f62 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Wed, 30 Mar 2022 15:47:41 +0200 Subject: [PATCH 11/15] test_all_test script save results in separate files --- scripts/test_all_tests.sh | 82 ++++++++++++++------------ tests/test_rtinference/rtinference.cpp | 8 +-- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index 0da9b6b..e44f7e8 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -cd build +#cd build RED='\033[1;31m' GREEN='\033[1;32m' @@ -29,24 +29,28 @@ function print_output { } +out_dir=results out_file=results.log -rm $out_file +rm -rf $out_dir/ +mkdir -p $out_dir function test_net { - ./test_$1 &>> $out_file + ./test_$1 &> $out_dir/$1_${TKDNN_MODE}_build_$out_file print_output $? $1 - ./test_rtinference $1*.rt $TKDNN_BATCHSIZE &>> $out_file + ./test_rtinference $1*.rt 1 &> $out_dir/$1_${TKDNN_MODE}_inference_batch1_$out_file + print_output $? "infer $1" + ./test_rtinference $1*.rt $TKDNN_BATCHSIZE &> $out_dir/$1_${TKDNN_MODE}_inference_batch${TKDNN_BATCHSIZE}_$out_file print_output $? "batched $1" } -modes=( 1 ) # only FP32 -# modes=( 1 2 ) # FP32 and FP16 +# modes=( 1 ) # only FP32 +modes=( 1 2 ) # FP32 and FP16 # modes=( 1 2 3 ) # FP32, FP16 and INT8 for i in "${modes[@]}" do - rm *rt + rm -f *rt if [ $i -eq 1 ] then export TKDNN_MODE=FP32 @@ -73,37 +77,37 @@ do # print_output $? imuodom test_net yolo4 - test_net yolo4_320 - test_net yolo4_320_coco2 - test_net yolo4_512 - test_net yolo4_608 - test_net yolo4-csp - test_net yolo4x - test_net yolo4_berkeley - test_net yolo4_berkeley_f1 - test_net yolo4tiny - test_net yolo4tiny_512 - test_net yolo3 - test_net yolo3_berkeley - test_net yolo3_coco4 - test_net yolo3_flir - test_net yolo3_512 - test_net yolo3tiny - test_net yolo3tiny_512 - test_net yolo2 - test_net yolo2_voc - #test_net yolo2tiny - test_net csresnext50-panet-spp - #test_net csresnext50-panet-spp_berkeley - test_net resnet101_cnet - test_net dla34_cnet - test_net dla34_cnet3d - test_net mobilenetv2ssd - test_net mobilenetv2ssd512 - test_net bdd-mobilenetv2ssd - test_net dla34_ctrack - test_net shelfnet - test_net shelfnet_berkeley + # test_net yolo4_320 + # test_net yolo4_320_coco2 + # test_net yolo4_512 + # test_net yolo4_608 + # test_net yolo4-csp + # test_net yolo4x + # test_net yolo4_berkeley + # test_net yolo4_berkeley_f1 + # test_net yolo4tiny + # test_net yolo4tiny_512 + # test_net yolo3 + # test_net yolo3_berkeley + # test_net yolo3_coco4 + # test_net yolo3_flir + # test_net yolo3_512 + # test_net yolo3tiny + # test_net yolo3tiny_512 + # test_net yolo2 + # test_net yolo2_voc + # test_net yolo2tiny + # test_net csresnext50-panet-spp + # test_net csresnext50-panet-spp_berkeley + # test_net resnet101_cnet + # test_net dla34_cnet + # test_net dla34_cnet3d + # test_net mobilenetv2ssd + # test_net mobilenetv2ssd512 + # test_net bdd-mobilenetv2ssd + # test_net dla34_ctrack + # test_net shelfnet + # test_net shelfnet_berkeley done -echo "If errors occured, check logfile $out_file" +echo "If errors occured, check logfiles in directory: $out_dir" diff --git a/tests/test_rtinference/rtinference.cpp b/tests/test_rtinference/rtinference.cpp index 22c333e..bcdc739 100644 --- a/tests/test_rtinference/rtinference.cpp +++ b/tests/test_rtinference/rtinference.cpp @@ -1,6 +1,6 @@ #include #include -#include "tkdnn.h" +#include "tkDNN/tkdnn.h" #include /* srand, rand */ @@ -66,11 +66,11 @@ int main(int argc, char *argv[]) { } } - double min = *std::min_element(stats.begin(), stats.end())/BATCH_SIZE; - double max = *std::max_element(stats.begin(), stats.end())/BATCH_SIZE; + double min = *std::min_element(stats.begin(), stats.end()); ///BATCH_SIZE; + double max = *std::max_element(stats.begin(), stats.end()); ///BATCH_SIZE; double mean =0; for(int i=0; i Date: Wed, 30 Mar 2022 15:52:30 +0200 Subject: [PATCH 12/15] compile with tensorrt7 --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1af836d..13db7b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,8 +78,7 @@ endif() #------------------------------------------------------------------------------- # CUDA #------------------------------------------------------------------------------- - -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'") +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}" --compiler-options '-fPIC') find_package(CUDNN REQUIRED) From fa9db167b88ac4161d81b1478b01168768c13613 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Wed, 30 Mar 2022 17:48:02 +0200 Subject: [PATCH 13/15] version 0.7 --- include/tkDNN/tkdnn.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/tkDNN/tkdnn.h b/include/tkDNN/tkdnn.h index 26aaa0b..b129e6e 100644 --- a/include/tkDNN/tkdnn.h +++ b/include/tkDNN/tkdnn.h @@ -5,4 +5,4 @@ #include "Layer.h" #include "NetworkRT.h" -#define TKDNN_VERSION 500 +#define TKDNN_VERSION 700 From 5e71b992658670ad8a86c7447981e58fc763e950 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Wed, 30 Mar 2022 20:46:51 +0200 Subject: [PATCH 14/15] YoloRT save bias, mask and clasesName into RT file --- demo/demo/demo.cpp | 11 +----- demo/demo/map.cpp | 16 +++------ demo/demoConfig.yaml | 10 +----- docs/demo.md | 2 -- include/tkDNN/CenternetDetection.h | 2 +- include/tkDNN/DetectionNN.h | 2 +- include/tkDNN/MobilenetDetection.h | 2 +- include/tkDNN/NetworkRT.h | 5 +-- include/tkDNN/Yolo3Detection.h | 3 +- include/tkDNN/pluginsRT/YoloRT.h | 5 +-- src/CenternetDetection.cpp | 2 +- src/MobilenetDetection.cpp | 2 +- src/NetworkRT.cpp | 16 +++++++++ src/Yolo3Detection.cpp | 50 +++++++++------------------ src/pluginsRT/YoloRT.cpp | 55 +++++++++++++++++++++++++++++- 15 files changed, 104 insertions(+), 79 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index c857086..b6f78d7 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -39,12 +39,8 @@ int main(int argc, char *argv[]) { #ifdef __linux__ std::string input = YAMLgetConf(conf, "input", "../demo/yolo_test.mp4"); - std::string cfgPath = YAMLgetConf(conf,"cfg_input", "../tests/darknet/cfg/yolo4tiny.cfg"); - std::string namePath = YAMLgetConf(conf,"name_input","../tests/darknet/names/coco.names"); #elif _WIN32 std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); - std::string cfgPath = YAMLgetConf(conf,"cfg_win_input","..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg"); - std::string namePath = YAMLgetConf(conf,"name_win_input","..\\..\\..\\tests\\darknet\\names\\coco.names"); #endif if(!fileExist(input.c_str())) FatalError("The given input video does not exist."); @@ -90,12 +86,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - if(ntype == 'c' || ntype == 'm'){ - cfgPath = ""; - namePath = ""; - } - - detNN->init(net,cfgPath,namePath,n_classes,n_batch,conf_thresh); + detNN->init(net,n_classes,n_batch,conf_thresh); // open video stream cv::VideoCapture cap(input); diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 70bab07..308475f 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -45,8 +45,6 @@ int main(int argc, char *argv[]) bool verbose; int classes, map_points, map_levels; float map_step, IoU_thresh, conf_thresh; - std::string cfg_path = "../tests/darknet/cfg/yolo4tiny.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; double vm_total = 0, rss_total = 0; double vm, rss; @@ -56,17 +54,13 @@ int main(int argc, char *argv[]) if(argc > 2) ntype = argv[2][0]; if(argc > 3) - cfg_path = argv[3]; + labels_path = argv[3]; if(argc > 4) - name_path = argv[4]; + config_filename = argv[4]; if(argc > 5) - labels_path = argv[5]; + n_batches = atoi(argv[5]); if(argc > 6) - config_filename = argv[6]; - if(argc > 7) - n_batches = atoi(argv[7]); - if(argc > 8) - confidence_thresh = atof(argv[8]); + confidence_thresh = atof(argv[6]); std::cout<<"conf t: "<init(net,cfg_path,name_path,n_classes, 1, conf_thresh); + detNN->init(net,n_classes, 1, conf_thresh); //read images std::ifstream all_labels(labels_path); diff --git a/demo/demoConfig.yaml b/demo/demoConfig.yaml index 194d466..5785852 100644 --- a/demo/demoConfig.yaml +++ b/demo/demoConfig.yaml @@ -2,16 +2,8 @@ input : "../demo/yolo_test.mp4" win_input : "..\\..\\..\\demo\\yolo_test.mp4" -#cfg input -cfg_input : "../tests/darknet/cfg/yolo4tiny.cfg" -cfg_win_input : "..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg" - -#name input -name_input : "../tests/darknet/names/coco.names" -name_win_input : "..\\..\\..\\tests\\darknet\\names\\coco.names" - # network config -net : "yolo4tiny_fp32.rt" +net : "yolo4_berkeley_fp32.rt" ntype : 'y' n_classes : 80 n_batch : 1 diff --git a/docs/demo.md b/docs/demo.md index bf6b79d..5961acb 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -46,8 +46,6 @@ The config file is a yaml file with the following attributes: * ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed. * ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1) * ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1) -* ```cfg_input``` (for linux) \ ```cfg_win_input``` (for windows) is the location of the cfg path of the network for mobilenet and centernet networks use ```" "``` -* ```name_input``` (for linux) \ ```name_win_input``` (for windows) is the location of the name path of the network for mobilenet and centernet networks use ```" "``` N.B. By default it is used FP32 inference diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 07c80cd..3c8cfbb 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,7 +73,7 @@ public: CenternetDetection() {}; ~CenternetDetection() {}; - bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 3a757ef..a8c81f7 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -87,7 +87,7 @@ class DetectionNN { * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h index ec35b20..58512d5 100644 --- a/include/tkDNN/MobilenetDetection.h +++ b/include/tkDNN/MobilenetDetection.h @@ -65,7 +65,7 @@ public: MobilenetDetection() {}; ~MobilenetDetection() {}; - bool init(const std::string& tensor_path, const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches=1, const float conf_thresh=0.3); + bool init(const std::string& tensor_path,const int n_classes, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index a7e67c2..a422134 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -30,10 +30,6 @@ namespace tk { namespace dnn { -using namespace nvinfer1; - - - class NetworkRT { public: @@ -57,6 +53,7 @@ public: dnnType *output; cudaStream_t stream; + std::vector yolo_plugins; // yolo layers in network NetworkRT(Network *net, const char *name); virtual ~NetworkRT(); diff --git a/include/tkDNN/Yolo3Detection.h b/include/tkDNN/Yolo3Detection.h index 5a29d9c..b49915a 100644 --- a/include/tkDNN/Yolo3Detection.h +++ b/include/tkDNN/Yolo3Detection.h @@ -19,13 +19,12 @@ private: tk::dnn::Yolo* getYoloLayer(int n=0); cv::Mat bgr_h; - std::vector noYolos; public: Yolo3Detection() {}; ~Yolo3Detection() {}; - bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path,const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index 352a0cb..f0e5733 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -5,7 +5,6 @@ #include #include "../kernels.h" #include -#include #define YOLORT_CLASSNAME_W 256 @@ -80,8 +79,10 @@ namespace nvinfer1 { float nms_thresh; int nms_kind; int new_coords; - int NUM = 0; + std::vector classesNames; + std::vector mask; + std::vector bias; int entry_index(int batch, int location, int entry) { diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index eba8ce4..c18e08c 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -3,7 +3,7 @@ namespace tk { namespace dnn { -bool CenternetDetection::init(const std::string& tensor_path, const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches, const float conf_thresh){ +bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index 94c90c7..13b836f 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -126,7 +126,7 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){ return iou; } -bool MobilenetDetection::init(const std::string& tensor_path, const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches, const float conf_thresh){ +bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); imageSize = netRT->input_dim.h; diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index c7427d1..26489bf 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -15,6 +15,9 @@ using namespace nvinfer1; +extern std::mutex gYoloPlugins_mutex; +extern std::vector gYoloPlugins; + // Logger for info/warning/errors class Logger : public ILogger { void log(Severity severity, const char* msg) NOEXCEPT override { @@ -826,6 +829,12 @@ IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Yolo *l) { mPluginAttributes.emplace_back(PluginField("nms_thresh",&l->nms_thresh,PluginFieldType::kFLOAT32,1)); mPluginAttributes.emplace_back(PluginField("nms_kins",&l->nsm_kind,PluginFieldType::kINT32,1)); mPluginAttributes.emplace_back(PluginField("new_coords",&l->new_coords,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("mask",l->mask_h,PluginFieldType::kFLOAT32,l->n_masks)); + mPluginAttributes.emplace_back(PluginField("bias",l->bias_h,PluginFieldType::kFLOAT32,l->n_masks*2*l->num)); + for(int i=0; iclasses; i++) { + mPluginAttributes.emplace_back(PluginField("class_name",l->classesNames[i].data(),PluginFieldType::kCHAR,l->classesNames[i].size())); + } + mFC.nbFields = mPluginAttributes.size(); mFC.fields = mPluginAttributes.data(); auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC); @@ -1001,7 +1010,14 @@ bool NetworkRT::deserialize(const char *filename) { } runtimeRT = createInferRuntime(loggerRT); + + gYoloPlugins_mutex.lock(); + gYoloPlugins.clear(); engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size); + yolo_plugins = gYoloPlugins; + gYoloPlugins.clear(); + gYoloPlugins_mutex.unlock(); + std::cout<input_dim; idim.n = nBatches; - std::vector yolosLine = noYolosLine(cfg_path); - noYolos = yolosLine; - int channels,height,width; - loadYoloInitInfo(channels,width,height,cfg_path); - - - - if(yolosLine.size() < 2 ) { + if(netRT->yolo_plugins.size() < 2 ) { FatalError("this is not yolo3"); } + for(int i=0; iyolo_plugins.size(); i++) { + nvinfer1::YoloRT *yRT = netRT->yolo_plugins[i]; + classes = yRT->classes; + num = yRT->num; + nMasks = yRT->n_masks; - for(int i=0; i maskTemp,anchorsTemp; - std::vector classNamesTemp; - int nms_kind,coords,numTemp; - float nmsthresh; - loadYoloInfo(cfg_path,yolosLine[i],maskTemp,anchorsTemp,numTemp,classes,nmsthresh,nms_kind,coords); - classNamesTemp = darknetReadNames(name_path); - num = numTemp/maskTemp.size(); - nMasks = maskTemp.size(); - dnnType* maskTempF; - dnnType* biasTempF; - maskTempF = maskTemp.data(); - biasTempF = anchorsTemp.data(); // make a yolo layer to interpret predictions yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias yolo[i]->mask_h = new dnnType[nMasks]; yolo[i]->bias_h = new dnnType[num*nMasks*2]; - memcpy(yolo[i]->mask_h, maskTempF, sizeof(dnnType)*nMasks); - memcpy(yolo[i]->bias_h, biasTempF, sizeof(dnnType)*num*nMasks*2); - auto dim = netRT->engineRT->getBindingDimensions(i+1); - yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, dim.d[0], dim.d[1], dim.d[2]); - yolo[i]->classesNames = classNamesTemp; - yolo[i]->nms_thresh = nmsthresh; - yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) nms_kind; - yolo[i]->new_coords = coords; + memcpy(yolo[i]->mask_h, yRT->mask.data(), sizeof(dnnType)*nMasks); + memcpy(yolo[i]->bias_h, yRT->bias.data(), sizeof(dnnType)*num*nMasks*2); + yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w); + yolo[i]->classesNames = yRT->classesNames; + yolo[i]->nms_thresh = yRT->nms_thresh; + yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) yRT->nms_kind; + yolo[i]->new_coords = yRT->new_coords; } dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); @@ -112,12 +96,12 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ //get yolo outputs - if(noYolos.size() < 2){ + if(netRT->yolo_plugins.size() < 2){ FatalError("YOLOS WRONG!!"); } std::vector rt_out; //dnnType *rt_out[netRT->pluginFactory->n_yolos]; - for(int i=0; iyolo_plugins.size(); i++) rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi); float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w); @@ -125,7 +109,7 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ // compute dets nDets = 0; - for(int i=0; iyolo_plugins.size(); i++) { yolo[i]->dstData = rt_out[i]; yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords); } diff --git a/src/pluginsRT/YoloRT.cpp b/src/pluginsRT/YoloRT.cpp index 55cb462..590acb3 100644 --- a/src/pluginsRT/YoloRT.cpp +++ b/src/pluginsRT/YoloRT.cpp @@ -1,8 +1,13 @@ #include #include +#include using namespace nvinfer1; +// used to retrive Yolo plugin during network deserialization +std::mutex gYoloPlugins_mutex; +std::vector gYoloPlugins; + std::vector YoloRTPluginCreator::mPluginAttributes; PluginFieldCollection YoloRTPluginCreator::mFC{}; @@ -22,6 +27,10 @@ YoloRT::YoloRT(int classes, int num, int c,int h,int w,int n_masks, float scale_ this->nms_thresh = nms_thresh; this->nms_kind = nms_kind; this->new_coords = new_coords; + + bias.clear(); + mask.clear(); + classesNames.clear(); } YoloRT::YoloRT(const void *data, size_t length) { @@ -36,7 +45,24 @@ YoloRT::YoloRT(const void *data, size_t length) { c = readBUF(buf); h = readBUF(buf); w = readBUF(buf); + + mask.resize(n_masks); + for(int i=0; i(buf); + bias.resize(n_masks*2*num); + for(int i=0; i(buf); + + // save classes names + classesNames.resize(classes); + for(int i=0; i(buf); + classesNames[i] = std::string(tmp); + } assert(buf == bufCheck + length); + gYoloPlugins.push_back(this); } YoloRT::~YoloRT() {} @@ -126,7 +152,7 @@ int32_t YoloRT::enqueue(int32_t batchSize, const void *const *inputs, void **out size_t YoloRT::getSerializationSize() const NOEXCEPT { - return 8 * sizeof(int) + 2 * sizeof(float) ; + return 8 * sizeof(int) + 2 * sizeof(float) + n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char); } bool YoloRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT { @@ -145,6 +171,19 @@ void YoloRT::serialize(void *buffer) const NOEXCEPT { writeBUF(buf, c); //std::cout << "C : " << c << std::endl; writeBUF(buf, h); //std::cout << "H : " << h << std::endl; writeBUF(buf, w); //std::cout << "C : " << c << std::endl; + for (int i = 0; i < n_masks; i++) + writeBUF(buf, mask[i]); //std::cout << "mask[i] : " << mask[i] << std::endl; + for (int i = 0; i < n_masks * 2 * num; i++) + writeBUF(buf, bias[i]); //std::cout << "bias[i] : " << bias[i] << std::endl; + + // save classes names + for(int i=0; imask = mask; + p->bias = bias; + p->classesNames = classesNames; p->setPluginNamespace(mPluginNamespace.c_str()); return p; } @@ -235,6 +277,17 @@ IPluginV2Ext *YoloRTPluginCreator::createPlugin(const char *name, const PluginFi int nms_kind = *(static_cast(fields[8].data)); int new_coords = *(static_cast(fields[9].data)); auto *pluginObj = new YoloRT(classes,num,c,h,w,n_masks,scaleXY,nmsThresh,nms_kind,new_coords); + + // fill additional data + pluginObj->mask.resize(fields[10].length*sizeof(float)); + memcpy(pluginObj->mask.data(), fields[10].data, fields[10].length*sizeof(float)); + pluginObj->bias.resize(fields[11].length*sizeof(float)); + memcpy(pluginObj->bias.data(), fields[11].data, fields[11].length*sizeof(float)); + pluginObj->classesNames.resize(classes); + for(int i=0; iclassesNames[i].resize(fields[12+i].length); + memcpy(&pluginObj->classesNames[i][0], fields[12+i].data, fields[12+i].length*sizeof(char)); + } return pluginObj; } From d4f7b4ad8b21f1af78e1bada3e0368c0c1304ad9 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Wed, 30 Mar 2022 22:14:06 +0200 Subject: [PATCH 15/15] remove using namespace useless --- include/tkDNN/pluginsRT/ReshapeRT.h | 1 - include/tkDNN/utils.h | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/tkDNN/pluginsRT/ReshapeRT.h b/include/tkDNN/pluginsRT/ReshapeRT.h index e56c79c..65d577d 100644 --- a/include/tkDNN/pluginsRT/ReshapeRT.h +++ b/include/tkDNN/pluginsRT/ReshapeRT.h @@ -5,7 +5,6 @@ #include #include #include -using namespace tk::dnn; namespace nvinfer1 { diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index 055ea67..607697a 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -27,10 +27,12 @@ -#if NV_TENSORRT_MAJOR > 7 -#define NOEXCEPT noexcept -#else -#define NOEXCEPT +#ifndef NOEXCEPT + #if NV_TENSORRT_MAJOR > 7 + #define NOEXCEPT noexcept + #else + #define NOEXCEPT + #endif #endif