From 20303ac32ef8e28ca0ff94be633478d7755ae307 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Tue, 9 Jun 2020 20:53:05 +0200 Subject: [PATCH 1/6] cudnn8 compile --- src/LSTM.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 6ecf4fd..511fbee 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -86,7 +86,11 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig // RNN descriptors checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc)); - checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, +#if CUDNN_MAJOR > 7 + checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle, +#else + checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, +#endif rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), From be9e327aef7fb9c251ab31bca5f7a923b31395e2 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 9 Jun 2020 21:01:19 +0200 Subject: [PATCH 2/6] Fix minor, update readme Signed-off-by: Micaela Verucchi --- .gitignore | 2 +- README.md | 9 +++++++++ demo/demo/map.cpp | 4 ++-- scripts/install_OpenCV4.sh | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d62c96d..b56526f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ build/ *.pk *.table demo/COCO_val2017 -demo/BDD100k_val \ No newline at end of file +demo/BDD100K_val \ No newline at end of file diff --git a/README.md b/README.md index fdfc877..1175465 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,18 @@ tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU. The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. + +If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . + +``` Accepted paper @ IRC 2020, will soon been published. M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020) +Accepted paper @ ETFA 2020, will soon been published. +M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri +"A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) +``` + ## Index - [tkDNN](#tkdnn) - [Index](#index) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 60aa7fa..d724db0 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -153,7 +153,7 @@ int main(int argc, char *argv[]) std::ofstream myfile; if(write_dets) - myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("000"))); + myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("labels/") + 7)); // save detections labels for(auto d:detected_bbox){ @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) f.det.push_back(b); if(write_dets) - myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n"; if(show)// draw rectangle for detection cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); diff --git a/scripts/install_OpenCV4.sh b/scripts/install_OpenCV4.sh index 8862cdc..f57c87b 100644 --- a/scripts/install_OpenCV4.sh +++ b/scripts/install_OpenCV4.sh @@ -62,5 +62,5 @@ make -j4 sudo make install sudo ldconfig -cd '~/Downloads/opencv4/lib/python3.6/site-packages' +cd ~/Downloads/opencv4/lib/python3.6/site-packages ln -s /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-aarch64-linux-gnu.so cv2.so From ab6d2d1766ea6761e79715a4c3c2eb60a19dc399 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 12 Jun 2020 11:39:46 +0200 Subject: [PATCH 3/6] Update README Signed-off-by: Micaela Verucchi --- README.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1175465..19b98a2 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,40 @@ The main goal of this project is to exploit NVIDIA boards as much as possible to If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . ``` -Accepted paper @ IRC 2020, will soon been published. +Accepted paper @ IRC 2020, will soon be published. M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020) -Accepted paper @ ETFA 2020, will soon been published. +Accepted paper @ ETFA 2020, will soon be published. M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri "A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) ``` +## Results +Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on + * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); + * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); + * Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 ); + * Jetson Nano, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ). + +| Platform | Network | FP32, B=1 | FP32, B=4 | FP16, B=1 | FP16, B=4 | INT8, B=1 | INT8, B=4 | +| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | +| RTX 2080Ti | yolo4 320 | 118,59 |237,31 | 207,81 | 443,32 | 262,37 | 530,93 | +| RTX 2080Ti | yolo4 416 | 104,81 |162,86 | 169,06 | 293,78 | 206,93 | 353,26 | +| RTX 2080Ti | yolo4 512 | 92,98 |132,43 | 140,36 | 215,17 | 165,35 | 254,96 | +| RTX 2080Ti | yolo4 608 | 63,77 |81,53 | 111,39 | 152,89 | 127,79 | 184,72 | +| AGX Xavier | yolo4 320 | 26,78 |32,05 | 57,14 | 79,05 | 73,15 | 97,56 | +| AGX Xavier | yolo4 416 | 19,96 |21,52 | 41,01 | 49,00 | 50,81 | 60,61 | +| AGX Xavier | yolo4 512 | 16,58 |16,98 | 31,12 | 33,84 | 37,82 | 41,28 | +| AGX Xavier | yolo4 608 | 9,45 |10,13 | 21,92 | 23,36 | 27,05 | 28,93 | +| Tx2 | yolo4 320 | 11,18 | 12,07 | 15,32 | 16,31 | - | - | +| Tx2 | yolo4 416 | 7,30 | 7,58 | 9,45 | 9,90 | - | - | +| Tx2 | yolo4 512 | 5,96 | 5,95 | 7,22 | 7,23 | - | - | +| Tx2 | yolo4 608 | 3,63 | 3,65 | 4,67 | 4,70 | - | - | +| Nano | yolo4 320 | 4,23 | 4,55 | 6,14 | 6,53 | - | - | +| Nano | yolo4 416 | 2,88 | 3,00 | 3,90 | 4,04 | - | - | +| Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - | +| Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - | + ## Index - [tkDNN](#tkdnn) - [Index](#index) From 567dc0f75d5951eb677f95a14e8d3d9216d90634 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sun, 14 Jun 2020 12:57:29 +0200 Subject: [PATCH 4/6] cmake cudnn fix --- CMakeLists.txt | 2 + cmake/FindCUDNN.cmake | 91 +++++++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 376c175..4a64372 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ SET(CUDA_SEPARABLE_COMPILATION ON) set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) find_package(CUDNN REQUIRED) +include_directories(${CUDNN_INCLUDE_DIR}) + # compile file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu") diff --git a/cmake/FindCUDNN.cmake b/cmake/FindCUDNN.cmake index f240fcb..583b4a6 100644 --- a/cmake/FindCUDNN.cmake +++ b/cmake/FindCUDNN.cmake @@ -1,33 +1,66 @@ -# Find the header files +# find the library +if(CUDA_FOUND) + find_cuda_helper_libs(cudnn) + set(CUDNN_LIBRARY ${CUDA_cudnn_LIBRARY} CACHE FILEPATH "location of the cuDNN library") + unset(CUDA_cudnn_LIBRARY CACHE) -find_path(CUDNN_INCLUDE_DIR - ${CMAKE_SYSROOT}/usr/local/include - ${CMAKE_SYSROOT}/usr/include - /usr/local/nvidia/tensorrt/include/ - NO_DEFAULT_PATH -) + find_cuda_helper_libs(nvinfer) + set(NVINFER_LIBRARY ${CUDA_nvinfer_LIBRARY} CACHE FILEPATH "location of the nvinfer library") + unset(CUDA_nvinfer_LIBRARY CACHE) +endif() -set(OLD_ROOT ${CMAKE_FIND_ROOT_PATH}) -list(APPEND CMAKE_FIND_ROOT_PATH /) -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.7) -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.5) -find_library(CUDNN_LIB - NAMES cudnn - PATHS - /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib - /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/ +# find the include +if(CUDNN_LIBRARY) + find_path(CUDNN_INCLUDE_DIR + cudnn.h + PATHS ${CUDA_TOOLKIT_INCLUDE} + DOC "location of cudnn.h" NO_DEFAULT_PATH -) -find_library(CUDNN_NVLIB - NAMES "nvinfer" - PATHS - /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib - /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/ - NO_DEFAULT_PATH -) -set(CMAKE_FIND_ROOT_PATH ${OLD_ROOT}) + ) -set(CUDNN_LIBRARIES ${CUDNN_LIB} ${CUDNN_NVLIB}) -message("-- Found CUDNN: " ${CUDNN_LIB}) -message("-- Found NVINFER: " ${CUDNN_NVLIB}) -set(CUDNN_FOUND true) + if(NOT CUDNN_INCLUDE_DIR) + find_path(CUDNN_INCLUDE_DIR + cudnn.h + DOC "location of cudnn.h" + ) + endif() + + message("-- Found CUDNN: " ${CUDNN_LIBRARY}) + message("-- Found CUDNN include: " ${CUDNN_INCLUDE_DIR}) +endif() + +if(NVINFER_LIBRARY) + find_path(NVINFER_INCLUDE_DIR + NvInfer.h + PATHS ${CUDA_TOOLKIT_INCLUDE} + DOC "location of NvInfer.h" + NO_DEFAULT_PATH + ) + + if(NOT NVINFER_INCLUDE_DIR) + find_path(NVINFER_INCLUDE_DIR + NvInfer.h + DOC "location of NvInfer.h" + ) + endif() + + message("-- Found NVINFER: " ${NVINFER_LIBRARY}) + message("-- Found NVINFER include: " ${NVINFER_INCLUDE_DIR}) +endif() + + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CUDNN + FOUND_VAR CUDNN_FOUND + REQUIRED_VARS + CUDNN_LIBRARY + CUDNN_INCLUDE_DIR + VERSION_VAR CUDNN_VERSION +) + +if(CUDNN_FOUND) + set(CUDNN_LIBRARIES ${CUDNN_LIBRARY} ${NVINFER_LIBRARY}) + set(CUDNN_INCLUDE_DIRS ${CUDNN_INCLUDE_DIR} ${NVINFER_INCLUDE_DIR}) +endif() + +set(CUDNN_FOUND true) \ No newline at end of file From cbfc8ea4f2d8d52763b36e2c8ad2db4d4aa9d621 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sun, 14 Jun 2020 13:01:45 +0200 Subject: [PATCH 5/6] serialize fix --- src/NetworkRT.cpp | 2 +- tests/darknet/yolo3.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 9f53b06..6e86de1 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -595,7 +595,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) { bool NetworkRT::serialize(const char *filename) { - std::ofstream p(filename); + std::ofstream p(filename, std::ios::binary); if (!p) { FatalError("could not open plan output file"); return false; diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index ea53b84..464c3cf 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -5,19 +5,19 @@ #include "DarknetParser.h" int main() { - std::string bin_path = "yolo3"; + std::string bin_path = "yolov3-sppx"; std::vector input_bins = { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "/debug/layer82_out.bin", - bin_path + "/debug/layer94_out.bin", - bin_path + "/debug/layer106_out.bin" + bin_path + "/debug/layer89_out.bin", + bin_path + "/debug/layer101_out.bin", + bin_path + "/debug/layer113_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolo3.cfg"; - std::string name_path = "../tests/darknet/names/coco.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + std::string cfg_path = "../tests/darknet/cfg/yolov3-sppx.cfg"; + std::string name_path = "../tests/darknet/names/coco4.names"; + //downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); From 6dff675db7a9f58798535dfce8029b2dddac174b Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sun, 14 Jun 2020 13:03:48 +0200 Subject: [PATCH 6/6] fix wrong commit --- tests/darknet/yolo3.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index 464c3cf..e73d226 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -5,19 +5,19 @@ #include "DarknetParser.h" int main() { - std::string bin_path = "yolov3-sppx"; + std::string bin_path = "yolo3"; std::vector input_bins = { bin_path + "/layers/input.bin" }; std::vector output_bins = { - bin_path + "/debug/layer89_out.bin", - bin_path + "/debug/layer101_out.bin", - bin_path + "/debug/layer113_out.bin" + bin_path + "/debug/layer82_out.bin", + bin_path + "/debug/layer94_out.bin", + bin_path + "/debug/layer106_out.bin" }; std::string wgs_path = bin_path + "/layers"; - std::string cfg_path = "../tests/darknet/cfg/yolov3-sppx.cfg"; - std::string name_path = "../tests/darknet/names/coco4.names"; - //downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); + std::string cfg_path = "../tests/darknet/cfg/yolo3.cfg"; + std::string name_path = "../tests/darknet/names/coco.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); @@ -31,4 +31,4 @@ int main() { delete net; delete netRT; return ret; -} +} \ No newline at end of file