diff --git a/.gitignore b/.gitignore index d62c96d..b56526f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ build/ *.pk *.table demo/COCO_val2017 -demo/BDD100k_val \ No newline at end of file +demo/BDD100K_val \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 376c175..4a64372 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ SET(CUDA_SEPARABLE_COMPILATION ON) set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) find_package(CUDNN REQUIRED) +include_directories(${CUDNN_INCLUDE_DIR}) + # compile file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu") diff --git a/README.md b/README.md index fdfc877..19b98a2 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,44 @@ tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU. The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. -Accepted paper @ IRC 2020, will soon been published. + +If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . + +``` +Accepted paper @ IRC 2020, will soon be published. M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020) +Accepted paper @ ETFA 2020, will soon be published. +M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri +"A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) +``` + +## Results +Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on + * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); + * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); + * Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 ); + * Jetson Nano, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ). + +| Platform | Network | FP32, B=1 | FP32, B=4 | FP16, B=1 | FP16, B=4 | INT8, B=1 | INT8, B=4 | +| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | +| RTX 2080Ti | yolo4 320 | 118,59 |237,31 | 207,81 | 443,32 | 262,37 | 530,93 | +| RTX 2080Ti | yolo4 416 | 104,81 |162,86 | 169,06 | 293,78 | 206,93 | 353,26 | +| RTX 2080Ti | yolo4 512 | 92,98 |132,43 | 140,36 | 215,17 | 165,35 | 254,96 | +| RTX 2080Ti | yolo4 608 | 63,77 |81,53 | 111,39 | 152,89 | 127,79 | 184,72 | +| AGX Xavier | yolo4 320 | 26,78 |32,05 | 57,14 | 79,05 | 73,15 | 97,56 | +| AGX Xavier | yolo4 416 | 19,96 |21,52 | 41,01 | 49,00 | 50,81 | 60,61 | +| AGX Xavier | yolo4 512 | 16,58 |16,98 | 31,12 | 33,84 | 37,82 | 41,28 | +| AGX Xavier | yolo4 608 | 9,45 |10,13 | 21,92 | 23,36 | 27,05 | 28,93 | +| Tx2 | yolo4 320 | 11,18 | 12,07 | 15,32 | 16,31 | - | - | +| Tx2 | yolo4 416 | 7,30 | 7,58 | 9,45 | 9,90 | - | - | +| Tx2 | yolo4 512 | 5,96 | 5,95 | 7,22 | 7,23 | - | - | +| Tx2 | yolo4 608 | 3,63 | 3,65 | 4,67 | 4,70 | - | - | +| Nano | yolo4 320 | 4,23 | 4,55 | 6,14 | 6,53 | - | - | +| Nano | yolo4 416 | 2,88 | 3,00 | 3,90 | 4,04 | - | - | +| Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - | +| Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - | + ## Index - [tkDNN](#tkdnn) - [Index](#index) diff --git a/cmake/FindCUDNN.cmake b/cmake/FindCUDNN.cmake index f240fcb..583b4a6 100644 --- a/cmake/FindCUDNN.cmake +++ b/cmake/FindCUDNN.cmake @@ -1,33 +1,66 @@ -# Find the header files +# find the library +if(CUDA_FOUND) + find_cuda_helper_libs(cudnn) + set(CUDNN_LIBRARY ${CUDA_cudnn_LIBRARY} CACHE FILEPATH "location of the cuDNN library") + unset(CUDA_cudnn_LIBRARY CACHE) -find_path(CUDNN_INCLUDE_DIR - ${CMAKE_SYSROOT}/usr/local/include - ${CMAKE_SYSROOT}/usr/include - /usr/local/nvidia/tensorrt/include/ - NO_DEFAULT_PATH -) + find_cuda_helper_libs(nvinfer) + set(NVINFER_LIBRARY ${CUDA_nvinfer_LIBRARY} CACHE FILEPATH "location of the nvinfer library") + unset(CUDA_nvinfer_LIBRARY CACHE) +endif() -set(OLD_ROOT ${CMAKE_FIND_ROOT_PATH}) -list(APPEND CMAKE_FIND_ROOT_PATH /) -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.7) -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.5) -find_library(CUDNN_LIB - NAMES cudnn - PATHS - /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib - /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/ +# find the include +if(CUDNN_LIBRARY) + find_path(CUDNN_INCLUDE_DIR + cudnn.h + PATHS ${CUDA_TOOLKIT_INCLUDE} + DOC "location of cudnn.h" NO_DEFAULT_PATH -) -find_library(CUDNN_NVLIB - NAMES "nvinfer" - PATHS - /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib - /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/ - NO_DEFAULT_PATH -) -set(CMAKE_FIND_ROOT_PATH ${OLD_ROOT}) + ) -set(CUDNN_LIBRARIES ${CUDNN_LIB} ${CUDNN_NVLIB}) -message("-- Found CUDNN: " ${CUDNN_LIB}) -message("-- Found NVINFER: " ${CUDNN_NVLIB}) -set(CUDNN_FOUND true) + if(NOT CUDNN_INCLUDE_DIR) + find_path(CUDNN_INCLUDE_DIR + cudnn.h + DOC "location of cudnn.h" + ) + endif() + + message("-- Found CUDNN: " ${CUDNN_LIBRARY}) + message("-- Found CUDNN include: " ${CUDNN_INCLUDE_DIR}) +endif() + +if(NVINFER_LIBRARY) + find_path(NVINFER_INCLUDE_DIR + NvInfer.h + PATHS ${CUDA_TOOLKIT_INCLUDE} + DOC "location of NvInfer.h" + NO_DEFAULT_PATH + ) + + if(NOT NVINFER_INCLUDE_DIR) + find_path(NVINFER_INCLUDE_DIR + NvInfer.h + DOC "location of NvInfer.h" + ) + endif() + + message("-- Found NVINFER: " ${NVINFER_LIBRARY}) + message("-- Found NVINFER include: " ${NVINFER_INCLUDE_DIR}) +endif() + + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CUDNN + FOUND_VAR CUDNN_FOUND + REQUIRED_VARS + CUDNN_LIBRARY + CUDNN_INCLUDE_DIR + VERSION_VAR CUDNN_VERSION +) + +if(CUDNN_FOUND) + set(CUDNN_LIBRARIES ${CUDNN_LIBRARY} ${NVINFER_LIBRARY}) + set(CUDNN_INCLUDE_DIRS ${CUDNN_INCLUDE_DIR} ${NVINFER_INCLUDE_DIR}) +endif() + +set(CUDNN_FOUND true) \ No newline at end of file diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 60aa7fa..d724db0 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -153,7 +153,7 @@ int main(int argc, char *argv[]) std::ofstream myfile; if(write_dets) - myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("000"))); + myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("labels/") + 7)); // save detections labels for(auto d:detected_bbox){ @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) f.det.push_back(b); if(write_dets) - myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n"; if(show)// draw rectangle for detection cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); diff --git a/scripts/install_OpenCV4.sh b/scripts/install_OpenCV4.sh index 8862cdc..f57c87b 100644 --- a/scripts/install_OpenCV4.sh +++ b/scripts/install_OpenCV4.sh @@ -62,5 +62,5 @@ make -j4 sudo make install sudo ldconfig -cd '~/Downloads/opencv4/lib/python3.6/site-packages' +cd ~/Downloads/opencv4/lib/python3.6/site-packages ln -s /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-aarch64-linux-gnu.so cv2.so diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 6ecf4fd..511fbee 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -86,7 +86,11 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig // RNN descriptors checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc)); - checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, +#if CUDNN_MAJOR > 7 + checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle, +#else + checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, +#endif rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 9f53b06..6e86de1 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -595,7 +595,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) { bool NetworkRT::serialize(const char *filename) { - std::ofstream p(filename); + std::ofstream p(filename, std::ios::binary); if (!p) { FatalError("could not open plan output file"); return false; diff --git a/tests/darknet/yolo3.cpp b/tests/darknet/yolo3.cpp index ea53b84..e73d226 100644 --- a/tests/darknet/yolo3.cpp +++ b/tests/darknet/yolo3.cpp @@ -31,4 +31,4 @@ int main() { delete net; delete netRT; return ret; -} +} \ No newline at end of file