From b12cf0d7c2599f36e7e564ba19868c6885ecca6d Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 13 Jul 2020 19:50:00 +0200 Subject: [PATCH 01/11] docker --- docker/Dockerfile | 7 ++++++ docker/Dockerfile.base | 57 ++++++++++++++++++++++++++++++++++++++++++ docker/README.md | 21 ++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.base create mode 100644 docker/README.md diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..3c9fb61 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,7 @@ +FROM ceccocats/tkdnn:latest +LABEL maintainer "Francesco Gatti" + +RUN cd && git clone https://github.com/ceccocats/tkDNN.git && cd tkDNN && mkdir build && cd build \ + && cmake .. && make -j12 + + diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 0000000..e61b0d3 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,57 @@ +FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04 +LABEL maintainer "Francesco Gatti" + +ADD nv-tensorrt-repo-ubuntu1804-cuda10.2-trt7.0.0.11-ga-20191216_1-1_amd64.deb /tmp/trt.deb +RUN apt-get update && dpkg -i /tmp/trt.deb && rm /tmp/trt.deb && apt-get update +RUN apt install -y libnvinfer7=7.0.0-1+cuda10.2 libnvinfer-dev=7.0.0-1+cuda10.2 +RUN DEBIAN_FRONTEND=noninteractive apt install -y git wget libeigen3-dev libyaml-cpp-dev +RUN cd /tmp && \ + wget https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh && \ + chmod +x cmake-3.17.3-Linux-x86_64.sh && \ + ./cmake-3.17.3-Linux-x86_64.sh --prefix=/usr/local --exclude-subdir --skip-license && \ + rm ./cmake-3.17.3-Linux-x86_64.sh + +RUN echo "INSTALL OPENCV" +RUN apt-get install -y build-essential \ + unzip \ + pkg-config \ + libjpeg-dev \ + libpng-dev \ + libtiff-dev \ + libavcodec-dev \ + libavformat-dev \ + libswscale-dev \ + libv4l-dev \ + libxvidcore-dev \ + libx264-dev \ + libgtk-3-dev \ + libatlas-base-dev \ + gfortran \ + libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev \ + libdc1394-22-dev \ + libavresample-dev +RUN cd && wget https://github.com/opencv/opencv/archive/4.3.0.tar.gz && tar -xf 4.3.0.tar.gz && rm *.tar.gz +RUN cd && wget https://github.com/opencv/opencv_contrib/archive/4.3.0.tar.gz && tar -xf 4.3.0.tar.gz && rm *.tar.gz +RUN cd && \ + cd opencv-4.3.0 && mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D INSTALL_PYTHON_EXAMPLES=OFF \ + -D INSTALL_C_EXAMPLES=OFF \ + -D OPENCV_EXTRA_MODULES_PATH='~/opencv_contrib-4.3.0/modules' \ + -D BUILD_EXAMPLES=OFF \ + -D WITH_CUDA=ON \ + -D CUDA_ARCH_BIN=7.2 \ + -D CUDA_ARCH_PTX="" \ + -D ENABLE_FAST_MATH=ON \ + -D CUDA_FAST_MATH=ON \ + -D WITH_CUBLAS=ON \ + -D WITH_LIBV4L=ON \ + -D WITH_GSTREAMER=ON \ + -D WITH_GSTREAMER_0_10=OFF \ + -D WITH_TBB=ON \ + ../ && make -j12 && make install +RUN apt clean + + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..a3edf3c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,21 @@ +# Use the prebuilt image +``` +# build image +docker build -t tkdnn:build -f Dockerfile-f Dockerfile . +``` + +# Build Base Docker image +``` +# make nvidia docker working +# follow this guide: https://github.com/NVIDIA/nvidia-docker + +# dowload tensorrt +# from: https://developer.nvidia.com/compute/machine-learning/tensorrt/secure/7.0/7.0.0.11/local_repo/nv-tensorrt-repo-ubuntu1804-cuda10.2-trt7.0.0.11-ga-20191216_1-1_amd64.deb + +# build image +docker build -t ceccocats/tkdnn:latest -f Dockerfile.base . + +# run image +docker run -ti --gpus all --rm ceccocats/tkdnn:latest bash +``` + From b2df9fc1107ca63301df42a0fbd0483189eda7d9 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 13 Jul 2020 19:51:09 +0200 Subject: [PATCH 02/11] Update README.md --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index a3edf3c..aec202a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,7 +1,7 @@ # Use the prebuilt image ``` # build image -docker build -t tkdnn:build -f Dockerfile-f Dockerfile . +docker build -t tkdnn:build -f Dockerfile . ``` # Build Base Docker image From c4aad7fe95e0f8ed45fd175d565d1926d2880f6c Mon Sep 17 00:00:00 2001 From: tk Date: Thu, 16 Jul 2020 18:16:09 +0200 Subject: [PATCH 03/11] Patch for CUDNN 8.0.1 Signed-off-by: tk --- include/tkDNN/Layer.h | 4 ++-- src/Conv2d.cpp | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index bd544b2..f2ec56d 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -273,8 +273,8 @@ public: protected: cudnnFilterDescriptor_t filterDesc; cudnnConvolutionDescriptor_t convDesc; - cudnnConvolutionFwdAlgo_t algo; - cudnnConvolutionBwdDataAlgo_t bwAlgo; + cudnnConvolutionFwdAlgoPerf_t algo; + cudnnConvolutionBwdDataAlgoPerf_t bwAlgo; cudnnTensorDescriptor_t biasTensorDesc; void initCUDNN(bool back = false); diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index 4704c66..595fec7 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -63,23 +63,23 @@ void Conv2d::initCUDNN(bool back) { workSpace = NULL; ws_sizeInBytes = 0; if(back) { - checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm(net->cudnnHandle, - filterDesc, dstTensor, convDesc, srcTensor, - CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST, 0, &bwAlgo) ); + checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm_v7(net->cudnnHandle, + filterDesc, dstTensor, convDesc, srcTensor, 1, 0, &bwAlgo) ); checkCUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(net->cudnnHandle, - filterDesc, dstTensor, convDesc, srcTensor, - bwAlgo, &ws_sizeInBytes)); + filterDesc, dstTensor, convDesc, srcTensor, + bwAlgo.algo, &ws_sizeInBytes)); + // invert tensors srcTensorDesc = dstTensor; dstTensorDesc = srcTensor; } else { - checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle, - srcTensor, filterDesc, convDesc, dstTensor, - CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) ); - checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, - srcTensor, filterDesc, convDesc, dstTensor, - algo, &ws_sizeInBytes)); + checkCUDNN( cudnnGetConvolutionForwardAlgorithm_v7(net->cudnnHandle, + srcTensor, filterDesc, convDesc, dstTensor, + 1, 0, &algo) ); + checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, + srcTensor, filterDesc, convDesc, dstTensor, + algo.algo, &ws_sizeInBytes)); } } @@ -91,12 +91,12 @@ void Conv2d::inferCUDNN(dnnType* srcData, bool back) { checkCUDNN(cudnnConvolutionBackwardData(net->cudnnHandle, &alpha, filterDesc, data_d, srcTensorDesc, srcData, - convDesc, bwAlgo, workSpace, ws_sizeInBytes, + convDesc, bwAlgo.algo, workSpace, ws_sizeInBytes, &beta, dstTensorDesc, dstData)); } else { checkCUDNN(cudnnConvolutionForward(net->cudnnHandle, &alpha, srcTensorDesc, srcData, filterDesc, - data_d, convDesc, algo, workSpace, ws_sizeInBytes, + data_d, convDesc, algo.algo, workSpace, ws_sizeInBytes, &beta, dstTensorDesc, dstData)); } From 6a68f19b2ceb61542fe87533fb193f696d30c664 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 16 Jul 2020 18:37:37 +0200 Subject: [PATCH 04/11] Fix patch from @ahmedius2 , tkDNN now supports CUDNN 8.0.1 (Fix #74) Signed-off-by: Micaela Verucchi Francesco Gatti --- src/Conv2d.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index 595fec7..b57cf58 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -62,9 +62,10 @@ void Conv2d::initCUDNN(bool back) { // init workspace workSpace = NULL; ws_sizeInBytes = 0; + int algo_count = 0; if(back) { checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm_v7(net->cudnnHandle, - filterDesc, dstTensor, convDesc, srcTensor, 1, 0, &bwAlgo) ); + filterDesc, dstTensor, convDesc, srcTensor, 1, &algo_count, &bwAlgo) ); checkCUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(net->cudnnHandle, filterDesc, dstTensor, convDesc, srcTensor, bwAlgo.algo, &ws_sizeInBytes)); @@ -74,13 +75,17 @@ void Conv2d::initCUDNN(bool back) { srcTensorDesc = dstTensor; dstTensorDesc = srcTensor; } else { + checkCUDNN( cudnnGetConvolutionForwardAlgorithm_v7(net->cudnnHandle, srcTensor, filterDesc, convDesc, dstTensor, - 1, 0, &algo) ); + 1, &algo_count, &algo) ); checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle, srcTensor, filterDesc, convDesc, dstTensor, algo.algo, &ws_sizeInBytes)); } + + if(algo_count < 1) + FatalError("Cannot retrieve convolutional algo"); } void Conv2d::inferCUDNN(dnnType* srcData, bool back) { From f4970d1e6faab505c2caf1d6833cf7490a971a0e Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Fri, 17 Jul 2020 14:37:10 +0200 Subject: [PATCH 05/11] Update README.md --- README.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3ff6fe7..a1b5b16 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # tkDNN -tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU. +tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier, Nano and several discrete GPUs. The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. -If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it or refer to https://hipert.unimore.it/ . +If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ . ``` Accepted paper @ IRC 2020, will soon be published. @@ -175,15 +175,25 @@ All models from darknet are now parsed directly from cfg, you still need to expo mish -## Run the demo +## Run the demo +This is an example using yolov4. -To run the an object detection demo follow these steps (example with yolov3): +To run the an object detection first create the .rt file by running: ``` -rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files -./test_yolo3 # run the yolo test (is slow) -./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y +rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files +./test_yolo4 # run the yolo test (is slow) ``` -In general the demo program takes 4 parameters: +If you get problems in the creation, try to check the error activating the debug of TensorRT in this way: +``` +cmake .. -DDEBUG=True +make +``` + +Once you have succesfully created your rt file, run the demo: +``` +./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y +``` +In general the demo program takes 6 parameters: ``` ./demo ``` @@ -197,6 +207,7 @@ where N.b. By default it is used FP32 inference + ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) ### FP16 inference From 3a0802d70c7a6286ac1ca871efc2498550295aa5 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Mon, 27 Jul 2020 13:45:42 +0200 Subject: [PATCH 06/11] Resolve detection objects pick by prob threshold. Before this it will only pick the last object with prob > thresh wich is absolutely wrong Now it picks all the objects with prob > thesh. fixes #94 --- src/Yolo3Detection.cpp | 47 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index c76af20..27f393f 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -113,34 +113,35 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ int x1 = (b.x+b.w/2.); int y0 = (b.y-b.h/2.); int y1 = (b.y+b.h/2.); - int obj_class = -1; - float prob = 0; + for(int c=0; c= confThreshold) { - obj_class = c; - prob = dets[j].prob[c]; + int obj_class = c; + float prob = dets[j].prob[c]; + + // convert to image coords + x0 = x_ratio*x0; + x1 = x_ratio*x1; + y0 = y_ratio*y0; + y1 = y_ratio*y1; + + tk::dnn::box res; + res.cl = obj_class; + res.prob = prob; + res.x = x0; + res.y = y0; + res.w = x1 - x0; + res.h = y1 - y0; + + // FIXME: this shuld be useless + // if(mAP) + // for(int c=0; c= 0) { - // convert to image coords - x0 = x_ratio*x0; - x1 = x_ratio*x1; - y0 = y_ratio*y0; - y1 = y_ratio*y1; - - tk::dnn::box res; - res.cl = obj_class; - res.prob = prob; - res.x = x0; - res.y = y0; - res.w = x1 - x0; - res.h = y1 - y0; - if(mAP) - for(int c=0; c Date: Mon, 27 Jul 2020 13:52:39 +0200 Subject: [PATCH 07/11] fix coords convert --- src/Yolo3Detection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index 27f393f..606c6d1 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -114,17 +114,17 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ int y0 = (b.y-b.h/2.); int y1 = (b.y+b.h/2.); + // convert to image coords + x0 = x_ratio*x0; + x1 = x_ratio*x1; + y0 = y_ratio*y0; + y1 = y_ratio*y1; + for(int c=0; c= confThreshold) { int obj_class = c; float prob = dets[j].prob[c]; - // convert to image coords - x0 = x_ratio*x0; - x1 = x_ratio*x1; - y0 = y_ratio*y0; - y1 = y_ratio*y1; - tk::dnn::box res; res.cl = obj_class; res.prob = prob; From f778e1aa998f894654b24c0ab9ad759c0eb14019 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 5 Aug 2020 19:55:10 +0200 Subject: [PATCH 08/11] Fixed boxes to float, add conf thresh as param Signed-off-by: Micaela Verucchi --- README.md | 3 ++- demo/config.yaml | 2 +- demo/demo/demo.cpp | 5 ++++- demo/demo/map.cpp | 2 +- include/tkDNN/CenternetDetection.h | 2 +- include/tkDNN/DetectionNN.h | 2 +- include/tkDNN/MobilenetDetection.h | 2 +- include/tkDNN/Yolo3Detection.h | 2 +- src/CenternetDetection.cpp | 3 ++- src/MobilenetDetection.cpp | 3 ++- src/Yolo3Detection.cpp | 11 ++++++----- 11 files changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a1b5b16..d9b5755 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ Once you have succesfully created your rt file, run the demo: ``` ./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y ``` -In general the demo program takes 6 parameters: +In general the demo program takes 7 parameters: ``` ./demo ``` @@ -204,6 +204,7 @@ where * ``````is the number of classes the network is trained on * `````` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). * `````` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1) +* `````` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed. N.b. By default it is used FP32 inference diff --git a/demo/config.yaml b/demo/config.yaml index e6f91a7..31ac599 100644 --- a/demo/config.yaml +++ b/demo/config.yaml @@ -3,5 +3,5 @@ map_points : 101 #number of recall points (0 for all, 101 for COCO, 11 Pascal map_levels : 10 #number of IoU step for the AP map_step : 0.05 #step of IoU IoU_thresh : 0.5 #starting IoU threshold -conf_thresh : 0.0 #threshold on the condifence of the bbox +conf_thresh : 0.001 #threshold on the condifence of the bbox verbose : false #print on screen information diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 76b451d..9f50d0b 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -40,6 +40,9 @@ int main(int argc, char *argv[]) { bool show = true; if(argc > 6) show = atoi(argv[6]); + float conf_thresh=0.3; + if(argc > 7) + conf_thresh = atof(argv[7]); if(n_batch < 1 || n_batch > 64) FatalError("Batch dim not supported"); @@ -69,7 +72,7 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes, n_batch); + detNN->init(net, n_classes, n_batch, conf_thresh); gRun = true; diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index d724db0..356e35a 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) default: FatalError("Network type not allowed (3rd parameter)\n"); } - detNN->init(net, n_classes); + detNN->init(net, n_classes, 1, conf_thresh); //read images std::ifstream all_labels(labels_path); diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 227cb78..3c8cfbb 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,7 +73,7 @@ public: CenternetDetection() {}; ~CenternetDetection() {}; - bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 030cf8f..ba42834 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -84,7 +84,7 @@ class DetectionNN { * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h index cabd7eb..9a5fedc 100644 --- a/include/tkDNN/MobilenetDetection.h +++ b/include/tkDNN/MobilenetDetection.h @@ -65,7 +65,7 @@ public: MobilenetDetection() {}; ~MobilenetDetection() {}; - bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1); + bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/Yolo3Detection.h b/include/tkDNN/Yolo3Detection.h index 6d38514..100a720 100644 --- a/include/tkDNN/Yolo3Detection.h +++ b/include/tkDNN/Yolo3Detection.h @@ -24,7 +24,7 @@ public: Yolo3Detection() {}; ~Yolo3Detection() {}; - bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 9d8df38..394e24a 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -3,11 +3,12 @@ namespace tk { namespace dnn { -bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){ +bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; nBatches = n_batches; + confThreshold = conf_thresh; dim = netRT->input_dim; diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index c905fea..3c54e28 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -126,12 +126,13 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){ return iou; } -bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){ +bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); imageSize = netRT->input_dim.h; classes = n_classes; nBatches = n_batches; + confThreshold = conf_thresh; SSDSpec specs[N_SSDSPEC]; diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index 606c6d1..e9b0064 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -3,13 +3,14 @@ namespace tk { namespace dnn { -bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) { +bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { //convert network to tensorRT std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); nBatches = n_batches; + confThreshold = conf_thresh; tk::dnn::dataDim_t idim = netRT->input_dim; idim.n = nBatches; @@ -109,10 +110,10 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ detected.clear(); for(int j=0; j Date: Thu, 6 Aug 2020 11:05:26 +0200 Subject: [PATCH 09/11] Update README.md --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d9b5755..ea55785 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capoton "A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020) ``` -## Results +## FPS Results Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); @@ -40,6 +40,20 @@ Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesio | Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - | | Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - | +## MAP Results +Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 + +| | CodaLab | CodaLab | CodaLab | CodaLab | tkDNN map | tkDNN map | +| -------------------- | :-----------: | :-------: | :-----------: | :---------: | :-----------: | :-------: | +| | **tkDNN** | **tkDNN** | **darknet** | **darknet** | **tkDNN** | **tkDNN** | +| | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 | +| Yolov3 (416x416) | 0.381 | 0.675 | 0.380 | 0.675 | 0.372 | 0.663 | +| yolov4 (416x416) | 0.468 | 0.705 | 0.471 | 0.710 | 0.459 | 0.695 | +| yolov3tiny (416x416) | 0.096 | 0.202 | 0.096 | 0.201 | 0.093 | 0.198 | +| yolov4tiny (416x416) | 0.202 | 0.400 | 0.201 | 0.400 | 0.197 | 0.395 | +| Cnet-dla34 (512x512) | 0.366 | 0.543 | \- | \- | 0.361 | 0.535 | +| mv2SSD (512x512) | 0.226 | 0.381 | \- | \- | 0.223 | 0.378 | + ## Index - [tkDNN](#tkdnn) - [Index](#index) From df5443e017f9390b3f282a5c18ecf335ffafc5f8 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Thu, 6 Aug 2020 15:53:57 +0200 Subject: [PATCH 10/11] Fix boxes also for Centernet Signed-off-by: Micaela Verucchi --- src/CenternetDetection.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 394e24a..46757f4 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -372,10 +372,10 @@ void CenternetDetection::postprocess(const int bi, const bool mAP){ // std::cout<<"th: "< Date: Fri, 11 Sep 2020 09:13:59 +0200 Subject: [PATCH 11/11] Fix typos (#107) Signed-off-by: micaela --- README.md | 14 +++++++------- include/tkDNN/DetectionNN.h | 8 ++++---- include/tkDNN/ImuOdom.h | 4 ++-- include/tkDNN/Layer.h | 20 ++++++++++---------- include/tkDNN/Network.h | 8 ++++---- include/tkDNN/NetworkRT.h | 2 +- include/tkDNN/evaluation.h | 8 ++++---- include/tkDNN/pluginsRT/DeformableConvRT.h | 2 +- include/tkDNN/test.h | 2 +- src/DarknetParser.cpp | 2 +- src/DeformConv2d.cpp | 2 +- src/Dense.cpp | 2 +- src/LSTM.cpp | 10 +++++----- src/LayerWgs.cpp | 2 +- src/MulAdd.cpp | 2 +- src/NetworkRT.cpp | 2 +- src/Region.cpp | 2 +- src/Shortcut.cpp | 2 +- src/evaluation.cpp | 6 +++--- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index ea55785..f17f75e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capoton ``` ## FPS Results -Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on +Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); * Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 ); @@ -169,7 +169,7 @@ tkDNN implement and easy parser for darknet cfg files, a network can be converte tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names"); net->print(); ``` -All models from darknet are now parsed directly from cfg, you still need to export the weights with the descripted tools in the previus section. +All models from darknet are now parsed directly from cfg, you still need to export the weights with the described tools in the previous section.
Supported layers convolutional @@ -203,7 +203,7 @@ cmake .. -DDEBUG=True make ``` -Once you have succesfully created your rt file, run the demo: +Once you have successfully created your rt file, run the demo: ``` ./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y ``` @@ -247,7 +247,7 @@ You should provide image_list.txt and label_list.txt, using training images. How ``` bash scripts/download_validation.sh COCO ``` -to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD insted of COCO to download BDD validation. +to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD instead of COCO to download BDD validation. Then a complete example using yolo3 and COCO dataset would be: ``` @@ -269,8 +269,8 @@ N.B. export TKDNN_BATCHSIZE=2 # build tensorRT files ``` -This will create a TensorRT file with the desidered **max** batch size. -The test will still run with a batch of 1, but the created tensorRT can manage the desidered batch size. +This will create a TensorRT file with the desired **max** batch size. +The test will still run with a batch of 1, but the created tensorRT can manage the desired batch size. ### Test batch Inference This will test the network with random input and check if the output of each batch is the same. @@ -316,7 +316,7 @@ cd build ./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` -This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate). +This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to submit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate). ## Existing tests and supported networks diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index ba42834..0498d41 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -76,10 +76,10 @@ class DetectionNN { ~DetectionNN(){}; /** - * Method used to inialize the class, allocate memory and compute + * Method used to initialize the class, allocate memory and compute * needed data. * - * @param tensor_path path to the rt file og the NN. + * @param tensor_path path to the rt file of the NN. * @param n_classes number of classes for the given dataset. * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. @@ -141,9 +141,9 @@ class DetectionNN { } /** - * Method to draw boundixg boxes and labels on a frame. + * Method to draw bounding boxes and labels on a frame. * - * @param frames orginal frame to draw bounding box on. + * @param frames original frame to draw bounding box on. */ void draw(std::vector& frames) { tk::dnn::box b; diff --git a/include/tkDNN/ImuOdom.h b/include/tkDNN/ImuOdom.h index 58def96..6d8d4cb 100644 --- a/include/tkDNN/ImuOdom.h +++ b/include/tkDNN/ImuOdom.h @@ -44,7 +44,7 @@ class ImuOdom { virtual ~ImuOdom() {} /** - * Method used for inizialize the class + * Method used for initialize the class * * @return Success of the initialization */ @@ -141,7 +141,7 @@ class ImuOdom { //odomPOS = odomPOS + deltaP.cast(); // V2 odomROT = odomROT * q.normalized().toRotationMatrix(); - // compute euler + // compute Euler auto newEULER = odomROT.eulerAngles(0, 1, 2); for(int i=0; i<3; i++) { while( fabs(newEULER(i) - odomEULER(i)) > M_PI_2 ) { diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index f2ec56d..790a431 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -171,7 +171,7 @@ public: /** - Input layer (it doesnt need weigths) + Input layer (it doesn't need weights) */ class Input : public Layer { @@ -207,7 +207,7 @@ public: /** - Avaible activation functions + Available activation functions */ typedef enum { ACTIVATION_ELU = 100, @@ -216,7 +216,7 @@ typedef enum { } tkdnnActivationMode_t; /** - Activation layer (it doesnt need weigths) + Activation layer (it doesn't need weights) */ class Activation : public Layer { @@ -318,9 +318,9 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); const bool bidirectional = true; /**> is the net bidir */ - bool returnSeq = false; /**> if false return only the result of last timestep */ + bool returnSeq = false; /**> if false return only the result of last timestamp */ int stateSize = 0; /**> number of hidden states */ - int seqLen = 0; /**> number of timesteps */ + int seqLen = 0; /**> number of timestamp */ int numLayers = 1; /**> number of internal layers */ protected: @@ -367,7 +367,7 @@ public: /** - Deformable Convolutionl 2d layer + Deformable Convolutional 2d layer */ class DeformConv2d : public LayerWgs { @@ -449,7 +449,7 @@ protected: /** - Avaible pooling functions (padding on tkDNN is not supported) + Available pooling functions (padding on tkDNN is not supported) */ typedef enum { POOLING_MAX = 0, @@ -460,7 +460,7 @@ typedef enum { /** Pooling layer - currenty supported only 2d pooing (also on 3d input) + currently supported only 2d pooing (also on 3d input) */ class Pooling : public Layer { @@ -526,7 +526,7 @@ public: /** Reorg layer - Mantain same dimension but change C*H*W distribution + Maintains same dimension but change C*H*W distribution */ class Reorg : public Layer { @@ -559,7 +559,7 @@ public: /** Upsample layer - Mantain same dimension but change C*H*W distribution + Maintains same dimension but change C*H*W distribution */ class Upsample : public Layer { diff --git a/include/tkDNN/Network.h b/include/tkDNN/Network.h index 2d95215..b78acff 100644 --- a/include/tkDNN/Network.h +++ b/include/tkDNN/Network.h @@ -7,12 +7,12 @@ namespace tk { namespace dnn { /** - Data rapresentation beetween layers + Data representation between layers n = batch size c = channels - h = heigth (lines) + h = height (lines) w = width (rows) - l = lenght (3rd dimension) + l = length (3rd dimension) */ struct dataDim_t { @@ -43,7 +43,7 @@ public: void releaseLayers(); /** - Do inferece for every added layer + Do inference for every added layer */ dnnType* infer(dataDim_t &dim, dnnType* data); diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 66b4f3d..4c6c816 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -91,7 +91,7 @@ public: } /** - Do inferece + Do inference */ dnnType* infer(dataDim_t &dim, dnnType* data); void enqueue(int batchSize = 1); diff --git a/include/tkDNN/evaluation.h b/include/tkDNN/evaluation.h index 8907d9d..128eba0 100644 --- a/include/tkDNN/evaluation.h +++ b/include/tkDNN/evaluation.h @@ -73,12 +73,12 @@ double computeMap( std::vector &images,const int classes, * all the recall levels are evaluated, otherwise only * map_point recall levels are used. For COCO evaluation * 101 points are used. - * @param map_step step used to increment IoU theshold + * @param map_step step used to increment IoU threshold * @param map_levels number of IoU step to perform * @param verbose is set to true, prints on screen additional info * @param write_on_file if set to true, the results produced by this function * are written on file - * @param net name of the considerd neural network + * @param net name of the considered neural network * * @return mAP IoU_tresh:IoU_tresh+map_step*map_levels (e.g. mAP 0.5:0.95 when * map_step=0.05 and map_levels=10) @@ -89,7 +89,7 @@ double computeMapNIoULevels(std::vector &images,const int classes, const int map_levels=10, const bool verbose=false, const bool write_on_file = false, std::string net = ""); /** - * This method computes the numper of True Positive (TP), False Positive (FP), + * This method computes the number of True Positive (TP), False Positive (FP), * False Negative (FN), precision, recall and f1-score. * Those values are computer over all the detections, over all the classes. * @@ -101,7 +101,7 @@ double computeMapNIoULevels(std::vector &images,const int classes, * @param verbose is set to true, prints on screen additional info * @param write_on_file if set to true, the results produced by this function * are written on file - * @param net name of the considerd neural network + * @param net name of the considered neural network */ void computeTPFPFN( std::vector &images,const int classes, const float IoU_thresh=0.5, const float conf_thresh=0.3, diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index bff6370..225a24e 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -89,7 +89,7 @@ public: for(int b=0; b input_bins, std::vector } if(output_bins.size() != outputs.size()) { std::cout< netLayers; std::ifstream if_cfg(cfg_file); diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index b161a22..dbb71e1 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -95,7 +95,7 @@ dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) { // split conv2d outputs into offset and mask checkCuda(cudaMemcpy(offset, output_conv, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - // kernel sigmoide + // kernel sigmoid activationSIGMOIDForward(mask, mask, chunk_dim); // deformable convolution diff --git a/src/Dense.cpp b/src/Dense.cpp index b6a9af2..4371d06 100644 --- a/src/Dense.cpp +++ b/src/Dense.cpp @@ -37,7 +37,7 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) { // place bias into dstData checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) ); - //do matrix moltiplication + //do matrix multiplication checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T, dim_x, dim_y, &alpha, diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 511fbee..7b87711 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -133,7 +133,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig output_dim = input_dim; output_dim.c = stateSize*(bidirectional ? 2 : 1); - // if retunseq is disabled only the last timestep is returned + // if retunseq is disabled only the last timestamp is returned if(!returnSeq) { output_dim.h = 1; output_dim.w = 1; @@ -254,7 +254,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { rnnDesc, seqLen, // number of time steps (nT) x_desc_vec_.data(), // input array of desc (nT*nC_in) - srcF, // input pointer + srcF, // input pointer hx_desc_, // initial hidden state desc hx_ptr, // initial hidden state pointer cx_desc_, // initial cell state desc @@ -281,7 +281,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { rnnDesc, seqLen, // number of time steps (nT) x_desc_vec_.data(), // input array of desc (nT*nC_in) - srcB, // input pointer + srcB, // input pointer hx_desc_, // initial hidden state desc hx_ptr, // initial hidden state pointer cx_desc_, // initial cell state desc @@ -289,7 +289,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { w_desc_, // weights desc wb_ptr, // weights pointer y_desc_vec_.data(), // output desc (nT*nC_out) - dstB_NR, // output pointer + dstB_NR, // output pointer hy_desc_, // final hidden state desc hy_ptr, // final hidden state pointer cy_desc_, // final cell state desc @@ -307,7 +307,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { one_output_dim.c*sizeof(dnnType), cudaMemcpyDeviceToDevice)); } - // if retunseq is disabled only the last timestep is returned + // if retunseq is disabled only the last timestamp is returned if(returnSeq) { // forward transpose matrixTranspose(net->cublasHandle, dstF, dstData, diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index 4afb7cc..a761327 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -105,7 +105,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, float2half(tmp_d, variance16_d, b_size); cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); - //conver scales + //convert scales float2half(scales_d, scales16_d, b_size); cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost); diff --git a/src/MulAdd.cpp b/src/MulAdd.cpp index 0c2a962..25cec8d 100644 --- a/src/MulAdd.cpp +++ b/src/MulAdd.cpp @@ -12,7 +12,7 @@ MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) { int size = input_dim.tot(); - // create a vector with all value setted to add + // create a vector with all value set to add dnnType *add_vector_h = new dnnType[size]; for(int i=0; igetBindingIndex("data"); buf_output_idx = engineRT->getBindingIndex("out"); - std::cout<<"input idex = "< output index = "< output index = "<getBindingDimensions(buf_input_idx); diff --git a/src/Region.cpp b/src/Region.cpp index 65bb786..7c26208 100644 --- a/src/Region.cpp +++ b/src/Region.cpp @@ -63,7 +63,7 @@ dnnType* Region::infer(dataDim_t &dim, dnnType* srcData) { } -/* Intepret class */ +/* Interpret class */ RegionInterpret::RegionInterpret(dataDim_t input_dim, dataDim_t output_dim, int classes, int coords, int num, float thresh, std::string fname_weights) { diff --git a/src/Shortcut.cpp b/src/Shortcut.cpp index 78a2f23..2c7a4f4 100644 --- a/src/Shortcut.cpp +++ b/src/Shortcut.cpp @@ -13,7 +13,7 @@ Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) { if( /*backLayer->output_dim.c != input_dim.c ||*/ backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) - FatalError("Shortcut dim missmatch"); + FatalError("Shortcut dim mismatch"); } Shortcut::~Shortcut() { diff --git a/src/evaluation.cpp b/src/evaluation.cpp index 58c951d..f23c380 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -63,7 +63,7 @@ double computeMap( std::vector &images,const int classes, int gt_checked = 0; - // for each detection comput IoU with groundtruth and match detetcion and + // for each detection compute IoU with groundtruth and match detetcion and // groundtruth with IoU greater than IoU_thresh for(auto &img:images){ for(size_t i=0; i &images,const int classes, } } - //compute average precision for each class. Two methods are avaible, + //compute average precision for each class. Two methods are available, //based on map_points required double mean_average_precision = 0; double last_recall, last_precision, delta_recall; @@ -287,7 +287,7 @@ void computeTPFPFN( std::vector &images,const int classes, } } - //count all TP, FP, FN and compute precsion, recall and f1-score + //count all TP, FP, FN and compute precision, recall and f1-score double avg_precision = 0, avg_recall = 0, f1_score = 0; int TP = 0, FP = 0, FN = 0; for(size_t i=0; i