From 07eae338fe3f87ab77b2fd2f9a46810d3af0946d Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 16 Jan 2020 18:21:34 +0100 Subject: [PATCH 001/482] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3b16ec3..6a0a7ff 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,4 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo # launch detection on a demo video ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` +![Demo preview](https://github.com/ceccocats/tkDNN/releases/download/v0.4/demo.gif) -- 2.52.0 From 2f57ba122201a256e1853382b4aa7543c6faaa7d Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 16 Jan 2020 18:21:34 +0100 Subject: [PATCH 002/482] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3b16ec3..6a0a7ff 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,4 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo # launch detection on a demo video ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` +![Demo preview](https://github.com/ceccocats/tkDNN/releases/download/v0.4/demo.gif) -- 2.52.0 From c3aadb089bbbb665f0399c30be69e26aaccf990e Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 16 Jan 2020 18:24:35 +0100 Subject: [PATCH 003/482] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a0a7ff..3d956a5 100644 --- a/README.md +++ b/README.md @@ -49,4 +49,4 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo # launch detection on a demo video ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` -![Demo preview](https://github.com/ceccocats/tkDNN/releases/download/v0.4/demo.gif) +![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) -- 2.52.0 From 146e1442495919de5627b7e4f3ad2adbacd8943c Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 16 Jan 2020 18:24:35 +0100 Subject: [PATCH 004/482] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a0a7ff..3d956a5 100644 --- a/README.md +++ b/README.md @@ -49,4 +49,4 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo # launch detection on a demo video ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` -![Demo preview](https://github.com/ceccocats/tkDNN/releases/download/v0.4/demo.gif) +![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) -- 2.52.0 From 6a3b261bc99adc38f2d2521e02a07fe9131c43b0 Mon Sep 17 00:00:00 2001 From: luca <228618@studenti.unimore.it> Date: Mon, 20 Jan 2020 14:51:38 +0100 Subject: [PATCH 005/482] compile with tensorrt 5 --- src/NetworkRT.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index d4fdf90..f27f70d 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -275,18 +275,18 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { if(l->act_mode == ACTIVATION_LEAKY) { //std::cout<<"New plugin LEAKY\n"; - /* +#if NV_TENSORRT_MAJOR < 6 // plugin version IPlugin *plugin = new ActivationLeakyRT(); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; - */ - +#else IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU); lRT->setAlpha(0.1); checkNULL(lRT); return lRT; +#endif } else if(l->act_mode == CUDNN_ACTIVATION_RELU) { IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kRELU); -- 2.52.0 From 8a4d1cac17f14f14c7db6bf3402c10bc83faeccb Mon Sep 17 00:00:00 2001 From: luca <228618@studenti.unimore.it> Date: Mon, 20 Jan 2020 14:51:38 +0100 Subject: [PATCH 006/482] compile with tensorrt 5 --- src/NetworkRT.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index d4fdf90..f27f70d 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -275,18 +275,18 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { if(l->act_mode == ACTIVATION_LEAKY) { //std::cout<<"New plugin LEAKY\n"; - /* +#if NV_TENSORRT_MAJOR < 6 // plugin version IPlugin *plugin = new ActivationLeakyRT(); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; - */ - +#else IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU); lRT->setAlpha(0.1); checkNULL(lRT); return lRT; +#endif } else if(l->act_mode == CUDNN_ACTIVATION_RELU) { IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kRELU); -- 2.52.0 From 07da54efe946948c669ab9f77b0b84e4509978b8 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 21 Jan 2020 12:50:18 +0100 Subject: [PATCH 007/482] Centernet: fix pooling problem, add centrnet demo Signed-off-by: Micaela Verucchi Signed-off-by: Davide Sapienza --- CMakeLists.txt | 5 +- demo/demo/demo_centernet.cpp | 88 +++ demo/demo/{demo.cpp => demo_yolo3.cpp} | 50 +- include/tkDNN/CenternetDetection.h | 3 + include/tkDNN/pluginsRT/ActivationSigmoidRT.h | 60 ++ include/tkDNN/pluginsRT/DeformableConvRT.h | 1 - src/CenternetDetection.cpp | 251 ++----- src/NetworkRT.cpp | 3 +- src/Pooling.cpp | 4 +- src/kernels/activation_sigmoid.cu | 16 +- src/sorting.cu | 2 +- tests/resnet101_cnet/resnet101_cnet.cpp | 681 +----------------- 12 files changed, 229 insertions(+), 935 deletions(-) create mode 100644 demo/demo/demo_centernet.cpp rename demo/demo/{demo.cpp => demo_yolo3.cpp} (57%) create mode 100644 include/tkDNN/pluginsRT/ActivationSigmoidRT.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7af66c0..8b847d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,9 +105,12 @@ target_link_libraries(test_resnet101_cnet tkDNN) add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) -add_executable(yolo3_demo demo/demo/demo.cpp) +add_executable(yolo3_demo demo/demo/demo_yolo3.cpp) target_link_libraries(yolo3_demo tkDNN) +add_executable(centernet_demo demo/demo/demo_centernet.cpp) +target_link_libraries(centernet_demo tkDNN) + #------------------------------------------------------------------------------- # Install diff --git a/demo/demo/demo_centernet.cpp b/demo/demo/demo_centernet.cpp new file mode 100644 index 0000000..0f9c86e --- /dev/null +++ b/demo/demo/demo_centernet.cpp @@ -0,0 +1,88 @@ +#include +#include +#include /* srand, rand */ +#include +#include +#include "utils.h" + +#include +#include +#include +#include + +#include "CenternetDetection.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + char *net = "resnet101_cnet.rt"; + if(argc > 1) + net = argv[1]; + char *input = "../demo/yolo_test.mp4"; + if(argc > 2) + input = argv[2]; + + tk::dnn::CenternetDetection cnet; + cnet.init(net); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + cv::Mat dnn_input; + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + while(gRun) { + cap >> frame; + if(!frame.data) { + break; + } + + // this will be resized to the net format + dnn_input = frame.clone(); + // TODO: async infer + cnet.update(dnn_input); + // draw dets + frame = cnet.draw(dnn_input); + + cv::imshow("detection", frame); + cv::waitKey(1); + if(SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + + + std::cout< #include -// #include "Yolo3Detection.h" -#include "CenternetDetection.h" +#include "Yolo3Detection.h" bool gRun; bool SAVE_RESULT = false; @@ -27,15 +26,14 @@ int main(int argc, char *argv[]) { signal(SIGINT, sig_handler); - char *net = "resnet101_cnet.rt"; + char *net = "yolo3_berkeley.rt"; if(argc > 1) net = argv[1]; char *input = "../demo/yolo_test.mp4"; if(argc > 2) input = argv[2]; - // tk::dnn::Yolo3Detection yolo; - tk::dnn::CenternetDetection yolo; + tk::dnn::Yolo3Detection yolo; yolo.init(net); gRun = true; @@ -69,30 +67,28 @@ int main(int argc, char *argv[]) { // TODO: async infer yolo.update(dnn_input); - frame = yolo.draw(dnn_input); - // // draw dets - // for(int i=0; iclassesNames[b.cl]; - // float prob = b.prob; + // draw dets + for(int i=0; iclassesNames[b.cl]; + float prob = b.prob; - // // std::cout< coco_class_name; + // keep track of inference times (ms) + std::vector stats; + CenternetDetection() {} virtual ~CenternetDetection() {} diff --git a/include/tkDNN/pluginsRT/ActivationSigmoidRT.h b/include/tkDNN/pluginsRT/ActivationSigmoidRT.h new file mode 100644 index 0000000..4435f08 --- /dev/null +++ b/include/tkDNN/pluginsRT/ActivationSigmoidRT.h @@ -0,0 +1,60 @@ +#include +#include "../kernels.h" + +class ActivationSigmoidRT : public IPlugin { + +public: + ActivationSigmoidRT() { + + + } + + ~ActivationSigmoidRT(){ + + } + + int getNbOutputs() const override { + return 1; + } + + Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { + return inputs[0]; + } + + void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { + size = 1; + for(int i=0; i(inputs[0]), + reinterpret_cast(outputs[0]), size, stream); + return 0; + } + + + virtual size_t getSerializationSize() override { + return 1*sizeof(int); + } + + virtual void serialize(void* buffer) override { + char *buf = reinterpret_cast(buffer); + tk::dnn::writeBUF(buf, size); + } + + int size; +}; diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index b236095..dca1020 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -90,7 +90,6 @@ public: } virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { -std::cout<<"LOL\n"; dnnType *srcData = (dnnType*)reinterpret_cast(inputs[0]); dnnType *output_conv = (dnnType*)reinterpret_cast(inputs[1]); diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 59bf827..fbc9832 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -229,58 +229,50 @@ void CenternetDetection::update(cv::Mat &imageORIG) { src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); - // std::cout<<"src: "<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME gett affine trans: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + + resize(imageORIG, imageF, cv::Size(new_width, new_height)); sz = imageF.size(); std::cout<<"size: "<(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - cv::warpAffine(imageF, imageF, trans, cv::Size(inp_width, inp_height), cv::INTER_LINEAR ); + + cv::warpAffine(imageF, imageF, trans, cv::Size(inp_width, inp_height), cv::INTER_LINEAR ); end_t = std::chrono::steady_clock::now(); std::cout << " TIME warpAffine: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + sz = imageF.size(); std::cout<<"size: "<(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - std::cout<<"mean: "<(end_t - step_t).count() << " us" << std::endl; - step_t = end_t; - - //split channels - cv::split(imageF,bgr);//split source end_t = std::chrono::steady_clock::now(); - std::cout << " TIME split: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME convert: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + dim2 = dim; + + //split channels + cv::split(imageF,bgr);//split source + for(int i=0; i<3; i++){ bgr[i] = bgr[i] - mean[i]; bgr[i] = bgr[i] / stddev[i]; } - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME mean std: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - //write channels for(int i=0; iinfer(dim2, input_d); TIMER_STOP dim2.print(); + + stats.push_back(t_ns); } // checkResult(dim2.tot(), input_h, input); - std::cout<<" --- pre-process ---\n"; - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; + step_t = std::chrono::steady_clock::now(); + // ------------------------------------ process -------------------------------------------- rt_out[0] = (dnnType *)netRT->buffersRT[1]; rt_out[1] = (dnnType *)netRT->buffersRT[2]; rt_out[2] = (dnnType *)netRT->buffersRT[3]; rt_out[3] = (dnnType *)netRT->buffersRT[4]; - activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); checkCuda( cudaDeviceSynchronize() ); - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME sigmoid : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0]); - float *prova; - checkCuda( cudaMallocHost(&prova, K*sizeof(float)) ); - checkCuda( cudaMemcpy(prova, rt_out[0], K*sizeof(float), cudaMemcpyDeviceToHost) ); - std::cout<<"heat:\n"; - for(int i=0; i toll || hm_h[i]-hmax_h[i] < -toll){ - // hm_h[i] = 0.0f; - // } - // } - // checkCuda( cudaFreeHost(hmax_h) ); - std::cout<<" --- hmax ---\n"; end_t = std::chrono::steady_clock::now(); - std::cout << " TIME : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME threshold: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; // ----------- nms end // ----------- topk - - // thrust::device_vector ids_d; - // int ids[dim_hm.h * dim_hm.w]; - // for(int i=0; i ids2( dim_hm.h * dim_hm.w ); - // for(int i=0; i dim_hm.h * dim_hm.w){ printf ("Error topk (K is too large)\n"); return; } - checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); - // checkCuda( cudaMemcpy(ids_2d, ids_2, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); - - - // sortAndTopKonDevice(rt_out[0], ids_2d, topk_scores, topk_inds_ , topk_ys_ , topk_xs_ ,dim_hm.h * dim_hm.w, K, dim_hm.c); - // checkCuda( cudaDeviceSynchronize() ); - // for(int i=0; ioutput_dim.h * hm->output_dim.w elements for each channel and sort it. Then find the first 100 elements - // // memcpy(ids2, ids, dim_hm.h * dim_hm.w); - // sort(rt_out[0]+ i * dim_hm.h * dim_hm.w, - // rt_out[0]+ i * dim_hm.h * dim_hm.w + dim_hm.h * dim_hm.w, - // ids_d); - // // end_t = std::chrono::steady_clock::now(); - // // std::cout << " TIME sort channel "<(end_t - step_t).count() << " ms" << std::endl; - // // step_t = end_t; - // topk(rt_out[0]+ i * dim_hm.h * dim_hm.w, ids_d, K, topk_scores + i*K, - // topk_inds_ + i*K, topk_ys_ + i*K, topk_xs_ + i*K); - // // checkCuda( cudaMemcpy(ids2, ids2_d, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyDeviceToHost) ); - - // // for (int j=0; j(end_t - step_t).count() << " ms" << std::endl; - // // step_t = end_t; - - // } - // checkCuda( cudaFree(ids_d )); - std::cout<<" --- a 100 ---\n"; - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME sort topk on 80 channel: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - // final - - // sort(topk_scores, - // topk_scores + dim_hm.c * K, - // topk_inds_); sort(rt_out[0], rt_out[0]+dim_hm.tot(), ids_d); checkCuda( cudaDeviceSynchronize() ); - int *topk_inds; - checkCuda( cudaMallocHost(&topk_inds, K*sizeof(int)) ); - // checkCuda( cudaMemcpy(topk_inds, ids_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - // for(int i=0; i(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME sort: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - // topk(topk_scores, topk_inds_, K, scores_d, - // topk_inds_d, topk_ys_d, topk_xs_d); topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d); - checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaDeviceSynchronize() ); + end_t = std::chrono::steady_clock::now(); - std::cout << " TIME topk channel: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME topk: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - - - checkCuda( cudaMemcpy(topk_inds, topk_inds_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - for(int i=0; i(end_t - step_t).count() << " ms" << std::endl; + step_t = end_t; + checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - std::cout<<"\ntopk_ids: \n"; - checkCuda( cudaMemcpy(topk_inds, topk_inds_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - for(int i=0; i(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; + // ----------- topk end - // dnnType *reg_aus; - // checkCuda( cudaMallocHost(®_aus, dim_reg.tot()*sizeof(dnnType)) ); - // checkCuda( cudaMemcpy(reg_aus, rt_out[3], dim_reg.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - - // for(int i = 0; i < K; i++){ - // topk_xs[i] = topk_xs[i] + reg_aus[topk_inds[i]]; - // topk_ys[i] = topk_ys[i] + reg_aus[topk_inds[i]+dim_reg.h*dim_reg.w]; - // } topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3]); // checkCuda( cudaDeviceSynchronize() ); + end_t = std::chrono::steady_clock::now(); std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - // checkCuda( cudaFreeHost(reg_aus) ); - - // dnnType *wh_aus; - // checkCuda( cudaMemcpy(wh_aus, rt_out[2], dim_wh.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d); // checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); - // for(int i = 0; i < K; i++){ - // bboxes[i * 4] = topk_xs[i] - wh_aus[topk_inds[i]] / 2; - // bboxes[i * 4 + 1] = topk_ys[i] - wh_aus[topk_inds[i]+dim_reg.h*dim_reg.w] / 2; - // bboxes[i * 4 + 2] = topk_xs[i] + wh_aus[topk_inds[i]] / 2; - // bboxes[i * 4 + 3] = topk_ys[i] + wh_aus[topk_inds[i]+dim_reg.h*dim_reg.w] / 2; - // } - // for(int i = 0; i < K; i++){ - // std::cout<<"-----\n(x0, y0) = ("<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME bboxes: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - // servono [bboxes, scores, clses] - // checkCuda( cudaDeviceSynchronize() ); - std::cout<<" --- process ---\n"; - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; // ---------------------------------- post-process ----------------------------------------- // --------- ctdet_post_process @@ -548,12 +384,13 @@ void CenternetDetection::update(cv::Mat &imageORIG) { cv::Mat trans2(cv::Size(3,2), CV_32F); trans2 = cv::getAffineTransform( dst, src ); + end_t = std::chrono::steady_clock::now(); std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + cv::Mat new_pt1(cv::Size(1,2), CV_32F); - cv::Mat new_pt2(cv::Size(1,2), CV_32F); - + cv::Mat new_pt2(cv::Size(1,2), CV_32F); for(int i = 0; i(0,0)=static_cast(trans2.at(0,0))*bbx0[i] + @@ -570,23 +407,18 @@ void CenternetDetection::update(cv::Mat &imageORIG) { static_cast(trans2.at(1,1))*bby1[i] + static_cast(trans2.at(1,2))*1.0; - // std::cout<<"\n new: "<(0,0); target_coords[i*4+1] = new_pt1.at(0,1); target_coords[i*4+2] = new_pt2.at(0,0); target_coords[i*4+3] = new_pt2.at(0,1); - // std::cout<(0,0)<<", "<(0,1)<<", "<(0,0)<<", "<(0,1)< thresh){ - std::cout<<"th: "<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME detections: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + std::cout<<"TOTAL: \n"; TIMER_STOP } diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index bec7b17..0f2d621 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -290,8 +290,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { if(l->pool_mode == tkdnnPoolingMode_t::POOLING_AVERAGE_EXCLUDE_PADDING) ptype = PoolingType::kMAX_AVERAGE_BLEND; - // if(l->input_dim.h % 2 == 1 && l->input_dim.w % 2 == 1) - if(l->input_dim.h == l->output_dim.h && l->input_dim.w == l->output_dim.w) + if(l->paddingH == 0 && l->paddingW == 0 && l->input_dim.h == l->output_dim.h && l->input_dim.w == l->output_dim.w) { IPlugin *plugin = new ResizeLayerRT( l->output_dim.c,l->output_dim.h+1,l->output_dim.w+1 ); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); diff --git a/src/Pooling.cpp b/src/Pooling.cpp index 1ad0664..cc498df 100644 --- a/src/Pooling.cpp +++ b/src/Pooling.cpp @@ -55,8 +55,8 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW, int padW = paddingW == 0? winW -1 : paddingW; if(final){ - h = (h + padH - winH)/strideH +1 +1; - w = (w + padW - winW)/strideW +1 +1; + h = (h + 2*paddingH - winH)/strideH +1 ; + w = (w + 2*paddingW - winW)/strideW +1; } else{ h = (h + padH - winH)/strideH +1; diff --git a/src/kernels/activation_sigmoid.cu b/src/kernels/activation_sigmoid.cu index ba6c997..400a948 100644 --- a/src/kernels/activation_sigmoid.cu +++ b/src/kernels/activation_sigmoid.cu @@ -1,21 +1,13 @@ #include "kernels.h" - -__device__ -__forceinline__ -double sigmoid (double a) -{ - return 1.0 / (1.0 + exp (-a)); -} +#include __global__ void activation_sigmoid(dnnType *input, dnnType *output, int size) { - int stride = gridDim.x * blockDim.x; - int tid = blockDim.x * blockIdx.x + threadIdx.x; - for (int i = tid; i < size; i += stride) { - output[i] = sigmoid (input[i]); - } + int i = blockDim.x * blockIdx.x + threadIdx.x; + if(i < size) + output[i] = 1.0f / (1.0f + exp (-input[i])); } diff --git a/src/sorting.cu b/src/sorting.cu index e007be1..fd62cec 100644 --- a/src/sorting.cu +++ b/src/sorting.cu @@ -45,7 +45,7 @@ struct threshold : public thrust::binary_function { __host__ __device__ float operator()(float x, float y) { - float toll = 1e-6; + double toll = 1e-6; if(fabsf(x-y)>toll) return 0.0f; else diff --git a/tests/resnet101_cnet/resnet101_cnet.cpp b/tests/resnet101_cnet/resnet101_cnet.cpp index 061e4b6..e05f517 100644 --- a/tests/resnet101_cnet/resnet101_cnet.cpp +++ b/tests/resnet101_cnet/resnet101_cnet.cpp @@ -183,53 +183,6 @@ const char *output_bin[]={ "../tests/resnet101_cnet/debug/wh.bin", "../tests/resnet101_cnet/debug/reg.bin"}; - - -std::vector sort_indexes(const std::vector &v) { - - // initialize original index locations - std::vector idx(v.size()); - iota(idx.begin(), idx.end(), 0); - - // sort indexes based on comparing values in v - sort(idx.begin(), idx.end(), - [&v](size_t i1, size_t i2) {return v[i1] > v[i2];}); - - return idx; -} - -float _colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} }; -float get_color(int c, int x, int max) -{ - float ratio = ((float)x/max)*5; - int i = floor(ratio); - int j = ceil(ratio); - ratio -= i; - float r = (1-ratio) * _colors[i % 6][c % 3] + ratio*_colors[j % 6][c % 3]; - //printf("%f\n", r); - return r; -} - -int computeDetections(dnnType *hm_d, dnnType *wh_d, dnnType *reg_d, int hm_dim, int wh_dim, int reg_dim, bool cat_spec_wh, int k){ - // _nms - int kernel = 3; - int pad = (kernel - 1)/2; - std::cout<<"computeDetections\n"; - // dnnType *hmax; - // tk::dnn::Pooling maxpool(&hmax, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX) - // = (dnnType *) - - // net.functional.max_pool2d( - // heat, (kernel, kernel), stride=1, padding=pad) - // keep = (hmax == heat).float() - // return heat * keep -} - -int process(dnnType *hm_d, dnnType *wh_d, dnnType *reg_d, int hm_dim, int wh_dim, int reg_dim){ - std::cout<<"process\n"; - // computeDetections(hm_d, wh_d, reg_d, hm_dim, wh_dim, reg_dim, false, 100); -} - int main() { @@ -450,637 +403,5 @@ int main() checkResult(odim, rt_out, out); std::cout << "CUDNN vs TRT "; checkResult(odim, cudnn_out, rt_out); - } - - TIMER_START - - // -------- transofrm compose - cv::Mat imageOrig = cv::imread("/media/davide/DATA/shared_home/Projects/Professionale/repos/photo_2020-01-14_09-56-07.jpg"); - cv::Mat imageF; - imageOrig.convertTo(imageF, CV_32FC3, 1/255.0); - cv::Mat image; - cv::Size sz = imageF.size(); - std::cout<<"image: "<output_dim.tot()*sizeof(dnnType)) ); - - dnnType *rt_out[4]; - rt_out[0] = (dnnType *)netRT.buffersRT[1]; - rt_out[1] = (dnnType *)netRT.buffersRT[2]; - rt_out[2] = (dnnType *)netRT.buffersRT[3]; - rt_out[3] = (dnnType *)netRT.buffersRT[4]; - - // checkCuda( cudaMemcpy(hm_h, rt_out[0], hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"hm\n"; - hm->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<output_dim.tot()); - checkCuda( cudaDeviceSynchronize() ); - - - checkCuda( cudaMemcpy(hm_h, rt_out[0], hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"hm\n"; - hm->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<infer(hmax->input_dim.tot(), rt_out[0]); - // keep = (hmax == heat).float() - // return heat * keep - - dnnType *hmax_h; - checkCuda( cudaMallocHost(&hmax_h, hmax->output_dim.tot()*sizeof(dnnType)) ); - checkCuda( cudaMemcpy(hmax_h, rt_out[1], hmax->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - - std::cout<<"hmax\n"; - hmax->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<output_dim.print(); - std::cout<<"hmax:\n"; - hmax->output_dim.print(); - // return 0; - float toll = 0.000001; - for(int i=0; i < hm->output_dim.tot(); i++){ - if(hm_h[i]-hmax_h[i] > toll || hm_h[i]-hmax_h[i] < -toll){ - hm_h[i] = 0.0f; - } - } - // std::cout<<"\n"; - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<dstData, hm_h, hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice) ); - checkCuda( cudaFreeHost(hmax_h) ); - // ----------- nms end - // ----------- topk - int K = 100; - int width = 56; // TODO - float *topk_scores; - int *topk_inds_; - float *topk_ys_; - float *topk_xs_; - std::cout<<"mah: "<output_dim.c * K<output_dim.c * K *sizeof(float)) ); - checkCuda( cudaMallocHost(&topk_inds_, hm->output_dim.c * K *sizeof(int)) ); - checkCuda( cudaMallocHost(&topk_ys_, hm->output_dim.c * K *sizeof(float)) ); - checkCuda( cudaMallocHost(&topk_xs_, hm->output_dim.c * K *sizeof(float)) ); - std::cout<<"1\n"; - dnnType *hm_aus; - checkCuda( cudaMallocHost(&hm_aus, hm->output_dim.h * hm->output_dim.w *sizeof(dnnType)) ); - std::cout<<"2\n"; - int count; - std::vector v = {2.0, 3.0, 9.0}; - for (auto i: sort_indexes(v)) { - std::cout << i<< "--" <output_dim.c; i++){ - count = 0; - // get the hm->output_dim.h * hm->output_dim.w elements for each channel and sort it. Then find the first 100 elements - checkCuda( cudaMemcpy(hm_aus, hm_h + i * hm->output_dim.h * hm->output_dim.w, - hm->output_dim.h * hm->output_dim.w * sizeof(dnnType), cudaMemcpyHostToHost) ); - // std::cout<<"top scores: "<output_dim.h * hm->output_dim.w<<"\n"; - // for(int k=0; koutput_dim.h * hm->output_dim.w; k++) - // std::cout< my_vector {arr, arr + arr_length} - std::vector my_vector{hm_aus, hm_aus + hm->output_dim.h * hm->output_dim.w}; - for (auto j: sort_indexes(my_vector)) { - // std::cout <<"j: "< "<< hm_aus[j] << std::endl; - topk_scores[i*K + count] = hm_aus[j]; - topk_inds_[i*K +count] = j; - topk_ys_[i*K +count] = (int)(j / width); - topk_xs_[i*K +count] = (int)(j % width); - if(++count == K) - break; - } - } - std::cout<<"topk_xs_[0]: "<output_dim.c * K; i++) - std::cout< my_vector{topk_scores, topk_scores + hm->output_dim.c * K }; - for (auto j: sort_indexes(my_vector)) { - // std::cout <<"j: "< "<< hm_aus[j] << std::endl; - scores[count] = topk_scores[j]; - clses[count] = (int)(j / K); - topk_inds[count] = topk_inds_[j]; - topk_ys[count] = topk_ys_[j]; - topk_xs[count] = topk_xs_[j]; - if(++count == K) - break; - } - checkCuda( cudaFreeHost(topk_scores) ); - checkCuda( cudaFreeHost(topk_inds_) ); - checkCuda( cudaFreeHost(topk_ys_) ); - checkCuda( cudaFreeHost(topk_xs_) ); - std::cout<<"5\n"; - // ----------- topk end - std::cout<<"topk_xs[0]: "<output_dim.tot()*sizeof(dnnType)) ); - checkCuda( cudaMemcpy(reg_aus, rt_out[3], reg->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"reg:\n"; - reg->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<output_dim.h*reg->output_dim.w]; - } - std::cout<<"topk_xs[0]: "<output_dim.tot()*sizeof(dnnType)) ); - checkCuda( cudaMallocHost(&bboxes, 4 * K *sizeof(dnnType)) ); - checkCuda( cudaMemcpy(wh_aus, rt_out[2], wh->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"7\n"; - for(int i = 0; i< K; i++) - std::cout<output_dim.h*reg->output_dim.w] / 2; - bboxes[i * 4 + 2] = topk_xs[i] + wh_aus[topk_inds[i]] / 2; - bboxes[i * 4 + 3] = topk_ys[i] + wh_aus[topk_inds[i]+reg->output_dim.h*reg->output_dim.w] / 2; - } - ////////////////// fin qui ok - - checkCuda( cudaFreeHost(wh_aus) ); - checkCuda( cudaFreeHost(topk_inds) ); - checkCuda( cudaFreeHost(topk_ys) ); - checkCuda( cudaFreeHost(topk_xs) ); - - std::cout<<"8\n"; - float *detections; - std::cout<<"bboxes:\n"; - for(int i = 0; i < K+1; i++){ - std::cout<(0,0)=c[0]; - src.at(0,1)=c[1]; - src.at(1,0)=c[0]; - src.at(1,1)=c[1] + s[0] * -0.5; - dst.at(0,0)=width * 0.5; - dst.at(0,1)=width * 0.5; - dst.at(1,0)=width * 0.5; - dst.at(1,1)=width * 0.5 + width * -0.5; - - src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); - src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); - dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); - dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); - std::cout<<"src: "<(0,0)<<" - "<(0,1)<<" - "<(0,2)<<"\n"<(1,0)<<" - "<(1,1)<<" - "<(1,2)<(0,0)=detections[i*4]; - // new_pt1.at(0,1)=detections[i*4+1]; - // new_pt1.at(0,2)=1.0; - // new_pt1 << detections[i], detections[i+K], 1.0; - // std::cout<<"----\ni: "<(0,0)=static_cast(trans2.at(0,0))*detections[i*4] + - static_cast(trans2.at(0,1))*detections[i*4+1] + - static_cast(trans2.at(0,2))*1.0; - new_pt1.at(0,1)=static_cast(trans2.at(1,0))*detections[i*4] + - static_cast(trans2.at(1,1))*detections[i*4+1] + - static_cast(trans2.at(1,2))*1.0; - - new_pt2.at(0,0)=static_cast(trans2.at(0,0))*detections[i*4+2] + - static_cast(trans2.at(0,1))*detections[i*4+3] + - static_cast(trans2.at(0,2))*1.0; - new_pt2.at(0,1)=static_cast(trans2.at(1,0))*detections[i*4+2] + - static_cast(trans2.at(1,1))*detections[i*4+3] + - static_cast(trans2.at(1,2))*1.0; - - - // std::cout<<"\n new: "<(0,0); - target_coords[i*4+1] = new_pt1.at(0,1); - target_coords[i*4+2] = new_pt2.at(0,0); - target_coords[i*4+3] = new_pt2.at(0,1); - // std::cout<(0,0)<<", "<(0,1)<<", "<(0,0)<<", "<(0,1)< coco_class_name(coco_class_name_, std::end( coco_class_name_ )); - int num_classes = 80; - float vis_threshold = 0.3; - // int *classes; - std::vector detected; - // checkCuda( cudaMallocHost(&classes, K *sizeof(int)) ); - // checkCuda( cudaMemcpy(classes, detections + 5 * K *sizeof(dnnType), K *sizeof(dnnType), cudaMemcpyHostToHost) ); - for(int i = 0; i i+1 (1:80); - - if(scores[j] > vis_threshold){ - std::cout<<"th: "< Date: Tue, 21 Jan 2020 12:50:18 +0100 Subject: [PATCH 008/482] Centernet: fix pooling problem, add centrnet demo Signed-off-by: Micaela Verucchi Signed-off-by: Davide Sapienza --- CMakeLists.txt | 5 +- demo/demo/demo_centernet.cpp | 88 +++ demo/demo/{demo.cpp => demo_yolo3.cpp} | 50 +- include/tkDNN/CenternetDetection.h | 3 + include/tkDNN/pluginsRT/ActivationSigmoidRT.h | 60 ++ include/tkDNN/pluginsRT/DeformableConvRT.h | 1 - src/CenternetDetection.cpp | 251 ++----- src/NetworkRT.cpp | 3 +- src/Pooling.cpp | 4 +- src/kernels/activation_sigmoid.cu | 16 +- src/sorting.cu | 2 +- tests/resnet101_cnet/resnet101_cnet.cpp | 681 +----------------- 12 files changed, 229 insertions(+), 935 deletions(-) create mode 100644 demo/demo/demo_centernet.cpp rename demo/demo/{demo.cpp => demo_yolo3.cpp} (57%) create mode 100644 include/tkDNN/pluginsRT/ActivationSigmoidRT.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7af66c0..8b847d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,9 +105,12 @@ target_link_libraries(test_resnet101_cnet tkDNN) add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) -add_executable(yolo3_demo demo/demo/demo.cpp) +add_executable(yolo3_demo demo/demo/demo_yolo3.cpp) target_link_libraries(yolo3_demo tkDNN) +add_executable(centernet_demo demo/demo/demo_centernet.cpp) +target_link_libraries(centernet_demo tkDNN) + #------------------------------------------------------------------------------- # Install diff --git a/demo/demo/demo_centernet.cpp b/demo/demo/demo_centernet.cpp new file mode 100644 index 0000000..0f9c86e --- /dev/null +++ b/demo/demo/demo_centernet.cpp @@ -0,0 +1,88 @@ +#include +#include +#include /* srand, rand */ +#include +#include +#include "utils.h" + +#include +#include +#include +#include + +#include "CenternetDetection.h" + +bool gRun; +bool SAVE_RESULT = false; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + std::cout<<"detection\n"; + signal(SIGINT, sig_handler); + + + char *net = "resnet101_cnet.rt"; + if(argc > 1) + net = argv[1]; + char *input = "../demo/yolo_test.mp4"; + if(argc > 2) + input = argv[2]; + + tk::dnn::CenternetDetection cnet; + cnet.init(net); + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + cv::Mat dnn_input; + cv::namedWindow("detection", cv::WINDOW_NORMAL); + + while(gRun) { + cap >> frame; + if(!frame.data) { + break; + } + + // this will be resized to the net format + dnn_input = frame.clone(); + // TODO: async infer + cnet.update(dnn_input); + // draw dets + frame = cnet.draw(dnn_input); + + cv::imshow("detection", frame); + cv::waitKey(1); + if(SAVE_RESULT) + resultVideo << frame; + } + + std::cout<<"detection end\n"; + + + std::cout< #include -// #include "Yolo3Detection.h" -#include "CenternetDetection.h" +#include "Yolo3Detection.h" bool gRun; bool SAVE_RESULT = false; @@ -27,15 +26,14 @@ int main(int argc, char *argv[]) { signal(SIGINT, sig_handler); - char *net = "resnet101_cnet.rt"; + char *net = "yolo3_berkeley.rt"; if(argc > 1) net = argv[1]; char *input = "../demo/yolo_test.mp4"; if(argc > 2) input = argv[2]; - // tk::dnn::Yolo3Detection yolo; - tk::dnn::CenternetDetection yolo; + tk::dnn::Yolo3Detection yolo; yolo.init(net); gRun = true; @@ -69,30 +67,28 @@ int main(int argc, char *argv[]) { // TODO: async infer yolo.update(dnn_input); - frame = yolo.draw(dnn_input); - // // draw dets - // for(int i=0; iclassesNames[b.cl]; - // float prob = b.prob; + // draw dets + for(int i=0; iclassesNames[b.cl]; + float prob = b.prob; - // // std::cout< coco_class_name; + // keep track of inference times (ms) + std::vector stats; + CenternetDetection() {} virtual ~CenternetDetection() {} diff --git a/include/tkDNN/pluginsRT/ActivationSigmoidRT.h b/include/tkDNN/pluginsRT/ActivationSigmoidRT.h new file mode 100644 index 0000000..4435f08 --- /dev/null +++ b/include/tkDNN/pluginsRT/ActivationSigmoidRT.h @@ -0,0 +1,60 @@ +#include +#include "../kernels.h" + +class ActivationSigmoidRT : public IPlugin { + +public: + ActivationSigmoidRT() { + + + } + + ~ActivationSigmoidRT(){ + + } + + int getNbOutputs() const override { + return 1; + } + + Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override { + return inputs[0]; + } + + void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { + size = 1; + for(int i=0; i(inputs[0]), + reinterpret_cast(outputs[0]), size, stream); + return 0; + } + + + virtual size_t getSerializationSize() override { + return 1*sizeof(int); + } + + virtual void serialize(void* buffer) override { + char *buf = reinterpret_cast(buffer); + tk::dnn::writeBUF(buf, size); + } + + int size; +}; diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index b236095..dca1020 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -90,7 +90,6 @@ public: } virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { -std::cout<<"LOL\n"; dnnType *srcData = (dnnType*)reinterpret_cast(inputs[0]); dnnType *output_conv = (dnnType*)reinterpret_cast(inputs[1]); diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 59bf827..fbc9832 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -229,58 +229,50 @@ void CenternetDetection::update(cv::Mat &imageORIG) { src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); - // std::cout<<"src: "<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME gett affine trans: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + + resize(imageORIG, imageF, cv::Size(new_width, new_height)); sz = imageF.size(); std::cout<<"size: "<(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - cv::warpAffine(imageF, imageF, trans, cv::Size(inp_width, inp_height), cv::INTER_LINEAR ); + + cv::warpAffine(imageF, imageF, trans, cv::Size(inp_width, inp_height), cv::INTER_LINEAR ); end_t = std::chrono::steady_clock::now(); std::cout << " TIME warpAffine: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + sz = imageF.size(); std::cout<<"size: "<(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - std::cout<<"mean: "<(end_t - step_t).count() << " us" << std::endl; - step_t = end_t; - - //split channels - cv::split(imageF,bgr);//split source end_t = std::chrono::steady_clock::now(); - std::cout << " TIME split: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME convert: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + dim2 = dim; + + //split channels + cv::split(imageF,bgr);//split source + for(int i=0; i<3; i++){ bgr[i] = bgr[i] - mean[i]; bgr[i] = bgr[i] / stddev[i]; } - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME mean std: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - //write channels for(int i=0; iinfer(dim2, input_d); TIMER_STOP dim2.print(); + + stats.push_back(t_ns); } // checkResult(dim2.tot(), input_h, input); - std::cout<<" --- pre-process ---\n"; - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; + step_t = std::chrono::steady_clock::now(); + // ------------------------------------ process -------------------------------------------- rt_out[0] = (dnnType *)netRT->buffersRT[1]; rt_out[1] = (dnnType *)netRT->buffersRT[2]; rt_out[2] = (dnnType *)netRT->buffersRT[3]; rt_out[3] = (dnnType *)netRT->buffersRT[4]; - activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot()); checkCuda( cudaDeviceSynchronize() ); - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME sigmoid : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0]); - float *prova; - checkCuda( cudaMallocHost(&prova, K*sizeof(float)) ); - checkCuda( cudaMemcpy(prova, rt_out[0], K*sizeof(float), cudaMemcpyDeviceToHost) ); - std::cout<<"heat:\n"; - for(int i=0; i toll || hm_h[i]-hmax_h[i] < -toll){ - // hm_h[i] = 0.0f; - // } - // } - // checkCuda( cudaFreeHost(hmax_h) ); - std::cout<<" --- hmax ---\n"; end_t = std::chrono::steady_clock::now(); - std::cout << " TIME : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME threshold: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; // ----------- nms end // ----------- topk - - // thrust::device_vector ids_d; - // int ids[dim_hm.h * dim_hm.w]; - // for(int i=0; i ids2( dim_hm.h * dim_hm.w ); - // for(int i=0; i dim_hm.h * dim_hm.w){ printf ("Error topk (K is too large)\n"); return; } - checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); - // checkCuda( cudaMemcpy(ids_2d, ids_2, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) ); - - - // sortAndTopKonDevice(rt_out[0], ids_2d, topk_scores, topk_inds_ , topk_ys_ , topk_xs_ ,dim_hm.h * dim_hm.w, K, dim_hm.c); - // checkCuda( cudaDeviceSynchronize() ); - // for(int i=0; ioutput_dim.h * hm->output_dim.w elements for each channel and sort it. Then find the first 100 elements - // // memcpy(ids2, ids, dim_hm.h * dim_hm.w); - // sort(rt_out[0]+ i * dim_hm.h * dim_hm.w, - // rt_out[0]+ i * dim_hm.h * dim_hm.w + dim_hm.h * dim_hm.w, - // ids_d); - // // end_t = std::chrono::steady_clock::now(); - // // std::cout << " TIME sort channel "<(end_t - step_t).count() << " ms" << std::endl; - // // step_t = end_t; - // topk(rt_out[0]+ i * dim_hm.h * dim_hm.w, ids_d, K, topk_scores + i*K, - // topk_inds_ + i*K, topk_ys_ + i*K, topk_xs_ + i*K); - // // checkCuda( cudaMemcpy(ids2, ids2_d, dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyDeviceToHost) ); - - // // for (int j=0; j(end_t - step_t).count() << " ms" << std::endl; - // // step_t = end_t; - - // } - // checkCuda( cudaFree(ids_d )); - std::cout<<" --- a 100 ---\n"; - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME sort topk on 80 channel: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - // final - - // sort(topk_scores, - // topk_scores + dim_hm.c * K, - // topk_inds_); sort(rt_out[0], rt_out[0]+dim_hm.tot(), ids_d); checkCuda( cudaDeviceSynchronize() ); - int *topk_inds; - checkCuda( cudaMallocHost(&topk_inds, K*sizeof(int)) ); - // checkCuda( cudaMemcpy(topk_inds, ids_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - // for(int i=0; i(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME sort: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - // topk(topk_scores, topk_inds_, K, scores_d, - // topk_inds_d, topk_ys_d, topk_xs_d); topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d); - checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaDeviceSynchronize() ); + end_t = std::chrono::steady_clock::now(); - std::cout << " TIME topk channel: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME topk: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - - - checkCuda( cudaMemcpy(topk_inds, topk_inds_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - for(int i=0; i(end_t - step_t).count() << " ms" << std::endl; + step_t = end_t; + checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - std::cout<<"\ntopk_ids: \n"; - checkCuda( cudaMemcpy(topk_inds, topk_inds_d, K*sizeof(int), cudaMemcpyDeviceToHost) ); - for(int i=0; i(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; + // ----------- topk end - // dnnType *reg_aus; - // checkCuda( cudaMallocHost(®_aus, dim_reg.tot()*sizeof(dnnType)) ); - // checkCuda( cudaMemcpy(reg_aus, rt_out[3], dim_reg.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - - // for(int i = 0; i < K; i++){ - // topk_xs[i] = topk_xs[i] + reg_aus[topk_inds[i]]; - // topk_ys[i] = topk_ys[i] + reg_aus[topk_inds[i]+dim_reg.h*dim_reg.w]; - // } topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3]); // checkCuda( cudaDeviceSynchronize() ); + end_t = std::chrono::steady_clock::now(); std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - // checkCuda( cudaFreeHost(reg_aus) ); - - // dnnType *wh_aus; - // checkCuda( cudaMemcpy(wh_aus, rt_out[2], dim_wh.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d); // checkCuda( cudaDeviceSynchronize() ); + checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); - // for(int i = 0; i < K; i++){ - // bboxes[i * 4] = topk_xs[i] - wh_aus[topk_inds[i]] / 2; - // bboxes[i * 4 + 1] = topk_ys[i] - wh_aus[topk_inds[i]+dim_reg.h*dim_reg.w] / 2; - // bboxes[i * 4 + 2] = topk_xs[i] + wh_aus[topk_inds[i]] / 2; - // bboxes[i * 4 + 3] = topk_ys[i] + wh_aus[topk_inds[i]+dim_reg.h*dim_reg.w] / 2; - // } - // for(int i = 0; i < K; i++){ - // std::cout<<"-----\n(x0, y0) = ("<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME bboxes: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - // servono [bboxes, scores, clses] - // checkCuda( cudaDeviceSynchronize() ); - std::cout<<" --- process ---\n"; - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME : " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; // ---------------------------------- post-process ----------------------------------------- // --------- ctdet_post_process @@ -548,12 +384,13 @@ void CenternetDetection::update(cv::Mat &imageORIG) { cv::Mat trans2(cv::Size(3,2), CV_32F); trans2 = cv::getAffineTransform( dst, src ); + end_t = std::chrono::steady_clock::now(); std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + cv::Mat new_pt1(cv::Size(1,2), CV_32F); - cv::Mat new_pt2(cv::Size(1,2), CV_32F); - + cv::Mat new_pt2(cv::Size(1,2), CV_32F); for(int i = 0; i(0,0)=static_cast(trans2.at(0,0))*bbx0[i] + @@ -570,23 +407,18 @@ void CenternetDetection::update(cv::Mat &imageORIG) { static_cast(trans2.at(1,1))*bby1[i] + static_cast(trans2.at(1,2))*1.0; - // std::cout<<"\n new: "<(0,0); target_coords[i*4+1] = new_pt1.at(0,1); target_coords[i*4+2] = new_pt2.at(0,0); target_coords[i*4+3] = new_pt2.at(0,1); - // std::cout<(0,0)<<", "<(0,1)<<", "<(0,0)<<", "<(0,1)< thresh){ - std::cout<<"th: "<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME detections: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; + std::cout<<"TOTAL: \n"; TIMER_STOP } diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index bec7b17..0f2d621 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -290,8 +290,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { if(l->pool_mode == tkdnnPoolingMode_t::POOLING_AVERAGE_EXCLUDE_PADDING) ptype = PoolingType::kMAX_AVERAGE_BLEND; - // if(l->input_dim.h % 2 == 1 && l->input_dim.w % 2 == 1) - if(l->input_dim.h == l->output_dim.h && l->input_dim.w == l->output_dim.w) + if(l->paddingH == 0 && l->paddingW == 0 && l->input_dim.h == l->output_dim.h && l->input_dim.w == l->output_dim.w) { IPlugin *plugin = new ResizeLayerRT( l->output_dim.c,l->output_dim.h+1,l->output_dim.w+1 ); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); diff --git a/src/Pooling.cpp b/src/Pooling.cpp index 1ad0664..cc498df 100644 --- a/src/Pooling.cpp +++ b/src/Pooling.cpp @@ -55,8 +55,8 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW, int padW = paddingW == 0? winW -1 : paddingW; if(final){ - h = (h + padH - winH)/strideH +1 +1; - w = (w + padW - winW)/strideW +1 +1; + h = (h + 2*paddingH - winH)/strideH +1 ; + w = (w + 2*paddingW - winW)/strideW +1; } else{ h = (h + padH - winH)/strideH +1; diff --git a/src/kernels/activation_sigmoid.cu b/src/kernels/activation_sigmoid.cu index ba6c997..400a948 100644 --- a/src/kernels/activation_sigmoid.cu +++ b/src/kernels/activation_sigmoid.cu @@ -1,21 +1,13 @@ #include "kernels.h" - -__device__ -__forceinline__ -double sigmoid (double a) -{ - return 1.0 / (1.0 + exp (-a)); -} +#include __global__ void activation_sigmoid(dnnType *input, dnnType *output, int size) { - int stride = gridDim.x * blockDim.x; - int tid = blockDim.x * blockIdx.x + threadIdx.x; - for (int i = tid; i < size; i += stride) { - output[i] = sigmoid (input[i]); - } + int i = blockDim.x * blockIdx.x + threadIdx.x; + if(i < size) + output[i] = 1.0f / (1.0f + exp (-input[i])); } diff --git a/src/sorting.cu b/src/sorting.cu index e007be1..fd62cec 100644 --- a/src/sorting.cu +++ b/src/sorting.cu @@ -45,7 +45,7 @@ struct threshold : public thrust::binary_function { __host__ __device__ float operator()(float x, float y) { - float toll = 1e-6; + double toll = 1e-6; if(fabsf(x-y)>toll) return 0.0f; else diff --git a/tests/resnet101_cnet/resnet101_cnet.cpp b/tests/resnet101_cnet/resnet101_cnet.cpp index 061e4b6..e05f517 100644 --- a/tests/resnet101_cnet/resnet101_cnet.cpp +++ b/tests/resnet101_cnet/resnet101_cnet.cpp @@ -183,53 +183,6 @@ const char *output_bin[]={ "../tests/resnet101_cnet/debug/wh.bin", "../tests/resnet101_cnet/debug/reg.bin"}; - - -std::vector sort_indexes(const std::vector &v) { - - // initialize original index locations - std::vector idx(v.size()); - iota(idx.begin(), idx.end(), 0); - - // sort indexes based on comparing values in v - sort(idx.begin(), idx.end(), - [&v](size_t i1, size_t i2) {return v[i1] > v[i2];}); - - return idx; -} - -float _colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} }; -float get_color(int c, int x, int max) -{ - float ratio = ((float)x/max)*5; - int i = floor(ratio); - int j = ceil(ratio); - ratio -= i; - float r = (1-ratio) * _colors[i % 6][c % 3] + ratio*_colors[j % 6][c % 3]; - //printf("%f\n", r); - return r; -} - -int computeDetections(dnnType *hm_d, dnnType *wh_d, dnnType *reg_d, int hm_dim, int wh_dim, int reg_dim, bool cat_spec_wh, int k){ - // _nms - int kernel = 3; - int pad = (kernel - 1)/2; - std::cout<<"computeDetections\n"; - // dnnType *hmax; - // tk::dnn::Pooling maxpool(&hmax, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX) - // = (dnnType *) - - // net.functional.max_pool2d( - // heat, (kernel, kernel), stride=1, padding=pad) - // keep = (hmax == heat).float() - // return heat * keep -} - -int process(dnnType *hm_d, dnnType *wh_d, dnnType *reg_d, int hm_dim, int wh_dim, int reg_dim){ - std::cout<<"process\n"; - // computeDetections(hm_d, wh_d, reg_d, hm_dim, wh_dim, reg_dim, false, 100); -} - int main() { @@ -450,637 +403,5 @@ int main() checkResult(odim, rt_out, out); std::cout << "CUDNN vs TRT "; checkResult(odim, cudnn_out, rt_out); - } - - TIMER_START - - // -------- transofrm compose - cv::Mat imageOrig = cv::imread("/media/davide/DATA/shared_home/Projects/Professionale/repos/photo_2020-01-14_09-56-07.jpg"); - cv::Mat imageF; - imageOrig.convertTo(imageF, CV_32FC3, 1/255.0); - cv::Mat image; - cv::Size sz = imageF.size(); - std::cout<<"image: "<output_dim.tot()*sizeof(dnnType)) ); - - dnnType *rt_out[4]; - rt_out[0] = (dnnType *)netRT.buffersRT[1]; - rt_out[1] = (dnnType *)netRT.buffersRT[2]; - rt_out[2] = (dnnType *)netRT.buffersRT[3]; - rt_out[3] = (dnnType *)netRT.buffersRT[4]; - - // checkCuda( cudaMemcpy(hm_h, rt_out[0], hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"hm\n"; - hm->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<output_dim.tot()); - checkCuda( cudaDeviceSynchronize() ); - - - checkCuda( cudaMemcpy(hm_h, rt_out[0], hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"hm\n"; - hm->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<infer(hmax->input_dim.tot(), rt_out[0]); - // keep = (hmax == heat).float() - // return heat * keep - - dnnType *hmax_h; - checkCuda( cudaMallocHost(&hmax_h, hmax->output_dim.tot()*sizeof(dnnType)) ); - checkCuda( cudaMemcpy(hmax_h, rt_out[1], hmax->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - - std::cout<<"hmax\n"; - hmax->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<output_dim.print(); - std::cout<<"hmax:\n"; - hmax->output_dim.print(); - // return 0; - float toll = 0.000001; - for(int i=0; i < hm->output_dim.tot(); i++){ - if(hm_h[i]-hmax_h[i] > toll || hm_h[i]-hmax_h[i] < -toll){ - hm_h[i] = 0.0f; - } - } - // std::cout<<"\n"; - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<dstData, hm_h, hm->output_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice) ); - checkCuda( cudaFreeHost(hmax_h) ); - // ----------- nms end - // ----------- topk - int K = 100; - int width = 56; // TODO - float *topk_scores; - int *topk_inds_; - float *topk_ys_; - float *topk_xs_; - std::cout<<"mah: "<output_dim.c * K<output_dim.c * K *sizeof(float)) ); - checkCuda( cudaMallocHost(&topk_inds_, hm->output_dim.c * K *sizeof(int)) ); - checkCuda( cudaMallocHost(&topk_ys_, hm->output_dim.c * K *sizeof(float)) ); - checkCuda( cudaMallocHost(&topk_xs_, hm->output_dim.c * K *sizeof(float)) ); - std::cout<<"1\n"; - dnnType *hm_aus; - checkCuda( cudaMallocHost(&hm_aus, hm->output_dim.h * hm->output_dim.w *sizeof(dnnType)) ); - std::cout<<"2\n"; - int count; - std::vector v = {2.0, 3.0, 9.0}; - for (auto i: sort_indexes(v)) { - std::cout << i<< "--" <output_dim.c; i++){ - count = 0; - // get the hm->output_dim.h * hm->output_dim.w elements for each channel and sort it. Then find the first 100 elements - checkCuda( cudaMemcpy(hm_aus, hm_h + i * hm->output_dim.h * hm->output_dim.w, - hm->output_dim.h * hm->output_dim.w * sizeof(dnnType), cudaMemcpyHostToHost) ); - // std::cout<<"top scores: "<output_dim.h * hm->output_dim.w<<"\n"; - // for(int k=0; koutput_dim.h * hm->output_dim.w; k++) - // std::cout< my_vector {arr, arr + arr_length} - std::vector my_vector{hm_aus, hm_aus + hm->output_dim.h * hm->output_dim.w}; - for (auto j: sort_indexes(my_vector)) { - // std::cout <<"j: "< "<< hm_aus[j] << std::endl; - topk_scores[i*K + count] = hm_aus[j]; - topk_inds_[i*K +count] = j; - topk_ys_[i*K +count] = (int)(j / width); - topk_xs_[i*K +count] = (int)(j % width); - if(++count == K) - break; - } - } - std::cout<<"topk_xs_[0]: "<output_dim.c * K; i++) - std::cout< my_vector{topk_scores, topk_scores + hm->output_dim.c * K }; - for (auto j: sort_indexes(my_vector)) { - // std::cout <<"j: "< "<< hm_aus[j] << std::endl; - scores[count] = topk_scores[j]; - clses[count] = (int)(j / K); - topk_inds[count] = topk_inds_[j]; - topk_ys[count] = topk_ys_[j]; - topk_xs[count] = topk_xs_[j]; - if(++count == K) - break; - } - checkCuda( cudaFreeHost(topk_scores) ); - checkCuda( cudaFreeHost(topk_inds_) ); - checkCuda( cudaFreeHost(topk_ys_) ); - checkCuda( cudaFreeHost(topk_xs_) ); - std::cout<<"5\n"; - // ----------- topk end - std::cout<<"topk_xs[0]: "<output_dim.tot()*sizeof(dnnType)) ); - checkCuda( cudaMemcpy(reg_aus, rt_out[3], reg->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"reg:\n"; - reg->output_dim.print(); - // for(int i=0; ioutput_dim.tot(); i++ ){ - // std::cout<output_dim.h*reg->output_dim.w]; - } - std::cout<<"topk_xs[0]: "<output_dim.tot()*sizeof(dnnType)) ); - checkCuda( cudaMallocHost(&bboxes, 4 * K *sizeof(dnnType)) ); - checkCuda( cudaMemcpy(wh_aus, rt_out[2], wh->output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); - std::cout<<"7\n"; - for(int i = 0; i< K; i++) - std::cout<output_dim.h*reg->output_dim.w] / 2; - bboxes[i * 4 + 2] = topk_xs[i] + wh_aus[topk_inds[i]] / 2; - bboxes[i * 4 + 3] = topk_ys[i] + wh_aus[topk_inds[i]+reg->output_dim.h*reg->output_dim.w] / 2; - } - ////////////////// fin qui ok - - checkCuda( cudaFreeHost(wh_aus) ); - checkCuda( cudaFreeHost(topk_inds) ); - checkCuda( cudaFreeHost(topk_ys) ); - checkCuda( cudaFreeHost(topk_xs) ); - - std::cout<<"8\n"; - float *detections; - std::cout<<"bboxes:\n"; - for(int i = 0; i < K+1; i++){ - std::cout<(0,0)=c[0]; - src.at(0,1)=c[1]; - src.at(1,0)=c[0]; - src.at(1,1)=c[1] + s[0] * -0.5; - dst.at(0,0)=width * 0.5; - dst.at(0,1)=width * 0.5; - dst.at(1,0)=width * 0.5; - dst.at(1,1)=width * 0.5 + width * -0.5; - - src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); - src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); - dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); - dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); - std::cout<<"src: "<(0,0)<<" - "<(0,1)<<" - "<(0,2)<<"\n"<(1,0)<<" - "<(1,1)<<" - "<(1,2)<(0,0)=detections[i*4]; - // new_pt1.at(0,1)=detections[i*4+1]; - // new_pt1.at(0,2)=1.0; - // new_pt1 << detections[i], detections[i+K], 1.0; - // std::cout<<"----\ni: "<(0,0)=static_cast(trans2.at(0,0))*detections[i*4] + - static_cast(trans2.at(0,1))*detections[i*4+1] + - static_cast(trans2.at(0,2))*1.0; - new_pt1.at(0,1)=static_cast(trans2.at(1,0))*detections[i*4] + - static_cast(trans2.at(1,1))*detections[i*4+1] + - static_cast(trans2.at(1,2))*1.0; - - new_pt2.at(0,0)=static_cast(trans2.at(0,0))*detections[i*4+2] + - static_cast(trans2.at(0,1))*detections[i*4+3] + - static_cast(trans2.at(0,2))*1.0; - new_pt2.at(0,1)=static_cast(trans2.at(1,0))*detections[i*4+2] + - static_cast(trans2.at(1,1))*detections[i*4+3] + - static_cast(trans2.at(1,2))*1.0; - - - // std::cout<<"\n new: "<(0,0); - target_coords[i*4+1] = new_pt1.at(0,1); - target_coords[i*4+2] = new_pt2.at(0,0); - target_coords[i*4+3] = new_pt2.at(0,1); - // std::cout<(0,0)<<", "<(0,1)<<", "<(0,0)<<", "<(0,1)< coco_class_name(coco_class_name_, std::end( coco_class_name_ )); - int num_classes = 80; - float vis_threshold = 0.3; - // int *classes; - std::vector detected; - // checkCuda( cudaMallocHost(&classes, K *sizeof(int)) ); - // checkCuda( cudaMemcpy(classes, detections + 5 * K *sizeof(dnnType), K *sizeof(dnnType), cudaMemcpyHostToHost) ); - for(int i = 0; i i+1 (1:80); - - if(scores[j] > vis_threshold){ - std::cout<<"th: "< Date: Fri, 24 Jan 2020 19:30:32 +0100 Subject: [PATCH 009/482] Implement DLA34, CUDNN and TensorRT work. Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/dla34/dla34.cpp | 350 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 tests/dla34/dla34.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b847d0..b5e24df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,9 @@ target_link_libraries(test_resnet101 tkDNN) add_executable(test_resnet101_cnet tests/resnet101_cnet/resnet101_cnet.cpp) target_link_libraries(test_resnet101_cnet tkDNN) +add_executable(test_dla34 tests/dla34/dla34.cpp) +target_link_libraries(test_dla34 tkDNN) + ################################################################################ diff --git a/tests/dla34/dla34.cpp b/tests/dla34/dla34.cpp new file mode 100644 index 0000000..15f7b3f --- /dev/null +++ b/tests/dla34/dla34.cpp @@ -0,0 +1,350 @@ +#include +#include "tkdnn.h" + +const char *input_bin = "../tests/dla34/debug/input.bin"; +const char *conv1_bin = "../tests/dla34/layers/features-init_block-conv1-conv.bin"; +const char *conv2_bin = "../tests/dla34/layers/features-init_block-conv2-conv.bin"; +const char *conv3_bin = "../tests/dla34/layers/features-init_block-conv3-conv.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "../tests/dla34/layers/features-stage1-tree1-body-conv1-conv.bin"; +const char *s1_t1_conv2_bin = "../tests/dla34/layers/features-stage1-tree1-body-conv2-conv.bin"; +const char *s1_t1_project = "../tests/dla34/layers/features-stage1-tree1-project_conv-conv.bin"; +const char *s1_t2_conv1_bin = "../tests/dla34/layers/features-stage1-tree2-body-conv1-conv.bin"; +const char *s1_t2_conv2_bin = "../tests/dla34/layers/features-stage1-tree2-body-conv2-conv.bin"; +const char *s1_root_conv1_bin = "../tests/dla34/layers/features-stage1-root-conv-conv.bin"; +const char *s2_t1_t1_conv1_bin = "../tests/dla34/layers/features-stage2-tree1-tree1-body-conv1-conv.bin"; +const char *s2_t1_t1_conv2_bin = "../tests/dla34/layers/features-stage2-tree1-tree1-body-conv2-conv.bin"; +const char *s2_t1_t1_project = "../tests/dla34/layers/features-stage2-tree1-tree1-project_conv-conv.bin"; +const char *s2_t1_t2_conv1_bin = "../tests/dla34/layers/features-stage2-tree1-tree2-body-conv1-conv.bin"; +const char *s2_t1_t2_conv2_bin = "../tests/dla34/layers/features-stage2-tree1-tree2-body-conv2-conv.bin"; +const char *s2_t1_root_conv1_bin = "../tests/dla34/layers/features-stage2-tree1-root-conv-conv.bin"; +const char *s2_t2_t1_conv1_bin = "../tests/dla34/layers/features-stage2-tree2-tree1-body-conv1-conv.bin"; +const char *s2_t2_t1_conv2_bin = "../tests/dla34/layers/features-stage2-tree2-tree1-body-conv2-conv.bin"; +const char *s2_t2_t2_conv1_bin = "../tests/dla34/layers/features-stage2-tree2-tree2-body-conv1-conv.bin"; +const char *s2_t2_t2_conv2_bin = "../tests/dla34/layers/features-stage2-tree2-tree2-body-conv2-conv.bin"; +const char *s2_t2_root_conv1_bin = "../tests/dla34/layers/features-stage2-tree2-root-conv-conv.bin"; +const char *s3_t1_t1_conv1_bin = "../tests/dla34/layers/features-stage3-tree1-tree1-body-conv1-conv.bin"; +const char *s3_t1_t1_conv2_bin = "../tests/dla34/layers/features-stage3-tree1-tree1-body-conv2-conv.bin"; +const char *s3_t1_t1_project = "../tests/dla34/layers/features-stage3-tree1-tree1-project_conv-conv.bin"; +const char *s3_t1_t2_conv1_bin = "../tests/dla34/layers/features-stage3-tree1-tree2-body-conv1-conv.bin"; +const char *s3_t1_t2_conv2_bin = "../tests/dla34/layers/features-stage3-tree1-tree2-body-conv2-conv.bin"; +const char *s3_t1_root_conv1_bin = "../tests/dla34/layers/features-stage3-tree1-root-conv-conv.bin"; +const char *s3_t2_t1_conv1_bin = "../tests/dla34/layers/features-stage3-tree2-tree1-body-conv1-conv.bin"; +const char *s3_t2_t1_conv2_bin = "../tests/dla34/layers/features-stage3-tree2-tree1-body-conv2-conv.bin"; +const char *s3_t2_t2_conv1_bin = "../tests/dla34/layers/features-stage3-tree2-tree2-body-conv1-conv.bin"; +const char *s3_t2_t2_conv2_bin = "../tests/dla34/layers/features-stage3-tree2-tree2-body-conv2-conv.bin"; +const char *s3_t2_root_conv1_bin = "../tests/dla34/layers/features-stage3-tree2-root-conv-conv.bin"; +const char *s4_t1_conv1_bin = "../tests/dla34/layers/features-stage4-tree1-body-conv1-conv.bin"; +const char *s4_t1_conv2_bin = "../tests/dla34/layers/features-stage4-tree1-body-conv2-conv.bin"; +const char *s4_t1_project = "../tests/dla34/layers/features-stage4-tree1-project_conv-conv.bin"; +const char *s4_t2_conv1_bin = "../tests/dla34/layers/features-stage4-tree2-body-conv1-conv.bin"; +const char *s4_t2_conv2_bin = "../tests/dla34/layers/features-stage4-tree2-body-conv2-conv.bin"; +const char *s4_root_conv1_bin = "../tests/dla34/layers/features-stage4-root-conv-conv.bin"; + +//final +const char *fc_bin = "../tests/dla34/layers/output.bin"; + +const char *output_bin = "../tests/dla34/debug/output.bin"; + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 224, 224, 1); + tk::dnn::Network net(dim); + tk::dnn::Layer *last1, *last2, *last3, *last4; + + + tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + + last1 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { last1 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_root_relu; + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { last1 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + + last1 = &s2_t2_root_relu; + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { last1 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_root_relu; + // level 4 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { last1 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + //final + tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + tk::dnn::Dense fc(&net, 1000, fc_bin); + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, "dla34.rt"); + + + tk::dnn::dataDim_t out_dim; + out_dim = net.layers[net.num_layers-1]->output_dim; + dnnType *cudnn_out, *rt_out; + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + cudnn_out = net.layers[net.num_layers-1]->dstData; + + + // printDeviceVector(64, cudnn_out, true); + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + rt_out = (dnnType *)netRT.buffersRT[1]; + + + printCenteredTitle(std::string(" RESNET CHECK RESULTS ").c_str(), '=', 30); + dnnType *out, *out_h; + int odim = out_dim.tot(); + readBinaryFile(output_bin, odim, &out_h, &out); + + std::cout << "CUDNN vs correct"; + checkResult(odim, cudnn_out, out); + std::cout << "TRT vs correct"; + checkResult(odim, rt_out, out); + std::cout << "CUDNN vs TRT "; + checkResult(odim, cudnn_out, rt_out); + + return 0; +} -- 2.52.0 From fe85c2688831f255601f9f62be55a821545886af Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 24 Jan 2020 19:30:32 +0100 Subject: [PATCH 010/482] Implement DLA34, CUDNN and TensorRT work. Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/dla34/dla34.cpp | 350 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 tests/dla34/dla34.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b847d0..b5e24df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,9 @@ target_link_libraries(test_resnet101 tkDNN) add_executable(test_resnet101_cnet tests/resnet101_cnet/resnet101_cnet.cpp) target_link_libraries(test_resnet101_cnet tkDNN) +add_executable(test_dla34 tests/dla34/dla34.cpp) +target_link_libraries(test_dla34 tkDNN) + ################################################################################ diff --git a/tests/dla34/dla34.cpp b/tests/dla34/dla34.cpp new file mode 100644 index 0000000..15f7b3f --- /dev/null +++ b/tests/dla34/dla34.cpp @@ -0,0 +1,350 @@ +#include +#include "tkdnn.h" + +const char *input_bin = "../tests/dla34/debug/input.bin"; +const char *conv1_bin = "../tests/dla34/layers/features-init_block-conv1-conv.bin"; +const char *conv2_bin = "../tests/dla34/layers/features-init_block-conv2-conv.bin"; +const char *conv3_bin = "../tests/dla34/layers/features-init_block-conv3-conv.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "../tests/dla34/layers/features-stage1-tree1-body-conv1-conv.bin"; +const char *s1_t1_conv2_bin = "../tests/dla34/layers/features-stage1-tree1-body-conv2-conv.bin"; +const char *s1_t1_project = "../tests/dla34/layers/features-stage1-tree1-project_conv-conv.bin"; +const char *s1_t2_conv1_bin = "../tests/dla34/layers/features-stage1-tree2-body-conv1-conv.bin"; +const char *s1_t2_conv2_bin = "../tests/dla34/layers/features-stage1-tree2-body-conv2-conv.bin"; +const char *s1_root_conv1_bin = "../tests/dla34/layers/features-stage1-root-conv-conv.bin"; +const char *s2_t1_t1_conv1_bin = "../tests/dla34/layers/features-stage2-tree1-tree1-body-conv1-conv.bin"; +const char *s2_t1_t1_conv2_bin = "../tests/dla34/layers/features-stage2-tree1-tree1-body-conv2-conv.bin"; +const char *s2_t1_t1_project = "../tests/dla34/layers/features-stage2-tree1-tree1-project_conv-conv.bin"; +const char *s2_t1_t2_conv1_bin = "../tests/dla34/layers/features-stage2-tree1-tree2-body-conv1-conv.bin"; +const char *s2_t1_t2_conv2_bin = "../tests/dla34/layers/features-stage2-tree1-tree2-body-conv2-conv.bin"; +const char *s2_t1_root_conv1_bin = "../tests/dla34/layers/features-stage2-tree1-root-conv-conv.bin"; +const char *s2_t2_t1_conv1_bin = "../tests/dla34/layers/features-stage2-tree2-tree1-body-conv1-conv.bin"; +const char *s2_t2_t1_conv2_bin = "../tests/dla34/layers/features-stage2-tree2-tree1-body-conv2-conv.bin"; +const char *s2_t2_t2_conv1_bin = "../tests/dla34/layers/features-stage2-tree2-tree2-body-conv1-conv.bin"; +const char *s2_t2_t2_conv2_bin = "../tests/dla34/layers/features-stage2-tree2-tree2-body-conv2-conv.bin"; +const char *s2_t2_root_conv1_bin = "../tests/dla34/layers/features-stage2-tree2-root-conv-conv.bin"; +const char *s3_t1_t1_conv1_bin = "../tests/dla34/layers/features-stage3-tree1-tree1-body-conv1-conv.bin"; +const char *s3_t1_t1_conv2_bin = "../tests/dla34/layers/features-stage3-tree1-tree1-body-conv2-conv.bin"; +const char *s3_t1_t1_project = "../tests/dla34/layers/features-stage3-tree1-tree1-project_conv-conv.bin"; +const char *s3_t1_t2_conv1_bin = "../tests/dla34/layers/features-stage3-tree1-tree2-body-conv1-conv.bin"; +const char *s3_t1_t2_conv2_bin = "../tests/dla34/layers/features-stage3-tree1-tree2-body-conv2-conv.bin"; +const char *s3_t1_root_conv1_bin = "../tests/dla34/layers/features-stage3-tree1-root-conv-conv.bin"; +const char *s3_t2_t1_conv1_bin = "../tests/dla34/layers/features-stage3-tree2-tree1-body-conv1-conv.bin"; +const char *s3_t2_t1_conv2_bin = "../tests/dla34/layers/features-stage3-tree2-tree1-body-conv2-conv.bin"; +const char *s3_t2_t2_conv1_bin = "../tests/dla34/layers/features-stage3-tree2-tree2-body-conv1-conv.bin"; +const char *s3_t2_t2_conv2_bin = "../tests/dla34/layers/features-stage3-tree2-tree2-body-conv2-conv.bin"; +const char *s3_t2_root_conv1_bin = "../tests/dla34/layers/features-stage3-tree2-root-conv-conv.bin"; +const char *s4_t1_conv1_bin = "../tests/dla34/layers/features-stage4-tree1-body-conv1-conv.bin"; +const char *s4_t1_conv2_bin = "../tests/dla34/layers/features-stage4-tree1-body-conv2-conv.bin"; +const char *s4_t1_project = "../tests/dla34/layers/features-stage4-tree1-project_conv-conv.bin"; +const char *s4_t2_conv1_bin = "../tests/dla34/layers/features-stage4-tree2-body-conv1-conv.bin"; +const char *s4_t2_conv2_bin = "../tests/dla34/layers/features-stage4-tree2-body-conv2-conv.bin"; +const char *s4_root_conv1_bin = "../tests/dla34/layers/features-stage4-root-conv-conv.bin"; + +//final +const char *fc_bin = "../tests/dla34/layers/output.bin"; + +const char *output_bin = "../tests/dla34/debug/output.bin"; + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 224, 224, 1); + tk::dnn::Network net(dim); + tk::dnn::Layer *last1, *last2, *last3, *last4; + + + tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + + last1 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { last1 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_root_relu; + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { last1 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + + last1 = &s2_t2_root_relu; + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { last1 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_root_relu; + // level 4 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { last1 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + //final + tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + tk::dnn::Dense fc(&net, 1000, fc_bin); + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, "dla34.rt"); + + + tk::dnn::dataDim_t out_dim; + out_dim = net.layers[net.num_layers-1]->output_dim; + dnnType *cudnn_out, *rt_out; + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + cudnn_out = net.layers[net.num_layers-1]->dstData; + + + // printDeviceVector(64, cudnn_out, true); + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + rt_out = (dnnType *)netRT.buffersRT[1]; + + + printCenteredTitle(std::string(" RESNET CHECK RESULTS ").c_str(), '=', 30); + dnnType *out, *out_h; + int odim = out_dim.tot(); + readBinaryFile(output_bin, odim, &out_h, &out); + + std::cout << "CUDNN vs correct"; + checkResult(odim, cudnn_out, out); + std::cout << "TRT vs correct"; + checkResult(odim, rt_out, out); + std::cout << "CUDNN vs TRT "; + checkResult(odim, cudnn_out, rt_out); + + return 0; +} -- 2.52.0 From 0d6acaec41da521f2969c58222a50685296aae55 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 5 Feb 2020 14:32:58 +0100 Subject: [PATCH 011/482] Add grouped convolutions in CUDNN and tensorRT. Signed-off-by: Davide Sapienza --- include/tkDNN/Layer.h | 9 +++++---- src/Conv2d.cpp | 16 +++++++++------- src/DeformConv2d.cpp | 4 ++-- src/LayerWgs.cpp | 9 +++++++-- src/NetworkRT.cpp | 2 ++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 2d84a0e..7716393 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -85,7 +85,7 @@ class LayerWgs : public Layer { public: LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt, - std::string fname_weights, bool batchnorm = false, bool additional_bias = false, bool final = false); + std::string fname_weights, bool batchnorm = false, bool additional_bias = false, bool final = false, bool deConv = false, int groups = 1); virtual ~LayerWgs(); int inputs, outputs; @@ -165,7 +165,7 @@ class Conv2d : public LayerWgs { public: Conv2d( Network *net, int out_ch, int kernelH, int kernelW, int strideH, int strideW, int paddingH, int paddingW, - std::string fname_weights, bool batchnorm = false, bool deConv = false, bool final = false); + std::string fname_weights, bool batchnorm = false, bool deConv = false, bool final = false, int groups = 1); virtual ~Conv2d(); virtual layerType_t getLayerType() { return LAYER_CONV2D; }; @@ -173,6 +173,7 @@ public: int kernelH, kernelW, strideH, strideW, paddingH, paddingW; bool deConv; + int groups; protected: cudnnFilterDescriptor_t filterDesc; @@ -196,8 +197,8 @@ class DeConv2d : public Conv2d { public: DeConv2d( Network *net, int out_ch, int kernelH, int kernelW, int strideH, int strideW, int paddingH, int paddingW, - std::string fname_weights, bool batchnorm = false) : - Conv2d(net, out_ch, kernelH, kernelW, strideH, strideW, paddingH, paddingW, fname_weights, batchnorm, true) {} + std::string fname_weights, bool batchnorm = false, int groups = 1) : + Conv2d(net, out_ch, kernelH, kernelW, strideH, strideW, paddingH, paddingW, fname_weights, batchnorm, true, false, groups) {} virtual ~DeConv2d() {} virtual layerType_t getLayerType() { return LAYER_DECONV2D; }; diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index f984f67..18e907d 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -17,8 +17,6 @@ void Conv2d::initCUDNN(bool back) { idim = output_dim; odim = input_dim; } - //idim.print(); - //odim.print(); checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) ); checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) ); @@ -29,7 +27,7 @@ void Conv2d::initCUDNN(bool back) { net->tensorFormat, net->dataType, idim.n, idim.c, idim.h, idim.w) ); checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc, - net->dataType, net->tensorFormat, odim.c, idim.c, + net->dataType, net->tensorFormat, odim.c, idim.c/groups, kernelH, kernelW) ); checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc, @@ -38,16 +36,20 @@ void Conv2d::initCUDNN(bool back) { 1,1, // upscale CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) ); + checkCUDNN( cudnnSetConvolutionGroupCount(convDesc, + groups) ); + // check dimension of convolution output dataDim_t tmpdim; checkCUDNN( cudnnGetConvolution2dForwardOutputDim( convDesc, srcTensor, filterDesc, &tmpdim.n, &tmpdim.c, &tmpdim.h, &tmpdim.w) ); + if(odim.n != tmpdim.n || odim.c != tmpdim.c || odim.h != tmpdim.h || odim.w != tmpdim.w) { std::cout<<"tkdim input: "; idim.print(); std::cout<<"tkdim output: "; odim.print(); std::cout<<"cudnndim: "; tmpdim.print(); - FatalError("Eror conv dimension mismatch"); + FatalError("Error conv dimension mismatch"); } checkCUDNN( cudnnSetTensor4dDescriptor(dstTensor, @@ -119,11 +121,10 @@ void Conv2d::inferCUDNN(dnnType* srcData, bool back) { Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, int strideH, int strideW, int paddingH, int paddingW, - std::string fname_weights, bool batchnorm, bool deConv, bool final) : + std::string fname_weights, bool batchnorm, bool deConv, bool final, int groups) : LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, - fname_weights, batchnorm, false, final) { - + fname_weights, batchnorm, false, final, deConv, groups) { this->kernelH = kernelH; this->kernelW = kernelW; this->strideH = strideH; @@ -131,6 +132,7 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, this->paddingH = paddingH; this->paddingW = paddingW; this->deConv = deConv; + this->groups = groups; if(!deConv) { output_dim.n = input_dim.n; diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index 8997593..40b6b9e 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -76,8 +76,8 @@ DeformConv2d::~DeformConv2d() { checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) ); checkCuda( cudaFree(dstData) ); - checkCuda( cudaFreeHost(ones_d1) ); - checkCuda( cudaFreeHost(ones_d2) ); + checkCuda( cudaFree(ones_d1) ); + checkCuda( cudaFree(ones_d2) ); checkCuda( cudaFree(offset) ); checkCuda( cudaFree(mask) ); checkCuda( cudaFree(output_conv) ); diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index a18fd53..41ca038 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -8,8 +8,13 @@ namespace tk { namespace dnn { LayerWgs::LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kl, - std::string fname_weights, bool batchnorm, bool additional_bias, bool final) : Layer(net, final) { - + std::string fname_weights, bool batchnorm, bool additional_bias, bool final, bool deConv, int groups) : Layer(net, final) { + + if(deConv) + inputs = inputs/groups; + else + outputs = outputs/groups; + this->inputs = inputs; this->outputs = outputs; this->weights_path = std::string(fname_weights); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 0f2d621..2b7377d 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -245,6 +245,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { checkNULL(lRTconv); lRTconv->setStride(DimsHW{l->strideH, l->strideW}); lRTconv->setPadding(DimsHW{l->paddingH, l->paddingW}); + lRTconv->setNbGroups(l->groups); lRT = (ILayer*) lRTconv; } else { IDeconvolutionLayer *lRTconv = networkRT->addDeconvolution(*input, @@ -252,6 +253,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { checkNULL(lRTconv); lRTconv->setStride(DimsHW{l->strideH, l->strideW}); lRTconv->setPadding(DimsHW{l->paddingH, l->paddingW}); + lRTconv->setNbGroups(l->groups); lRT = (ILayer*) lRTconv; Dims d = lRTconv->getOutput(0)->getDimensions(); -- 2.52.0 From 4616be073831cecccce0ca2ce6852eac9b4e9e96 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 5 Feb 2020 14:32:58 +0100 Subject: [PATCH 012/482] Add grouped convolutions in CUDNN and tensorRT. Signed-off-by: Davide Sapienza --- include/tkDNN/Layer.h | 9 +++++---- src/Conv2d.cpp | 16 +++++++++------- src/DeformConv2d.cpp | 4 ++-- src/LayerWgs.cpp | 9 +++++++-- src/NetworkRT.cpp | 2 ++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 2d84a0e..7716393 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -85,7 +85,7 @@ class LayerWgs : public Layer { public: LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt, - std::string fname_weights, bool batchnorm = false, bool additional_bias = false, bool final = false); + std::string fname_weights, bool batchnorm = false, bool additional_bias = false, bool final = false, bool deConv = false, int groups = 1); virtual ~LayerWgs(); int inputs, outputs; @@ -165,7 +165,7 @@ class Conv2d : public LayerWgs { public: Conv2d( Network *net, int out_ch, int kernelH, int kernelW, int strideH, int strideW, int paddingH, int paddingW, - std::string fname_weights, bool batchnorm = false, bool deConv = false, bool final = false); + std::string fname_weights, bool batchnorm = false, bool deConv = false, bool final = false, int groups = 1); virtual ~Conv2d(); virtual layerType_t getLayerType() { return LAYER_CONV2D; }; @@ -173,6 +173,7 @@ public: int kernelH, kernelW, strideH, strideW, paddingH, paddingW; bool deConv; + int groups; protected: cudnnFilterDescriptor_t filterDesc; @@ -196,8 +197,8 @@ class DeConv2d : public Conv2d { public: DeConv2d( Network *net, int out_ch, int kernelH, int kernelW, int strideH, int strideW, int paddingH, int paddingW, - std::string fname_weights, bool batchnorm = false) : - Conv2d(net, out_ch, kernelH, kernelW, strideH, strideW, paddingH, paddingW, fname_weights, batchnorm, true) {} + std::string fname_weights, bool batchnorm = false, int groups = 1) : + Conv2d(net, out_ch, kernelH, kernelW, strideH, strideW, paddingH, paddingW, fname_weights, batchnorm, true, false, groups) {} virtual ~DeConv2d() {} virtual layerType_t getLayerType() { return LAYER_DECONV2D; }; diff --git a/src/Conv2d.cpp b/src/Conv2d.cpp index f984f67..18e907d 100644 --- a/src/Conv2d.cpp +++ b/src/Conv2d.cpp @@ -17,8 +17,6 @@ void Conv2d::initCUDNN(bool back) { idim = output_dim; odim = input_dim; } - //idim.print(); - //odim.print(); checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) ); checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) ); @@ -29,7 +27,7 @@ void Conv2d::initCUDNN(bool back) { net->tensorFormat, net->dataType, idim.n, idim.c, idim.h, idim.w) ); checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc, - net->dataType, net->tensorFormat, odim.c, idim.c, + net->dataType, net->tensorFormat, odim.c, idim.c/groups, kernelH, kernelW) ); checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc, @@ -38,16 +36,20 @@ void Conv2d::initCUDNN(bool back) { 1,1, // upscale CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) ); + checkCUDNN( cudnnSetConvolutionGroupCount(convDesc, + groups) ); + // check dimension of convolution output dataDim_t tmpdim; checkCUDNN( cudnnGetConvolution2dForwardOutputDim( convDesc, srcTensor, filterDesc, &tmpdim.n, &tmpdim.c, &tmpdim.h, &tmpdim.w) ); + if(odim.n != tmpdim.n || odim.c != tmpdim.c || odim.h != tmpdim.h || odim.w != tmpdim.w) { std::cout<<"tkdim input: "; idim.print(); std::cout<<"tkdim output: "; odim.print(); std::cout<<"cudnndim: "; tmpdim.print(); - FatalError("Eror conv dimension mismatch"); + FatalError("Error conv dimension mismatch"); } checkCUDNN( cudnnSetTensor4dDescriptor(dstTensor, @@ -119,11 +121,10 @@ void Conv2d::inferCUDNN(dnnType* srcData, bool back) { Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, int strideH, int strideW, int paddingH, int paddingW, - std::string fname_weights, bool batchnorm, bool deConv, bool final) : + std::string fname_weights, bool batchnorm, bool deConv, bool final, int groups) : LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, - fname_weights, batchnorm, false, final) { - + fname_weights, batchnorm, false, final, deConv, groups) { this->kernelH = kernelH; this->kernelW = kernelW; this->strideH = strideH; @@ -131,6 +132,7 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, this->paddingH = paddingH; this->paddingW = paddingW; this->deConv = deConv; + this->groups = groups; if(!deConv) { output_dim.n = input_dim.n; diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index 8997593..40b6b9e 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -76,8 +76,8 @@ DeformConv2d::~DeformConv2d() { checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) ); checkCuda( cudaFree(dstData) ); - checkCuda( cudaFreeHost(ones_d1) ); - checkCuda( cudaFreeHost(ones_d2) ); + checkCuda( cudaFree(ones_d1) ); + checkCuda( cudaFree(ones_d2) ); checkCuda( cudaFree(offset) ); checkCuda( cudaFree(mask) ); checkCuda( cudaFree(output_conv) ); diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index a18fd53..41ca038 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -8,8 +8,13 @@ namespace tk { namespace dnn { LayerWgs::LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kl, - std::string fname_weights, bool batchnorm, bool additional_bias, bool final) : Layer(net, final) { - + std::string fname_weights, bool batchnorm, bool additional_bias, bool final, bool deConv, int groups) : Layer(net, final) { + + if(deConv) + inputs = inputs/groups; + else + outputs = outputs/groups; + this->inputs = inputs; this->outputs = outputs; this->weights_path = std::string(fname_weights); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 0f2d621..2b7377d 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -245,6 +245,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { checkNULL(lRTconv); lRTconv->setStride(DimsHW{l->strideH, l->strideW}); lRTconv->setPadding(DimsHW{l->paddingH, l->paddingW}); + lRTconv->setNbGroups(l->groups); lRT = (ILayer*) lRTconv; } else { IDeconvolutionLayer *lRTconv = networkRT->addDeconvolution(*input, @@ -252,6 +253,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { checkNULL(lRTconv); lRTconv->setStride(DimsHW{l->strideH, l->strideW}); lRTconv->setPadding(DimsHW{l->paddingH, l->paddingW}); + lRTconv->setNbGroups(l->groups); lRT = (ILayer*) lRTconv; Dims d = lRTconv->getOutput(0)->getDimensions(); -- 2.52.0 From 507d30bf5214935c0868efefb3e8ab3be11e3c9b Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 5 Feb 2020 14:38:26 +0100 Subject: [PATCH 013/482] Implement CenterNet based on DLA34, CUDNN and TensorRT work. Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/dla34_cnet/dla34_cnet.cpp | 527 ++++++++++++++++++++++++++++++++ 2 files changed, 530 insertions(+) create mode 100644 tests/dla34_cnet/dla34_cnet.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b5e24df..e4a5b89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,9 @@ target_link_libraries(test_resnet101_cnet tkDNN) add_executable(test_dla34 tests/dla34/dla34.cpp) target_link_libraries(test_dla34 tkDNN) +add_executable(test_dla34_cnet tests/dla34_cnet/dla34_cnet.cpp) +target_link_libraries(test_dla34_cnet tkDNN) + ################################################################################ diff --git a/tests/dla34_cnet/dla34_cnet.cpp b/tests/dla34_cnet/dla34_cnet.cpp new file mode 100644 index 0000000..bccbc09 --- /dev/null +++ b/tests/dla34_cnet/dla34_cnet.cpp @@ -0,0 +1,527 @@ +#include +#include "tkdnn.h" + +const char *input_bin = "../tests/dla34_cnet/debug/input.bin"; +const char *conv1_bin = "../tests/dla34_cnet/layers/base-base_layer-0.bin"; +const char *conv2_bin = "../tests/dla34_cnet/layers/base-level0-0.bin"; +const char *conv3_bin = "../tests/dla34_cnet/layers/base-level1-0.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "../tests/dla34_cnet/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "../tests/dla34_cnet/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "../tests/dla34_cnet/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "../tests/dla34_cnet/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "../tests/dla34_cnet/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "../tests/dla34_cnet/layers/base-level5-root-conv.bin"; + +//final +// const char *fc_bin = "../tests/dla34_cnet/layers/output.bin"; + +const char *ida_0_p_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; + +const char *ida_1_p_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; + +const char *ida_2_p_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; + +const char *ida_up_p_1_dcn_bin = "../tests/dla34_cnet/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "../tests/dla34_cnet/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "../tests/dla34_cnet/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "../tests/dla34_cnet/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "../tests/dla34_cnet/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "../tests/dla34_cnet/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "../tests/dla34_cnet/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "../tests/dla34_cnet/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "../tests/dla34_cnet/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "../tests/dla34_cnet/layers/ida_up-node_2-conv-conv_offset_mask.bin"; + +const char *hm_conv1_bin = "../tests/dla34_cnet/layers/hm-0.bin"; +const char *hm_conv2_bin = "../tests/dla34_cnet/layers/hm-2.bin"; +const char *wh_conv1_bin = "../tests/dla34_cnet/layers/wh-0.bin"; +const char *wh_conv2_bin = "../tests/dla34_cnet/layers/wh-2.bin"; +const char *reg_conv1_bin = "../tests/dla34_cnet/layers/reg-0.bin"; +const char *reg_conv2_bin = "../tests/dla34_cnet/layers/reg-2.bin"; + +const char *output_bin[]={ +"../tests/dla34_cnet/debug/hm.bin", +"../tests/dla34_cnet/debug/wh.bin", +"../tests/dla34_cnet/debug/reg.bin"}; + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 224, 224, 1); + tk::dnn::Network net(dim); + tk::dnn::Layer *last1, *last2, *last3, *last4; + tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2; + + tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + base1 = &relu2; + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + base2 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { base2 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base3 = &s1_root_relu; + + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base4 = &s2_t2_root_relu; + + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base5 = &s3_t2_root_relu; + + // level 5 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { base5 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base6 = &s4_root_relu; + + //final + // tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + // tk::dnn::Dense fc(&net, 1000, fc_bin); + + //ida 0 + tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true); + tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256); + tk::dnn::Shortcut ida_0_shortcut(&net, base5); + tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true); + tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida1 = &ida_0_n_1_relu; + + //ida1-1 + tk::dnn::Layer *route_ida1_layers_1[1] = { base5 }; + tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1); + + tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true); + tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut1(&net, base4); + tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true); + tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_1 = &ida_1_n_1_relu; + + //ida1-2 + tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 }; + tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1); + + tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true); + tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1); + tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true); + tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_2 = &ida_1_n_2_relu; + + //ida2-1 + tk::dnn::Layer *route_ida2_layers_1[1] = { base4 }; + tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1); + + tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true); + tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut1(&net, base3); + tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true); + tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_1 = &ida_2_n_1_relu; + + //ida2-2 + tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 }; + tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1); + + tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true); + tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1); + tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true); + tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_2 = &ida_2_n_2_relu; + + //ida2-3 + tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 }; + tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1); + + tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true); + tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2); + tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true); + tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_3 = &ida_2_n_3_relu; + + //idaup-1 + tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 }; + tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1); + + tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true); + tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3); + tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true); + tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_1 = &idaup_n_1_relu; + + //idaup-2 + tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 }; + tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1); + + tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true); + tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1); + tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true); + tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_2 = &idaup_n_2_relu; + + tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 }; + + // hm + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 80, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false, false, true); + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX, true); + + // // wh + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false, false, true); + + // // reg + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false, false, true); + + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, "dla34_cnet.rt"); + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + tk::dnn::Layer *outs[3] = { hm, wh, reg }; + int out_count = 1; + for(int i=0; i<3; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout << "CUDNN vs correct"; + checkResult(odim, cudnn_out, out); + std::cout << "TRT vs correct"; + checkResult(odim, rt_out, out); + std::cout << "CUDNN vs TRT "; + checkResult(odim, cudnn_out, rt_out); + } + + return 0; +} -- 2.52.0 From c695d8c5d7a94e1dff531272eb8e9ccd7db42289 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Wed, 5 Feb 2020 14:38:26 +0100 Subject: [PATCH 014/482] Implement CenterNet based on DLA34, CUDNN and TensorRT work. Signed-off-by: Davide Sapienza --- CMakeLists.txt | 3 + tests/dla34_cnet/dla34_cnet.cpp | 527 ++++++++++++++++++++++++++++++++ 2 files changed, 530 insertions(+) create mode 100644 tests/dla34_cnet/dla34_cnet.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b5e24df..e4a5b89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,9 @@ target_link_libraries(test_resnet101_cnet tkDNN) add_executable(test_dla34 tests/dla34/dla34.cpp) target_link_libraries(test_dla34 tkDNN) +add_executable(test_dla34_cnet tests/dla34_cnet/dla34_cnet.cpp) +target_link_libraries(test_dla34_cnet tkDNN) + ################################################################################ diff --git a/tests/dla34_cnet/dla34_cnet.cpp b/tests/dla34_cnet/dla34_cnet.cpp new file mode 100644 index 0000000..bccbc09 --- /dev/null +++ b/tests/dla34_cnet/dla34_cnet.cpp @@ -0,0 +1,527 @@ +#include +#include "tkdnn.h" + +const char *input_bin = "../tests/dla34_cnet/debug/input.bin"; +const char *conv1_bin = "../tests/dla34_cnet/layers/base-base_layer-0.bin"; +const char *conv2_bin = "../tests/dla34_cnet/layers/base-level0-0.bin"; +const char *conv3_bin = "../tests/dla34_cnet/layers/base-level1-0.bin"; +// s - stage, t - tree +const char *s1_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level2-tree1-conv1.bin"; +const char *s1_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level2-tree1-conv2.bin"; +const char *s1_t1_project = "../tests/dla34_cnet/layers/base-level2-project-0.bin"; +const char *s1_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level2-tree2-conv1.bin"; +const char *s1_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level2-tree2-conv2.bin"; +const char *s1_root_conv1_bin = "../tests/dla34_cnet/layers/base-level2-root-conv.bin"; +const char *s2_t1_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree1-conv1.bin"; +const char *s2_t1_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree1-conv2.bin"; +const char *s2_t1_t1_project = "../tests/dla34_cnet/layers/base-level3-tree1-project-0.bin"; +const char *s2_t1_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree2-conv1.bin"; +const char *s2_t1_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree1-tree2-conv2.bin"; +const char *s2_t1_root_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree1-root-conv.bin"; +const char *s2_t2_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree1-conv1.bin"; +const char *s2_t2_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree1-conv2.bin"; +const char *s2_t2_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree2-conv1.bin"; +const char *s2_t2_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level3-tree2-tree2-conv2.bin"; +const char *s2_t2_root_conv1_bin = "../tests/dla34_cnet/layers/base-level3-tree2-root-conv.bin"; +const char *s3_t1_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree1-conv1.bin"; +const char *s3_t1_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree1-conv2.bin"; +const char *s3_t1_t1_project = "../tests/dla34_cnet/layers/base-level4-tree1-project-0.bin"; +const char *s3_t1_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree2-conv1.bin"; +const char *s3_t1_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree1-tree2-conv2.bin"; +const char *s3_t1_root_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree1-root-conv.bin"; +const char *s3_t2_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree1-conv1.bin"; +const char *s3_t2_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree1-conv2.bin"; +const char *s3_t2_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree2-conv1.bin"; +const char *s3_t2_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level4-tree2-tree2-conv2.bin"; +const char *s3_t2_root_conv1_bin = "../tests/dla34_cnet/layers/base-level4-tree2-root-conv.bin"; +const char *s4_t1_conv1_bin = "../tests/dla34_cnet/layers/base-level5-tree1-conv1.bin"; +const char *s4_t1_conv2_bin = "../tests/dla34_cnet/layers/base-level5-tree1-conv2.bin"; +const char *s4_t1_project = "../tests/dla34_cnet/layers/base-level5-project-0.bin"; +const char *s4_t2_conv1_bin = "../tests/dla34_cnet/layers/base-level5-tree2-conv1.bin"; +const char *s4_t2_conv2_bin = "../tests/dla34_cnet/layers/base-level5-tree2-conv2.bin"; +const char *s4_root_conv1_bin = "../tests/dla34_cnet/layers/base-level5-root-conv.bin"; + +//final +// const char *fc_bin = "../tests/dla34_cnet/layers/output.bin"; + +const char *ida_0_p_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-proj_1-conv.bin"; +const char *ida_0_p_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin"; +const char *ida_0_up_1_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-up_1.bin"; +const char *ida_0_n_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-node_1-conv.bin"; +const char *ida_0_n_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin"; + +const char *ida_1_p_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_1-conv.bin"; +const char *ida_1_p_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin"; +const char *ida_1_up_1_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-up_1.bin"; +const char *ida_1_n_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_1-conv.bin"; +const char *ida_1_n_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin"; +const char *ida_1_p_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_2-conv.bin"; +const char *ida_1_p_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin"; +const char *ida_1_up_2_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-up_2.bin"; +const char *ida_1_n_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_2-conv.bin"; +const char *ida_1_n_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin"; + +const char *ida_2_p_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_1-conv.bin"; +const char *ida_2_p_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin"; +const char *ida_2_up_1_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-up_1.bin"; +const char *ida_2_n_1_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_1-conv.bin"; +const char *ida_2_n_1_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin"; +const char *ida_2_p_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_2-conv.bin"; +const char *ida_2_p_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin"; +const char *ida_2_up_2_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-up_2.bin"; +const char *ida_2_n_2_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_2-conv.bin"; +const char *ida_2_n_2_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin"; +const char *ida_2_p_3_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_3-conv.bin"; +const char *ida_2_p_3_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin"; +const char *ida_2_up_3_deconv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-up_3.bin"; +const char *ida_2_n_3_dcn_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_3-conv.bin"; +const char *ida_2_n_3_conv_bin = "../tests/dla34_cnet/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin"; + +const char *ida_up_p_1_dcn_bin = "../tests/dla34_cnet/layers/ida_up-proj_1-conv.bin"; +const char *ida_up_p_1_conv_bin = "../tests/dla34_cnet/layers/ida_up-proj_1-conv-conv_offset_mask.bin"; +const char *ida_up_up_1_deconv_bin = "../tests/dla34_cnet/layers/ida_up-up_1.bin"; +const char *ida_up_n_1_dcn_bin = "../tests/dla34_cnet/layers/ida_up-node_1-conv.bin"; +const char *ida_up_n_1_conv_bin = "../tests/dla34_cnet/layers/ida_up-node_1-conv-conv_offset_mask.bin"; +const char *ida_up_p_2_dcn_bin = "../tests/dla34_cnet/layers/ida_up-proj_2-conv.bin"; +const char *ida_up_p_2_conv_bin = "../tests/dla34_cnet/layers/ida_up-proj_2-conv-conv_offset_mask.bin"; +const char *ida_up_up_2_deconv_bin = "../tests/dla34_cnet/layers/ida_up-up_2.bin"; +const char *ida_up_n_2_dcn_bin = "../tests/dla34_cnet/layers/ida_up-node_2-conv.bin"; +const char *ida_up_n_2_conv_bin = "../tests/dla34_cnet/layers/ida_up-node_2-conv-conv_offset_mask.bin"; + +const char *hm_conv1_bin = "../tests/dla34_cnet/layers/hm-0.bin"; +const char *hm_conv2_bin = "../tests/dla34_cnet/layers/hm-2.bin"; +const char *wh_conv1_bin = "../tests/dla34_cnet/layers/wh-0.bin"; +const char *wh_conv2_bin = "../tests/dla34_cnet/layers/wh-2.bin"; +const char *reg_conv1_bin = "../tests/dla34_cnet/layers/reg-0.bin"; +const char *reg_conv2_bin = "../tests/dla34_cnet/layers/reg-2.bin"; + +const char *output_bin[]={ +"../tests/dla34_cnet/debug/hm.bin", +"../tests/dla34_cnet/debug/wh.bin", +"../tests/dla34_cnet/debug/reg.bin"}; + +int main() +{ + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 224, 224, 1); + tk::dnn::Network net(dim); + tk::dnn::Layer *last1, *last2, *last3, *last4; + tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2; + + tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true); + tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true); + tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU); + base1 = &relu2; + + tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true); + tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU); + base2 = &relu3; + + // level 2 + // tree 1 + tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true); + tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true); + last2 = &s1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s1_t1_layers[1] = { base2 }; + tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1); + // downsample + tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + // project + tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true); + + tk::dnn::Shortcut s1_t1_s1(&net, last2); + tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s1_t1_relu; + // tree 2 + tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true); + tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true); + + tk::dnn::Shortcut s1_t2_s1(&net, last1); + tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2); + tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true); + tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base3 = &s1_root_relu; + + // level 3 + // tree 1 + // tree 1 + tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true); + tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true); + last2 = &s2_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 }; + tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s2_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true); + + tk::dnn::Shortcut s2_t1_t1_s1(&net, last2); + tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true); + tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t1_t2_s1(&net, last1); + tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t1_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2); + tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true); + tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t1_root_relu; + last3 = &s2_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true); + tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s2_t2_t1_s1(&net, last1); + tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s2_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true); + tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s2_t2_t2_s1(&net, last1); + tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s2_t2_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4); + tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true); + tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base4 = &s2_t2_root_relu; + + // level 4 + // tree 1 + // tree 1 + tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true); + tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true); + last2 = &s3_t1_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 }; + tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1); + // downsample + tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s3_t1_t1_maxpool1; + // project + tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true); + + tk::dnn::Shortcut s3_t1_t1_s1(&net, last2); + tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true); + tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t1_t2_s1(&net, last1); + tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t1_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 }; + tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2); + tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true); + tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t1_root_relu; + last3 = &s3_t1_root_relu; + // tree 2 + // tree 1 + tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true); + tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true); + tk::dnn::Shortcut s3_t2_t1_s1(&net, last1); + tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s3_t2_t1_relu; + + // tree 2 + tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true); + tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true); + + tk::dnn::Shortcut s3_t2_t2_s1(&net, last1); + tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s3_t2_t2_relu; + + // root + // join last1 and net in single input 256, 56, 56 + tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3}; + tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4); + tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true); + tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base5 = &s3_t2_root_relu; + + // level 5 + // tree 1 + tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true); + tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true); + last2 = &s4_t1_conv2; + + // get the basicblock input and apply maxpool conv2d and relu + tk::dnn::Layer *route_s4_t1_layers[1] = { base5 }; + tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1); + // downsample + tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX); + last4 = &s4_t1_maxpool1; + // project + tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true); + + tk::dnn::Shortcut s4_t1_s1(&net, last2); + tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU); + + last1 = &s4_t1_relu; + + // tree 2 + tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true); + tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU); + + tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true); + + tk::dnn::Shortcut s4_t2_s1(&net, last1); + tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU); + last2 = &s4_t2_relu; + + // root + // join last1 and net in single input 128, 56, 56 + tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 }; + tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3); + tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true); + tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU); + + base6 = &s4_root_relu; + + //final + // tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE); + // tk::dnn::Dense fc(&net, 1000, fc_bin); + + //ida 0 + tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true); + tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256); + tk::dnn::Shortcut ida_0_shortcut(&net, base5); + tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true); + tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida1 = &ida_0_n_1_relu; + + //ida1-1 + tk::dnn::Layer *route_ida1_layers_1[1] = { base5 }; + tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1); + + tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true); + tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut1(&net, base4); + tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true); + tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_1 = &ida_1_n_1_relu; + + //ida1-2 + tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 }; + tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1); + + tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true); + tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128); + tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1); + tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true); + tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida2_2 = &ida_1_n_2_relu; + + //ida2-1 + tk::dnn::Layer *route_ida2_layers_1[1] = { base4 }; + tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1); + + tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true); + tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut1(&net, base3); + tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true); + tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_1 = &ida_2_n_1_relu; + + //ida2-2 + tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 }; + tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1); + + tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true); + tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1); + tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true); + tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_2 = &ida_2_n_2_relu; + + //ida2-3 + tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 }; + tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1); + + tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true); + tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64); + tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2); + tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true); + tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU); + ida3_3 = &ida_2_n_3_relu; + + //idaup-1 + tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 }; + tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1); + + tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true); + tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3); + tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true); + tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_1 = &idaup_n_1_relu; + + //idaup-2 + tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 }; + tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1); + + tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true); + tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64); + tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1); + tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true); + tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU); + idaup_2 = &idaup_n_2_relu; + + tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 }; + + // hm + tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false); + tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 80, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false, false, true); + int kernel = 3; + int pad = (kernel - 1)/2; + tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX, true); + + // // wh + tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false); + tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false, false, true); + + // // reg + tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1); + tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false); + tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false, false, true); + + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + //printDeviceVector(64, data, true); + + //print network model + net.print(); + + //convert network to tensorRT + tk::dnn::NetworkRT netRT(&net, "dla34_cnet.rt"); + + tk::dnn::dataDim_t dim1 = dim; //input dim + printCenteredTitle(" CUDNN inference ", '=', 30); + { + dim1.print(); + TIMER_START + net.infer(dim1, data); + TIMER_STOP + dim1.print(); + } + + tk::dnn::dataDim_t dim2 = dim; + printCenteredTitle(" TENSORRT inference ", '=', 30); + { + dim2.print(); + TIMER_START + netRT.infer(dim2, data); + TIMER_STOP + dim2.print(); + } + + tk::dnn::Layer *outs[3] = { hm, wh, reg }; + int out_count = 1; + for(int i=0; i<3; i++) { + printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i+out_count]; + // there is the maxpool. It isn't an output but it is necessary for the process section + if(i==0) + out_count ++; + + std::cout << "CUDNN vs correct"; + checkResult(odim, cudnn_out, out); + std::cout << "TRT vs correct"; + checkResult(odim, rt_out, out); + std::cout << "CUDNN vs TRT "; + checkResult(odim, cudnn_out, rt_out); + } + + return 0; +} -- 2.52.0 From ef18be7b7b6f6e8a58416329474e278e9320c42e Mon Sep 17 00:00:00 2001 From: xavier Date: Wed, 5 Feb 2020 18:39:13 +0100 Subject: [PATCH 015/482] Add mAP computation and demo Signed-off-by: xavier --- CMakeLists.txt | 3 + demo/demo/map.cpp | 434 ++++++++++++++++++++++++++++++++++++++++++ include/tkDNN/Layer.h | 2 +- 3 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 demo/demo/map.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b5e24df..eae11be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,6 +114,9 @@ target_link_libraries(yolo3_demo tkDNN) add_executable(centernet_demo demo/demo/demo_centernet.cpp) target_link_libraries(centernet_demo tkDNN) +add_executable(map_demo demo/demo/map.cpp) +target_link_libraries(map_demo tkDNN) + #------------------------------------------------------------------------------- # Install diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp new file mode 100644 index 0000000..15825f7 --- /dev/null +++ b/demo/demo/map.cpp @@ -0,0 +1,434 @@ + +#include +#include +#include /* srand, rand */ +#include +#include +#include "utils.h" + +#include +#include +#include +#include + +#include "Yolo3Detection.h" +#include "CenternetDetection.h" + +#include + +struct BoundigBox : public tk::dnn::box +{ + friend std::ostream& operator<<(std::ostream& os, const BoundigBox& bb); + int unique_truth_index = -1; + int truth_flag = 0; +}; + +bool boxComparison (const BoundigBox& a,const BoundigBox& b) +{ + return (a.prob>b.prob); +} + + +std::ostream& operator<<(std::ostream& os, const BoundigBox& bb) +{ + os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << + ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< + bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<<"\n"; + return os; +} + +struct Frame +{ + void print() const + { + std::cout<<"labels filename: "< gt; + std::vector det; +}; + +void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) +{ + filename.replace(filename.find(l_folder),l_folder.length(),i_folder); + filename.replace(filename.find(l_ext),l_ext.length(),i_ext); +} + +float overlap(float x1, float w1, float x2, float w2) +{ + float l1 = x1 - w1/2; + float l2 = x2 - w2/2; + float left = l1 > l2 ? l1 : l2; + float r1 = x1 + w1/2; + float r2 = x2 + w2/2; + float right = r1 < r2 ? r1 : r2; + return right - left; +} + +float boxIntersection(const BoundigBox &a, const BoundigBox &b) +{ + float w = overlap(a.x, a.w, b.x, b.w); + float h = overlap(a.y, a.h, b.y, b.h); + if(w < 0 || h < 0) + return 0; + float area = w*h; + return area; +} + +float boxUnion(const BoundigBox &a, const BoundigBox &b) +{ + float i = boxIntersection(a, b); + float u = a.w*a.h + b.w*b.h - i; + return u; +} + +float boxIoU(const BoundigBox &a, const BoundigBox &b) +{ + float I = boxIntersection(a, b); + // std::cout<<"I: "< &images,const int classes,const int IoU_thresh, const int map_points, const bool verbose=false) +{ + std::cout<<"Computing mAP"< truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + // std::vector avg_iou_per_class(classes,0); + // std::vector tp_for_thresh_per_class(classes,0); + // std::vector fp_for_thresh_per_class(classes,0); + + + + //count groundtruth and detections in total and for each class + for(auto i:images) + { + for(auto gt:i.gt) + truth_classes_count[gt.cl]++; + for(auto det:i.det) + dets_classes_count[det.cl]++; + detections_count += i.det.size(); + groundtruths_count += i.gt.size(); + } + + std::cout<<"gt_count: "< all_dets; + std::vector all_gts; + + int gt_checked = 0; + + // for each detection comput IoU with groundtruth and match detetcion and + // groundtruth with IoU greater than IoU_thresh + for(auto &img:images) + { + for(size_t i=0; i 0) + { + float maxIoU = 0; + int truth_index = -1; + for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) + { + maxIoU = currentIoU; + truth_index = j; + } + } + // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) + { + img.det[i].unique_truth_index = truth_index + gt_checked; + img.det[i].truth_flag = 1; + } + } + + all_dets.push_back(img.det[i]); + } + gt_checked += img.gt.size(); + } + + if(verbose) + { + for(auto img:images) + img.print(); + std::cout<<"\n\n\n\n"; + } + + //sort all detections by descending value of confidence + std::sort(all_dets.begin(), all_dets.end(), boxComparison); + std::vector truth_flags(groundtruths_count,0); + + if(verbose) + for(auto d:all_dets) + std::cout<> pr( classes, std::vector(detections_count)); + for(int rank = 0; rank< detections_count; ++rank) + { + if (rank > 0) + { + for (int class_id = 0; class_id < classes; ++class_id) + { + pr[class_id][rank].tp = pr[class_id][rank - 1].tp; + pr[class_id][rank].fp = pr[class_id][rank - 1].fp; + } + } + + //if it was detected and never detected before + if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) + { + truth_flags[all_dets[rank].unique_truth_index] = 1; + pr[all_dets[rank].cl][rank].tp++; // true-positive + } + else + { + pr[all_dets[rank].cl][rank].fp++; // false-positive + } + + for (int i = 0; i < classes; ++i) + { + const int tp = pr[i][rank].tp; + const int fp = pr[i][rank].fp; + const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive + pr[i][rank].fn = fn; + + if ((tp + fp) > 0) + pr[i][rank].precision = (double)tp / (double)(tp + fp); + else + pr[i][rank].precision = 0; + + if ((tp + fn) > 0) + pr[i][rank].recall = (double)tp / (double)(tp + fn); + else + pr[i][rank].recall = 0; + + if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) + { // check for last rank + printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); + } + } + } + + if(verbose) + { + for(int i=0; i < pr.size(); i++) + { + std::cout<<"---------Class "<= 0; --rank) + { + delta_recall = last_recall - pr[i][rank].recall; + last_recall = pr[i][rank].recall; + + if (pr[i][rank].precision > last_precision) + last_precision = pr[i][rank].precision; + + avg_precision += delta_recall * last_precision; + } + } + else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points + { + for (int point = 0; point < map_points; ++point) { + cur_recall = point * 1.0 / ( map_points - 1 ); + cur_precision = 0; + for (int rank = 0; rank < detections_count; ++rank) + if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) + cur_precision = pr[i][rank].precision; + + avg_precision += cur_precision; + } + avg_precision = avg_precision / map_points; + } + + std::cout<<"Class: "< 1) + net = argv[1]; + char *labels_path = "/media/887E650E7E64F67A/val2014/all_labels.txt"; + if(argc > 2) + labels_path = argv[2]; + + networkType_t ntype = YOLO; + bool show = false; + + tk::dnn::Yolo3Detection yolo; + tk::dnn::CenternetDetection cnet; + + switch(ntype) + { + case YOLO: + yolo.init(net); + break; + case CENTERNET: + cnet.init(net); + break; + default: + FatalError("Network type not allowed "); + } + + std::ifstream all_labels(labels_path); + std::string l_filename; + std::vector images; + + std::cout<<"Reading groundtruth and generating detections"< detected_bbox; + switch(ntype) + { + case YOLO: + yolo.update(dnn_input); + detected_bbox = yolo.detected; + break; + case CENTERNET: + cnet.update(dnn_input); + detected_bbox = cnet.detected; + break; + default: + FatalError("Network type not allowed "); + } + + // save detections labels + for(auto d:detected_bbox) + { + //convert detected bb in the same format as label + /// / / / + BoundigBox b; + b.x = (d.x + d.w/2) / width; + b.y = (d.y + d.h/2) / height; + b.w = d.w / width; + b.h = d.h / height; + b.prob = d.prob; + b.cl = d.cl; + f.det.push_back(b); + + if(show)// draw rectangle for detection + cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(255, 0, 0), 2); + } + + // read and save groundtruth labels + std::ifstream labels(l_filename); + for(std::string line; std::getline(labels, line); ) + { + std::istringstream in(line); + BoundigBox b; + in >> b.cl >> b.x >> b.y >> b.w >> b.h; + b.prob = 1; + b.truth_flag = 1; + f.gt.push_back(b); + + if(show)// draw rectangle for groundtruth + cv::rectangle(frame, cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2); + } + + images.push_back(f); + + if(show) + { + cv::imshow("detection", frame); + cv::waitKey(0); + } + } + + std::cout<<"Done."< Date: Wed, 5 Feb 2020 18:39:13 +0100 Subject: [PATCH 016/482] Add mAP computation and demo Signed-off-by: xavier --- CMakeLists.txt | 3 + demo/demo/map.cpp | 434 ++++++++++++++++++++++++++++++++++++++++++ include/tkDNN/Layer.h | 2 +- 3 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 demo/demo/map.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b5e24df..eae11be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,6 +114,9 @@ target_link_libraries(yolo3_demo tkDNN) add_executable(centernet_demo demo/demo/demo_centernet.cpp) target_link_libraries(centernet_demo tkDNN) +add_executable(map_demo demo/demo/map.cpp) +target_link_libraries(map_demo tkDNN) + #------------------------------------------------------------------------------- # Install diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp new file mode 100644 index 0000000..15825f7 --- /dev/null +++ b/demo/demo/map.cpp @@ -0,0 +1,434 @@ + +#include +#include +#include /* srand, rand */ +#include +#include +#include "utils.h" + +#include +#include +#include +#include + +#include "Yolo3Detection.h" +#include "CenternetDetection.h" + +#include + +struct BoundigBox : public tk::dnn::box +{ + friend std::ostream& operator<<(std::ostream& os, const BoundigBox& bb); + int unique_truth_index = -1; + int truth_flag = 0; +}; + +bool boxComparison (const BoundigBox& a,const BoundigBox& b) +{ + return (a.prob>b.prob); +} + + +std::ostream& operator<<(std::ostream& os, const BoundigBox& bb) +{ + os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << + ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< + bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<<"\n"; + return os; +} + +struct Frame +{ + void print() const + { + std::cout<<"labels filename: "< gt; + std::vector det; +}; + +void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) +{ + filename.replace(filename.find(l_folder),l_folder.length(),i_folder); + filename.replace(filename.find(l_ext),l_ext.length(),i_ext); +} + +float overlap(float x1, float w1, float x2, float w2) +{ + float l1 = x1 - w1/2; + float l2 = x2 - w2/2; + float left = l1 > l2 ? l1 : l2; + float r1 = x1 + w1/2; + float r2 = x2 + w2/2; + float right = r1 < r2 ? r1 : r2; + return right - left; +} + +float boxIntersection(const BoundigBox &a, const BoundigBox &b) +{ + float w = overlap(a.x, a.w, b.x, b.w); + float h = overlap(a.y, a.h, b.y, b.h); + if(w < 0 || h < 0) + return 0; + float area = w*h; + return area; +} + +float boxUnion(const BoundigBox &a, const BoundigBox &b) +{ + float i = boxIntersection(a, b); + float u = a.w*a.h + b.w*b.h - i; + return u; +} + +float boxIoU(const BoundigBox &a, const BoundigBox &b) +{ + float I = boxIntersection(a, b); + // std::cout<<"I: "< &images,const int classes,const int IoU_thresh, const int map_points, const bool verbose=false) +{ + std::cout<<"Computing mAP"< truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + // std::vector avg_iou_per_class(classes,0); + // std::vector tp_for_thresh_per_class(classes,0); + // std::vector fp_for_thresh_per_class(classes,0); + + + + //count groundtruth and detections in total and for each class + for(auto i:images) + { + for(auto gt:i.gt) + truth_classes_count[gt.cl]++; + for(auto det:i.det) + dets_classes_count[det.cl]++; + detections_count += i.det.size(); + groundtruths_count += i.gt.size(); + } + + std::cout<<"gt_count: "< all_dets; + std::vector all_gts; + + int gt_checked = 0; + + // for each detection comput IoU with groundtruth and match detetcion and + // groundtruth with IoU greater than IoU_thresh + for(auto &img:images) + { + for(size_t i=0; i 0) + { + float maxIoU = 0; + int truth_index = -1; + for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) + { + maxIoU = currentIoU; + truth_index = j; + } + } + // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) + { + img.det[i].unique_truth_index = truth_index + gt_checked; + img.det[i].truth_flag = 1; + } + } + + all_dets.push_back(img.det[i]); + } + gt_checked += img.gt.size(); + } + + if(verbose) + { + for(auto img:images) + img.print(); + std::cout<<"\n\n\n\n"; + } + + //sort all detections by descending value of confidence + std::sort(all_dets.begin(), all_dets.end(), boxComparison); + std::vector truth_flags(groundtruths_count,0); + + if(verbose) + for(auto d:all_dets) + std::cout<> pr( classes, std::vector(detections_count)); + for(int rank = 0; rank< detections_count; ++rank) + { + if (rank > 0) + { + for (int class_id = 0; class_id < classes; ++class_id) + { + pr[class_id][rank].tp = pr[class_id][rank - 1].tp; + pr[class_id][rank].fp = pr[class_id][rank - 1].fp; + } + } + + //if it was detected and never detected before + if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) + { + truth_flags[all_dets[rank].unique_truth_index] = 1; + pr[all_dets[rank].cl][rank].tp++; // true-positive + } + else + { + pr[all_dets[rank].cl][rank].fp++; // false-positive + } + + for (int i = 0; i < classes; ++i) + { + const int tp = pr[i][rank].tp; + const int fp = pr[i][rank].fp; + const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive + pr[i][rank].fn = fn; + + if ((tp + fp) > 0) + pr[i][rank].precision = (double)tp / (double)(tp + fp); + else + pr[i][rank].precision = 0; + + if ((tp + fn) > 0) + pr[i][rank].recall = (double)tp / (double)(tp + fn); + else + pr[i][rank].recall = 0; + + if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) + { // check for last rank + printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); + } + } + } + + if(verbose) + { + for(int i=0; i < pr.size(); i++) + { + std::cout<<"---------Class "<= 0; --rank) + { + delta_recall = last_recall - pr[i][rank].recall; + last_recall = pr[i][rank].recall; + + if (pr[i][rank].precision > last_precision) + last_precision = pr[i][rank].precision; + + avg_precision += delta_recall * last_precision; + } + } + else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points + { + for (int point = 0; point < map_points; ++point) { + cur_recall = point * 1.0 / ( map_points - 1 ); + cur_precision = 0; + for (int rank = 0; rank < detections_count; ++rank) + if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) + cur_precision = pr[i][rank].precision; + + avg_precision += cur_precision; + } + avg_precision = avg_precision / map_points; + } + + std::cout<<"Class: "< 1) + net = argv[1]; + char *labels_path = "/media/887E650E7E64F67A/val2014/all_labels.txt"; + if(argc > 2) + labels_path = argv[2]; + + networkType_t ntype = YOLO; + bool show = false; + + tk::dnn::Yolo3Detection yolo; + tk::dnn::CenternetDetection cnet; + + switch(ntype) + { + case YOLO: + yolo.init(net); + break; + case CENTERNET: + cnet.init(net); + break; + default: + FatalError("Network type not allowed "); + } + + std::ifstream all_labels(labels_path); + std::string l_filename; + std::vector images; + + std::cout<<"Reading groundtruth and generating detections"< detected_bbox; + switch(ntype) + { + case YOLO: + yolo.update(dnn_input); + detected_bbox = yolo.detected; + break; + case CENTERNET: + cnet.update(dnn_input); + detected_bbox = cnet.detected; + break; + default: + FatalError("Network type not allowed "); + } + + // save detections labels + for(auto d:detected_bbox) + { + //convert detected bb in the same format as label + /// / / / + BoundigBox b; + b.x = (d.x + d.w/2) / width; + b.y = (d.y + d.h/2) / height; + b.w = d.w / width; + b.h = d.h / height; + b.prob = d.prob; + b.cl = d.cl; + f.det.push_back(b); + + if(show)// draw rectangle for detection + cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(255, 0, 0), 2); + } + + // read and save groundtruth labels + std::ifstream labels(l_filename); + for(std::string line; std::getline(labels, line); ) + { + std::istringstream in(line); + BoundigBox b; + in >> b.cl >> b.x >> b.y >> b.w >> b.h; + b.prob = 1; + b.truth_flag = 1; + f.gt.push_back(b); + + if(show)// draw rectangle for groundtruth + cv::rectangle(frame, cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2); + } + + images.push_back(f); + + if(show) + { + cv::imshow("detection", frame); + cv::waitKey(0); + } + } + + std::cout<<"Done."< Date: Thu, 6 Feb 2020 18:08:06 +0100 Subject: [PATCH 017/482] Change CenterNet input dimension. This commit changes the image input dimension, it updates the CenterNet detection class. Signed-off-by: Davide Sapienza --- include/tkDNN/CenternetDetection.h | 6 +++--- src/CenternetDetection.cpp | 17 +++++++++++------ src/DeformConv2d.cpp | 9 ++++++--- tests/dla34_cnet/dla34_cnet.cpp | 2 +- tests/resnet101_cnet/resnet101_cnet.cpp | 2 +- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 46af73b..1868839 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,14 +73,14 @@ class CenternetDetection { //processing float toll = 0.000001; int K = 100; - int width = 56; // TODO + int width = 128;//56; // TODO public: dnnType *rt_out[4]; - float inp_height = 224;//512; - float inp_width = 224;//512; + float inp_height = 512;//224;//512; + float inp_width = 512;//224;//512; int classes = 80; int num = 0; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index fbc9832..c6c4bc7 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -18,7 +18,7 @@ bool CenternetDetection::init(std::string tensor_path) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); - dim = tk::dnn::dataDim_t(1, 3, 224, 224, 1); + dim = tk::dnn::dataDim_t(1, 3, 512, 512, 1); const char *coco_class_name_[] = { "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", @@ -43,9 +43,12 @@ bool CenternetDetection::init(std::string tensor_path) { checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); - dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1); - dim_wh = tk::dnn::dataDim_t(1, 2, 56, 56, 1); - dim_reg = tk::dnn::dataDim_t(1, 2, 56, 56, 1); + // dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1); + // dim_wh = tk::dnn::dataDim_t(1, 2, 56, 56, 1); + // dim_reg = tk::dnn::dataDim_t(1, 2, 56, 56, 1); + dim_hm = tk::dnn::dataDim_t(1, 80, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); @@ -93,6 +96,8 @@ bool CenternetDetection::init(std::string tensor_path) { mean << 0.408, 0.447, 0.47; stddev << 0.289, 0.274, 0.278; + // mean << 0.485, 0.456, 0.406; + // stddev << 0.229, 0.224, 0.225; } void CenternetDetection::testdog() { @@ -104,8 +109,8 @@ void CenternetDetection::testdog() { imageORIG.convertTo(imageF, CV_32FC3, 1/255.0); sz = imageF.size(); std::cout<<"image: "< Date: Thu, 6 Feb 2020 18:08:06 +0100 Subject: [PATCH 018/482] Change CenterNet input dimension. This commit changes the image input dimension, it updates the CenterNet detection class. Signed-off-by: Davide Sapienza --- include/tkDNN/CenternetDetection.h | 6 +++--- src/CenternetDetection.cpp | 17 +++++++++++------ src/DeformConv2d.cpp | 9 ++++++--- tests/dla34_cnet/dla34_cnet.cpp | 2 +- tests/resnet101_cnet/resnet101_cnet.cpp | 2 +- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 46af73b..1868839 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,14 +73,14 @@ class CenternetDetection { //processing float toll = 0.000001; int K = 100; - int width = 56; // TODO + int width = 128;//56; // TODO public: dnnType *rt_out[4]; - float inp_height = 224;//512; - float inp_width = 224;//512; + float inp_height = 512;//224;//512; + float inp_width = 512;//224;//512; int classes = 80; int num = 0; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index fbc9832..c6c4bc7 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -18,7 +18,7 @@ bool CenternetDetection::init(std::string tensor_path) { std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); - dim = tk::dnn::dataDim_t(1, 3, 224, 224, 1); + dim = tk::dnn::dataDim_t(1, 3, 512, 512, 1); const char *coco_class_name_[] = { "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", @@ -43,9 +43,12 @@ bool CenternetDetection::init(std::string tensor_path) { checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); - dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1); - dim_wh = tk::dnn::dataDim_t(1, 2, 56, 56, 1); - dim_reg = tk::dnn::dataDim_t(1, 2, 56, 56, 1); + // dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1); + // dim_wh = tk::dnn::dataDim_t(1, 2, 56, 56, 1); + // dim_reg = tk::dnn::dataDim_t(1, 2, 56, 56, 1); + dim_hm = tk::dnn::dataDim_t(1, 80, 128, 128, 1); + dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1); + dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1); checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) ); checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) ); @@ -93,6 +96,8 @@ bool CenternetDetection::init(std::string tensor_path) { mean << 0.408, 0.447, 0.47; stddev << 0.289, 0.274, 0.278; + // mean << 0.485, 0.456, 0.406; + // stddev << 0.229, 0.224, 0.225; } void CenternetDetection::testdog() { @@ -104,8 +109,8 @@ void CenternetDetection::testdog() { imageORIG.convertTo(imageF, CV_32FC3, 1/255.0); sz = imageF.size(); std::cout<<"image: "< Date: Thu, 6 Feb 2020 23:09:12 +0100 Subject: [PATCH 019/482] Fix memory leak This commit moves cublasCreate out from dcn_v2_cuda_forward to save some milliseconds and it adds cublasDestroy (cause of memory leak). Signed-off-by: Davide Sapienza --- include/tkDNN/Layer.h | 3 +++ include/tkDNN/kernels.h | 3 ++- include/tkDNN/pluginsRT/DeformableConvRT.h | 18 ++++++++++++++++-- src/DeformConv2d.cpp | 11 +++++++++-- src/NetworkRT.cpp | 2 +- src/kernels/deformable_conv.cu | 12 +++--------- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 7716393..f9da061 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -229,6 +229,9 @@ public: dnnType *offset, *mask; dnnType *output_conv; + cublasStatus_t stat; + cublasHandle_t handle; + protected: cudnnTensorDescriptor_t biasTensorDesc; diff --git a/include/tkDNN/kernels.h b/include/tkDNN/kernels.h index d7d5d05..94c5dcb 100644 --- a/include/tkDNN/kernels.h +++ b/include/tkDNN/kernels.h @@ -38,7 +38,8 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, const int dilation_h, const int dilation_w, const int deformable_group, float *data_col); -void dcn_v2_cuda_forward(float *input, float *weight, +void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, + float *input, float *weight, float *bias, float *ones, float *offset, float *mask, float *output, float *columns, diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index dca1020..f4d0b30 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -52,10 +52,21 @@ public: checkCuda( cudaMemcpy(mask, deformable->mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToDevice) ); checkCuda( cudaMemcpy(ones_d2, deformable->ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToDevice) ); } + stat = cublasCreate(&handle); + if (stat != CUBLAS_STATUS_SUCCESS) { + printf ("CUBLAS initialization failed\n"); + return; + } } ~DeformableConvRT(){ - + checkCuda( cudaFree(data_d) ); + checkCuda( cudaFree(bias2_d) ); + checkCuda( cudaFree(ones_d1) ); + checkCuda( cudaFree(offset) ); + checkCuda( cudaFree(mask) ); + checkCuda( cudaFree(ones_d2) ); + cublasDestroy(handle); } int getNbOutputs() const override { @@ -100,7 +111,8 @@ public: activationSIGMOIDForward(mask, mask, chunk_dim); // deformable convolution - dcn_v2_cuda_forward(srcData, data_d, + dcn_v2_cuda_forward(stat, handle, + srcData, data_d, bias2_d, ones_d1, offset, mask, reinterpret_cast(outputs[0]), ones_d2, @@ -172,6 +184,8 @@ public: free(aus); } + cublasStatus_t stat; + cublasHandle_t handle; int i_n, i_c, i_h, i_w; int o_n, o_c, o_h, o_w; int size; diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index 957ef24..8cf1a6d 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -9,6 +9,11 @@ namespace tk { namespace dnn { void DeformConv2d::initCUDNN() { + stat = cublasCreate(&handle); + if (stat != CUBLAS_STATUS_SUCCESS) { + printf ("CUBLAS initialization failed\n"); + return; + } checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) ); checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc, net->tensorFormat, net->dataType, @@ -84,6 +89,7 @@ DeformConv2d::~DeformConv2d() { checkCuda( cudaFree(offset) ); checkCuda( cudaFree(mask) ); checkCuda( cudaFree(output_conv) ); + cublasDestroy(handle); } dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) { @@ -95,9 +101,10 @@ dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) { checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); // kernel sigmoide activationSIGMOIDForward(mask, mask, chunk_dim); - + // deformable convolution - dcn_v2_cuda_forward(srcData, this->data_d, + dcn_v2_cuda_forward(stat, handle, + srcData, this->data_d, this->bias2_d, ones_d1, offset, mask, dstData, ones_d2, diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 2b7377d..5162d48 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -453,7 +453,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) { IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin); checkNULL(lRT); lRT->setName( ("Deformable" + std::to_string(l->id)).c_str() ); - + delete(inputs); // batchnorm void *bias_b, *power_b, *mean_b, *variance_b, *scales_b; if(dtRT == DataType::kHALF) { diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index cb9ded0..0579620 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -138,7 +138,8 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, } -void dcn_v2_cuda_forward(float *input, float *weight, +void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, + float *input, float *weight, float *bias, float *ones, float *offset, float *mask, float *output, float *columns, @@ -151,14 +152,7 @@ void dcn_v2_cuda_forward(float *input, float *weight, const int out_n, const int out_c, const int out_h, const int out_w, const int chunk_dim, cudaStream_t stream) { - cublasStatus_t stat; - cublasHandle_t handle; - stat = cublasCreate(&handle); - if (stat != CUBLAS_STATUS_SUCCESS) { - printf ("CUBLAS initialization failed\n"); - return; - } - + // stat and handle have be moved out to preserve 2 - 6 milliseconds every 100. const int channels = in_c; const int height = in_h; const int width = in_w; -- 2.52.0 From 2503eba1735c510f3f9454b536399762fbd2ec11 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Thu, 6 Feb 2020 23:09:12 +0100 Subject: [PATCH 020/482] Fix memory leak This commit moves cublasCreate out from dcn_v2_cuda_forward to save some milliseconds and it adds cublasDestroy (cause of memory leak). Signed-off-by: Davide Sapienza --- include/tkDNN/Layer.h | 3 +++ include/tkDNN/kernels.h | 3 ++- include/tkDNN/pluginsRT/DeformableConvRT.h | 18 ++++++++++++++++-- src/DeformConv2d.cpp | 11 +++++++++-- src/NetworkRT.cpp | 2 +- src/kernels/deformable_conv.cu | 12 +++--------- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 7716393..f9da061 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -229,6 +229,9 @@ public: dnnType *offset, *mask; dnnType *output_conv; + cublasStatus_t stat; + cublasHandle_t handle; + protected: cudnnTensorDescriptor_t biasTensorDesc; diff --git a/include/tkDNN/kernels.h b/include/tkDNN/kernels.h index d7d5d05..94c5dcb 100644 --- a/include/tkDNN/kernels.h +++ b/include/tkDNN/kernels.h @@ -38,7 +38,8 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, const int dilation_h, const int dilation_w, const int deformable_group, float *data_col); -void dcn_v2_cuda_forward(float *input, float *weight, +void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, + float *input, float *weight, float *bias, float *ones, float *offset, float *mask, float *output, float *columns, diff --git a/include/tkDNN/pluginsRT/DeformableConvRT.h b/include/tkDNN/pluginsRT/DeformableConvRT.h index dca1020..f4d0b30 100644 --- a/include/tkDNN/pluginsRT/DeformableConvRT.h +++ b/include/tkDNN/pluginsRT/DeformableConvRT.h @@ -52,10 +52,21 @@ public: checkCuda( cudaMemcpy(mask, deformable->mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToDevice) ); checkCuda( cudaMemcpy(ones_d2, deformable->ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToDevice) ); } + stat = cublasCreate(&handle); + if (stat != CUBLAS_STATUS_SUCCESS) { + printf ("CUBLAS initialization failed\n"); + return; + } } ~DeformableConvRT(){ - + checkCuda( cudaFree(data_d) ); + checkCuda( cudaFree(bias2_d) ); + checkCuda( cudaFree(ones_d1) ); + checkCuda( cudaFree(offset) ); + checkCuda( cudaFree(mask) ); + checkCuda( cudaFree(ones_d2) ); + cublasDestroy(handle); } int getNbOutputs() const override { @@ -100,7 +111,8 @@ public: activationSIGMOIDForward(mask, mask, chunk_dim); // deformable convolution - dcn_v2_cuda_forward(srcData, data_d, + dcn_v2_cuda_forward(stat, handle, + srcData, data_d, bias2_d, ones_d1, offset, mask, reinterpret_cast(outputs[0]), ones_d2, @@ -172,6 +184,8 @@ public: free(aus); } + cublasStatus_t stat; + cublasHandle_t handle; int i_n, i_c, i_h, i_w; int o_n, o_c, o_h, o_w; int size; diff --git a/src/DeformConv2d.cpp b/src/DeformConv2d.cpp index 957ef24..8cf1a6d 100644 --- a/src/DeformConv2d.cpp +++ b/src/DeformConv2d.cpp @@ -9,6 +9,11 @@ namespace tk { namespace dnn { void DeformConv2d::initCUDNN() { + stat = cublasCreate(&handle); + if (stat != CUBLAS_STATUS_SUCCESS) { + printf ("CUBLAS initialization failed\n"); + return; + } checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) ); checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc, net->tensorFormat, net->dataType, @@ -84,6 +89,7 @@ DeformConv2d::~DeformConv2d() { checkCuda( cudaFree(offset) ); checkCuda( cudaFree(mask) ); checkCuda( cudaFree(output_conv) ); + cublasDestroy(handle); } dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) { @@ -95,9 +101,10 @@ dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) { checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); // kernel sigmoide activationSIGMOIDForward(mask, mask, chunk_dim); - + // deformable convolution - dcn_v2_cuda_forward(srcData, this->data_d, + dcn_v2_cuda_forward(stat, handle, + srcData, this->data_d, this->bias2_d, ones_d1, offset, mask, dstData, ones_d2, diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 2b7377d..5162d48 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -453,7 +453,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) { IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin); checkNULL(lRT); lRT->setName( ("Deformable" + std::to_string(l->id)).c_str() ); - + delete(inputs); // batchnorm void *bias_b, *power_b, *mean_b, *variance_b, *scales_b; if(dtRT == DataType::kHALF) { diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index cb9ded0..0579620 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -138,7 +138,8 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, } -void dcn_v2_cuda_forward(float *input, float *weight, +void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, + float *input, float *weight, float *bias, float *ones, float *offset, float *mask, float *output, float *columns, @@ -151,14 +152,7 @@ void dcn_v2_cuda_forward(float *input, float *weight, const int out_n, const int out_c, const int out_h, const int out_w, const int chunk_dim, cudaStream_t stream) { - cublasStatus_t stat; - cublasHandle_t handle; - stat = cublasCreate(&handle); - if (stat != CUBLAS_STATUS_SUCCESS) { - printf ("CUBLAS initialization failed\n"); - return; - } - + // stat and handle have be moved out to preserve 2 - 6 milliseconds every 100. const int channels = in_c; const int height = in_h; const int width = in_w; -- 2.52.0 From 9bc4ba0eff3903bbc5b1fa1af0d5c0c4182add93 Mon Sep 17 00:00:00 2001 From: xavier Date: Fri, 7 Feb 2020 09:09:40 +0100 Subject: [PATCH 021/482] Add mAP 0.5:0.95, other small fix Signed-off-by: xavier --- demo/demo/map.cpp | 73 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 15825f7..8e4ec3f 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -21,6 +21,14 @@ struct BoundigBox : public tk::dnn::box friend std::ostream& operator<<(std::ostream& os, const BoundigBox& bb); int unique_truth_index = -1; int truth_flag = 0; + float max_IoU = 0; + + void clear() + { + unique_truth_index = -1; + truth_flag = 0; + max_IoU = 0; + } }; bool boxComparison (const BoundigBox& a,const BoundigBox& b) @@ -33,7 +41,8 @@ std::ostream& operator<<(std::ostream& os, const BoundigBox& bb) { os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< - bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<<"\n"; + bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< + ", maxIoU: "<< bb.max_IoU<<"\n"; return os; } @@ -111,7 +120,7 @@ struct PR } }; -double computeMap(std::vector &images,const int classes,const int IoU_thresh, const int map_points, const bool verbose=false) +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false) { std::cout<<"Computing mAP"< &images,const int classes,const int IoU_thr // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) { + // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< &images,const int classes,const int IoU_thr mean_average_precision = mean_average_precision / classes; - std::cout<<"Classes: "< 1) net = argv[1]; - char *labels_path = "/media/887E650E7E64F67A/val2014/all_labels.txt"; + char type = 'y'; if(argc > 2) - labels_path = argv[2]; + type = argv[2][0]; + char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + if(argc > 3) + labels_path = argv[3]; + + + networkType_t ntype; + switch(type) + { + case 'y': + ntype = YOLO; + break; + case 'c': + ntype = CENTERNET; + break; + default: + FatalError("type not allowed (3rd parameter)"); + } - networkType_t ntype = YOLO; bool show = false; tk::dnn::Yolo3Detection yolo; @@ -342,8 +369,10 @@ int main(int argc, char *argv[]) if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); + std::vector detected_bbox; + int i=0; - while (std::getline(all_labels, l_filename) && i < 1000) + while (std::getline(all_labels, l_filename)) // && i < 1000) { Frame f; f.l_filename = l_filename; @@ -363,7 +392,9 @@ int main(int argc, char *argv[]) dnn_input = frame.clone(); //inference - std::vector detected_bbox; + + detected_bbox.clear(); + switch(ntype) { case YOLO: @@ -378,6 +409,9 @@ int main(int argc, char *argv[]) FatalError("Network type not allowed "); } + // std::ofstream myfile; + // myfile.open ("det/"+f.l_filename.substr(l_filename.find("000"))); + // save detections labels for(auto d:detected_bbox) { @@ -392,10 +426,14 @@ int main(int argc, char *argv[]) b.cl = d.cl; f.det.push_back(b); + // myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + if(show)// draw rectangle for detection - cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(255, 0, 0), 2); + cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); } + // myfile.close(); + // read and save groundtruth labels std::ifstream labels(l_filename); for(std::string line; std::getline(labels, line); ) @@ -423,11 +461,24 @@ int main(int argc, char *argv[]) std::cout<<"Done."< &images,const int classes,const int IoU_thresh, const int map_points, const bool verbose=false) +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false) { std::cout<<"Computing mAP"< &images,const int classes,const int IoU_thr // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) { + // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< &images,const int classes,const int IoU_thr mean_average_precision = mean_average_precision / classes; - std::cout<<"Classes: "< 1) net = argv[1]; - char *labels_path = "/media/887E650E7E64F67A/val2014/all_labels.txt"; + char type = 'y'; if(argc > 2) - labels_path = argv[2]; + type = argv[2][0]; + char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + if(argc > 3) + labels_path = argv[3]; + + + networkType_t ntype; + switch(type) + { + case 'y': + ntype = YOLO; + break; + case 'c': + ntype = CENTERNET; + break; + default: + FatalError("type not allowed (3rd parameter)"); + } - networkType_t ntype = YOLO; bool show = false; tk::dnn::Yolo3Detection yolo; @@ -342,8 +369,10 @@ int main(int argc, char *argv[]) if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); + std::vector detected_bbox; + int i=0; - while (std::getline(all_labels, l_filename) && i < 1000) + while (std::getline(all_labels, l_filename)) // && i < 1000) { Frame f; f.l_filename = l_filename; @@ -363,7 +392,9 @@ int main(int argc, char *argv[]) dnn_input = frame.clone(); //inference - std::vector detected_bbox; + + detected_bbox.clear(); + switch(ntype) { case YOLO: @@ -378,6 +409,9 @@ int main(int argc, char *argv[]) FatalError("Network type not allowed "); } + // std::ofstream myfile; + // myfile.open ("det/"+f.l_filename.substr(l_filename.find("000"))); + // save detections labels for(auto d:detected_bbox) { @@ -392,10 +426,14 @@ int main(int argc, char *argv[]) b.cl = d.cl; f.det.push_back(b); + // myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + if(show)// draw rectangle for detection - cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(255, 0, 0), 2); + cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); } + // myfile.close(); + // read and save groundtruth labels std::ifstream labels(l_filename); for(std::string line; std::getline(labels, line); ) @@ -423,11 +461,24 @@ int main(int argc, char *argv[]) std::cout<<"Done."< -struct BoundigBox : public tk::dnn::box -{ - friend std::ostream& operator<<(std::ostream& os, const BoundigBox& bb); - int unique_truth_index = -1; - int truth_flag = 0; - float max_IoU = 0; - - void clear() - { - unique_truth_index = -1; - truth_flag = 0; - max_IoU = 0; - } -}; - -bool boxComparison (const BoundigBox& a,const BoundigBox& b) -{ - return (a.prob>b.prob); -} - - -std::ostream& operator<<(std::ostream& os, const BoundigBox& bb) -{ - os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << - ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< - bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< - ", maxIoU: "<< bb.max_IoU<<"\n"; - return os; -} - -struct Frame -{ - void print() const - { - std::cout<<"labels filename: "< gt; - std::vector det; -}; - void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) { filename.replace(filename.find(l_folder),l_folder.length(),i_folder); filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } -float overlap(float x1, float w1, float x2, float w2) -{ - float l1 = x1 - w1/2; - float l2 = x2 - w2/2; - float left = l1 > l2 ? l1 : l2; - float r1 = x1 + w1/2; - float r2 = x2 + w2/2; - float right = r1 < r2 ? r1 : r2; - return right - left; -} - -float boxIntersection(const BoundigBox &a, const BoundigBox &b) -{ - float w = overlap(a.x, a.w, b.x, b.w); - float h = overlap(a.y, a.h, b.y, b.h); - if(w < 0 || h < 0) - return 0; - float area = w*h; - return area; -} - -float boxUnion(const BoundigBox &a, const BoundigBox &b) -{ - float i = boxIntersection(a, b); - float u = a.w*a.h + b.w*b.h - i; - return u; -} - -float boxIoU(const BoundigBox &a, const BoundigBox &b) -{ - float I = boxIntersection(a, b); - // std::cout<<"I: "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false) -{ - std::cout<<"Computing mAP"< truth_classes_count(classes,0); - std::vector dets_classes_count(classes,0); - // std::vector avg_iou_per_class(classes,0); - // std::vector tp_for_thresh_per_class(classes,0); - // std::vector fp_for_thresh_per_class(classes,0); - - - - //count groundtruth and detections in total and for each class - for(auto i:images) - { - for(auto gt:i.gt) - truth_classes_count[gt.cl]++; - for(auto det:i.det) - dets_classes_count[det.cl]++; - detections_count += i.det.size(); - groundtruths_count += i.gt.size(); - } - - std::cout<<"gt_count: "< all_dets; - std::vector all_gts; - - int gt_checked = 0; - - // for each detection comput IoU with groundtruth and match detetcion and - // groundtruth with IoU greater than IoU_thresh - for(auto &img:images) - { - for(size_t i=0; i 0) - { - float maxIoU = 0; - int truth_index = -1; - for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) - { - maxIoU = currentIoU; - truth_index = j; - } - } - // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) - { - // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< truth_flags(groundtruths_count,0); - - if(verbose) - for(auto d:all_dets) - std::cout<> pr( classes, std::vector(detections_count)); - for(int rank = 0; rank< detections_count; ++rank) - { - if (rank > 0) - { - for (int class_id = 0; class_id < classes; ++class_id) - { - pr[class_id][rank].tp = pr[class_id][rank - 1].tp; - pr[class_id][rank].fp = pr[class_id][rank - 1].fp; - } - } - - //if it was detected and never detected before - if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) - { - truth_flags[all_dets[rank].unique_truth_index] = 1; - pr[all_dets[rank].cl][rank].tp++; // true-positive - } - else - { - pr[all_dets[rank].cl][rank].fp++; // false-positive - } - - for (int i = 0; i < classes; ++i) - { - const int tp = pr[i][rank].tp; - const int fp = pr[i][rank].fp; - const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive - pr[i][rank].fn = fn; - - if ((tp + fp) > 0) - pr[i][rank].precision = (double)tp / (double)(tp + fp); - else - pr[i][rank].precision = 0; - - if ((tp + fn) > 0) - pr[i][rank].recall = (double)tp / (double)(tp + fn); - else - pr[i][rank].recall = 0; - - if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) - { // check for last rank - printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); - } - } - } - - if(verbose) - { - for(int i=0; i < pr.size(); i++) - { - std::cout<<"---------Class "<= 0; --rank) - { - delta_recall = last_recall - pr[i][rank].recall; - last_recall = pr[i][rank].recall; - - if (pr[i][rank].precision > last_precision) - last_precision = pr[i][rank].precision; - - avg_precision += delta_recall * last_precision; - } - } - else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points - { - for (int point = 0; point < map_points; ++point) { - cur_recall = point * 1.0 / ( map_points - 1 ); - cur_precision = 0; - for (int rank = 0; rank < detections_count; ++rank) - if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) - cur_precision = pr[i][rank].precision; - - avg_precision += cur_precision; - } - avg_precision = avg_precision / map_points; - } - - std::cout<<"Class: "< 1) net = argv[1]; - char type = 'y'; + char ntype = 'y'; if(argc > 2) - type = argv[2][0]; + ntype = argv[2][0]; + //path to txt file with all realpath of images labels char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; if(argc > 3) labels_path = argv[3]; - - networkType_t ntype; - switch(type) - { - case 'y': - ntype = YOLO; - break; - case 'c': - ntype = CENTERNET; - break; - default: - FatalError("type not allowed (3rd parameter)"); - } - bool show = false; + bool write_dets = false; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; + switch(ntype) { - case YOLO: + case 'y': yolo.init(net); break; - case CENTERNET: + case 'c': cnet.init(net); break; default: - FatalError("Network type not allowed "); + FatalError("Network type not allowed (3rd parameter)\n"); } + std::ifstream all_labels(labels_path); std::string l_filename; std::vector images; @@ -372,14 +70,14 @@ int main(int argc, char *argv[]) std::vector detected_bbox; int i=0; - while (std::getline(all_labels, l_filename)) // && i < 1000) + while (std::getline(all_labels, l_filename) && i < 1000) { + std::cout </ / / / - BoundigBox b; + BoundingBox b; b.x = (d.x + d.w/2) / width; b.y = (d.y + d.h/2) / height; b.w = d.w / width; @@ -426,20 +123,22 @@ int main(int argc, char *argv[]) b.cl = d.cl; f.det.push_back(b); - // myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + if(write_dets) + myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; if(show)// draw rectangle for detection cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); } - // myfile.close(); + if(write_dets) + myfile.close(); // read and save groundtruth labels std::ifstream labels(l_filename); for(std::string line; std::getline(labels, line); ) { std::istringstream in(line); - BoundigBox b; + BoundingBox b; in >> b.cl >> b.x >> b.y >> b.w >> b.h; b.prob = 1; b.truth_flag = 1; @@ -461,23 +160,14 @@ int main(int argc, char *argv[]) std::cout<<"Done."< gt; + std::vector det; + + void print() const; +}; + +struct PR +{ + double precision = 0; + double recall = 0; + int tp = 0, fp = 0, fn = 0; + + void print(); +}; + +float overlap(float x1, float w1, float x2, float w2); +float boxIntersection(const BoundingBox &a, const BoundingBox &b); +float boxUnion(const BoundingBox &a, const BoundingBox &b); +float boxIoU(const BoundingBox &a, const BoundingBox &b); + +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false); +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); + +#endif /*EVALUATION_H*/ \ No newline at end of file diff --git a/src/evaluation.cpp b/src/evaluation.cpp new file mode 100644 index 0000000..2b60449 --- /dev/null +++ b/src/evaluation.cpp @@ -0,0 +1,284 @@ +#include "evaluation.h" + + +void BoundingBox::clear() +{ + unique_truth_index = -1; + truth_flag = 0; + max_IoU = 0; +} + +bool boxComparison (const BoundingBox& a,const BoundingBox& b) +{ + return (a.prob>b.prob); +} + +std::ostream& operator<<(std::ostream& os, const BoundingBox& bb) +{ + os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << + ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< + bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< + ", maxIoU: "<< bb.max_IoU<<"\n"; + return os; +} + +void Frame::print() const +{ + std::cout<<"labels filename: "< l2 ? l1 : l2; + float r1 = x1 + w1/2; + float r2 = x2 + w2/2; + float right = r1 < r2 ? r1 : r2; + return right - left; +} + +float boxIntersection(const BoundingBox &a, const BoundingBox &b) +{ + float w = overlap(a.x, a.w, b.x, b.w); + float h = overlap(a.y, a.h, b.y, b.h); + if(w < 0 || h < 0) + return 0; + float area = w*h; + return area; +} + +float boxUnion(const BoundingBox &a, const BoundingBox &b) +{ + float i = boxIntersection(a, b); + float u = a.w*a.h + b.w*b.h - i; + return u; +} + +float boxIoU(const BoundingBox &a, const BoundingBox &b) +{ + float I = boxIntersection(a, b); + // std::cout<<"I: "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose) +{ + + std::cout<<"Computing mAP"< truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + + //count groundtruth and detections in total and for each class + for(auto i:images) + { + for(auto gt:i.gt) + truth_classes_count[gt.cl]++; + for(auto det:i.det) + dets_classes_count[det.cl]++; + detections_count += i.det.size(); + groundtruths_count += i.gt.size(); + } + + std::cout<<"gt_count: "< all_dets; + std::vector all_gts; + + int gt_checked = 0; + + // for each detection comput IoU with groundtruth and match detetcion and + // groundtruth with IoU greater than IoU_thresh + for(auto &img:images) + { + for(size_t i=0; i 0) + { + float maxIoU = 0; + int truth_index = -1; + for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) + { + maxIoU = currentIoU; + truth_index = j; + } + } + // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) + { + // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< truth_flags(groundtruths_count,0); + + if(verbose) + for(auto d:all_dets) + std::cout<> pr( classes, std::vector(detections_count)); + for(int rank = 0; rank< detections_count; ++rank) + { + if (rank > 0) + { + for (int class_id = 0; class_id < classes; ++class_id) + { + pr[class_id][rank].tp = pr[class_id][rank - 1].tp; + pr[class_id][rank].fp = pr[class_id][rank - 1].fp; + } + } + + //if it was detected and never detected before + if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) + { + truth_flags[all_dets[rank].unique_truth_index] = 1; + pr[all_dets[rank].cl][rank].tp++; // true-positive + } + else + { + pr[all_dets[rank].cl][rank].fp++; // false-positive + } + + for (int i = 0; i < classes; ++i) + { + const int tp = pr[i][rank].tp; + const int fp = pr[i][rank].fp; + const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive + pr[i][rank].fn = fn; + + if ((tp + fp) > 0) + pr[i][rank].precision = (double)tp / (double)(tp + fp); + else + pr[i][rank].precision = 0; + + if ((tp + fn) > 0) + pr[i][rank].recall = (double)tp / (double)(tp + fn); + else + pr[i][rank].recall = 0; + + if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) + { // check for last rank + printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); + } + } + } + + if(verbose) + { + for(int i=0; i < pr.size(); i++) + { + std::cout<<"---------Class "<= 0; --rank) + { + delta_recall = last_recall - pr[i][rank].recall; + last_recall = pr[i][rank].recall; + + if (pr[i][rank].precision > last_precision) + last_precision = pr[i][rank].precision; + + avg_precision += delta_recall * last_precision; + } + } + else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points + { + for (int point = 0; point < map_points; ++point) { + cur_recall = point * 1.0 / ( map_points - 1 ); + cur_precision = 0; + for (int rank = 0; rank < detections_count; ++rank) + if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) + cur_precision = pr[i][rank].precision; + + avg_precision += cur_precision; + } + avg_precision = avg_precision / map_points; + } + + std::cout<<"Class: "< &images,const int classes,const float i_IoU_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) +{ + double AP = 0; + float IoU_thresh = i_IoU_thresh; + for(int i=0; i Date: Fri, 7 Feb 2020 12:47:02 +0100 Subject: [PATCH 024/482] Refactoring map_demo, add evaluation.h and evaluation.cpp Signed-off-by: xavier --- demo/demo/map.cpp | 370 ++++--------------------------------------- include/evaluation.h | 51 ++++++ src/evaluation.cpp | 284 +++++++++++++++++++++++++++++++++ 3 files changed, 365 insertions(+), 340 deletions(-) create mode 100644 include/evaluation.h create mode 100644 src/evaluation.cpp diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 8e4ec3f..f7e9e3e 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -14,352 +14,50 @@ #include "Yolo3Detection.h" #include "CenternetDetection.h" +#include "evaluation.h" + #include -struct BoundigBox : public tk::dnn::box -{ - friend std::ostream& operator<<(std::ostream& os, const BoundigBox& bb); - int unique_truth_index = -1; - int truth_flag = 0; - float max_IoU = 0; - - void clear() - { - unique_truth_index = -1; - truth_flag = 0; - max_IoU = 0; - } -}; - -bool boxComparison (const BoundigBox& a,const BoundigBox& b) -{ - return (a.prob>b.prob); -} - - -std::ostream& operator<<(std::ostream& os, const BoundigBox& bb) -{ - os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << - ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< - bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< - ", maxIoU: "<< bb.max_IoU<<"\n"; - return os; -} - -struct Frame -{ - void print() const - { - std::cout<<"labels filename: "< gt; - std::vector det; -}; - void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) { filename.replace(filename.find(l_folder),l_folder.length(),i_folder); filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } -float overlap(float x1, float w1, float x2, float w2) -{ - float l1 = x1 - w1/2; - float l2 = x2 - w2/2; - float left = l1 > l2 ? l1 : l2; - float r1 = x1 + w1/2; - float r2 = x2 + w2/2; - float right = r1 < r2 ? r1 : r2; - return right - left; -} - -float boxIntersection(const BoundigBox &a, const BoundigBox &b) -{ - float w = overlap(a.x, a.w, b.x, b.w); - float h = overlap(a.y, a.h, b.y, b.h); - if(w < 0 || h < 0) - return 0; - float area = w*h; - return area; -} - -float boxUnion(const BoundigBox &a, const BoundigBox &b) -{ - float i = boxIntersection(a, b); - float u = a.w*a.h + b.w*b.h - i; - return u; -} - -float boxIoU(const BoundigBox &a, const BoundigBox &b) -{ - float I = boxIntersection(a, b); - // std::cout<<"I: "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false) -{ - std::cout<<"Computing mAP"< truth_classes_count(classes,0); - std::vector dets_classes_count(classes,0); - // std::vector avg_iou_per_class(classes,0); - // std::vector tp_for_thresh_per_class(classes,0); - // std::vector fp_for_thresh_per_class(classes,0); - - - - //count groundtruth and detections in total and for each class - for(auto i:images) - { - for(auto gt:i.gt) - truth_classes_count[gt.cl]++; - for(auto det:i.det) - dets_classes_count[det.cl]++; - detections_count += i.det.size(); - groundtruths_count += i.gt.size(); - } - - std::cout<<"gt_count: "< all_dets; - std::vector all_gts; - - int gt_checked = 0; - - // for each detection comput IoU with groundtruth and match detetcion and - // groundtruth with IoU greater than IoU_thresh - for(auto &img:images) - { - for(size_t i=0; i 0) - { - float maxIoU = 0; - int truth_index = -1; - for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) - { - maxIoU = currentIoU; - truth_index = j; - } - } - // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) - { - // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< truth_flags(groundtruths_count,0); - - if(verbose) - for(auto d:all_dets) - std::cout<> pr( classes, std::vector(detections_count)); - for(int rank = 0; rank< detections_count; ++rank) - { - if (rank > 0) - { - for (int class_id = 0; class_id < classes; ++class_id) - { - pr[class_id][rank].tp = pr[class_id][rank - 1].tp; - pr[class_id][rank].fp = pr[class_id][rank - 1].fp; - } - } - - //if it was detected and never detected before - if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) - { - truth_flags[all_dets[rank].unique_truth_index] = 1; - pr[all_dets[rank].cl][rank].tp++; // true-positive - } - else - { - pr[all_dets[rank].cl][rank].fp++; // false-positive - } - - for (int i = 0; i < classes; ++i) - { - const int tp = pr[i][rank].tp; - const int fp = pr[i][rank].fp; - const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive - pr[i][rank].fn = fn; - - if ((tp + fp) > 0) - pr[i][rank].precision = (double)tp / (double)(tp + fp); - else - pr[i][rank].precision = 0; - - if ((tp + fn) > 0) - pr[i][rank].recall = (double)tp / (double)(tp + fn); - else - pr[i][rank].recall = 0; - - if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) - { // check for last rank - printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); - } - } - } - - if(verbose) - { - for(int i=0; i < pr.size(); i++) - { - std::cout<<"---------Class "<= 0; --rank) - { - delta_recall = last_recall - pr[i][rank].recall; - last_recall = pr[i][rank].recall; - - if (pr[i][rank].precision > last_precision) - last_precision = pr[i][rank].precision; - - avg_precision += delta_recall * last_precision; - } - } - else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points - { - for (int point = 0; point < map_points; ++point) { - cur_recall = point * 1.0 / ( map_points - 1 ); - cur_precision = 0; - for (int rank = 0; rank < detections_count; ++rank) - if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) - cur_precision = pr[i][rank].precision; - - avg_precision += cur_precision; - } - avg_precision = avg_precision / map_points; - } - - std::cout<<"Class: "< 1) net = argv[1]; - char type = 'y'; + char ntype = 'y'; if(argc > 2) - type = argv[2][0]; + ntype = argv[2][0]; + //path to txt file with all realpath of images labels char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; if(argc > 3) labels_path = argv[3]; - - networkType_t ntype; - switch(type) - { - case 'y': - ntype = YOLO; - break; - case 'c': - ntype = CENTERNET; - break; - default: - FatalError("type not allowed (3rd parameter)"); - } - bool show = false; + bool write_dets = false; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; + switch(ntype) { - case YOLO: + case 'y': yolo.init(net); break; - case CENTERNET: + case 'c': cnet.init(net); break; default: - FatalError("Network type not allowed "); + FatalError("Network type not allowed (3rd parameter)\n"); } + std::ifstream all_labels(labels_path); std::string l_filename; std::vector images; @@ -372,14 +70,14 @@ int main(int argc, char *argv[]) std::vector detected_bbox; int i=0; - while (std::getline(all_labels, l_filename)) // && i < 1000) + while (std::getline(all_labels, l_filename) && i < 1000) { + std::cout </ / / / - BoundigBox b; + BoundingBox b; b.x = (d.x + d.w/2) / width; b.y = (d.y + d.h/2) / height; b.w = d.w / width; @@ -426,20 +123,22 @@ int main(int argc, char *argv[]) b.cl = d.cl; f.det.push_back(b); - // myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; + if(write_dets) + myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n"; if(show)// draw rectangle for detection cv::rectangle(frame, cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2); } - // myfile.close(); + if(write_dets) + myfile.close(); // read and save groundtruth labels std::ifstream labels(l_filename); for(std::string line; std::getline(labels, line); ) { std::istringstream in(line); - BoundigBox b; + BoundingBox b; in >> b.cl >> b.x >> b.y >> b.w >> b.h; b.prob = 1; b.truth_flag = 1; @@ -461,23 +160,14 @@ int main(int argc, char *argv[]) std::cout<<"Done."< gt; + std::vector det; + + void print() const; +}; + +struct PR +{ + double precision = 0; + double recall = 0; + int tp = 0, fp = 0, fn = 0; + + void print(); +}; + +float overlap(float x1, float w1, float x2, float w2); +float boxIntersection(const BoundingBox &a, const BoundingBox &b); +float boxUnion(const BoundingBox &a, const BoundingBox &b); +float boxIoU(const BoundingBox &a, const BoundingBox &b); + +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false); +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); + +#endif /*EVALUATION_H*/ \ No newline at end of file diff --git a/src/evaluation.cpp b/src/evaluation.cpp new file mode 100644 index 0000000..2b60449 --- /dev/null +++ b/src/evaluation.cpp @@ -0,0 +1,284 @@ +#include "evaluation.h" + + +void BoundingBox::clear() +{ + unique_truth_index = -1; + truth_flag = 0; + max_IoU = 0; +} + +bool boxComparison (const BoundingBox& a,const BoundingBox& b) +{ + return (a.prob>b.prob); +} + +std::ostream& operator<<(std::ostream& os, const BoundingBox& bb) +{ + os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y << + ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< + bb.truth_flag<< ", assignedGT: "<< bb.unique_truth_index<< + ", maxIoU: "<< bb.max_IoU<<"\n"; + return os; +} + +void Frame::print() const +{ + std::cout<<"labels filename: "< l2 ? l1 : l2; + float r1 = x1 + w1/2; + float r2 = x2 + w2/2; + float right = r1 < r2 ? r1 : r2; + return right - left; +} + +float boxIntersection(const BoundingBox &a, const BoundingBox &b) +{ + float w = overlap(a.x, a.w, b.x, b.w); + float h = overlap(a.y, a.h, b.y, b.h); + if(w < 0 || h < 0) + return 0; + float area = w*h; + return area; +} + +float boxUnion(const BoundingBox &a, const BoundingBox &b) +{ + float i = boxIntersection(a, b); + float u = a.w*a.h + b.w*b.h - i; + return u; +} + +float boxIoU(const BoundingBox &a, const BoundingBox &b) +{ + float I = boxIntersection(a, b); + // std::cout<<"I: "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose) +{ + + std::cout<<"Computing mAP"< truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + + //count groundtruth and detections in total and for each class + for(auto i:images) + { + for(auto gt:i.gt) + truth_classes_count[gt.cl]++; + for(auto det:i.det) + dets_classes_count[det.cl]++; + detections_count += i.det.size(); + groundtruths_count += i.gt.size(); + } + + std::cout<<"gt_count: "< all_dets; + std::vector all_gts; + + int gt_checked = 0; + + // for each detection comput IoU with groundtruth and match detetcion and + // groundtruth with IoU greater than IoU_thresh + for(auto &img:images) + { + for(size_t i=0; i 0) + { + float maxIoU = 0; + int truth_index = -1; + for(size_t j=0; j maxIoU && img.det[i].cl == img.gt[j].cl) + { + maxIoU = currentIoU; + truth_index = j; + } + } + // std::cout<<"det i:"< -1 && maxIoU > IoU_thresh) + { + // std::cout<<"(INSIDE) IoU thresh:"< IoU_thresh:"<<(maxIoU > IoU_thresh)< truth_flags(groundtruths_count,0); + + if(verbose) + for(auto d:all_dets) + std::cout<> pr( classes, std::vector(detections_count)); + for(int rank = 0; rank< detections_count; ++rank) + { + if (rank > 0) + { + for (int class_id = 0; class_id < classes; ++class_id) + { + pr[class_id][rank].tp = pr[class_id][rank - 1].tp; + pr[class_id][rank].fp = pr[class_id][rank - 1].fp; + } + } + + //if it was detected and never detected before + if (all_dets[rank].truth_flag == 1 && truth_flags[all_dets[rank].unique_truth_index] == 0) + { + truth_flags[all_dets[rank].unique_truth_index] = 1; + pr[all_dets[rank].cl][rank].tp++; // true-positive + } + else + { + pr[all_dets[rank].cl][rank].fp++; // false-positive + } + + for (int i = 0; i < classes; ++i) + { + const int tp = pr[i][rank].tp; + const int fp = pr[i][rank].fp; + const int fn = truth_classes_count[i] - tp; // false-negative = objects - true-positive + pr[i][rank].fn = fn; + + if ((tp + fp) > 0) + pr[i][rank].precision = (double)tp / (double)(tp + fp); + else + pr[i][rank].precision = 0; + + if ((tp + fn) > 0) + pr[i][rank].recall = (double)tp / (double)(tp + fn); + else + pr[i][rank].recall = 0; + + if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) + { // check for last rank + printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp); + } + } + } + + if(verbose) + { + for(int i=0; i < pr.size(); i++) + { + std::cout<<"---------Class "<= 0; --rank) + { + delta_recall = last_recall - pr[i][rank].recall; + last_recall = pr[i][rank].recall; + + if (pr[i][rank].precision > last_precision) + last_precision = pr[i][rank].precision; + + avg_precision += delta_recall * last_precision; + } + } + else //MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points + { + for (int point = 0; point < map_points; ++point) { + cur_recall = point * 1.0 / ( map_points - 1 ); + cur_precision = 0; + for (int rank = 0; rank < detections_count; ++rank) + if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) + cur_precision = pr[i][rank].precision; + + avg_precision += cur_precision; + } + avg_precision = avg_precision / map_points; + } + + std::cout<<"Class: "< &images,const int classes,const float i_IoU_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) +{ + double AP = 0; + float IoU_thresh = i_IoU_thresh; + for(int i=0; i Date: Fri, 7 Feb 2020 14:50:05 +0100 Subject: [PATCH 025/482] Remove mallocs and frees from the kernels Signed-off-by: Davide Sapienza --- include/sorting.h | 6 ++++-- include/tkDNN/CenternetDetection.h | 4 +++- src/CenternetDetection.cpp | 13 +++++++++---- src/sorting.cu | 20 ++++++-------------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/include/sorting.h b/include/sorting.h index d153c61..9c968f3 100644 --- a/include/sorting.h +++ b/include/sorting.h @@ -15,5 +15,7 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores, void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes); void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out); void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys); -void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin); -void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1); +void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, + float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out); +void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, + dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out); diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 1868839..e39644f 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -75,7 +75,9 @@ class CenternetDetection { int K = 100; int width = 128;//56; // TODO - + // pointer used in the kernels + float *src_out; + int *ids_out; public: dnnType *rt_out[4]; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index c6c4bc7..c707c6c 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -96,8 +96,13 @@ bool CenternetDetection::init(std::string tensor_path) { mean << 0.408, 0.447, 0.47; stddev << 0.289, 0.274, 0.278; - // mean << 0.485, 0.456, 0.406; - // stddev << 0.229, 0.224, 0.225; + + // Alloc array used in the kernel + checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + // checkCuda( cudaFree(src_out) ); + // checkCuda( cudaFree(ids_out) ); + } void CenternetDetection::testdog() { @@ -349,14 +354,14 @@ void CenternetDetection::update(cv::Mat &imageORIG) { // ----------- topk end - topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3]); + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); // checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d); + bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out); // checkCuda( cudaDeviceSynchronize() ); checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); diff --git a/src/sorting.cu b/src/sorting.cu index fd62cec..3ba5ac3 100644 --- a/src/sorting.cu +++ b/src/sorting.cu @@ -66,31 +66,25 @@ void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, co } -void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin){ - float *src_out; - checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); +void topKxyAddOffset(int * ids_begin, const int K, const int size, + int *intxs_begin, int *intys_begin, float *xs_begin, + float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out){ thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out); thrust::transform(thrust::device, intxs_begin, intxs_begin + K, src_out, xs_begin, thrust::plus()); - int *ids_out; - checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus()); thrust::gather(thrust::device, ids_out, ids_out+K, src_begin, src_out); thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus()); - checkCuda( cudaFree(src_out) ); - checkCuda( cudaFree(ids_out) ); } -void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1){ - float *src_out; - checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); +void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, + dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, + float *src_out, int *ids_out){ thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out); thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides()); // x0 thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx0, thrust::minus()); // x1 thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx1, thrust::plus()); - int *ids_out; - checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus()); thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out); thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides()); @@ -98,7 +92,5 @@ void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby0, thrust::minus()); // y1 thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby1, thrust::plus()); - checkCuda( cudaFree(src_out) ); - checkCuda( cudaFree(ids_out) ); } -- 2.52.0 From 9007e25a0064cd5916ab35e409877dfaf4000e82 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 7 Feb 2020 14:50:05 +0100 Subject: [PATCH 026/482] Remove mallocs and frees from the kernels Signed-off-by: Davide Sapienza --- include/sorting.h | 6 ++++-- include/tkDNN/CenternetDetection.h | 4 +++- src/CenternetDetection.cpp | 13 +++++++++---- src/sorting.cu | 20 ++++++-------------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/include/sorting.h b/include/sorting.h index d153c61..9c968f3 100644 --- a/include/sorting.h +++ b/include/sorting.h @@ -15,5 +15,7 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores, void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes); void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out); void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys); -void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin); -void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1); +void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, + float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out); +void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, + dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out); diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 1868839..e39644f 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -75,7 +75,9 @@ class CenternetDetection { int K = 100; int width = 128;//56; // TODO - + // pointer used in the kernels + float *src_out; + int *ids_out; public: dnnType *rt_out[4]; diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index c6c4bc7..c707c6c 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -96,8 +96,13 @@ bool CenternetDetection::init(std::string tensor_path) { mean << 0.408, 0.447, 0.47; stddev << 0.289, 0.274, 0.278; - // mean << 0.485, 0.456, 0.406; - // stddev << 0.229, 0.224, 0.225; + + // Alloc array used in the kernel + checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); + checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); + // checkCuda( cudaFree(src_out) ); + // checkCuda( cudaFree(ids_out) ); + } void CenternetDetection::testdog() { @@ -349,14 +354,14 @@ void CenternetDetection::update(cv::Mat &imageORIG) { // ----------- topk end - topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3]); + topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out); // checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; step_t = end_t; - bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d); + bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out); // checkCuda( cudaDeviceSynchronize() ); checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); diff --git a/src/sorting.cu b/src/sorting.cu index fd62cec..3ba5ac3 100644 --- a/src/sorting.cu +++ b/src/sorting.cu @@ -66,31 +66,25 @@ void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, co } -void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin){ - float *src_out; - checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); +void topKxyAddOffset(int * ids_begin, const int K, const int size, + int *intxs_begin, int *intys_begin, float *xs_begin, + float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out){ thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out); thrust::transform(thrust::device, intxs_begin, intxs_begin + K, src_out, xs_begin, thrust::plus()); - int *ids_out; - checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus()); thrust::gather(thrust::device, ids_out, ids_out+K, src_begin, src_out); thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus()); - checkCuda( cudaFree(src_out) ); - checkCuda( cudaFree(ids_out) ); } -void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1){ - float *src_out; - checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); +void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, + dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, + float *src_out, int *ids_out){ thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out); thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides()); // x0 thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx0, thrust::minus()); // x1 thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx1, thrust::plus()); - int *ids_out; - checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus()); thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out); thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides()); @@ -98,7 +92,5 @@ void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby0, thrust::minus()); // y1 thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby1, thrust::plus()); - checkCuda( cudaFree(src_out) ); - checkCuda( cudaFree(ids_out) ); } -- 2.52.0 From 5d07e98a9cadcacafd4532c249eae7956183a6ad Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 10:54:04 +0100 Subject: [PATCH 027/482] Update ResNet101 weights exporter Signed-off-by: Davide Sapienza --- tests/resnet101/resnet101_weightsexporter.py | 139 +++++++++---------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/tests/resnet101/resnet101_weightsexporter.py b/tests/resnet101/resnet101_weightsexporter.py index 140154e..e10a037 100644 --- a/tests/resnet101/resnet101_weightsexporter.py +++ b/tests/resnet101/resnet101_weightsexporter.py @@ -2,15 +2,26 @@ import torch import urllib from PIL import Image from torchvision import transforms -from torchsummary import summary import numpy as np import struct +import os +from pytorchcv.model_provider import get_model as ptcv_get_model +from torch.autograd import Variable + +from torchsummary import summary import torch.nn as nn +from torch.jit import trace + +def create_folders(): + if not os.path.exists('debug'): + os.makedirs('debug') + if not os.path.exists('layers'): + os.makedirs('layers') + def bin_write(f, data): data =data.flatten() - # print(data) fmt = 'f'*len(data) bin = struct.pack(fmt, *data) f.write(bin) @@ -18,38 +29,46 @@ def bin_write(f, data): def hook(module, input, output): setattr(module, "_value_hook", output) +def load_ex_image(model): + # Download an example image from the pytorch website + url, filename = ( + "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") + try: + urllib.URLopener().retrieve(url, filename) + except: + urllib.request.urlretrieve(url, filename) + + # sample execution (requires torchvision) + input_image = Image.open(filename) + print("input_image: ",input_image.size) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + input_tensor = preprocess(input_image) + print("input_tensor: ",input_tensor.shape) + # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) + # move the input and model to GPU for speed if available + if torch.cuda.is_available(): + input_batch = input_batch.to('cuda') + model.to('cuda') + + return model, input_batch -def print_wb(model, folder): - for name, param in model.named_parameters(): - # print ("Layer", name) - t = name.split('.')[0:-1] - arg = name.split('.')[-1] - t = '-'.join(t) - print (" type: ", t) - - if arg == 'weight': - w = param.data.numpy() - print (" weights shape:", np.shape(w)) - w.tofile(folder + "/" + t + ".bin", format="f") - elif arg == 'bias': - b = param.data.numpy() - print (" bias shape:", np.shape(b)) - b.tofile(folder + "/" + t + ".bias.bin", format="f") - else: - print("Ops!") - - -def print_wb_output(model, input_batch): - for n, m in model.named_modules(): - m.register_forward_hook(hook) - +def exp_input(model, input_batch): + # Export the input batch model(input_batch) - i = input_batch.data.numpy() + i = input_batch.cpu().data.numpy() i = np.array(i, dtype=np.float32) - print(i.shape) i.tofile("debug/input.bin", format="f") + print("input: ", i.shape) +def print_wb_output(model): f = None for n, m in model.named_modules(): in_output = m._value_hook @@ -57,7 +76,9 @@ def print_wb_output(model, input_batch): o = np.array(o, dtype=np.float32) t = '-'.join(n.split('.')) o.tofile("debug/" + t + ".bin", format="f") - + print('------- ', n, ' ------') + print("debug ",o.shape) + if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)): continue @@ -66,14 +87,8 @@ def print_wb_output(model, input_batch): print("open file: ", file_name) f = open(file_name, mode='wb') - print(n, ' ----------------------------------------------------------------') - # print(m._parameters) - #print(m.type) - w = np.array([]) b = np.array([]) - - if 'weight' in m._parameters and m._parameters['weight'] is not None: w = m._parameters['weight'].data.numpy() w = np.array(w, dtype=np.float32) @@ -83,9 +98,6 @@ def print_wb_output(model, input_batch): b = m._parameters['bias'].data.numpy() b = np.array(b, dtype=np.float32) print (" bias shape:", np.shape(b)) - # else: - # b = np.zeros(w.shape[0], dtype=np.float32) - # print (" bias shape:", np.shape(b)) if 'BatchNorm2d' in str(m.type): b = m._parameters['bias'].data.numpy() @@ -96,30 +108,24 @@ def print_wb_output(model, input_batch): rm = np.array(rm, dtype=np.float32) rv = m.running_var.data.numpy() rv = np.array(rv, dtype=np.float32) - #s.tofile(f, format="f") bin_write(f,b) bin_write(f,s) bin_write(f,rm) bin_write(f,rv) + print (" b shape:", np.shape(b)) print (" s shape:", np.shape(s)) print (" rm shape:", np.shape(rm)) print (" rv shape:", np.shape(rv)) else: - - # w.tofile(f, format="f") bin_write(f,w) - # print("w- ",w) if b.size > 0: - # b.tofile(f, format="f") bin_write(f,b) - # print("b - ",b) if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type): f.close() print("close file") f = None - # return @@ -130,34 +136,27 @@ if __name__ == '__main__': model = torch.hub.load('pytorch/vision', 'resnet101', pretrained=True) model.eval() - # Download an example image from the pytorch website - url, filename = ("https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") - try: urllib.URLopener().retrieve(url, filename) - except: urllib.request.urlretrieve(url, filename) - - # sample execution (requires torchvision) - input_image = Image.open(filename) - preprocess = transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - ]) - input_tensor = preprocess(input_image) - input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model - - # move the input and model to GPU for speed if available - if torch.cuda.is_available(): - input_batch = input_batch.to('cuda') - model.to('cuda') - + # load an example image and load it on model + model, input_batch = load_ex_image(model) + model.eval() with torch.no_grad(): output = model(input_batch) - # Tensor of shape 1000, with confidence scores over Imagenet's 1000 classes - # print(output) + # create folders debug and layers if do not exist + create_folders() + # add output attribute to the layers + for n, m in model.named_modules(): + m.register_forward_hook(hook) - print_wb_output(model, input_batch) + # export input bin + exp_input(model, input_batch) - # print(list(model.children())) + print_wb_output(model) + + with open("resnet101.txt", 'w') as f: + for item in list(model.children()): + f.write("%s\n" % item) + + summary(model, (3, 224, 224)) + # print(trace(model, input_batch)) -- 2.52.0 From 62fe82ce9e905cf815eb74b02b4b5ffd47d0253e Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 10:54:04 +0100 Subject: [PATCH 028/482] Update ResNet101 weights exporter Signed-off-by: Davide Sapienza --- tests/resnet101/resnet101_weightsexporter.py | 139 +++++++++---------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/tests/resnet101/resnet101_weightsexporter.py b/tests/resnet101/resnet101_weightsexporter.py index 140154e..e10a037 100644 --- a/tests/resnet101/resnet101_weightsexporter.py +++ b/tests/resnet101/resnet101_weightsexporter.py @@ -2,15 +2,26 @@ import torch import urllib from PIL import Image from torchvision import transforms -from torchsummary import summary import numpy as np import struct +import os +from pytorchcv.model_provider import get_model as ptcv_get_model +from torch.autograd import Variable + +from torchsummary import summary import torch.nn as nn +from torch.jit import trace + +def create_folders(): + if not os.path.exists('debug'): + os.makedirs('debug') + if not os.path.exists('layers'): + os.makedirs('layers') + def bin_write(f, data): data =data.flatten() - # print(data) fmt = 'f'*len(data) bin = struct.pack(fmt, *data) f.write(bin) @@ -18,38 +29,46 @@ def bin_write(f, data): def hook(module, input, output): setattr(module, "_value_hook", output) +def load_ex_image(model): + # Download an example image from the pytorch website + url, filename = ( + "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") + try: + urllib.URLopener().retrieve(url, filename) + except: + urllib.request.urlretrieve(url, filename) + + # sample execution (requires torchvision) + input_image = Image.open(filename) + print("input_image: ",input_image.size) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + input_tensor = preprocess(input_image) + print("input_tensor: ",input_tensor.shape) + # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) + # move the input and model to GPU for speed if available + if torch.cuda.is_available(): + input_batch = input_batch.to('cuda') + model.to('cuda') + + return model, input_batch -def print_wb(model, folder): - for name, param in model.named_parameters(): - # print ("Layer", name) - t = name.split('.')[0:-1] - arg = name.split('.')[-1] - t = '-'.join(t) - print (" type: ", t) - - if arg == 'weight': - w = param.data.numpy() - print (" weights shape:", np.shape(w)) - w.tofile(folder + "/" + t + ".bin", format="f") - elif arg == 'bias': - b = param.data.numpy() - print (" bias shape:", np.shape(b)) - b.tofile(folder + "/" + t + ".bias.bin", format="f") - else: - print("Ops!") - - -def print_wb_output(model, input_batch): - for n, m in model.named_modules(): - m.register_forward_hook(hook) - +def exp_input(model, input_batch): + # Export the input batch model(input_batch) - i = input_batch.data.numpy() + i = input_batch.cpu().data.numpy() i = np.array(i, dtype=np.float32) - print(i.shape) i.tofile("debug/input.bin", format="f") + print("input: ", i.shape) +def print_wb_output(model): f = None for n, m in model.named_modules(): in_output = m._value_hook @@ -57,7 +76,9 @@ def print_wb_output(model, input_batch): o = np.array(o, dtype=np.float32) t = '-'.join(n.split('.')) o.tofile("debug/" + t + ".bin", format="f") - + print('------- ', n, ' ------') + print("debug ",o.shape) + if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)): continue @@ -66,14 +87,8 @@ def print_wb_output(model, input_batch): print("open file: ", file_name) f = open(file_name, mode='wb') - print(n, ' ----------------------------------------------------------------') - # print(m._parameters) - #print(m.type) - w = np.array([]) b = np.array([]) - - if 'weight' in m._parameters and m._parameters['weight'] is not None: w = m._parameters['weight'].data.numpy() w = np.array(w, dtype=np.float32) @@ -83,9 +98,6 @@ def print_wb_output(model, input_batch): b = m._parameters['bias'].data.numpy() b = np.array(b, dtype=np.float32) print (" bias shape:", np.shape(b)) - # else: - # b = np.zeros(w.shape[0], dtype=np.float32) - # print (" bias shape:", np.shape(b)) if 'BatchNorm2d' in str(m.type): b = m._parameters['bias'].data.numpy() @@ -96,30 +108,24 @@ def print_wb_output(model, input_batch): rm = np.array(rm, dtype=np.float32) rv = m.running_var.data.numpy() rv = np.array(rv, dtype=np.float32) - #s.tofile(f, format="f") bin_write(f,b) bin_write(f,s) bin_write(f,rm) bin_write(f,rv) + print (" b shape:", np.shape(b)) print (" s shape:", np.shape(s)) print (" rm shape:", np.shape(rm)) print (" rv shape:", np.shape(rv)) else: - - # w.tofile(f, format="f") bin_write(f,w) - # print("w- ",w) if b.size > 0: - # b.tofile(f, format="f") bin_write(f,b) - # print("b - ",b) if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type): f.close() print("close file") f = None - # return @@ -130,34 +136,27 @@ if __name__ == '__main__': model = torch.hub.load('pytorch/vision', 'resnet101', pretrained=True) model.eval() - # Download an example image from the pytorch website - url, filename = ("https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") - try: urllib.URLopener().retrieve(url, filename) - except: urllib.request.urlretrieve(url, filename) - - # sample execution (requires torchvision) - input_image = Image.open(filename) - preprocess = transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - ]) - input_tensor = preprocess(input_image) - input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model - - # move the input and model to GPU for speed if available - if torch.cuda.is_available(): - input_batch = input_batch.to('cuda') - model.to('cuda') - + # load an example image and load it on model + model, input_batch = load_ex_image(model) + model.eval() with torch.no_grad(): output = model(input_batch) - # Tensor of shape 1000, with confidence scores over Imagenet's 1000 classes - # print(output) + # create folders debug and layers if do not exist + create_folders() + # add output attribute to the layers + for n, m in model.named_modules(): + m.register_forward_hook(hook) - print_wb_output(model, input_batch) + # export input bin + exp_input(model, input_batch) - # print(list(model.children())) + print_wb_output(model) + + with open("resnet101.txt", 'w') as f: + for item in list(model.children()): + f.write("%s\n" % item) + + summary(model, (3, 224, 224)) + # print(trace(model, input_batch)) -- 2.52.0 From 932de41e385884f4f8ae986e6c903e3904a23c77 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 10:56:33 +0100 Subject: [PATCH 029/482] Add DLA34 weights exporter Signed-off-by: Davide Sapienza --- tests/dla34/dla34_weightsexporter.py | 162 +++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/dla34/dla34_weightsexporter.py diff --git a/tests/dla34/dla34_weightsexporter.py b/tests/dla34/dla34_weightsexporter.py new file mode 100644 index 0000000..4a412df --- /dev/null +++ b/tests/dla34/dla34_weightsexporter.py @@ -0,0 +1,162 @@ +import torch +import urllib +from PIL import Image +from torchvision import transforms +import numpy as np +import struct +import os + +from pytorchcv.model_provider import get_model as ptcv_get_model +from torch.autograd import Variable + +from torchsummary import summary +import torch.nn as nn + +from torch.jit import trace + +def create_folders(): + if not os.path.exists('debug'): + os.makedirs('debug') + if not os.path.exists('layers'): + os.makedirs('layers') + +def bin_write(f, data): + data =data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + +def hook(module, input, output): + setattr(module, "_value_hook", output) + +def load_ex_image(model): + # Download an example image from the pytorch website + url, filename = ( + "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") + try: + urllib.URLopener().retrieve(url, filename) + except: + urllib.request.urlretrieve(url, filename) + + # sample execution (requires torchvision) + input_image = Image.open(filename) + print("input_image: ",input_image.size) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + input_tensor = preprocess(input_image) + print("input_tensor: ",input_tensor.shape) + # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) + + # move the input and model to GPU for speed if available + if torch.cuda.is_available(): + input_batch = input_batch.to('cuda') + model.to('cuda') + + return model, input_batch + +def exp_input(model, input_batch): + # Export the input batch + model(input_batch) + i = input_batch.cpu().data.numpy() + i = np.array(i, dtype=np.float32) + i.tofile("debug/input.bin", format="f") + print("input: ", i.shape) + +def print_wb_output(model): + f = None + for n, m in model.named_modules(): + m.eval() + if 'DLAResBlock' in str(m.type): + continue + + in_output = m._value_hook + o = in_output.data.numpy() + o = np.array(o, dtype=np.float32) + + t = '-'.join(n.split('.')) + o.tofile("debug/" + t + ".bin", format="f") + print('------- ', n, ' ------') + print("debug ",o.shape) + + if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)): + continue + + if ' of Conv2d' in str(m.type) or ' of Linear' in str(m.type): + file_name = "layers/" + t + ".bin" + print("open file: ", file_name) + f = open(file_name, mode='wb') + + w = np.array([]) + b = np.array([]) + if 'weight' in m._parameters and m._parameters['weight'] is not None: + w = m._parameters['weight'].data.numpy() + w = np.array(w, dtype=np.float32) + print (" weights shape:", np.shape(w)) + + if 'bias' in m._parameters and m._parameters['bias'] is not None: + b = m._parameters['bias'].data.numpy() + b = np.array(b, dtype=np.float32) + print (" bias shape:", np.shape(b)) + + if 'BatchNorm2d' in str(m.type): + b = m._parameters['bias'].data.numpy() + b = np.array(b, dtype=np.float32) + s = m._parameters['weight'].data.numpy() + s = np.array(s, dtype=np.float32) + rm = m.running_mean.data.numpy() + rm = np.array(rm, dtype=np.float32) + rv = m.running_var.data.numpy() + rv = np.array(rv, dtype=np.float32) + bin_write(f,b) + bin_write(f,s) + bin_write(f,rm) + bin_write(f,rv) + print (" b shape:", np.shape(b)) + print (" s shape:", np.shape(s)) + print (" rm shape:", np.shape(rm)) + print (" rv shape:", np.shape(rv)) + + else: + bin_write(f,w) + if b.size > 0 and b is not None: + bin_write(f,b) + + if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type): + f.close() + print("close file") + f = None + +if __name__ == '__main__': + model = ptcv_get_model("dla34", pretrained=True) + model.eval() + + # load an example image and load it on model + model, input_batch = load_ex_image(model) + model.eval() + with torch.no_grad(): + output = model(input_batch) + + # create folders debug and layers if do not exist + create_folders() + + # add output attribute to the layers + for n, m in model.named_modules(): + m.register_forward_hook(hook) + + # export input bin + exp_input(model, input_batch) + + print_wb_output(model) + + with open("dla34.txt", 'w') as f: + for item in list(model.children()): + f.write("%s\n" % item) + + summary(model, (3, 224, 224)) + # print(trace(model, input_batch)) -- 2.52.0 From e72aa348a0b571e4044a5c67141cce4fc18d462f Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 10:56:33 +0100 Subject: [PATCH 030/482] Add DLA34 weights exporter Signed-off-by: Davide Sapienza --- tests/dla34/dla34_weightsexporter.py | 162 +++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/dla34/dla34_weightsexporter.py diff --git a/tests/dla34/dla34_weightsexporter.py b/tests/dla34/dla34_weightsexporter.py new file mode 100644 index 0000000..4a412df --- /dev/null +++ b/tests/dla34/dla34_weightsexporter.py @@ -0,0 +1,162 @@ +import torch +import urllib +from PIL import Image +from torchvision import transforms +import numpy as np +import struct +import os + +from pytorchcv.model_provider import get_model as ptcv_get_model +from torch.autograd import Variable + +from torchsummary import summary +import torch.nn as nn + +from torch.jit import trace + +def create_folders(): + if not os.path.exists('debug'): + os.makedirs('debug') + if not os.path.exists('layers'): + os.makedirs('layers') + +def bin_write(f, data): + data =data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + +def hook(module, input, output): + setattr(module, "_value_hook", output) + +def load_ex_image(model): + # Download an example image from the pytorch website + url, filename = ( + "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg") + try: + urllib.URLopener().retrieve(url, filename) + except: + urllib.request.urlretrieve(url, filename) + + # sample execution (requires torchvision) + input_image = Image.open(filename) + print("input_image: ",input_image.size) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + input_tensor = preprocess(input_image) + print("input_tensor: ",input_tensor.shape) + # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) + + # move the input and model to GPU for speed if available + if torch.cuda.is_available(): + input_batch = input_batch.to('cuda') + model.to('cuda') + + return model, input_batch + +def exp_input(model, input_batch): + # Export the input batch + model(input_batch) + i = input_batch.cpu().data.numpy() + i = np.array(i, dtype=np.float32) + i.tofile("debug/input.bin", format="f") + print("input: ", i.shape) + +def print_wb_output(model): + f = None + for n, m in model.named_modules(): + m.eval() + if 'DLAResBlock' in str(m.type): + continue + + in_output = m._value_hook + o = in_output.data.numpy() + o = np.array(o, dtype=np.float32) + + t = '-'.join(n.split('.')) + o.tofile("debug/" + t + ".bin", format="f") + print('------- ', n, ' ------') + print("debug ",o.shape) + + if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)): + continue + + if ' of Conv2d' in str(m.type) or ' of Linear' in str(m.type): + file_name = "layers/" + t + ".bin" + print("open file: ", file_name) + f = open(file_name, mode='wb') + + w = np.array([]) + b = np.array([]) + if 'weight' in m._parameters and m._parameters['weight'] is not None: + w = m._parameters['weight'].data.numpy() + w = np.array(w, dtype=np.float32) + print (" weights shape:", np.shape(w)) + + if 'bias' in m._parameters and m._parameters['bias'] is not None: + b = m._parameters['bias'].data.numpy() + b = np.array(b, dtype=np.float32) + print (" bias shape:", np.shape(b)) + + if 'BatchNorm2d' in str(m.type): + b = m._parameters['bias'].data.numpy() + b = np.array(b, dtype=np.float32) + s = m._parameters['weight'].data.numpy() + s = np.array(s, dtype=np.float32) + rm = m.running_mean.data.numpy() + rm = np.array(rm, dtype=np.float32) + rv = m.running_var.data.numpy() + rv = np.array(rv, dtype=np.float32) + bin_write(f,b) + bin_write(f,s) + bin_write(f,rm) + bin_write(f,rv) + print (" b shape:", np.shape(b)) + print (" s shape:", np.shape(s)) + print (" rm shape:", np.shape(rm)) + print (" rv shape:", np.shape(rv)) + + else: + bin_write(f,w) + if b.size > 0 and b is not None: + bin_write(f,b) + + if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type): + f.close() + print("close file") + f = None + +if __name__ == '__main__': + model = ptcv_get_model("dla34", pretrained=True) + model.eval() + + # load an example image and load it on model + model, input_batch = load_ex_image(model) + model.eval() + with torch.no_grad(): + output = model(input_batch) + + # create folders debug and layers if do not exist + create_folders() + + # add output attribute to the layers + for n, m in model.named_modules(): + m.register_forward_hook(hook) + + # export input bin + exp_input(model, input_batch) + + print_wb_output(model) + + with open("dla34.txt", 'w') as f: + for item in list(model.children()): + f.write("%s\n" % item) + + summary(model, (3, 224, 224)) + # print(trace(model, input_batch)) -- 2.52.0 From da8ba7d8ce01939e9d35cba101e813cec8f59dd7 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 10:58:04 +0100 Subject: [PATCH 031/482] Add Anaconda environment for ResNet101 and DLA34 weights exporter Signed-off-by: Davide Sapienza --- tests/dla34/env_dla34.yml | 60 +++++++++++++++++++++++++++++++ tests/resnet101/env_resnet101.yml | 56 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 tests/dla34/env_dla34.yml create mode 100644 tests/resnet101/env_resnet101.yml diff --git a/tests/dla34/env_dla34.yml b/tests/dla34/env_dla34.yml new file mode 100644 index 0000000..4e27db2 --- /dev/null +++ b/tests/dla34/env_dla34.yml @@ -0,0 +1,60 @@ +name: dla34 +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _pytorch_select=0.2=gpu_0 + - blas=1.0=mkl + - ca-certificates=2019.10.16=0 + - certifi=2019.9.11=py36_0 + - cffi=1.13.1=py36h2e261b9_0 + - cudatoolkit=10.0.130=0 + - cudnn=7.6.0=cuda10.0_0 + - freetype=2.9.1=h8a8886c_1 + - intel-openmp=2019.4=243 + - jpeg=9b=h024ee3a_2 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=9.1.0=hdf63c60_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.1.0=hdf63c60_0 + - libtiff=4.0.10=h2733197_2 + - mkl=2019.4=243 + - mkl-service=2.3.0=py36he904b0f_0 + - mkl_fft=1.0.14=py36ha843d7b_0 + - mkl_random=1.1.0=py36hd6b4f25_0 + - ncurses=6.1=he6710b0_1 + - ninja=1.9.0=py36hfd86e86_0 + - numpy=1.17.2=py36haad9e8e_0 + - numpy-base=1.17.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openssl=1.1.1d=h7b6447c_3 + - pillow=6.2.0=py36h34e0f95_0 + - pip=19.3.1=py36_0 + - pycparser=2.19=py36_0 + - python=3.6.9=h265db76_0 + - readline=7.0=h7b6447c_5 + - setuptools=41.6.0=py36_0 + - six=1.12.0=py36_0 + - sqlite=3.30.1=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - wheel=0.33.6=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - chardet==3.0.4 + - decorator==4.4.1 + - idna==2.8 + - lxml==4.4.2 + - networkx==2.4 + - nltk==3.4.5 + - pytorchcv==0.0.55 + - requests==2.22.0 + - summary==0.2.0 + - torch==1.3.0 + - torchsummary==1.5.1 + - torchvision==0.4.1 + - urllib3==1.25.8 + diff --git a/tests/resnet101/env_resnet101.yml b/tests/resnet101/env_resnet101.yml new file mode 100644 index 0000000..35987c3 --- /dev/null +++ b/tests/resnet101/env_resnet101.yml @@ -0,0 +1,56 @@ +name: resnet101 +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _pytorch_select=0.2=gpu_0 + - blas=1.0=mkl + - ca-certificates=2019.10.16=0 + - certifi=2019.9.11=py36_0 + - cffi=1.13.1=py36h2e261b9_0 + - cudatoolkit=10.0.130=0 + - cudnn=7.6.0=cuda10.0_0 + - freetype=2.9.1=h8a8886c_1 + - intel-openmp=2019.4=243 + - jpeg=9b=h024ee3a_2 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=9.1.0=hdf63c60_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.1.0=hdf63c60_0 + - libtiff=4.0.10=h2733197_2 + - mkl=2019.4=243 + - mkl-service=2.3.0=py36he904b0f_0 + - mkl_fft=1.0.14=py36ha843d7b_0 + - mkl_random=1.1.0=py36hd6b4f25_0 + - ncurses=6.1=he6710b0_1 + - ninja=1.9.0=py36hfd86e86_0 + - numpy=1.17.2=py36haad9e8e_0 + - numpy-base=1.17.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openssl=1.1.1d=h7b6447c_3 + - pillow=6.2.0=py36h34e0f95_0 + - pip=19.3.1=py36_0 + - pycparser=2.19=py36_0 + - python=3.6.9=h265db76_0 + - pytorch=1.2.0=cuda100py36h938c94c_0 + - readline=7.0=h7b6447c_5 + - setuptools=41.6.0=py36_0 + - six=1.12.0=py36_0 + - sqlite=3.30.1=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - wheel=0.33.6=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - chardet==3.0.4 + - idna==2.8 + - pytorchcv==0.0.55 + - requests==2.22.0 + - torch==1.3.0 + - torchsummary==1.5.1 + - torchvision==0.4.1 + - urllib3==1.25.8 + -- 2.52.0 From c02238ddc8cde8b62b5c2bf97d584a20e6be98e6 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 10:58:04 +0100 Subject: [PATCH 032/482] Add Anaconda environment for ResNet101 and DLA34 weights exporter Signed-off-by: Davide Sapienza --- tests/dla34/env_dla34.yml | 60 +++++++++++++++++++++++++++++++ tests/resnet101/env_resnet101.yml | 56 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 tests/dla34/env_dla34.yml create mode 100644 tests/resnet101/env_resnet101.yml diff --git a/tests/dla34/env_dla34.yml b/tests/dla34/env_dla34.yml new file mode 100644 index 0000000..4e27db2 --- /dev/null +++ b/tests/dla34/env_dla34.yml @@ -0,0 +1,60 @@ +name: dla34 +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _pytorch_select=0.2=gpu_0 + - blas=1.0=mkl + - ca-certificates=2019.10.16=0 + - certifi=2019.9.11=py36_0 + - cffi=1.13.1=py36h2e261b9_0 + - cudatoolkit=10.0.130=0 + - cudnn=7.6.0=cuda10.0_0 + - freetype=2.9.1=h8a8886c_1 + - intel-openmp=2019.4=243 + - jpeg=9b=h024ee3a_2 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=9.1.0=hdf63c60_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.1.0=hdf63c60_0 + - libtiff=4.0.10=h2733197_2 + - mkl=2019.4=243 + - mkl-service=2.3.0=py36he904b0f_0 + - mkl_fft=1.0.14=py36ha843d7b_0 + - mkl_random=1.1.0=py36hd6b4f25_0 + - ncurses=6.1=he6710b0_1 + - ninja=1.9.0=py36hfd86e86_0 + - numpy=1.17.2=py36haad9e8e_0 + - numpy-base=1.17.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openssl=1.1.1d=h7b6447c_3 + - pillow=6.2.0=py36h34e0f95_0 + - pip=19.3.1=py36_0 + - pycparser=2.19=py36_0 + - python=3.6.9=h265db76_0 + - readline=7.0=h7b6447c_5 + - setuptools=41.6.0=py36_0 + - six=1.12.0=py36_0 + - sqlite=3.30.1=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - wheel=0.33.6=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - chardet==3.0.4 + - decorator==4.4.1 + - idna==2.8 + - lxml==4.4.2 + - networkx==2.4 + - nltk==3.4.5 + - pytorchcv==0.0.55 + - requests==2.22.0 + - summary==0.2.0 + - torch==1.3.0 + - torchsummary==1.5.1 + - torchvision==0.4.1 + - urllib3==1.25.8 + diff --git a/tests/resnet101/env_resnet101.yml b/tests/resnet101/env_resnet101.yml new file mode 100644 index 0000000..35987c3 --- /dev/null +++ b/tests/resnet101/env_resnet101.yml @@ -0,0 +1,56 @@ +name: resnet101 +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _pytorch_select=0.2=gpu_0 + - blas=1.0=mkl + - ca-certificates=2019.10.16=0 + - certifi=2019.9.11=py36_0 + - cffi=1.13.1=py36h2e261b9_0 + - cudatoolkit=10.0.130=0 + - cudnn=7.6.0=cuda10.0_0 + - freetype=2.9.1=h8a8886c_1 + - intel-openmp=2019.4=243 + - jpeg=9b=h024ee3a_2 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=9.1.0=hdf63c60_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.1.0=hdf63c60_0 + - libtiff=4.0.10=h2733197_2 + - mkl=2019.4=243 + - mkl-service=2.3.0=py36he904b0f_0 + - mkl_fft=1.0.14=py36ha843d7b_0 + - mkl_random=1.1.0=py36hd6b4f25_0 + - ncurses=6.1=he6710b0_1 + - ninja=1.9.0=py36hfd86e86_0 + - numpy=1.17.2=py36haad9e8e_0 + - numpy-base=1.17.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openssl=1.1.1d=h7b6447c_3 + - pillow=6.2.0=py36h34e0f95_0 + - pip=19.3.1=py36_0 + - pycparser=2.19=py36_0 + - python=3.6.9=h265db76_0 + - pytorch=1.2.0=cuda100py36h938c94c_0 + - readline=7.0=h7b6447c_5 + - setuptools=41.6.0=py36_0 + - six=1.12.0=py36_0 + - sqlite=3.30.1=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - wheel=0.33.6=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - chardet==3.0.4 + - idna==2.8 + - pytorchcv==0.0.55 + - requests==2.22.0 + - torch==1.3.0 + - torchsummary==1.5.1 + - torchvision==0.4.1 + - urllib3==1.25.8 + -- 2.52.0 From 34ecb6a1bbc9e82c17a3da628198fda8e302e253 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 11:19:39 +0100 Subject: [PATCH 033/482] Update README.md Signed-off-by: Davide Sapienza --- README.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/README.md b/README.md index 3d956a5..12bb124 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,36 @@ make during the cmake configuration it will be dowloaded the weights needed for running the tests +## DLA34 and ResNet101 weights +To get weights and outputs needed for running the tests you can use the Python +script and the Anaconda environment included in the repository. + +Create Anaconda environment and activate it: +``` +conda env create -f file_name.yml +source activate env_name +``` +Run the Python script inside the environment. + +## CenterNet weights +To get the weights needed for running the tests: + +* clone the forked repository by the original CenterNet: +``` +git clone https://github.com/sapienzadavide/CenterNet.git +``` +* follow the instruction in the README.md and INSTALL.md +* copy the weigths and outputs from /path/to/CenterNet/src/ in ./test/centernet-path/ . For example: +``` +cp /path/to/CenterNet/src/layers_dla/* ./test/dla34_cnet/layers/ +cp /path/to/CenterNet/src/debug_dla/* ./test/dla34_cnet/debug/ +``` +or +``` +cp /path/to/CenterNet/src/layers_resdcn/* ./test/resnet101_cnet/layers/ +cp /path/to/CenterNet/src/debug_resdcn/* ./test/resnet101_cnet/debug/ +``` + ## Test Assumiung you have correctly builded the library these are the test ready to exec: * test_simple: a simple convolutional and dense network (CUDNN only) @@ -35,6 +65,11 @@ Assumiung you have correctly builded the library these are the test ready to exe * test_yolo: YOLO detection network (CUDNN and TENSORRT) * test_yolo_tiny: smaller version of YOLO (CUDNN and TENSRRT) * test_yolo3_berkeley: our yolo3 version trained with BDD100K dateset +* test_resnet101: ResNet101 network (CUDNN and TENSORRT) +* test_resnet101_cnet: CenterNet detection based on ResNet101 (CUDNN and TENSORRT) +* test_dla34: DLA34 network (CUDNN and TENSORRT) +* test_dla34_cnet: CenterNet detection based on DLA34 (CUDNN and TENSORRT) + ## yolo3 berkeley demo detection For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process: @@ -50,3 +85,31 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) + + +## CenterNet (DLA34, ResNet101) demo detection +For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process: +``` +export TKDNN_MODE=FP16 # set the half floating point optimization +``` + +For CenterNet based on ResNet101: +``` +rm resnet101_cnet.rt # be sure to delete(or move) old tensorRT files +./test_resnet101_cnet # run the yolo test (is slow) +# with f16 inference the result will be a bit incorrect +``` + +For CenterNet based on DLA34: +``` +rm dla34_cnet.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet # run the yolo test (is slow) +# with f16 inference the result will be a bit incorrect +``` + +this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used for live detection: +``` +./centernet_demo # launch detection on a demo video +./centernet_demo resnet101_cnet.rt /dev/video0 # launch detection on device 0 +./centernet_demo dla34_cnet.rt /dev/video0 # launch detection on device 0 +``` \ No newline at end of file -- 2.52.0 From b06dc286e974f1fb9892dba4bf8424aa5dfa2154 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Mon, 10 Feb 2020 11:19:39 +0100 Subject: [PATCH 034/482] Update README.md Signed-off-by: Davide Sapienza --- README.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/README.md b/README.md index 3d956a5..12bb124 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,36 @@ make during the cmake configuration it will be dowloaded the weights needed for running the tests +## DLA34 and ResNet101 weights +To get weights and outputs needed for running the tests you can use the Python +script and the Anaconda environment included in the repository. + +Create Anaconda environment and activate it: +``` +conda env create -f file_name.yml +source activate env_name +``` +Run the Python script inside the environment. + +## CenterNet weights +To get the weights needed for running the tests: + +* clone the forked repository by the original CenterNet: +``` +git clone https://github.com/sapienzadavide/CenterNet.git +``` +* follow the instruction in the README.md and INSTALL.md +* copy the weigths and outputs from /path/to/CenterNet/src/ in ./test/centernet-path/ . For example: +``` +cp /path/to/CenterNet/src/layers_dla/* ./test/dla34_cnet/layers/ +cp /path/to/CenterNet/src/debug_dla/* ./test/dla34_cnet/debug/ +``` +or +``` +cp /path/to/CenterNet/src/layers_resdcn/* ./test/resnet101_cnet/layers/ +cp /path/to/CenterNet/src/debug_resdcn/* ./test/resnet101_cnet/debug/ +``` + ## Test Assumiung you have correctly builded the library these are the test ready to exec: * test_simple: a simple convolutional and dense network (CUDNN only) @@ -35,6 +65,11 @@ Assumiung you have correctly builded the library these are the test ready to exe * test_yolo: YOLO detection network (CUDNN and TENSORRT) * test_yolo_tiny: smaller version of YOLO (CUDNN and TENSRRT) * test_yolo3_berkeley: our yolo3 version trained with BDD100K dateset +* test_resnet101: ResNet101 network (CUDNN and TENSORRT) +* test_resnet101_cnet: CenterNet detection based on ResNet101 (CUDNN and TENSORRT) +* test_dla34: DLA34 network (CUDNN and TENSORRT) +* test_dla34_cnet: CenterNet detection based on DLA34 (CUDNN and TENSORRT) + ## yolo3 berkeley demo detection For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process: @@ -50,3 +85,31 @@ this will genereate a yolo3_berkeley.rt file that can be used for live detection ./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0 ``` ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif) + + +## CenterNet (DLA34, ResNet101) demo detection +For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process: +``` +export TKDNN_MODE=FP16 # set the half floating point optimization +``` + +For CenterNet based on ResNet101: +``` +rm resnet101_cnet.rt # be sure to delete(or move) old tensorRT files +./test_resnet101_cnet # run the yolo test (is slow) +# with f16 inference the result will be a bit incorrect +``` + +For CenterNet based on DLA34: +``` +rm dla34_cnet.rt # be sure to delete(or move) old tensorRT files +./test_dla34_cnet # run the yolo test (is slow) +# with f16 inference the result will be a bit incorrect +``` + +this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used for live detection: +``` +./centernet_demo # launch detection on a demo video +./centernet_demo resnet101_cnet.rt /dev/video0 # launch detection on device 0 +./centernet_demo dla34_cnet.rt /dev/video0 # launch detection on device 0 +``` \ No newline at end of file -- 2.52.0 From c7cb3e75713f482271f31ba3f546b105da1b2766 Mon Sep 17 00:00:00 2001 From: xavier Date: Mon, 10 Feb 2020 18:19:08 +0100 Subject: [PATCH 035/482] Add avg precision, recall, f1score computation, other minors Signed-off-by: xavier --- demo/demo/map.cpp | 5 ++- include/evaluation.h | 6 ++- src/evaluation.cpp | 99 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index f7e9e3e..bbbd77a 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -164,11 +164,14 @@ int main(int argc, char *argv[]) int map_levels = 10; float map_step = 0.05; float IoU_thresh = 0.5; + float conf_thresh = 0.3; bool verbose = false; - double AP = computeMapNIoULevels(images,classes,IoU_thresh, map_points, map_step, map_levels, verbose); + double AP = computeMapNIoULevels(images,classes,IoU_thresh,conf_thresh, map_points, map_step, map_levels, verbose); std::cout<<"mAP "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false); -double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh=0.3, const int map_points=101, const bool verbose=false); +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); + +void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh=0.5, const float conf_thresh=0.3, bool verbose=false); #endif /*EVALUATION_H*/ \ No newline at end of file diff --git a/src/evaluation.cpp b/src/evaluation.cpp index 2b60449..65f7708 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -77,11 +77,8 @@ float boxIoU(const BoundingBox &a, const BoundingBox &b) } /* Credits to https://github.com/AlexeyAB/darknet/blob/master/src/detector.c*/ -double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose) +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, const int map_points, const bool verbose) { - - std::cout<<"Computing mAP"< &images,const int classes,const float IoU_t groundtruths_count += i.gt.size(); } - std::cout<<"gt_count: "< all_dets; std::vector all_gts; @@ -117,7 +117,7 @@ double computeMap(std::vector &images,const int classes,const float IoU_t { for(size_t i=0; i 0) + if(img.det[i].prob > conf_thresh) { float maxIoU = 0; int truth_index = -1; @@ -257,17 +257,18 @@ double computeMap(std::vector &images,const int classes,const float IoU_t avg_precision = avg_precision / map_points; } - std::cout<<"Class: "< &images,const int classes,const float i_IoU_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh, const float conf_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) { double AP = 0; float IoU_thresh = i_IoU_thresh; @@ -276,9 +277,85 @@ double computeMapNIoULevels(std::vector &images,const int classes,const f for(auto& img:images) for(auto & d:img.det) d.clear(); - AP += computeMap(images,classes,IoU_thresh,map_points, verbose); + AP += computeMap(images,classes,IoU_thresh,conf_thresh,map_points, verbose); IoU_thresh +=map_step; } AP/=map_levels; return AP; +} + +void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, bool verbose) +{ + std::vector truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + std::vector pr( classes); + + for(auto &img:images) + { + for(auto& tc: truth_classes_count) + tc = 0; + for(auto& dc: dets_classes_count) + dc = 0; + + std::vector det_assigned(img.det.size(), false); + for(size_t j=0; j conf_thresh) + { + float currentIoU = boxIoU(img.det[i], img.gt[j]); + if(currentIoU > maxIoU && img.det[i].cl == img.gt[j].cl && !det_assigned[i]) + { + maxIoU = currentIoU; + det_index = i; + } + } + } + if(det_index > -1 && maxIoU > IoU_thresh && !det_assigned[det_index]) + { + img.det[det_index].unique_truth_index = j; + img.det[det_index].truth_flag = 1; + img.det[det_index].max_IoU = maxIoU; + det_assigned[det_index] = true; + dets_classes_count[img.det[det_index].cl]++; + } + } + + for(size_t i=0; i 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fp) : 0; + pr[i].recall = (pr[i].tp + pr[i].fn) > 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fn) : 0; + if(verbose) + std::cout<<"Class "< 0 ? 2 * ( avg_precision * avg_recall ) / ( avg_precision + avg_recall ) : 0; + + std::cout<<"avg precision: "< Date: Mon, 10 Feb 2020 18:19:08 +0100 Subject: [PATCH 036/482] Add avg precision, recall, f1score computation, other minors Signed-off-by: xavier --- demo/demo/map.cpp | 5 ++- include/evaluation.h | 6 ++- src/evaluation.cpp | 99 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index f7e9e3e..bbbd77a 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -164,11 +164,14 @@ int main(int argc, char *argv[]) int map_levels = 10; float map_step = 0.05; float IoU_thresh = 0.5; + float conf_thresh = 0.3; bool verbose = false; - double AP = computeMapNIoULevels(images,classes,IoU_thresh, map_points, map_step, map_levels, verbose); + double AP = computeMapNIoULevels(images,classes,IoU_thresh,conf_thresh, map_points, map_step, map_levels, verbose); std::cout<<"mAP "< &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose=false); -double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh=0.3, const int map_points=101, const bool verbose=false); +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); + +void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh=0.5, const float conf_thresh=0.3, bool verbose=false); #endif /*EVALUATION_H*/ \ No newline at end of file diff --git a/src/evaluation.cpp b/src/evaluation.cpp index 2b60449..65f7708 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -77,11 +77,8 @@ float boxIoU(const BoundingBox &a, const BoundingBox &b) } /* Credits to https://github.com/AlexeyAB/darknet/blob/master/src/detector.c*/ -double computeMap(std::vector &images,const int classes,const float IoU_thresh, const int map_points, const bool verbose) +double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, const int map_points, const bool verbose) { - - std::cout<<"Computing mAP"< &images,const int classes,const float IoU_t groundtruths_count += i.gt.size(); } - std::cout<<"gt_count: "< all_dets; std::vector all_gts; @@ -117,7 +117,7 @@ double computeMap(std::vector &images,const int classes,const float IoU_t { for(size_t i=0; i 0) + if(img.det[i].prob > conf_thresh) { float maxIoU = 0; int truth_index = -1; @@ -257,17 +257,18 @@ double computeMap(std::vector &images,const int classes,const float IoU_t avg_precision = avg_precision / map_points; } - std::cout<<"Class: "< &images,const int classes,const float i_IoU_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) +double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh, const float conf_thresh, const int map_points, const float map_step, const int map_levels, const bool verbose) { double AP = 0; float IoU_thresh = i_IoU_thresh; @@ -276,9 +277,85 @@ double computeMapNIoULevels(std::vector &images,const int classes,const f for(auto& img:images) for(auto & d:img.det) d.clear(); - AP += computeMap(images,classes,IoU_thresh,map_points, verbose); + AP += computeMap(images,classes,IoU_thresh,conf_thresh,map_points, verbose); IoU_thresh +=map_step; } AP/=map_levels; return AP; +} + +void computeTPFPFN(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, bool verbose) +{ + std::vector truth_classes_count(classes,0); + std::vector dets_classes_count(classes,0); + std::vector pr( classes); + + for(auto &img:images) + { + for(auto& tc: truth_classes_count) + tc = 0; + for(auto& dc: dets_classes_count) + dc = 0; + + std::vector det_assigned(img.det.size(), false); + for(size_t j=0; j conf_thresh) + { + float currentIoU = boxIoU(img.det[i], img.gt[j]); + if(currentIoU > maxIoU && img.det[i].cl == img.gt[j].cl && !det_assigned[i]) + { + maxIoU = currentIoU; + det_index = i; + } + } + } + if(det_index > -1 && maxIoU > IoU_thresh && !det_assigned[det_index]) + { + img.det[det_index].unique_truth_index = j; + img.det[det_index].truth_flag = 1; + img.det[det_index].max_IoU = maxIoU; + det_assigned[det_index] = true; + dets_classes_count[img.det[det_index].cl]++; + } + } + + for(size_t i=0; i 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fp) : 0; + pr[i].recall = (pr[i].tp + pr[i].fn) > 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fn) : 0; + if(verbose) + std::cout<<"Class "< 0 ? 2 * ( avg_precision * avg_recall ) / ( avg_precision + avg_recall ) : 0; + + std::cout<<"avg precision: "< Date: Tue, 11 Feb 2020 10:06:08 +0100 Subject: [PATCH 037/482] Optimize deformable kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off.by: Ignacio Sañudo Olmedo Signed-off-by: Davide Sapienza --- CMakeLists.txt | 1 + include/tkDNN/kernels.h | 12 +- src/kernels/deformable_conv.cu | 227 ++++++++++++++++++++++++--------- 3 files changed, 180 insertions(+), 60 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 27d99ea..a4a3caa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ endif() find_package(CUDA 9.0 REQUIRED) SET(CUDA_SEPARABLE_COMPILATION ON) #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'") +#set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) find_package(CUDNN REQUIRED) diff --git a/include/tkDNN/kernels.h b/include/tkDNN/kernels.h index 94c5dcb..f514ffc 100644 --- a/include/tkDNN/kernels.h +++ b/include/tkDNN/kernels.h @@ -30,14 +30,20 @@ void upsampleForward(dnnType* srcData, dnnType* dstData, void float2half(float* srcData, __half* dstData, int size, const cudaStream_t stream = cudaStream_t(0)); +// void modulated_deformable_im2col_cuda(cudaStream_t stream, +// const float *data_im, const float *data_offset, const float *data_mask, +// const int batch_size, const int channels, const int height_im, const int width_im, +// const int height_col, const int width_col, const int kernel_h, const int kenerl_w, +// const int pad_h, const int pad_w, const int stride_h, const int stride_w, +// const int dilation_h, const int dilation_w, +// const int deformable_group, float *data_col); void modulated_deformable_im2col_cuda(cudaStream_t stream, const float *data_im, const float *data_offset, const float *data_mask, const int batch_size, const int channels, const int height_im, const int width_im, - const int height_col, const int width_col, const int kernel_h, const int kenerl_w, - const int pad_h, const int pad_w, const int stride_h, const int stride_w, - const int dilation_h, const int dilation_w, + const int height_col, const int width_col, const int deformable_group, float *data_col); + void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, float *input, float *weight, float *bias, float *ones, diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index 0579620..88e62e9 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -9,7 +9,7 @@ i < (n); \ i += blockDim.x * gridDim.x) -const int CUDA_NUM_THREADS = 1024; +const int CUDA_NUM_THREADS = 512; inline int GET_BLOCKS(const int N) { return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; @@ -17,84 +17,89 @@ inline int GET_BLOCKS(const int N) __device__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width, - const int height, const int width, float h, float w) + const int height, const int width, float h, float w) { - int h_low = floor(h); - int w_low = floor(w); - int h_high = h_low + 1; - int w_high = w_low + 1; +int h_low = floor(h); +int w_low = floor(w); +int h_high = h_low + 1; +int w_high = w_low + 1; - float lh = h - h_low; - float lw = w - w_low; - float hh = 1 - lh, hw = 1 - lw; +float lh = h - h_low; +float lw = w - w_low; +float hh = 1 - lh, hw = 1 - lw; - float v1 = 0; - if (h_low >= 0 && w_low >= 0) - v1 = bottom_data[h_low * data_width + w_low]; - float v2 = 0; - if (h_low >= 0 && w_high <= width - 1) - v2 = bottom_data[h_low * data_width + w_high]; - float v3 = 0; - if (h_high <= height - 1 && w_low >= 0) - v3 = bottom_data[h_high * data_width + w_low]; - float v4 = 0; - if (h_high <= height - 1 && w_high <= width - 1) - v4 = bottom_data[h_high * data_width + w_high]; +float v1 = ( (h_low >= 0 && w_low >= 0) ? bottom_data[h_low * data_width + w_low]:0); +float v2 = ( (h_low >= 0 && w_high <= width - 1) ? bottom_data[h_low * data_width + w_high]:0); +float v3 = ( (h_high <= height - 1 && w_low >= 0) ? bottom_data[h_high * data_width + w_low]:0); +float v4 = ( (h_high <= height - 1 && w_high <= width - 1) ? bottom_data[h_high * data_width + w_high]:0); - float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; +float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; - float val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); - return val; +float val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); +return val; } __global__ void modulated_deformable_im2col_gpu_kernel(const int n, - const float *data_im, const float *data_offset, const float *data_mask, - const int height, const int width, const int kernel_h, const int kernel_w, - const int pad_h, const int pad_w, - const int stride_h, const int stride_w, - const int dilation_h, const int dilation_w, - const int channel_per_deformable_group, - const int batch_size, const int num_channels, const int deformable_group, - const int height_col, const int width_col, - float *data_col) + const float *data_im, const float *data_offset, const float *data_mask, + const int height, const int width, + const int batch_size, const int num_channels, const int deformable_group, + const int height_col, const int width_col, + float *data_col) { CUDA_KERNEL_LOOP(index, n) { + //If n is a power of 2, ( i / n ) is equivalent to ( i ≫ log2 n ) and ( i % n ) is equivalent to ( i & n - 1 ). + const int ind_on_w = index / width_col; + const int ind_on_w_on_h = ind_on_w / height_col; + const int kk = 3 * 3; // index index of output matrix const int w_col = index % width_col; - const int h_col = (index / width_col) % height_col; - const int b_col = (index / width_col / height_col) % batch_size; - const int c_im = (index / width_col / height_col) / batch_size; - const int c_col = c_im * kernel_h * kernel_w; + const int h_col = (ind_on_w) % height_col; + const int b_col = (ind_on_w_on_h) % batch_size; + const int c_im = (ind_on_w_on_h) / batch_size; + const int c_col = c_im * kk; // compute deformable group index - const int deformable_group_index = c_im / channel_per_deformable_group; + const int deformable_group_index = c_im / (int)(num_channels / deformable_group); - const int h_in = h_col * stride_h - pad_h; - const int w_in = w_col * stride_w - pad_w; + const int h_in = h_col - 1; + const int w_in = w_col - 1; + const int s_col = height_col * width_col; + const int s_col2 = 2 * s_col; float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; - const float *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; + const float *data_offset_ptr = data_offset + add_ptr + add_ptr; - const float *data_mask_ptr = data_mask + (b_col * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + const float *data_mask_ptr = data_mask + add_ptr; - for (int i = 0; i < kernel_h; ++i) + const int first_member = w_col + width_col * h_col; + float val = static_cast(0); + #pragma unroll + for (int i = 0; i < 3; ++i) { - for (int j = 0; j < kernel_w; ++j) + #pragma unroll + for (int j = 0; j < 3; ++j) { - const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; - const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; - const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const int iter_member = (i * 3 + j); + // const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_h_ptr = first_member + s_col2 * iter_member; + + // const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = s_col + first_member + s_col2 * iter_member; + + // const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = first_member + s_col * iter_member; + const float offset_h = data_offset_ptr[data_offset_h_ptr]; const float offset_w = data_offset_ptr[data_offset_w_ptr]; const float mask = data_mask_ptr[data_mask_hw_ptr]; - float val = static_cast(0); - const float h_im = h_in + i * dilation_h + offset_h; - const float w_im = w_in + j * dilation_w + offset_w; + const float h_im = offset_h + h_in + i; + const float w_im = offset_w + w_in + j; //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { - if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + if (h_im < height && w_im < width && h_im > -1 && w_im > -1) { //const float map_h = i * dilation_h + offset_h; //const float map_w = j * dilation_w + offset_w; @@ -104,7 +109,89 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); } *data_col_ptr = val * mask; - data_col_ptr += batch_size * height_col * width_col; + data_col_ptr += batch_size * s_col; + //data_col_ptr += height_col * width_col; + } + } + } +} + +__global__ void modulated_deformable_im2col_gpu_kernel2(const int n, + const float *data_im, const float *data_offset, const float *data_mask, + const int height, const int width, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, const int deformable_group, + const int height_col, const int width_col, + float *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + //If n is a power of 2, ( i / n ) is equivalent to ( i ≫ log2 n ) and ( i % n ) is equivalent to ( i & n - 1 ). + // printf("--- %d %d %d %d %d %d %d %d\n",kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w); + const int ind_on_w = index / width_col; + const int ind_on_w_on_h = ind_on_w / height_col; + const int kk = kernel_h * kernel_w; + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (ind_on_w) % height_col; + const int b_col = (ind_on_w_on_h) % batch_size; + const int c_im = (ind_on_w_on_h) / batch_size; + const int c_col = c_im * kk; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + const int s_col = height_col * width_col; + const int s_col2 = 2 * s_col; + + float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; + const float *data_offset_ptr = data_offset + add_ptr + add_ptr; + + const float *data_mask_ptr = data_mask + add_ptr; + + const int first_member = w_col + width_col * h_col; + float val = static_cast(0); + #pragma unroll + for (int i = 0; i < kernel_h; ++i) + { + #pragma unroll + for (int j = 0; j < kernel_w; ++j) + { + const int iter_member = (i * kernel_w + j); + // const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_h_ptr = first_member + s_col2 * iter_member; + + // const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = s_col + first_member + s_col2 * iter_member; + + // const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = first_member + s_col * iter_member; + + const float offset_h = data_offset_ptr[data_offset_h_ptr]; + const float offset_w = data_offset_ptr[data_offset_w_ptr]; + const float mask = data_mask_ptr[data_mask_hw_ptr]; + const float h_im = offset_h + h_in + i * dilation_h; + const float w_im = offset_w + w_in + j * dilation_w; + //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + if (h_im < height && w_im < width && h_im > -1 && w_im > -1) + { + //const float map_h = i * dilation_h + offset_h; + //const float map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * s_col; //data_col_ptr += height_col * width_col; } } @@ -113,6 +200,28 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, void modulated_deformable_im2col_cuda(cudaStream_t stream, + const float* data_im, const float* data_offset, const float* data_mask, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, + const int deformable_group, float* data_col) { + // num_axes should be smaller than block size + // const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + modulated_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, data_mask, height_im, width_im, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +void modulated_deformable_im2col_cuda2(cudaStream_t stream, const float* data_im, const float* data_offset, const float* data_mask, const int batch_size, const int channels, const int height_im, const int width_im, const int height_col, const int width_col, const int kernel_h, const int kenerl_w, @@ -122,7 +231,7 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, // num_axes should be smaller than block size const int channel_per_deformable_group = channels / deformable_group; const int num_kernels = channels * batch_size * height_col * width_col; - modulated_deformable_im2col_gpu_kernel + modulated_deformable_im2col_gpu_kernel2 <<>>( num_kernels, data_im, data_offset, data_mask, height_im, width_im, kernel_h, kenerl_w, @@ -137,7 +246,6 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, } - void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, float *input, float *weight, float *bias, float *ones, @@ -182,9 +290,14 @@ void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, input, offset, mask, 1, channels, height, width, - height_out, width_out, kernel_h, kernel_w, - pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, - deformable_group, columns); + height_out, width_out, deformable_group, columns); + // modulated_deformable_im2col_cuda2(stream, + // input, offset, + // mask, + // 1, channels, height, width, + // height_out, width_out, kernel_h, kernel_w, + // pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + // deformable_group, columns); //(k * m) x (m * n) // Y = WC -- 2.52.0 From 51ffcb1f50e3c9aa2b24f49be48d5d54daa71cb8 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 Feb 2020 10:06:08 +0100 Subject: [PATCH 038/482] Optimize deformable kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off.by: Ignacio Sañudo Olmedo Signed-off-by: Davide Sapienza --- CMakeLists.txt | 1 + include/tkDNN/kernels.h | 12 +- src/kernels/deformable_conv.cu | 227 ++++++++++++++++++++++++--------- 3 files changed, 180 insertions(+), 60 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 27d99ea..a4a3caa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ endif() find_package(CUDA 9.0 REQUIRED) SET(CUDA_SEPARABLE_COMPILATION ON) #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'") +#set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) find_package(CUDNN REQUIRED) diff --git a/include/tkDNN/kernels.h b/include/tkDNN/kernels.h index 94c5dcb..f514ffc 100644 --- a/include/tkDNN/kernels.h +++ b/include/tkDNN/kernels.h @@ -30,14 +30,20 @@ void upsampleForward(dnnType* srcData, dnnType* dstData, void float2half(float* srcData, __half* dstData, int size, const cudaStream_t stream = cudaStream_t(0)); +// void modulated_deformable_im2col_cuda(cudaStream_t stream, +// const float *data_im, const float *data_offset, const float *data_mask, +// const int batch_size, const int channels, const int height_im, const int width_im, +// const int height_col, const int width_col, const int kernel_h, const int kenerl_w, +// const int pad_h, const int pad_w, const int stride_h, const int stride_w, +// const int dilation_h, const int dilation_w, +// const int deformable_group, float *data_col); void modulated_deformable_im2col_cuda(cudaStream_t stream, const float *data_im, const float *data_offset, const float *data_mask, const int batch_size, const int channels, const int height_im, const int width_im, - const int height_col, const int width_col, const int kernel_h, const int kenerl_w, - const int pad_h, const int pad_w, const int stride_h, const int stride_w, - const int dilation_h, const int dilation_w, + const int height_col, const int width_col, const int deformable_group, float *data_col); + void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, float *input, float *weight, float *bias, float *ones, diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index 0579620..88e62e9 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -9,7 +9,7 @@ i < (n); \ i += blockDim.x * gridDim.x) -const int CUDA_NUM_THREADS = 1024; +const int CUDA_NUM_THREADS = 512; inline int GET_BLOCKS(const int N) { return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; @@ -17,84 +17,89 @@ inline int GET_BLOCKS(const int N) __device__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width, - const int height, const int width, float h, float w) + const int height, const int width, float h, float w) { - int h_low = floor(h); - int w_low = floor(w); - int h_high = h_low + 1; - int w_high = w_low + 1; +int h_low = floor(h); +int w_low = floor(w); +int h_high = h_low + 1; +int w_high = w_low + 1; - float lh = h - h_low; - float lw = w - w_low; - float hh = 1 - lh, hw = 1 - lw; +float lh = h - h_low; +float lw = w - w_low; +float hh = 1 - lh, hw = 1 - lw; - float v1 = 0; - if (h_low >= 0 && w_low >= 0) - v1 = bottom_data[h_low * data_width + w_low]; - float v2 = 0; - if (h_low >= 0 && w_high <= width - 1) - v2 = bottom_data[h_low * data_width + w_high]; - float v3 = 0; - if (h_high <= height - 1 && w_low >= 0) - v3 = bottom_data[h_high * data_width + w_low]; - float v4 = 0; - if (h_high <= height - 1 && w_high <= width - 1) - v4 = bottom_data[h_high * data_width + w_high]; +float v1 = ( (h_low >= 0 && w_low >= 0) ? bottom_data[h_low * data_width + w_low]:0); +float v2 = ( (h_low >= 0 && w_high <= width - 1) ? bottom_data[h_low * data_width + w_high]:0); +float v3 = ( (h_high <= height - 1 && w_low >= 0) ? bottom_data[h_high * data_width + w_low]:0); +float v4 = ( (h_high <= height - 1 && w_high <= width - 1) ? bottom_data[h_high * data_width + w_high]:0); - float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; +float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; - float val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); - return val; +float val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); +return val; } __global__ void modulated_deformable_im2col_gpu_kernel(const int n, - const float *data_im, const float *data_offset, const float *data_mask, - const int height, const int width, const int kernel_h, const int kernel_w, - const int pad_h, const int pad_w, - const int stride_h, const int stride_w, - const int dilation_h, const int dilation_w, - const int channel_per_deformable_group, - const int batch_size, const int num_channels, const int deformable_group, - const int height_col, const int width_col, - float *data_col) + const float *data_im, const float *data_offset, const float *data_mask, + const int height, const int width, + const int batch_size, const int num_channels, const int deformable_group, + const int height_col, const int width_col, + float *data_col) { CUDA_KERNEL_LOOP(index, n) { + //If n is a power of 2, ( i / n ) is equivalent to ( i ≫ log2 n ) and ( i % n ) is equivalent to ( i & n - 1 ). + const int ind_on_w = index / width_col; + const int ind_on_w_on_h = ind_on_w / height_col; + const int kk = 3 * 3; // index index of output matrix const int w_col = index % width_col; - const int h_col = (index / width_col) % height_col; - const int b_col = (index / width_col / height_col) % batch_size; - const int c_im = (index / width_col / height_col) / batch_size; - const int c_col = c_im * kernel_h * kernel_w; + const int h_col = (ind_on_w) % height_col; + const int b_col = (ind_on_w_on_h) % batch_size; + const int c_im = (ind_on_w_on_h) / batch_size; + const int c_col = c_im * kk; // compute deformable group index - const int deformable_group_index = c_im / channel_per_deformable_group; + const int deformable_group_index = c_im / (int)(num_channels / deformable_group); - const int h_in = h_col * stride_h - pad_h; - const int w_in = w_col * stride_w - pad_w; + const int h_in = h_col - 1; + const int w_in = w_col - 1; + const int s_col = height_col * width_col; + const int s_col2 = 2 * s_col; float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; - const float *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; + const float *data_offset_ptr = data_offset + add_ptr + add_ptr; - const float *data_mask_ptr = data_mask + (b_col * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + const float *data_mask_ptr = data_mask + add_ptr; - for (int i = 0; i < kernel_h; ++i) + const int first_member = w_col + width_col * h_col; + float val = static_cast(0); + #pragma unroll + for (int i = 0; i < 3; ++i) { - for (int j = 0; j < kernel_w; ++j) + #pragma unroll + for (int j = 0; j < 3; ++j) { - const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; - const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; - const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const int iter_member = (i * 3 + j); + // const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_h_ptr = first_member + s_col2 * iter_member; + + // const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = s_col + first_member + s_col2 * iter_member; + + // const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = first_member + s_col * iter_member; + const float offset_h = data_offset_ptr[data_offset_h_ptr]; const float offset_w = data_offset_ptr[data_offset_w_ptr]; const float mask = data_mask_ptr[data_mask_hw_ptr]; - float val = static_cast(0); - const float h_im = h_in + i * dilation_h + offset_h; - const float w_im = w_in + j * dilation_w + offset_w; + const float h_im = offset_h + h_in + i; + const float w_im = offset_w + w_in + j; //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { - if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + if (h_im < height && w_im < width && h_im > -1 && w_im > -1) { //const float map_h = i * dilation_h + offset_h; //const float map_w = j * dilation_w + offset_w; @@ -104,7 +109,89 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); } *data_col_ptr = val * mask; - data_col_ptr += batch_size * height_col * width_col; + data_col_ptr += batch_size * s_col; + //data_col_ptr += height_col * width_col; + } + } + } +} + +__global__ void modulated_deformable_im2col_gpu_kernel2(const int n, + const float *data_im, const float *data_offset, const float *data_mask, + const int height, const int width, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, const int deformable_group, + const int height_col, const int width_col, + float *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + //If n is a power of 2, ( i / n ) is equivalent to ( i ≫ log2 n ) and ( i % n ) is equivalent to ( i & n - 1 ). + // printf("--- %d %d %d %d %d %d %d %d\n",kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w); + const int ind_on_w = index / width_col; + const int ind_on_w_on_h = ind_on_w / height_col; + const int kk = kernel_h * kernel_w; + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (ind_on_w) % height_col; + const int b_col = (ind_on_w_on_h) % batch_size; + const int c_im = (ind_on_w_on_h) / batch_size; + const int c_col = c_im * kk; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + const int s_col = height_col * width_col; + const int s_col2 = 2 * s_col; + + float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; + const float *data_offset_ptr = data_offset + add_ptr + add_ptr; + + const float *data_mask_ptr = data_mask + add_ptr; + + const int first_member = w_col + width_col * h_col; + float val = static_cast(0); + #pragma unroll + for (int i = 0; i < kernel_h; ++i) + { + #pragma unroll + for (int j = 0; j < kernel_w; ++j) + { + const int iter_member = (i * kernel_w + j); + // const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_h_ptr = first_member + s_col2 * iter_member; + + // const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = s_col + first_member + s_col2 * iter_member; + + // const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = first_member + s_col * iter_member; + + const float offset_h = data_offset_ptr[data_offset_h_ptr]; + const float offset_w = data_offset_ptr[data_offset_w_ptr]; + const float mask = data_mask_ptr[data_mask_hw_ptr]; + const float h_im = offset_h + h_in + i * dilation_h; + const float w_im = offset_w + w_in + j * dilation_w; + //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + if (h_im < height && w_im < width && h_im > -1 && w_im > -1) + { + //const float map_h = i * dilation_h + offset_h; + //const float map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * s_col; //data_col_ptr += height_col * width_col; } } @@ -113,6 +200,28 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, void modulated_deformable_im2col_cuda(cudaStream_t stream, + const float* data_im, const float* data_offset, const float* data_mask, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, + const int deformable_group, float* data_col) { + // num_axes should be smaller than block size + // const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + modulated_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, data_mask, height_im, width_im, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +void modulated_deformable_im2col_cuda2(cudaStream_t stream, const float* data_im, const float* data_offset, const float* data_mask, const int batch_size, const int channels, const int height_im, const int width_im, const int height_col, const int width_col, const int kernel_h, const int kenerl_w, @@ -122,7 +231,7 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, // num_axes should be smaller than block size const int channel_per_deformable_group = channels / deformable_group; const int num_kernels = channels * batch_size * height_col * width_col; - modulated_deformable_im2col_gpu_kernel + modulated_deformable_im2col_gpu_kernel2 <<>>( num_kernels, data_im, data_offset, data_mask, height_im, width_im, kernel_h, kenerl_w, @@ -137,7 +246,6 @@ void modulated_deformable_im2col_cuda(cudaStream_t stream, } - void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, float *input, float *weight, float *bias, float *ones, @@ -182,9 +290,14 @@ void dcn_v2_cuda_forward(cublasStatus_t stat, cublasHandle_t handle, input, offset, mask, 1, channels, height, width, - height_out, width_out, kernel_h, kernel_w, - pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, - deformable_group, columns); + height_out, width_out, deformable_group, columns); + // modulated_deformable_im2col_cuda2(stream, + // input, offset, + // mask, + // 1, channels, height, width, + // height_out, width_out, kernel_h, kernel_w, + // pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + // deformable_group, columns); //(k * m) x (m * n) // Y = WC -- 2.52.0 From 73548b179ec3f53eb0e117b85266a0f51bce02fc Mon Sep 17 00:00:00 2001 From: xavier Date: Tue, 11 Feb 2020 10:32:42 +0100 Subject: [PATCH 039/482] Read parameters for mAP from yaml, add yampl-cpp dependency Signed-off-by: xavier --- CMakeLists.txt | 3 +- demo/demo/map.cpp | 75 +++++++++++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 27d99ea..a7eb12e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,7 @@ cuda_add_library(kernels SHARED ${tkdnn_CUSRC}) find_package(OpenCV REQUIRED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV") +find_package(yaml-cpp REQUIRED) #------------------------------------------------------------------------------- # Build Libraries @@ -118,7 +119,7 @@ add_executable(centernet_demo demo/demo/demo_centernet.cpp) target_link_libraries(centernet_demo tkDNN) add_executable(map_demo demo/demo/map.cpp) -target_link_libraries(map_demo tkDNN) +target_link_libraries(map_demo tkDNN yaml-cpp) #------------------------------------------------------------------------------- diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index bbbd77a..80c56dd 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -17,6 +17,8 @@ #include "evaluation.h" #include +#include + void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) { @@ -24,27 +26,53 @@ void convertFilename(std::string &filename,const std::string l_folder, const std filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } +void readParams(char* config_filename, std::string& net, char &ntype, std::string& labels_path, + bool &show, bool& write_dets, int& classes, int& n_images, + int& map_points, int& map_levels, float& map_step, + float& IoU_thresh, float& conf_thresh, bool& verbose) +{ + YAML::Node config = YAML::LoadFile(config_filename); + net = config["net"].as(); + ntype = config["ntype"].as(); + labels_path = config["labels_path"].as(); + show = config["show"].as(); + write_dets = config["write_dets"].as(); + classes = config["classes"].as(); + n_images = config["n_images"].as(); + map_points = config["map_points"].as(); + map_levels = config["map_levels"].as(); + map_step = config["map_step"].as(); + IoU_thresh = config["IoU_thresh"].as(); + conf_thresh = config["conf_thresh"].as(); + verbose = config["verbose"].as(); + +} + int main(int argc, char *argv[]) { - // char *net = "resnet101_cnet_FP32.rt"; - char *net = "yolo3.rt"; + + char *config_filename = "config.yaml"; if(argc > 1) - net = argv[1]; - char ntype = 'y'; + config_filename = argv[1]; + + char ntype; + std::string net, labels_path; + bool show, write_dets, verbose; + int classes, map_points, map_levels, n_images; + float map_step, IoU_thresh, conf_thresh; + + readParams( config_filename, net, ntype, labels_path, show, write_dets, + classes, n_images, map_points, map_levels, map_step, + IoU_thresh, conf_thresh, verbose); + if(argc > 2) - ntype = argv[2][0]; - //path to txt file with all realpath of images labels - char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + net = argv[2]; if(argc > 3) - labels_path = argv[3]; - - bool show = false; - bool write_dets = false; + ntype = argv[3][0]; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; - switch(ntype) { case 'y': @@ -57,29 +85,26 @@ int main(int argc, char *argv[]) FatalError("Network type not allowed (3rd parameter)\n"); } - std::ifstream all_labels(labels_path); std::string l_filename; std::vector images; + std::vector detected_bbox; std::cout<<"Reading groundtruth and generating detections"< detected_bbox; - - int i=0; - while (std::getline(all_labels, l_filename) && i < 1000) + for (int images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) { - std::cout < +#include + void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) { @@ -24,27 +26,53 @@ void convertFilename(std::string &filename,const std::string l_folder, const std filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } +void readParams(char* config_filename, std::string& net, char &ntype, std::string& labels_path, + bool &show, bool& write_dets, int& classes, int& n_images, + int& map_points, int& map_levels, float& map_step, + float& IoU_thresh, float& conf_thresh, bool& verbose) +{ + YAML::Node config = YAML::LoadFile(config_filename); + net = config["net"].as(); + ntype = config["ntype"].as(); + labels_path = config["labels_path"].as(); + show = config["show"].as(); + write_dets = config["write_dets"].as(); + classes = config["classes"].as(); + n_images = config["n_images"].as(); + map_points = config["map_points"].as(); + map_levels = config["map_levels"].as(); + map_step = config["map_step"].as(); + IoU_thresh = config["IoU_thresh"].as(); + conf_thresh = config["conf_thresh"].as(); + verbose = config["verbose"].as(); + +} + int main(int argc, char *argv[]) { - // char *net = "resnet101_cnet_FP32.rt"; - char *net = "yolo3.rt"; + + char *config_filename = "config.yaml"; if(argc > 1) - net = argv[1]; - char ntype = 'y'; + config_filename = argv[1]; + + char ntype; + std::string net, labels_path; + bool show, write_dets, verbose; + int classes, map_points, map_levels, n_images; + float map_step, IoU_thresh, conf_thresh; + + readParams( config_filename, net, ntype, labels_path, show, write_dets, + classes, n_images, map_points, map_levels, map_step, + IoU_thresh, conf_thresh, verbose); + if(argc > 2) - ntype = argv[2][0]; - //path to txt file with all realpath of images labels - char *labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + net = argv[2]; if(argc > 3) - labels_path = argv[3]; - - bool show = false; - bool write_dets = false; + ntype = argv[3][0]; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; - switch(ntype) { case 'y': @@ -57,29 +85,26 @@ int main(int argc, char *argv[]) FatalError("Network type not allowed (3rd parameter)\n"); } - std::ifstream all_labels(labels_path); std::string l_filename; std::vector images; + std::vector detected_bbox; std::cout<<"Reading groundtruth and generating detections"< detected_bbox; - - int i=0; - while (std::getline(all_labels, l_filename) && i < 1000) + for (int images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) { - std::cout < +``` +where +* : rt file of a choosen network on wich compute the mAP. +* : type of network. Right now only y(yolo) and c(centernet) are allowed +* : path to a text file containing all the paths of the groundtruth labels. It is important that all the labels of the groundtruth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the groundtruth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. +* : path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml + +Example: + +``` +./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml ``` \ No newline at end of file diff --git a/demo/config.yaml b/demo/config.yaml new file mode 100644 index 0000000..3577abf --- /dev/null +++ b/demo/config.yaml @@ -0,0 +1,7 @@ +classes : 80 #number of classes +map_points : 101 #number of recall points (0 for all, 101 for COCO, 11 PascalVOC) +map_levels : 10 #number of IoU step for the AP +map_step : 0.05 #step of IoU +IoU_thresh : 0.5 #starting IoU threshold +conf_thresh : 0.3 #threshold on the condifence of the bbox +verbose : false #print on screen information \ No newline at end of file diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 80c56dd..bc578d8 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -17,7 +17,7 @@ #include "evaluation.h" #include -#include + void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) @@ -26,49 +26,24 @@ void convertFilename(std::string &filename,const std::string l_folder, const std filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } -void readParams(char* config_filename, std::string& net, char &ntype, std::string& labels_path, - bool &show, bool& write_dets, int& classes, int& n_images, - int& map_points, int& map_levels, float& map_step, - float& IoU_thresh, float& conf_thresh, bool& verbose) -{ - YAML::Node config = YAML::LoadFile(config_filename); - net = config["net"].as(); - ntype = config["ntype"].as(); - labels_path = config["labels_path"].as(); - show = config["show"].as(); - write_dets = config["write_dets"].as(); - classes = config["classes"].as(); - n_images = config["n_images"].as(); - map_points = config["map_points"].as(); - map_levels = config["map_levels"].as(); - map_step = config["map_step"].as(); - IoU_thresh = config["IoU_thresh"].as(); - conf_thresh = config["conf_thresh"].as(); - verbose = config["verbose"].as(); - -} - int main(int argc, char *argv[]) { - - char *config_filename = "config.yaml"; + char ntype = 'y'; + char *config_filename = "../demo/config.yaml"; + char * net = "yolo3.rt"; + char * labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + bool show = false; + bool write_dets = false; + int n_images = 1000; + if(argc > 1) - config_filename = argv[1]; - - char ntype; - std::string net, labels_path; - bool show, write_dets, verbose; - int classes, map_points, map_levels, n_images; - float map_step, IoU_thresh, conf_thresh; - - readParams( config_filename, net, ntype, labels_path, show, write_dets, - classes, n_images, map_points, map_levels, map_step, - IoU_thresh, conf_thresh, verbose); - + net = argv[1]; if(argc > 2) - net = argv[2]; + ntype = argv[2][0]; if(argc > 3) - ntype = argv[3][0]; + labels_path = argv[3]; + if(argc > 3) + config_filename = argv[4]; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; @@ -183,6 +158,14 @@ int main(int argc, char *argv[]) } std::cout<<"Done."< #include +#include + #include "tkdnn.h" @@ -45,6 +47,10 @@ float boxIntersection(const BoundingBox &a, const BoundingBox &b); float boxUnion(const BoundingBox &a, const BoundingBox &b); float boxIoU(const BoundingBox &a, const BoundingBox &b); +void readParams(char* config_filename, int& classes, int& map_points, + int& map_levels, float& map_step, float& IoU_thresh, + float& conf_thresh, bool& verbose); + double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh=0.3, const int map_points=101, const bool verbose=false); double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); diff --git a/src/evaluation.cpp b/src/evaluation.cpp index 65f7708..c65ce50 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -76,6 +76,21 @@ float boxIoU(const BoundingBox &a, const BoundingBox &b) return I / U; } +void readParams(char* config_filename, int& classes, int& map_points, + int& map_levels, float& map_step, float& IoU_thresh, + float& conf_thresh, bool& verbose) +{ + YAML::Node config = YAML::LoadFile(config_filename); + classes = config["classes"].as(); + map_points = config["map_points"].as(); + map_levels = config["map_levels"].as(); + map_step = config["map_step"].as(); + IoU_thresh = config["IoU_thresh"].as(); + conf_thresh = config["conf_thresh"].as(); + verbose = config["verbose"].as(); + +} + /* Credits to https://github.com/AlexeyAB/darknet/blob/master/src/detector.c*/ double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, const int map_points, const bool verbose) { -- 2.52.0 From 1936e54870bffffc2705549d610109e746644d94 Mon Sep 17 00:00:00 2001 From: xavier Date: Tue, 11 Feb 2020 11:25:42 +0100 Subject: [PATCH 042/482] Change README.md, add config.yaml, refactoring Signed-off-by: xavier --- CMakeLists.txt | 6 ++--- README.md | 20 +++++++++++++++ demo/config.yaml | 7 +++++ demo/demo/map.cpp | 61 ++++++++++++++++---------------------------- include/evaluation.h | 6 +++++ src/evaluation.cpp | 15 +++++++++++ 6 files changed, 73 insertions(+), 42 deletions(-) create mode 100644 demo/config.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b2551e..b430195 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ endif() find_package(CUDA 9.0 REQUIRED) SET(CUDA_SEPARABLE_COMPILATION ON) #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'") -#set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) +set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32) find_package(CUDNN REQUIRED) @@ -39,7 +39,7 @@ find_package(yaml-cpp REQUIRED) # Build Libraries #------------------------------------------------------------------------------- file(GLOB tkdnn_SRC "src/*.cpp") -set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS}) +set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11") include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES}) @@ -120,7 +120,7 @@ add_executable(centernet_demo demo/demo/demo_centernet.cpp) target_link_libraries(centernet_demo tkDNN) add_executable(map_demo demo/demo/map.cpp) -target_link_libraries(map_demo tkDNN yaml-cpp) +target_link_libraries(map_demo tkDNN) #------------------------------------------------------------------------------- diff --git a/README.md b/README.md index 12bb124..b4ab4a6 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ this branch actually work on every NVIDIA GPU that support the dependencies: * CUDNN 7.603 * TENSORRT 6.01 * OPENCV 4.1 +* yaml-cpp 0.5.2 (sudo apt install libyaml-cpp-dev) ## Workflow The recommended workflow follow these step: @@ -112,4 +113,23 @@ this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used fo ./centernet_demo # launch detection on a demo video ./centernet_demo resnet101_cnet.rt /dev/video0 # launch detection on device 0 ./centernet_demo dla34_cnet.rt /dev/video0 # launch detection on device 0 +``` + +## mAP demo +To compute mAP, precision, recall and f1score, run the map_demo. + +The following parameters are needed: +``` +./map_demo +``` +where +* : rt file of a choosen network on wich compute the mAP. +* : type of network. Right now only y(yolo) and c(centernet) are allowed +* : path to a text file containing all the paths of the groundtruth labels. It is important that all the labels of the groundtruth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the groundtruth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. +* : path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml + +Example: + +``` +./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml ``` \ No newline at end of file diff --git a/demo/config.yaml b/demo/config.yaml new file mode 100644 index 0000000..3577abf --- /dev/null +++ b/demo/config.yaml @@ -0,0 +1,7 @@ +classes : 80 #number of classes +map_points : 101 #number of recall points (0 for all, 101 for COCO, 11 PascalVOC) +map_levels : 10 #number of IoU step for the AP +map_step : 0.05 #step of IoU +IoU_thresh : 0.5 #starting IoU threshold +conf_thresh : 0.3 #threshold on the condifence of the bbox +verbose : false #print on screen information \ No newline at end of file diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index 80c56dd..bc578d8 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -17,7 +17,7 @@ #include "evaluation.h" #include -#include + void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext) @@ -26,49 +26,24 @@ void convertFilename(std::string &filename,const std::string l_folder, const std filename.replace(filename.find(l_ext),l_ext.length(),i_ext); } -void readParams(char* config_filename, std::string& net, char &ntype, std::string& labels_path, - bool &show, bool& write_dets, int& classes, int& n_images, - int& map_points, int& map_levels, float& map_step, - float& IoU_thresh, float& conf_thresh, bool& verbose) -{ - YAML::Node config = YAML::LoadFile(config_filename); - net = config["net"].as(); - ntype = config["ntype"].as(); - labels_path = config["labels_path"].as(); - show = config["show"].as(); - write_dets = config["write_dets"].as(); - classes = config["classes"].as(); - n_images = config["n_images"].as(); - map_points = config["map_points"].as(); - map_levels = config["map_levels"].as(); - map_step = config["map_step"].as(); - IoU_thresh = config["IoU_thresh"].as(); - conf_thresh = config["conf_thresh"].as(); - verbose = config["verbose"].as(); - -} - int main(int argc, char *argv[]) { - - char *config_filename = "config.yaml"; + char ntype = 'y'; + char *config_filename = "../demo/config.yaml"; + char * net = "yolo3.rt"; + char * labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + bool show = false; + bool write_dets = false; + int n_images = 1000; + if(argc > 1) - config_filename = argv[1]; - - char ntype; - std::string net, labels_path; - bool show, write_dets, verbose; - int classes, map_points, map_levels, n_images; - float map_step, IoU_thresh, conf_thresh; - - readParams( config_filename, net, ntype, labels_path, show, write_dets, - classes, n_images, map_points, map_levels, map_step, - IoU_thresh, conf_thresh, verbose); - + net = argv[1]; if(argc > 2) - net = argv[2]; + ntype = argv[2][0]; if(argc > 3) - ntype = argv[3][0]; + labels_path = argv[3]; + if(argc > 3) + config_filename = argv[4]; tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; @@ -183,6 +158,14 @@ int main(int argc, char *argv[]) } std::cout<<"Done."< #include +#include + #include "tkdnn.h" @@ -45,6 +47,10 @@ float boxIntersection(const BoundingBox &a, const BoundingBox &b); float boxUnion(const BoundingBox &a, const BoundingBox &b); float boxIoU(const BoundingBox &a, const BoundingBox &b); +void readParams(char* config_filename, int& classes, int& map_points, + int& map_levels, float& map_step, float& IoU_thresh, + float& conf_thresh, bool& verbose); + double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh=0.3, const int map_points=101, const bool verbose=false); double computeMapNIoULevels(std::vector &images,const int classes,const float i_IoU_thresh=0.5, const float conf_thresh=0.3, const int map_points=101, const float map_step=0.05, const int map_levels=10, const bool verbose=false); diff --git a/src/evaluation.cpp b/src/evaluation.cpp index 65f7708..c65ce50 100644 --- a/src/evaluation.cpp +++ b/src/evaluation.cpp @@ -76,6 +76,21 @@ float boxIoU(const BoundingBox &a, const BoundingBox &b) return I / U; } +void readParams(char* config_filename, int& classes, int& map_points, + int& map_levels, float& map_step, float& IoU_thresh, + float& conf_thresh, bool& verbose) +{ + YAML::Node config = YAML::LoadFile(config_filename); + classes = config["classes"].as(); + map_points = config["map_points"].as(); + map_levels = config["map_levels"].as(); + map_step = config["map_step"].as(); + IoU_thresh = config["IoU_thresh"].as(); + conf_thresh = config["conf_thresh"].as(); + verbose = config["verbose"].as(); + +} + /* Credits to https://github.com/AlexeyAB/darknet/blob/master/src/detector.c*/ double computeMap(std::vector &images,const int classes,const float IoU_thresh, const float conf_thresh, const int map_points, const bool verbose) { -- 2.52.0 From 21535dd6b30c0dd7f4de9331c444290eb271340e Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 11 Feb 2020 11:28:20 +0100 Subject: [PATCH 043/482] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b4ab4a6..b798ce8 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,13 @@ The following parameters are needed: ./map_demo ``` where -* : rt file of a choosen network on wich compute the mAP. -* : type of network. Right now only y(yolo) and c(centernet) are allowed -* : path to a text file containing all the paths of the groundtruth labels. It is important that all the labels of the groundtruth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the groundtruth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. -* : path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml +* ``````: rt file of a choosen network on wich compute the mAP. +* ``````: type of network. Right now only y(yolo) and c(centernet) are allowed +* ``````: path to a text file containing all the paths of the groundtruth labels. It is important that all the labels of the groundtruth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the groundtruth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. +* ``````: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml Example: ``` ./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml -``` \ No newline at end of file +``` -- 2.52.0 From 6e2ff405f40e640823c1359a42aebe2d28a93820 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 11 Feb 2020 11:28:20 +0100 Subject: [PATCH 044/482] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b4ab4a6..b798ce8 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,13 @@ The following parameters are needed: ./map_demo ``` where -* : rt file of a choosen network on wich compute the mAP. -* : type of network. Right now only y(yolo) and c(centernet) are allowed -* : path to a text file containing all the paths of the groundtruth labels. It is important that all the labels of the groundtruth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the groundtruth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. -* : path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml +* ``````: rt file of a choosen network on wich compute the mAP. +* ``````: type of network. Right now only y(yolo) and c(centernet) are allowed +* ``````: path to a text file containing all the paths of the groundtruth labels. It is important that all the labels of the groundtruth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the groundtruth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. +* ``````: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml Example: ``` ./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml -``` \ No newline at end of file +``` -- 2.52.0 From 9c4ea524fc3c1ad90e28d918a547090db3443c4e Mon Sep 17 00:00:00 2001 From: xavier Date: Tue, 11 Feb 2020 14:56:54 +0100 Subject: [PATCH 045/482] Add script to download COCO val2017 Signed-off-by: xavier --- README.md | 10 ++++++++-- demo/demo/map.cpp | 4 ++-- download_validation.sh | 8 ++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 download_validation.sh diff --git a/README.md b/README.md index b4ab4a6..9219939 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,12 @@ this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used fo ## mAP demo To compute mAP, precision, recall and f1score, run the map_demo. -The following parameters are needed: +A validation set is needed. To download COCO_val2017 run (form the root folder): +``` +bash download_validation.sh +``` + +To compute the map, the following parameters are needed: ``` ./map_demo ``` @@ -131,5 +136,6 @@ where Example: ``` -./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml +cd build +./map_demo dla34_cnet.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` \ No newline at end of file diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index bc578d8..13636fb 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -31,7 +31,7 @@ int main(int argc, char *argv[]) char ntype = 'y'; char *config_filename = "../demo/config.yaml"; char * net = "yolo3.rt"; - char * labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + char * labels_path = "../demo/COCO_val2017/all_labels.txt"; bool show = false; bool write_dets = false; int n_images = 1000; @@ -42,7 +42,7 @@ int main(int argc, char *argv[]) ntype = argv[2][0]; if(argc > 3) labels_path = argv[3]; - if(argc > 3) + if(argc > 4) config_filename = argv[4]; tk::dnn::Yolo3Detection yolo; diff --git a/download_validation.sh b/download_validation.sh new file mode 100644 index 0000000..4e07aae --- /dev/null +++ b/download_validation.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd demo +wget https://cloud.hipert.unimore.it/s/LNxBDk4wzqXPL8c/download -O COCO_val2017.zip +unzip -d COCO_val2017 COCO_val2017.zip +rm COCO_val2017.zip +cd COCO_val2017/ +realpath labels/* > all_labels.txt -- 2.52.0 From 7e21b10aeec4d3f302e63517ae5a5ec672a06782 Mon Sep 17 00:00:00 2001 From: xavier Date: Tue, 11 Feb 2020 14:56:54 +0100 Subject: [PATCH 046/482] Add script to download COCO val2017 Signed-off-by: xavier --- README.md | 10 ++++++++-- demo/demo/map.cpp | 4 ++-- download_validation.sh | 8 ++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 download_validation.sh diff --git a/README.md b/README.md index b4ab4a6..9219939 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,12 @@ this will genereate resnet101_cnet.rt and dla34_cnet.rt file that can be used fo ## mAP demo To compute mAP, precision, recall and f1score, run the map_demo. -The following parameters are needed: +A validation set is needed. To download COCO_val2017 run (form the root folder): +``` +bash download_validation.sh +``` + +To compute the map, the following parameters are needed: ``` ./map_demo ``` @@ -131,5 +136,6 @@ where Example: ``` -./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml +cd build +./map_demo dla34_cnet.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` \ No newline at end of file diff --git a/demo/demo/map.cpp b/demo/demo/map.cpp index bc578d8..13636fb 100644 --- a/demo/demo/map.cpp +++ b/demo/demo/map.cpp @@ -31,7 +31,7 @@ int main(int argc, char *argv[]) char ntype = 'y'; char *config_filename = "../demo/config.yaml"; char * net = "yolo3.rt"; - char * labels_path = "/media/887E650E7E64F67A/val2017/all_labels2017.txt"; + char * labels_path = "../demo/COCO_val2017/all_labels.txt"; bool show = false; bool write_dets = false; int n_images = 1000; @@ -42,7 +42,7 @@ int main(int argc, char *argv[]) ntype = argv[2][0]; if(argc > 3) labels_path = argv[3]; - if(argc > 3) + if(argc > 4) config_filename = argv[4]; tk::dnn::Yolo3Detection yolo; diff --git a/download_validation.sh b/download_validation.sh new file mode 100644 index 0000000..4e07aae --- /dev/null +++ b/download_validation.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd demo +wget https://cloud.hipert.unimore.it/s/LNxBDk4wzqXPL8c/download -O COCO_val2017.zip +unzip -d COCO_val2017 COCO_val2017.zip +rm COCO_val2017.zip +cd COCO_val2017/ +realpath labels/* > all_labels.txt -- 2.52.0 From 217b3fbfb564a7a6c42f6c8ffc0289fc6ab3b092 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 11 Feb 2020 15:12:27 +0100 Subject: [PATCH 047/482] Update README.md --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 27d4af6..f8a3262 100644 --- a/README.md +++ b/README.md @@ -136,11 +136,6 @@ where Example: ``` -<<<<<<< HEAD cd build ./map_demo dla34_cnet.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` -======= -./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml -``` ->>>>>>> 6e2ff405f40e640823c1359a42aebe2d28a93820 -- 2.52.0 From 4c2d2a79658195a7e2fd2949218389eb7da6ca44 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 11 Feb 2020 15:12:27 +0100 Subject: [PATCH 048/482] Update README.md --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 27d4af6..f8a3262 100644 --- a/README.md +++ b/README.md @@ -136,11 +136,6 @@ where Example: ``` -<<<<<<< HEAD cd build ./map_demo dla34_cnet.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml ``` -======= -./map_demo dla34_cnet.rt c path/to/labelsCOCO_val2017.txt ../demo/config.yaml -``` ->>>>>>> 6e2ff405f40e640823c1359a42aebe2d28a93820 -- 2.52.0 From 40a4a55cd0fbf838fbb2f18ea329a8151f393434 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 Feb 2020 16:28:17 +0100 Subject: [PATCH 049/482] Fix a bug in the deformable kernel. There was a wrong variable initialization. Fixes: 51ffcb1f50e3c9aa2b2 ("Optimize deformable kernel") Signed-off-by: Davide Sapienza --- src/kernels/deformable_conv.cu | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index 88e62e9..672655f 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -67,16 +67,15 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, const int s_col = height_col * width_col; const int s_col2 = 2 * s_col; - float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + const int first_member = w_col + width_col * h_col; + // float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + float *data_col_ptr = data_col + first_member + s_col * (c_col * batch_size + b_col); //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; const float *data_offset_ptr = data_offset + add_ptr + add_ptr; - const float *data_mask_ptr = data_mask + add_ptr; - const int first_member = w_col + width_col * h_col; - float val = static_cast(0); #pragma unroll for (int i = 0; i < 3; ++i) { @@ -99,6 +98,7 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, const float h_im = offset_h + h_in + i; const float w_im = offset_w + w_in + j; //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + float val = static_cast(0); if (h_im < height && w_im < width && h_im > -1 && w_im > -1) { //const float map_h = i * dilation_h + offset_h; @@ -149,16 +149,16 @@ __global__ void modulated_deformable_im2col_gpu_kernel2(const int n, const int s_col = height_col * width_col; const int s_col2 = 2 * s_col; - float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + const int first_member = w_col + width_col * h_col; + // float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + float *data_col_ptr = data_col + first_member + s_col * (c_col * batch_size + b_col); //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; const float *data_offset_ptr = data_offset + add_ptr + add_ptr; const float *data_mask_ptr = data_mask + add_ptr; - - const int first_member = w_col + width_col * h_col; - float val = static_cast(0); + #pragma unroll for (int i = 0; i < kernel_h; ++i) { @@ -181,6 +181,7 @@ __global__ void modulated_deformable_im2col_gpu_kernel2(const int n, const float h_im = offset_h + h_in + i * dilation_h; const float w_im = offset_w + w_in + j * dilation_w; //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + float val = static_cast(0); if (h_im < height && w_im < width && h_im > -1 && w_im > -1) { //const float map_h = i * dilation_h + offset_h; -- 2.52.0 From 97b88ef52d96da859b4ad53527357bbde735a624 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Tue, 11 Feb 2020 16:28:17 +0100 Subject: [PATCH 050/482] Fix a bug in the deformable kernel. There was a wrong variable initialization. Fixes: 51ffcb1f50e3c9aa2b2 ("Optimize deformable kernel") Signed-off-by: Davide Sapienza --- src/kernels/deformable_conv.cu | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/kernels/deformable_conv.cu b/src/kernels/deformable_conv.cu index 88e62e9..672655f 100644 --- a/src/kernels/deformable_conv.cu +++ b/src/kernels/deformable_conv.cu @@ -67,16 +67,15 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, const int s_col = height_col * width_col; const int s_col2 = 2 * s_col; - float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + const int first_member = w_col + width_col * h_col; + // float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + float *data_col_ptr = data_col + first_member + s_col * (c_col * batch_size + b_col); //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; const float *data_offset_ptr = data_offset + add_ptr + add_ptr; - const float *data_mask_ptr = data_mask + add_ptr; - const int first_member = w_col + width_col * h_col; - float val = static_cast(0); #pragma unroll for (int i = 0; i < 3; ++i) { @@ -99,6 +98,7 @@ __global__ void modulated_deformable_im2col_gpu_kernel(const int n, const float h_im = offset_h + h_in + i; const float w_im = offset_w + w_in + j; //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + float val = static_cast(0); if (h_im < height && w_im < width && h_im > -1 && w_im > -1) { //const float map_h = i * dilation_h + offset_h; @@ -149,16 +149,16 @@ __global__ void modulated_deformable_im2col_gpu_kernel2(const int n, const int s_col = height_col * width_col; const int s_col2 = 2 * s_col; - float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + const int first_member = w_col + width_col * h_col; + // float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + float *data_col_ptr = data_col + first_member + s_col * (c_col * batch_size + b_col); //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col; const float *data_offset_ptr = data_offset + add_ptr + add_ptr; const float *data_mask_ptr = data_mask + add_ptr; - - const int first_member = w_col + width_col * h_col; - float val = static_cast(0); + #pragma unroll for (int i = 0; i < kernel_h; ++i) { @@ -181,6 +181,7 @@ __global__ void modulated_deformable_im2col_gpu_kernel2(const int n, const float h_im = offset_h + h_in + i * dilation_h; const float w_im = offset_w + w_in + j * dilation_w; //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + float val = static_cast(0); if (h_im < height && w_im < width && h_im > -1 && w_im > -1) { //const float map_h = i * dilation_h + offset_h; -- 2.52.0 From c876fa05ae0f1558889bb92643584eac9addf6a7 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 19:27:18 +0100 Subject: [PATCH 051/482] LSTM cudnn test --- .gitignore | 3 +- CMakeLists.txt | 3 + include/tkDNN/Layer.h | 76 +++++++++++++++++ src/LSTM.cpp | 142 ++++++++++++++++++++++++++++++++ src/utils.cpp | 3 +- tests/imuodom/imuodom.cpp | 70 ++++++++++++++++ tests/imuodom/infer.py | 74 +++++++++++++++++ tests/simple/test_model.py | 60 +++++++------- tests/simple/test_simple.cpp | 14 +--- tests/weights_exporter.py | 153 ++++++++++++++--------------------- 10 files changed, 464 insertions(+), 134 deletions(-) create mode 100644 src/LSTM.cpp create mode 100644 tests/imuodom/imuodom.cpp create mode 100644 tests/imuodom/infer.py diff --git a/.gitignore b/.gitignore index 02f2a8e..de78410 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ build/ *.h5 *.tar.gz *.weights -.idea/ \ No newline at end of file +.idea/ +*.hdf5 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b4b3c38..e3d8607 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,9 @@ target_link_libraries(test_yolo3_berkeley tkDNN) add_executable(test_yolo3_flir tests/yolo3_flir/yolo3_flir.cpp) target_link_libraries(test_yolo3_flir tkDNN) + +add_executable(test_imuodom tests/imuodom/imuodom.cpp) +target_link_libraries(test_imuodom tkDNN) ################################################################################ diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index ee73109..f15c781 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -9,8 +9,10 @@ namespace tk { namespace dnn { enum layerType_t { + LAYER_INPUT, LAYER_DENSE, LAYER_CONV2D, + LAYER_LSTM, LAYER_ACTIVATION, LAYER_FLATTEN, LAYER_MULADD, @@ -47,8 +49,10 @@ public: std::string getLayerName() { layerType_t type = getLayerType(); switch(type) { + case LAYER_INPUT: return "Input"; case LAYER_DENSE: return "Dense"; case LAYER_CONV2D: return "Conv2d"; + case LAYER_LSTM: return "LSTM"; case LAYER_ACTIVATION: return "Activation"; case LAYER_FLATTEN: return "Flatten"; case LAYER_MULADD: return "MulAdd"; @@ -105,6 +109,27 @@ public: }; +/** + Input layer (it doesnt need weigths) +*/ +class Input : public Layer { + +public: + + Input(Network *net, dataDim_t &dim, dnnType* srcData) : Layer(net) { + input_dim = dim; + output_dim = dim; + dstData = srcData; + } + virtual ~Input() {} + virtual layerType_t getLayerType() { return LAYER_INPUT; }; + + virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) { + return dstData; + } +}; + + /** Dense (full interconnection) layer */ @@ -148,6 +173,14 @@ protected: /** Convolutional 2D layer + + WEIGHTS shape: OUTCH, INCH, KH, KW ... + BIAS shape: OUTCH + + with BATCHNORM: + scales: OUTCH + means: OUTCH + variance: OUTCH */ class Conv2d : public LayerWgs { @@ -172,6 +205,49 @@ protected: size_t ws_sizeInBytes; }; +/** + Bidirectional LSTM layer + https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp + + numlayers = 1 # hardcoded as 1 + + PARAMS (numlayers*2): + layer0: + ( INCH, ? ) ??? + ( HIDDEN, ? ) ??? + ( HIDDEN * 8 ) ??? + layer2: + ( INCH, ? ) ??? + ( HIDDEN, ? ) ??? + ( HIDDEN * 8 ) ??? + + output shape: ( 2*HIDDEN, INH, INW ) +*/ +class LSTM : public Layer { + +public: + LSTM(Network *net, int hiddensize, std::string fname_weights); + virtual ~LSTM(); + virtual layerType_t getLayerType() { return LAYER_LSTM; }; + + virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + + int kernelH, kernelW, strideH, strideW, paddingH, paddingW; + +protected: + cudnnFilterDescriptor_t paramDesc; + cudnnTensorDescriptor_t hiddenStateTensorDesc, cellStateTensorDesc; + cudnnRNNDescriptor_t rnnDesc; + cudnnRNNDataDescriptor_t rnnDataDesc; + cudnnDropoutDescriptor_t dropDesc; + cudnnRNNAlgo_t algo; + + dnnType *hiddenStateData, *cellStateData; + dnnType *paramsSpace; + void* workSpace; + size_t ws_sizeInBytes; +}; + /** Flatten layer diff --git a/src/LSTM.cpp b/src/LSTM.cpp new file mode 100644 index 0000000..3176819 --- /dev/null +++ b/src/LSTM.cpp @@ -0,0 +1,142 @@ +#include + +#include "Layer.h" + +namespace tk { namespace dnn { + +LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : + Layer(net) { + + checkCUDNN( cudnnCreateFilterDescriptor(¶mDesc)); + checkCUDNN( cudnnCreateRNNDescriptor(&rnnDesc) ); + checkCUDNN( cudnnCreateRNNDataDescriptor(&rnnDataDesc) ); + checkCUDNN( cudnnCreateDropoutDescriptor(&dropDesc)); + + int n = input_dim.n; + int c = input_dim.c; + int h = input_dim.h; + int w = input_dim.w; + checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, + net->tensorFormat, net->dataType, n, 1, h, w) ); + + int numlayers = 1; + checkCUDNN( cudnnSetRNNDescriptor(net->cudnnHandle, rnnDesc, hiddensize, numlayers, dropDesc, + cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, + cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, + cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType) ); + + // find dimension of params + size_t params_size = 0; + checkCUDNN( cudnnGetRNNParamsSize(net->cudnnHandle, rnnDesc, srcTensorDesc, ¶ms_size, net->dataType) ); + std::cout<<"Params size bytes: "<dataType, net->tensorFormat, 3, dimW)); + checkCuda( cudaMalloc(¶msSpace, params_size) ); + + + int numlinearlayers = 8; + + for(int i=0; icudnnHandle, rnnDesc, + i, srcTensorDesc, paramDesc, paramsSpace, + j, linLayerMatDesc, (void **)&linLayerMat)); + + if(linLayerMat == nullptr) { + FatalError("LSTM No weights in hidden layer"); + } + + cudnnDataType_t dataType; + cudnnTensorFormat_t format; + int nbDims; + int filterDimA[3]; + checkCUDNN(cudnnGetFilterNdDescriptor(linLayerMatDesc, 3, &dataType, + &format, &nbDims, filterDimA)); + std::cout<<"Wgs Dims: "<cudnnHandle, rnnDesc, + i, srcTensorDesc, paramDesc, paramsSpace, + j, linLayerBiasDesc, (void **)&linLayerBias)); + + if(linLayerMat == nullptr) { + FatalError("LSTM No bias in hidden layer"); + } + + checkCUDNN(cudnnGetFilterNdDescriptor(linLayerBiasDesc, 3, &dataType, + &format, &nbDims, filterDimA)); + std::cout<<"bias Dims: "<tensorFormat, net->dataType, 2*n, c, h, w) ); + checkCuda( cudaMalloc(&hiddenStateData, 2*input_dim.tot()*sizeof(dnnType)) ); + checkCUDNN( cudnnCreateTensorDescriptor(&cellStateTensorDesc)); + checkCUDNN( cudnnSetTensor4dDescriptor(cellStateTensorDesc, + net->tensorFormat, net->dataType, 2*n, c, h, w) ); + checkCuda( cudaMalloc(&cellStateData, 2*input_dim.tot()*sizeof(dnnType)) ); + + + output_dim = input_dim; + output_dim.c = hiddensize*2; + checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc, + net->tensorFormat, net->dataType, output_dim.n, output_dim.c, output_dim.h, output_dim.w) ); + + + + + //allocate data for infer result + checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); +} + +LSTM::~LSTM() { + + checkCuda( cudaFree(dstData) ); +} + +dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { + + checkCUDNN(cudnnRNNForwardInference( + net->cudnnHandle, rnnDesc, 1, + &srcTensorDesc, srcData, + hiddenStateTensorDesc, hiddenStateData, + cellStateTensorDesc, cellStateData, + paramDesc, paramsSpace, + &dstTensorDesc, dstData, + hiddenStateTensorDesc, hiddenStateData, + cellStateTensorDesc, cellStateData, + workSpace, ws_sizeInBytes + )); + + return dstData; +} + +}} diff --git a/src/utils.cpp b/src/utils.cpp index e6f71f8..444318d 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -39,7 +39,8 @@ void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** dat *data_h = new dnnType[size]; if (!dataFile.read ((char*) *data_h, size_b)) { - error_s << "Error reading file " << fname; + error_s << "Error reading file " << fname << " with n of float: "< +#include "tkdnn.h" + +const char *i0_bin = "../tests/imuodom/layers/input0.bin"; +const char *i1_bin = "../tests/imuodom/layers/input1.bin"; +const char *i2_bin = "../tests/imuodom/layers/input2.bin"; +const char *o0_bin = "../tests/imuodom/layers/output0.bin"; +const char *o1_bin = "../tests/imuodom/layers/output1.bin"; +const char *output_bin = "../tests/imuodom/layers/output.bin"; + +const char *c0_bin = "../tests/imuodom/layers/conv1d_7.bin"; +const char *c1_bin = "../tests/imuodom/layers/conv1d_8.bin"; +const char *c2_bin = "../tests/imuodom/layers/conv1d_9.bin"; +const char *c3_bin = "../tests/imuodom/layers/conv1d_10.bin"; +const char *c4_bin = "../tests/imuodom/layers/conv1d_11.bin"; +const char *c5_bin = "../tests/imuodom/layers/conv1d_12.bin"; + +int main() { + + // Network layout + tk::dnn::dataDim_t dim0(1, 4, 1, 100); + tk::dnn::dataDim_t dim1(1, 3, 1, 100); + tk::dnn::dataDim_t dim2(1, 3, 1, 100); + + // Load input + dnnType *i0_d, *i1_d, *i2_d; + dnnType *i0_h, *i1_h, *i2_h; + readBinaryFile(i0_bin, dim0.tot(), &i0_h, &i0_d); + readBinaryFile(i1_bin, dim1.tot(), &i1_h, &i1_d); + readBinaryFile(i2_bin, dim2.tot(), &i2_h, &i2_d); + + tk::dnn::Network net(dim0); + tk::dnn::Input x0 (&net, dim0, i0_d); + tk::dnn::Conv2d x0_0(&net, 128, 1, 11, 1, 1, 0, 0, c0_bin); + tk::dnn::Conv2d x0_1(&net, 128, 1, 11, 1, 1, 0, 0, c1_bin); + tk::dnn::Pooling x0_2(&net, 1, 3, 1, 3, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX); + + tk::dnn::Input x1 (&net, dim1, i1_d); + tk::dnn::Conv2d x1_0(&net, 128, 1, 11, 1, 1, 0, 0, c2_bin); + tk::dnn::Conv2d x1_1(&net, 128, 1, 11, 1, 1, 0, 0, c3_bin); + tk::dnn::Pooling x1_2(&net, 1, 3, 1, 3, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX); + + tk::dnn::Input x2 (&net, dim2, i2_d); + tk::dnn::Conv2d x2_0(&net, 128, 1, 11, 1, 1, 0, 0, c4_bin); + tk::dnn::Conv2d x2_1(&net, 128, 1, 11, 1, 1, 0, 0, c5_bin); + tk::dnn::Pooling x2_2(&net, 1, 3, 1, 3, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX); + + tk::dnn::Layer *concat_l[3] = { &x0_2, &x1_2, &x2_2 }; + tk::dnn::Route concat (&net, concat_l, 3); + + tk::dnn::LSTM lstm0(&net, 128, "ciao"); + + net.print(); + + dnnType *data; + tk::dnn::dataDim_t dim; + + TIMER_START + // Inference + data = net.infer(dim, data); dim.print(); + TIMER_STOP + + // Print real test + std::cout<<"\n==== CHECK RESULT ====\n"; + dnnType *out; + dnnType *out_h; + readBinaryFile(output_bin, dim.tot(), &out_h, &out); + checkResult(dim.tot(), data, out); + return 0; +} diff --git a/tests/imuodom/infer.py b/tests/imuodom/infer.py new file mode 100644 index 0000000..9d9929d --- /dev/null +++ b/tests/imuodom/infer.py @@ -0,0 +1,74 @@ +import keras +from keras.models import load_model +import keras.backend.tensorflow_backend as KTF +import numpy as np +import argparse +import tensorflow as tf +import os +import random +import struct +from keras.models import Sequential, Model + +def bin_write(f, data): + data = data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + + +if __name__ == '__main__': + + + print("DATA FORMAT: ", keras.backend.image_data_format()) + + print("Load model: ", "ferrariS1.hdf5") + model = load_model("ferrariS1.hdf5") + model.summary() + + weights = model.get_weights() + + x_angle = np.random.rand(1,100,4) + x_gyro = np.random.rand(1,100,3) + x_acc = np.random.rand(1,100,3) + + [yhat_delta_p, yhat_delta_q] = model.predict([x_angle, x_gyro, x_acc], batch_size=1, verbose=1) + + layer_name = 'bidirectional_3' + intermediate_layer_model = Model(inputs=model.input, + outputs=model.get_layer(layer_name).output) + intermediate_output = intermediate_layer_model.predict([x_angle, x_gyro, x_acc]) + + x_angle = np.array([x_angle]) + x_gyro = np.array([x_gyro]) + x_acc = np.array([x_acc]) + intermediate_output = np.array([intermediate_output]) + + x_angle = x_angle.transpose(0, 3, 1, 2) + x_gyro = x_gyro.transpose(0, 3, 1, 2) + x_acc = x_acc.transpose(0, 3, 1, 2) + intermediate_output = intermediate_output.transpose(0, 3, 1, 2) + + print("x0: ", np.shape(x_angle)) + print("out: ",np.shape(intermediate_output)) + + x_angle = np.array(x_angle.flatten(), dtype=np.float32) + x_gyro = np.array(x_gyro.flatten(), dtype=np.float32) + x_acc = np.array(x_acc.flatten(), dtype=np.float32) + yhat_delta_p = np.array(yhat_delta_p.flatten(), dtype=np.float32) + yhat_delta_q = np.array(yhat_delta_q.flatten(), dtype=np.float32) + intermediate_output = np.array(intermediate_output.flatten(), dtype=np.float32) + + + f = open("layers/input0.bin", mode='wb') + bin_write(f, x_angle) + f = open("layers/input1.bin", mode='wb') + bin_write(f, x_gyro) + f = open("layers/input2.bin", mode='wb') + bin_write(f, x_acc) + f = open("layers/output0.bin", mode='wb') + bin_write(f, yhat_delta_p) + f = open("layers/output1.bin", mode='wb') + bin_write(f, yhat_delta_q) + f = open("layers/output.bin", mode='wb') + bin_write(f, intermediate_output) + diff --git a/tests/simple/test_model.py b/tests/simple/test_model.py index 60f17d8..3b1d053 100644 --- a/tests/simple/test_model.py +++ b/tests/simple/test_model.py @@ -1,43 +1,49 @@ import keras import numpy as np from keras.models import Sequential -from keras.layers import Input, Dense, Activation, Flatten, Dropout, ELU, Reshape, Lambda +from keras.layers import Input, Dense, Activation, Flatten, Dropout, ELU, Reshape, Lambda, Conv1D from keras.layers.convolutional import Convolution2D, Convolution3D from keras.layers.pooling import MaxPooling2D, MaxPooling3D, AveragePooling3D from keras.models import Sequential, Model from keras.layers import Cropping2D import keras.backend.tensorflow_backend as KTF +import struct +from keras.models import Sequential, Model -def dense_model(): - model = Sequential() +def bin_write(f, data): + data = data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + +def create_model(): + x1 = Input((6, 16), name='x1') + conv = Conv1D(4, 2)(x1) + model = Model([x1], [conv]) + model.summary() - model.add(Reshape((10, 10, 1), input_shape=(10, 10))) - model.add(Convolution2D(2, (4, 4), subsample=(2, 2), - bias_initializer='random_uniform', activation="relu")) - model.add(Convolution2D(4, (2, 2), subsample=(1, 1), - bias_initializer='random_uniform', activation="relu")) - model.add(Flatten()) - model.add(Dense(4, bias_initializer='random_uniform', activation="relu")) - sgd = keras.optimizers.Adam(lr=1e-4, decay=1e-8) - model.compile(optimizer=sgd, loss="mse") return model - if __name__ == '__main__': - print "DATA FORMAT: ", keras.backend.image_data_format() + print ("DATA FORMAT: ", keras.backend.image_data_format()) - model = dense_model() - model.save("net.h5") + model = create_model() + model.save("net.hdf5") - grid = np.random.rand(10,10) - X = grid[None,:,:] - i = np.array(grid.flatten(), dtype=np.float32) - print i - i.tofile("input.bin", format="f") - print "Input: ", X + x = np.random.rand(1,1,6,16) + r = model.predict( x[0], batch_size=1) + r = np.array([r]) + + x = x.transpose(0, 3, 1, 2) + r = r.transpose(0, 3, 1, 2) + print("in: ", np.shape(x)) + print("out: ", np.shape(r)) + + x = np.array(x.flatten(), dtype=np.float32) + f = open("input.bin", mode='wb') + bin_write(f, x) + + r = np.array(r.flatten(), dtype=np.float32) + f = open("output.bin", mode='wb') + bin_write(f, r) - r = model.predict( X, batch_size=1) - print np.shape(r) - print "Result: ", r - print "Result shape: ", np.shape(r) - r.tofile("output.bin", format="f") diff --git a/tests/simple/test_simple.cpp b/tests/simple/test_simple.cpp index a7628ea..3427d6d 100644 --- a/tests/simple/test_simple.cpp +++ b/tests/simple/test_simple.cpp @@ -2,23 +2,15 @@ #include "tkdnn.h" const char *input_bin = "../tests/simple/input.bin"; -const char *c0_bin = "../tests/simple/layers/c0.bin"; -const char *c1_bin = "../tests/simple/layers/c1.bin"; -const char *d2_bin = "../tests/simple/layers/d2.bin"; +const char *c0_bin = "../tests/simple/layers/conv1d_1.bin"; const char *output_bin = "../tests/simple/output.bin"; int main() { // Network layout - tk::dnn::dataDim_t dim(1, 1, 10, 10, 1); + tk::dnn::dataDim_t dim(1, 16, 1, 6); tk::dnn::Network net(dim); - tk::dnn::Conv2d l0(&net, 2, 4, 4, 2, 2, 0, 0, c0_bin); - tk::dnn::Activation l1(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d l2(&net, 4, 2, 2, 1, 1, 0, 0, c1_bin); - tk::dnn::Activation l3(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Flatten l4(&net); - tk::dnn::Dense l5(&net, 4, d2_bin); - tk::dnn::Activation l6(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d l0(&net, 4, 1, 2, 1, 1, 0, 0, c0_bin); // Load input dnnType *data; diff --git a/tests/weights_exporter.py b/tests/weights_exporter.py index df8bab3..912ef82 100644 --- a/tests/weights_exporter.py +++ b/tests/weights_exporter.py @@ -5,96 +5,51 @@ import numpy as np import argparse import tensorflow as tf import os -import msgpack -import lmdb import random +import struct +from keras.models import Sequential, Model -def export_dense(name, weights, bias): - print "######## EXPORT", name, "LAYER ########" - print "Original weighs:" - print weights - print bias, "\n" +def bin_write(f, data): + data = data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) - #input, filters - I, C = np.shape(weights) - B = np.shape(bias) - print "w shape: ", I, C - print "b shape: ", B +def export_layer(name, weights, bias): + print ("######## EXPORT", name, "LAYER ########") - wgs = [ [ j[i] for j in weights ] for i in xrange(C) ] - wgs = np.array(wgs, dtype=np.float32) - - print "REPOSITIONED WEIGHTS:" - print wgs + print("wgs pretranpose: ", np.shape(weights)) + # convert NHWC to NCHW + if(weights.ndim == 4): + weights = weights.transpose(3,2,0,1) + elif(weights.ndim == 3): + weights = weights.transpose(2,1,0) + else: + print("Ndim", weights.ndim) + raise("not implemented with dim" ) + print("weights: ", np.shape(weights)) + print("bias: ", np.shape(bias)) + + weights = np.array(weights.flatten(), dtype=np.float32) bias = np.array(bias, dtype=np.float32) - wgs.tofile(name + ".bin", format="f") - bias.tofile(name + ".bias.bin", format="f") - print "WEIGHTS saved\n" + print(len(weights) + len(bias)) -def export_conv2d(name, weights, bias): - print "######## EXPORT", name, "LAYER ########" - print "Original weighs:" - print weights - print bias, "\n" - - # height, width, input, filters - H, W, N, C = np.shape(weights) - B = np.shape(bias) - print "w shape: ", N, C, H, W - print "b shape: ", B + f = open(name + ".bin", mode='wb') + bin_write(f, weights) + bin_write(f, bias) + print ("WEIGHTS saved\n") - wgs = weights.transpose() - wgs = wgs.transpose(0, 1, 3, 2) - print "Final shape:", np.shape(wgs) - wgs = np.array(wgs.flatten(), dtype=np.float32) - - print "REPOSITIONED WEIGHTS:" - print wgs - - bias = np.array(bias, dtype=np.float32) - - wgs.tofile(name + ".bin", format="f") - bias.tofile(name + ".bias.bin", format="f") - print "WEIGHTS saved\n" - -def export_conv3d(name, weights, bias): - print "######## EXPORT", name, "LAYER ########" - print "Original weighs:" - print weights - print bias, "\n" - - print np.shape(weights) - # height, width, input, thickness, filters - H, W, T, N, C = np.shape(weights) - B = np.shape(bias) - print "w shape: ", T, C, H, W #thickness is number of images for cudnn - print "b shape: ", B - - wgs = weights.transpose() - wgs = wgs.transpose(0, 1, 4, 3, 2) - print "Final shape:", np.shape(wgs) - wgs = np.array(wgs.flatten(), dtype=np.float32) - - print "REPOSITIONED WEIGHTS:" - print wgs - - bias = np.array(bias, dtype=np.float32) - - wgs.tofile(name + ".bin", format="f") - bias.tofile(name + ".bias.bin", format="f") - print "WEIGHTS saved\n" - - -def get_session(gpu_fraction=0.5): - gpu_options = tf.GPUOptions(allow_growth=True) - #per_process_gpu_memory_fraction=gpu_fraction) - return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) +def export_bidir(name, weights): + print ("######## EXPORT", name, "LAYER ########") + + for w in weights: + print(np.shape(w)) #https://github.com/fchollet/keras/wiki/Converting-convolution-kernels-from-Theano-to-TensorFlow-and-vice-versa if __name__ == '__main__': - KTF.set_session(get_session()) + print("DATA FORMAT: ", keras.backend.image_data_format()) parser = argparse.ArgumentParser(description='KERAS WEIGHTS EXPORTER TO CUDNN') parser.add_argument('model',type=str, @@ -103,31 +58,41 @@ if __name__ == '__main__': args = parser.parse_args() - print "DATA FORMAT: ", keras.backend.image_data_format() + print("DATA FORMAT: ", keras.backend.image_data_format()) - print "Load model: ", args.model + print("Load model: ", args.model) model = load_model(args.model) + model.summary() + weights = model.get_weights() ws = np.shape(weights) - print "Weights shape:", ws + print("Weights shape:", ws) if not os.path.exists(args.output): os.makedirs(args.output) - num = 0 + name_num = 0 for l in model.layers: - name = l.name - if name.startswith("conv3d"): - export_conv3d(args.output + "/conv" + str(name_num), weights[num], weights[num+1]) - elif name.startswith("conv2d"): - export_conv2d(args.output + "/conv" + str(name_num), weights[num], weights[num+1]) - elif name.startswith("dense"): - export_dense(args.output + "/dense" + str(name_num), weights[num], weights[num+1]) - else: - print "skip:", name, "has no weights" - continue - name_num += 1 - num += 2 + print("\n\nNAME: ", l.name) + print("input: ", l.input_shape, " output: ", l.output_shape) + wgs = l.get_weights() + print("wgs num: ", len(wgs)) + + name = l.name + if name.startswith("conv3d"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("conv2d"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("conv1d"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("dense"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("bidirectional"): + export_bidir(args.output + "/" + name, wgs) + else: + print ("skip:", name, "has no weights") + continue + -- 2.52.0 From a9c0db0bf6abc3f21611b1c13f55aaba383159e5 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 19:27:18 +0100 Subject: [PATCH 052/482] LSTM cudnn test --- .gitignore | 3 +- CMakeLists.txt | 3 + include/tkDNN/Layer.h | 76 +++++++++++++++++ src/LSTM.cpp | 142 ++++++++++++++++++++++++++++++++ src/utils.cpp | 3 +- tests/imuodom/imuodom.cpp | 70 ++++++++++++++++ tests/imuodom/infer.py | 74 +++++++++++++++++ tests/simple/test_model.py | 60 +++++++------- tests/simple/test_simple.cpp | 14 +--- tests/weights_exporter.py | 153 ++++++++++++++--------------------- 10 files changed, 464 insertions(+), 134 deletions(-) create mode 100644 src/LSTM.cpp create mode 100644 tests/imuodom/imuodom.cpp create mode 100644 tests/imuodom/infer.py diff --git a/.gitignore b/.gitignore index 02f2a8e..de78410 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ build/ *.h5 *.tar.gz *.weights -.idea/ \ No newline at end of file +.idea/ +*.hdf5 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b4b3c38..e3d8607 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,9 @@ target_link_libraries(test_yolo3_berkeley tkDNN) add_executable(test_yolo3_flir tests/yolo3_flir/yolo3_flir.cpp) target_link_libraries(test_yolo3_flir tkDNN) + +add_executable(test_imuodom tests/imuodom/imuodom.cpp) +target_link_libraries(test_imuodom tkDNN) ################################################################################ diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index ee73109..f15c781 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -9,8 +9,10 @@ namespace tk { namespace dnn { enum layerType_t { + LAYER_INPUT, LAYER_DENSE, LAYER_CONV2D, + LAYER_LSTM, LAYER_ACTIVATION, LAYER_FLATTEN, LAYER_MULADD, @@ -47,8 +49,10 @@ public: std::string getLayerName() { layerType_t type = getLayerType(); switch(type) { + case LAYER_INPUT: return "Input"; case LAYER_DENSE: return "Dense"; case LAYER_CONV2D: return "Conv2d"; + case LAYER_LSTM: return "LSTM"; case LAYER_ACTIVATION: return "Activation"; case LAYER_FLATTEN: return "Flatten"; case LAYER_MULADD: return "MulAdd"; @@ -105,6 +109,27 @@ public: }; +/** + Input layer (it doesnt need weigths) +*/ +class Input : public Layer { + +public: + + Input(Network *net, dataDim_t &dim, dnnType* srcData) : Layer(net) { + input_dim = dim; + output_dim = dim; + dstData = srcData; + } + virtual ~Input() {} + virtual layerType_t getLayerType() { return LAYER_INPUT; }; + + virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) { + return dstData; + } +}; + + /** Dense (full interconnection) layer */ @@ -148,6 +173,14 @@ protected: /** Convolutional 2D layer + + WEIGHTS shape: OUTCH, INCH, KH, KW ... + BIAS shape: OUTCH + + with BATCHNORM: + scales: OUTCH + means: OUTCH + variance: OUTCH */ class Conv2d : public LayerWgs { @@ -172,6 +205,49 @@ protected: size_t ws_sizeInBytes; }; +/** + Bidirectional LSTM layer + https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp + + numlayers = 1 # hardcoded as 1 + + PARAMS (numlayers*2): + layer0: + ( INCH, ? ) ??? + ( HIDDEN, ? ) ??? + ( HIDDEN * 8 ) ??? + layer2: + ( INCH, ? ) ??? + ( HIDDEN, ? ) ??? + ( HIDDEN * 8 ) ??? + + output shape: ( 2*HIDDEN, INH, INW ) +*/ +class LSTM : public Layer { + +public: + LSTM(Network *net, int hiddensize, std::string fname_weights); + virtual ~LSTM(); + virtual layerType_t getLayerType() { return LAYER_LSTM; }; + + virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + + int kernelH, kernelW, strideH, strideW, paddingH, paddingW; + +protected: + cudnnFilterDescriptor_t paramDesc; + cudnnTensorDescriptor_t hiddenStateTensorDesc, cellStateTensorDesc; + cudnnRNNDescriptor_t rnnDesc; + cudnnRNNDataDescriptor_t rnnDataDesc; + cudnnDropoutDescriptor_t dropDesc; + cudnnRNNAlgo_t algo; + + dnnType *hiddenStateData, *cellStateData; + dnnType *paramsSpace; + void* workSpace; + size_t ws_sizeInBytes; +}; + /** Flatten layer diff --git a/src/LSTM.cpp b/src/LSTM.cpp new file mode 100644 index 0000000..3176819 --- /dev/null +++ b/src/LSTM.cpp @@ -0,0 +1,142 @@ +#include + +#include "Layer.h" + +namespace tk { namespace dnn { + +LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : + Layer(net) { + + checkCUDNN( cudnnCreateFilterDescriptor(¶mDesc)); + checkCUDNN( cudnnCreateRNNDescriptor(&rnnDesc) ); + checkCUDNN( cudnnCreateRNNDataDescriptor(&rnnDataDesc) ); + checkCUDNN( cudnnCreateDropoutDescriptor(&dropDesc)); + + int n = input_dim.n; + int c = input_dim.c; + int h = input_dim.h; + int w = input_dim.w; + checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, + net->tensorFormat, net->dataType, n, 1, h, w) ); + + int numlayers = 1; + checkCUDNN( cudnnSetRNNDescriptor(net->cudnnHandle, rnnDesc, hiddensize, numlayers, dropDesc, + cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, + cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, + cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType) ); + + // find dimension of params + size_t params_size = 0; + checkCUDNN( cudnnGetRNNParamsSize(net->cudnnHandle, rnnDesc, srcTensorDesc, ¶ms_size, net->dataType) ); + std::cout<<"Params size bytes: "<dataType, net->tensorFormat, 3, dimW)); + checkCuda( cudaMalloc(¶msSpace, params_size) ); + + + int numlinearlayers = 8; + + for(int i=0; icudnnHandle, rnnDesc, + i, srcTensorDesc, paramDesc, paramsSpace, + j, linLayerMatDesc, (void **)&linLayerMat)); + + if(linLayerMat == nullptr) { + FatalError("LSTM No weights in hidden layer"); + } + + cudnnDataType_t dataType; + cudnnTensorFormat_t format; + int nbDims; + int filterDimA[3]; + checkCUDNN(cudnnGetFilterNdDescriptor(linLayerMatDesc, 3, &dataType, + &format, &nbDims, filterDimA)); + std::cout<<"Wgs Dims: "<cudnnHandle, rnnDesc, + i, srcTensorDesc, paramDesc, paramsSpace, + j, linLayerBiasDesc, (void **)&linLayerBias)); + + if(linLayerMat == nullptr) { + FatalError("LSTM No bias in hidden layer"); + } + + checkCUDNN(cudnnGetFilterNdDescriptor(linLayerBiasDesc, 3, &dataType, + &format, &nbDims, filterDimA)); + std::cout<<"bias Dims: "<tensorFormat, net->dataType, 2*n, c, h, w) ); + checkCuda( cudaMalloc(&hiddenStateData, 2*input_dim.tot()*sizeof(dnnType)) ); + checkCUDNN( cudnnCreateTensorDescriptor(&cellStateTensorDesc)); + checkCUDNN( cudnnSetTensor4dDescriptor(cellStateTensorDesc, + net->tensorFormat, net->dataType, 2*n, c, h, w) ); + checkCuda( cudaMalloc(&cellStateData, 2*input_dim.tot()*sizeof(dnnType)) ); + + + output_dim = input_dim; + output_dim.c = hiddensize*2; + checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc, + net->tensorFormat, net->dataType, output_dim.n, output_dim.c, output_dim.h, output_dim.w) ); + + + + + //allocate data for infer result + checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); +} + +LSTM::~LSTM() { + + checkCuda( cudaFree(dstData) ); +} + +dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { + + checkCUDNN(cudnnRNNForwardInference( + net->cudnnHandle, rnnDesc, 1, + &srcTensorDesc, srcData, + hiddenStateTensorDesc, hiddenStateData, + cellStateTensorDesc, cellStateData, + paramDesc, paramsSpace, + &dstTensorDesc, dstData, + hiddenStateTensorDesc, hiddenStateData, + cellStateTensorDesc, cellStateData, + workSpace, ws_sizeInBytes + )); + + return dstData; +} + +}} diff --git a/src/utils.cpp b/src/utils.cpp index e6f71f8..444318d 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -39,7 +39,8 @@ void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** dat *data_h = new dnnType[size]; if (!dataFile.read ((char*) *data_h, size_b)) { - error_s << "Error reading file " << fname; + error_s << "Error reading file " << fname << " with n of float: "< +#include "tkdnn.h" + +const char *i0_bin = "../tests/imuodom/layers/input0.bin"; +const char *i1_bin = "../tests/imuodom/layers/input1.bin"; +const char *i2_bin = "../tests/imuodom/layers/input2.bin"; +const char *o0_bin = "../tests/imuodom/layers/output0.bin"; +const char *o1_bin = "../tests/imuodom/layers/output1.bin"; +const char *output_bin = "../tests/imuodom/layers/output.bin"; + +const char *c0_bin = "../tests/imuodom/layers/conv1d_7.bin"; +const char *c1_bin = "../tests/imuodom/layers/conv1d_8.bin"; +const char *c2_bin = "../tests/imuodom/layers/conv1d_9.bin"; +const char *c3_bin = "../tests/imuodom/layers/conv1d_10.bin"; +const char *c4_bin = "../tests/imuodom/layers/conv1d_11.bin"; +const char *c5_bin = "../tests/imuodom/layers/conv1d_12.bin"; + +int main() { + + // Network layout + tk::dnn::dataDim_t dim0(1, 4, 1, 100); + tk::dnn::dataDim_t dim1(1, 3, 1, 100); + tk::dnn::dataDim_t dim2(1, 3, 1, 100); + + // Load input + dnnType *i0_d, *i1_d, *i2_d; + dnnType *i0_h, *i1_h, *i2_h; + readBinaryFile(i0_bin, dim0.tot(), &i0_h, &i0_d); + readBinaryFile(i1_bin, dim1.tot(), &i1_h, &i1_d); + readBinaryFile(i2_bin, dim2.tot(), &i2_h, &i2_d); + + tk::dnn::Network net(dim0); + tk::dnn::Input x0 (&net, dim0, i0_d); + tk::dnn::Conv2d x0_0(&net, 128, 1, 11, 1, 1, 0, 0, c0_bin); + tk::dnn::Conv2d x0_1(&net, 128, 1, 11, 1, 1, 0, 0, c1_bin); + tk::dnn::Pooling x0_2(&net, 1, 3, 1, 3, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX); + + tk::dnn::Input x1 (&net, dim1, i1_d); + tk::dnn::Conv2d x1_0(&net, 128, 1, 11, 1, 1, 0, 0, c2_bin); + tk::dnn::Conv2d x1_1(&net, 128, 1, 11, 1, 1, 0, 0, c3_bin); + tk::dnn::Pooling x1_2(&net, 1, 3, 1, 3, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX); + + tk::dnn::Input x2 (&net, dim2, i2_d); + tk::dnn::Conv2d x2_0(&net, 128, 1, 11, 1, 1, 0, 0, c4_bin); + tk::dnn::Conv2d x2_1(&net, 128, 1, 11, 1, 1, 0, 0, c5_bin); + tk::dnn::Pooling x2_2(&net, 1, 3, 1, 3, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX); + + tk::dnn::Layer *concat_l[3] = { &x0_2, &x1_2, &x2_2 }; + tk::dnn::Route concat (&net, concat_l, 3); + + tk::dnn::LSTM lstm0(&net, 128, "ciao"); + + net.print(); + + dnnType *data; + tk::dnn::dataDim_t dim; + + TIMER_START + // Inference + data = net.infer(dim, data); dim.print(); + TIMER_STOP + + // Print real test + std::cout<<"\n==== CHECK RESULT ====\n"; + dnnType *out; + dnnType *out_h; + readBinaryFile(output_bin, dim.tot(), &out_h, &out); + checkResult(dim.tot(), data, out); + return 0; +} diff --git a/tests/imuodom/infer.py b/tests/imuodom/infer.py new file mode 100644 index 0000000..9d9929d --- /dev/null +++ b/tests/imuodom/infer.py @@ -0,0 +1,74 @@ +import keras +from keras.models import load_model +import keras.backend.tensorflow_backend as KTF +import numpy as np +import argparse +import tensorflow as tf +import os +import random +import struct +from keras.models import Sequential, Model + +def bin_write(f, data): + data = data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + + +if __name__ == '__main__': + + + print("DATA FORMAT: ", keras.backend.image_data_format()) + + print("Load model: ", "ferrariS1.hdf5") + model = load_model("ferrariS1.hdf5") + model.summary() + + weights = model.get_weights() + + x_angle = np.random.rand(1,100,4) + x_gyro = np.random.rand(1,100,3) + x_acc = np.random.rand(1,100,3) + + [yhat_delta_p, yhat_delta_q] = model.predict([x_angle, x_gyro, x_acc], batch_size=1, verbose=1) + + layer_name = 'bidirectional_3' + intermediate_layer_model = Model(inputs=model.input, + outputs=model.get_layer(layer_name).output) + intermediate_output = intermediate_layer_model.predict([x_angle, x_gyro, x_acc]) + + x_angle = np.array([x_angle]) + x_gyro = np.array([x_gyro]) + x_acc = np.array([x_acc]) + intermediate_output = np.array([intermediate_output]) + + x_angle = x_angle.transpose(0, 3, 1, 2) + x_gyro = x_gyro.transpose(0, 3, 1, 2) + x_acc = x_acc.transpose(0, 3, 1, 2) + intermediate_output = intermediate_output.transpose(0, 3, 1, 2) + + print("x0: ", np.shape(x_angle)) + print("out: ",np.shape(intermediate_output)) + + x_angle = np.array(x_angle.flatten(), dtype=np.float32) + x_gyro = np.array(x_gyro.flatten(), dtype=np.float32) + x_acc = np.array(x_acc.flatten(), dtype=np.float32) + yhat_delta_p = np.array(yhat_delta_p.flatten(), dtype=np.float32) + yhat_delta_q = np.array(yhat_delta_q.flatten(), dtype=np.float32) + intermediate_output = np.array(intermediate_output.flatten(), dtype=np.float32) + + + f = open("layers/input0.bin", mode='wb') + bin_write(f, x_angle) + f = open("layers/input1.bin", mode='wb') + bin_write(f, x_gyro) + f = open("layers/input2.bin", mode='wb') + bin_write(f, x_acc) + f = open("layers/output0.bin", mode='wb') + bin_write(f, yhat_delta_p) + f = open("layers/output1.bin", mode='wb') + bin_write(f, yhat_delta_q) + f = open("layers/output.bin", mode='wb') + bin_write(f, intermediate_output) + diff --git a/tests/simple/test_model.py b/tests/simple/test_model.py index 60f17d8..3b1d053 100644 --- a/tests/simple/test_model.py +++ b/tests/simple/test_model.py @@ -1,43 +1,49 @@ import keras import numpy as np from keras.models import Sequential -from keras.layers import Input, Dense, Activation, Flatten, Dropout, ELU, Reshape, Lambda +from keras.layers import Input, Dense, Activation, Flatten, Dropout, ELU, Reshape, Lambda, Conv1D from keras.layers.convolutional import Convolution2D, Convolution3D from keras.layers.pooling import MaxPooling2D, MaxPooling3D, AveragePooling3D from keras.models import Sequential, Model from keras.layers import Cropping2D import keras.backend.tensorflow_backend as KTF +import struct +from keras.models import Sequential, Model -def dense_model(): - model = Sequential() +def bin_write(f, data): + data = data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) + +def create_model(): + x1 = Input((6, 16), name='x1') + conv = Conv1D(4, 2)(x1) + model = Model([x1], [conv]) + model.summary() - model.add(Reshape((10, 10, 1), input_shape=(10, 10))) - model.add(Convolution2D(2, (4, 4), subsample=(2, 2), - bias_initializer='random_uniform', activation="relu")) - model.add(Convolution2D(4, (2, 2), subsample=(1, 1), - bias_initializer='random_uniform', activation="relu")) - model.add(Flatten()) - model.add(Dense(4, bias_initializer='random_uniform', activation="relu")) - sgd = keras.optimizers.Adam(lr=1e-4, decay=1e-8) - model.compile(optimizer=sgd, loss="mse") return model - if __name__ == '__main__': - print "DATA FORMAT: ", keras.backend.image_data_format() + print ("DATA FORMAT: ", keras.backend.image_data_format()) - model = dense_model() - model.save("net.h5") + model = create_model() + model.save("net.hdf5") - grid = np.random.rand(10,10) - X = grid[None,:,:] - i = np.array(grid.flatten(), dtype=np.float32) - print i - i.tofile("input.bin", format="f") - print "Input: ", X + x = np.random.rand(1,1,6,16) + r = model.predict( x[0], batch_size=1) + r = np.array([r]) + + x = x.transpose(0, 3, 1, 2) + r = r.transpose(0, 3, 1, 2) + print("in: ", np.shape(x)) + print("out: ", np.shape(r)) + + x = np.array(x.flatten(), dtype=np.float32) + f = open("input.bin", mode='wb') + bin_write(f, x) + + r = np.array(r.flatten(), dtype=np.float32) + f = open("output.bin", mode='wb') + bin_write(f, r) - r = model.predict( X, batch_size=1) - print np.shape(r) - print "Result: ", r - print "Result shape: ", np.shape(r) - r.tofile("output.bin", format="f") diff --git a/tests/simple/test_simple.cpp b/tests/simple/test_simple.cpp index a7628ea..3427d6d 100644 --- a/tests/simple/test_simple.cpp +++ b/tests/simple/test_simple.cpp @@ -2,23 +2,15 @@ #include "tkdnn.h" const char *input_bin = "../tests/simple/input.bin"; -const char *c0_bin = "../tests/simple/layers/c0.bin"; -const char *c1_bin = "../tests/simple/layers/c1.bin"; -const char *d2_bin = "../tests/simple/layers/d2.bin"; +const char *c0_bin = "../tests/simple/layers/conv1d_1.bin"; const char *output_bin = "../tests/simple/output.bin"; int main() { // Network layout - tk::dnn::dataDim_t dim(1, 1, 10, 10, 1); + tk::dnn::dataDim_t dim(1, 16, 1, 6); tk::dnn::Network net(dim); - tk::dnn::Conv2d l0(&net, 2, 4, 4, 2, 2, 0, 0, c0_bin); - tk::dnn::Activation l1(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Conv2d l2(&net, 4, 2, 2, 1, 1, 0, 0, c1_bin); - tk::dnn::Activation l3(&net, CUDNN_ACTIVATION_RELU); - tk::dnn::Flatten l4(&net); - tk::dnn::Dense l5(&net, 4, d2_bin); - tk::dnn::Activation l6(&net, CUDNN_ACTIVATION_RELU); + tk::dnn::Conv2d l0(&net, 4, 1, 2, 1, 1, 0, 0, c0_bin); // Load input dnnType *data; diff --git a/tests/weights_exporter.py b/tests/weights_exporter.py index df8bab3..912ef82 100644 --- a/tests/weights_exporter.py +++ b/tests/weights_exporter.py @@ -5,96 +5,51 @@ import numpy as np import argparse import tensorflow as tf import os -import msgpack -import lmdb import random +import struct +from keras.models import Sequential, Model -def export_dense(name, weights, bias): - print "######## EXPORT", name, "LAYER ########" - print "Original weighs:" - print weights - print bias, "\n" +def bin_write(f, data): + data = data.flatten() + fmt = 'f'*len(data) + bin = struct.pack(fmt, *data) + f.write(bin) - #input, filters - I, C = np.shape(weights) - B = np.shape(bias) - print "w shape: ", I, C - print "b shape: ", B +def export_layer(name, weights, bias): + print ("######## EXPORT", name, "LAYER ########") - wgs = [ [ j[i] for j in weights ] for i in xrange(C) ] - wgs = np.array(wgs, dtype=np.float32) - - print "REPOSITIONED WEIGHTS:" - print wgs + print("wgs pretranpose: ", np.shape(weights)) + # convert NHWC to NCHW + if(weights.ndim == 4): + weights = weights.transpose(3,2,0,1) + elif(weights.ndim == 3): + weights = weights.transpose(2,1,0) + else: + print("Ndim", weights.ndim) + raise("not implemented with dim" ) + print("weights: ", np.shape(weights)) + print("bias: ", np.shape(bias)) + + weights = np.array(weights.flatten(), dtype=np.float32) bias = np.array(bias, dtype=np.float32) - wgs.tofile(name + ".bin", format="f") - bias.tofile(name + ".bias.bin", format="f") - print "WEIGHTS saved\n" + print(len(weights) + len(bias)) -def export_conv2d(name, weights, bias): - print "######## EXPORT", name, "LAYER ########" - print "Original weighs:" - print weights - print bias, "\n" - - # height, width, input, filters - H, W, N, C = np.shape(weights) - B = np.shape(bias) - print "w shape: ", N, C, H, W - print "b shape: ", B + f = open(name + ".bin", mode='wb') + bin_write(f, weights) + bin_write(f, bias) + print ("WEIGHTS saved\n") - wgs = weights.transpose() - wgs = wgs.transpose(0, 1, 3, 2) - print "Final shape:", np.shape(wgs) - wgs = np.array(wgs.flatten(), dtype=np.float32) - - print "REPOSITIONED WEIGHTS:" - print wgs - - bias = np.array(bias, dtype=np.float32) - - wgs.tofile(name + ".bin", format="f") - bias.tofile(name + ".bias.bin", format="f") - print "WEIGHTS saved\n" - -def export_conv3d(name, weights, bias): - print "######## EXPORT", name, "LAYER ########" - print "Original weighs:" - print weights - print bias, "\n" - - print np.shape(weights) - # height, width, input, thickness, filters - H, W, T, N, C = np.shape(weights) - B = np.shape(bias) - print "w shape: ", T, C, H, W #thickness is number of images for cudnn - print "b shape: ", B - - wgs = weights.transpose() - wgs = wgs.transpose(0, 1, 4, 3, 2) - print "Final shape:", np.shape(wgs) - wgs = np.array(wgs.flatten(), dtype=np.float32) - - print "REPOSITIONED WEIGHTS:" - print wgs - - bias = np.array(bias, dtype=np.float32) - - wgs.tofile(name + ".bin", format="f") - bias.tofile(name + ".bias.bin", format="f") - print "WEIGHTS saved\n" - - -def get_session(gpu_fraction=0.5): - gpu_options = tf.GPUOptions(allow_growth=True) - #per_process_gpu_memory_fraction=gpu_fraction) - return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) +def export_bidir(name, weights): + print ("######## EXPORT", name, "LAYER ########") + + for w in weights: + print(np.shape(w)) #https://github.com/fchollet/keras/wiki/Converting-convolution-kernels-from-Theano-to-TensorFlow-and-vice-versa if __name__ == '__main__': - KTF.set_session(get_session()) + print("DATA FORMAT: ", keras.backend.image_data_format()) parser = argparse.ArgumentParser(description='KERAS WEIGHTS EXPORTER TO CUDNN') parser.add_argument('model',type=str, @@ -103,31 +58,41 @@ if __name__ == '__main__': args = parser.parse_args() - print "DATA FORMAT: ", keras.backend.image_data_format() + print("DATA FORMAT: ", keras.backend.image_data_format()) - print "Load model: ", args.model + print("Load model: ", args.model) model = load_model(args.model) + model.summary() + weights = model.get_weights() ws = np.shape(weights) - print "Weights shape:", ws + print("Weights shape:", ws) if not os.path.exists(args.output): os.makedirs(args.output) - num = 0 + name_num = 0 for l in model.layers: - name = l.name - if name.startswith("conv3d"): - export_conv3d(args.output + "/conv" + str(name_num), weights[num], weights[num+1]) - elif name.startswith("conv2d"): - export_conv2d(args.output + "/conv" + str(name_num), weights[num], weights[num+1]) - elif name.startswith("dense"): - export_dense(args.output + "/dense" + str(name_num), weights[num], weights[num+1]) - else: - print "skip:", name, "has no weights" - continue - name_num += 1 - num += 2 + print("\n\nNAME: ", l.name) + print("input: ", l.input_shape, " output: ", l.output_shape) + wgs = l.get_weights() + print("wgs num: ", len(wgs)) + + name = l.name + if name.startswith("conv3d"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("conv2d"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("conv1d"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("dense"): + export_layer(args.output + "/" + name, wgs[0], wgs[1]) + elif name.startswith("bidirectional"): + export_bidir(args.output + "/" + name, wgs) + else: + print ("skip:", name, "has no weights") + continue + -- 2.52.0 From e8280d3a6502ae9234f6f7f747cecfe71b49f75e Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 23:04:29 +0100 Subject: [PATCH 053/482] LSTM to be tested --- include/tkDNN/Layer.h | 40 ++++--- src/LSTM.cpp | 246 ++++++++++++++++++++++++------------------ 2 files changed, 170 insertions(+), 116 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index f15c781..c538fc9 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -207,9 +207,11 @@ protected: /** Bidirectional LSTM layer + + implementation info: https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp - - numlayers = 1 # hardcoded as 1 + https://github.com/Jeffery-Song/mxnet-test/blob/aab666faad44011f7a67b527b5f6c960367d0422/src/operator/cudnn_rnn-inl.h + https://stackoverflow.com/a/38737941 PARAMS (numlayers*2): layer0: @@ -221,7 +223,9 @@ protected: ( HIDDEN, ? ) ??? ( HIDDEN * 8 ) ??? - output shape: ( 2*HIDDEN, INH, INW ) + OUTPUT shape: + (N, C, 1, W) ---> LSTM(HIDDEN, returnSeq=True) ---> (N, 2*HIDDEN, 1, W) # W is seqLength + (N, C, 1, W) ---> LSTM(HIDDEN, returnSeq=False) ---> (N, 2*HIDDEN, 1, 1) */ class LSTM : public Layer { @@ -232,20 +236,28 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - int kernelH, kernelW, strideH, strideW, paddingH, paddingW; + const bool bidirectional = 1; /**> is the net bidir */ + int stateSize = 0; /**> number of hidden states */ + int seqLen = 0; /**> number of timesteps */ + int numLayers = 1; /**> number of internal layers */ protected: - cudnnFilterDescriptor_t paramDesc; - cudnnTensorDescriptor_t hiddenStateTensorDesc, cellStateTensorDesc; - cudnnRNNDescriptor_t rnnDesc; - cudnnRNNDataDescriptor_t rnnDataDesc; - cudnnDropoutDescriptor_t dropDesc; - cudnnRNNAlgo_t algo; + cudnnRNNDescriptor_t rnnDesc; + cudnnDropoutDescriptor_t dropoutDesc; + dnnType *dropout_states_, *work_space_; - dnnType *hiddenStateData, *cellStateData; - dnnType *paramsSpace; - void* workSpace; - size_t ws_sizeInBytes; + size_t workspace_byte_, reserve_space_byte_, dropout_byte_; + int workspace_size_, dropout_size_; + + std::vector x_desc_vec_, y_desc_vec_, dx_desc_vec_, dy_desc_vec_; + cudnnTensorDescriptor_t hx_desc_, cx_desc_; + cudnnTensorDescriptor_t hy_desc_, cy_desc_; + cudnnTensorDescriptor_t dhx_desc_, dcx_desc_; + cudnnTensorDescriptor_t dhy_desc_, dcy_desc_; + dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; + + cudnnFilterDescriptor_t w_desc_, dw_desc_; + dnnType *w_ptr, *dw_ptr; }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 3176819..be12cec 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -7,111 +7,143 @@ namespace tk { namespace dnn { LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : Layer(net) { - checkCUDNN( cudnnCreateFilterDescriptor(¶mDesc)); - checkCUDNN( cudnnCreateRNNDescriptor(&rnnDesc) ); - checkCUDNN( cudnnCreateRNNDataDescriptor(&rnnDataDesc) ); - checkCUDNN( cudnnCreateDropoutDescriptor(&dropDesc)); + int batchSize = input_dim.n; + int inputSize = input_dim.c; + seqLen = input_dim.w; + stateSize = hiddensize; - int n = input_dim.n; - int c = input_dim.c; - int h = input_dim.h; - int w = input_dim.w; - checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, - net->tensorFormat, net->dataType, n, 1, h, w) ); + std::cout<<"LSTM seqLen: "<cudnnHandle, rnnDesc, hiddensize, numlayers, dropDesc, - cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, - cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, - cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType) ); + // init Tensor Descriptors + std::vector x_vec(seqLen); + std::vector y_vec(seqLen); + std::vector dx_vec(seqLen); + std::vector dy_vec(seqLen); - // find dimension of params - size_t params_size = 0; - checkCUDNN( cudnnGetRNNParamsSize(net->cudnnHandle, rnnDesc, srcTensorDesc, ¶ms_size, net->dataType) ); - std::cout<<"Params size bytes: "<dataType, net->tensorFormat, 3, dimW)); - checkCuda( cudaMalloc(¶msSpace, params_size) ); + checkCUDNN(cudnnSetTensorNdDescriptor(x_vec[i], + net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dx_vec[i], + net->dataType, 3, dimA, strideA)); + dimA[0] = batchSize; + dimA[1] = bidirectional ? stateSize*2 : stateSize; + dimA[2] = 1; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; - - int numlinearlayers = 8; - - for(int i=0; icudnnHandle, rnnDesc, - i, srcTensorDesc, paramDesc, paramsSpace, - j, linLayerMatDesc, (void **)&linLayerMat)); - - if(linLayerMat == nullptr) { - FatalError("LSTM No weights in hidden layer"); - } - - cudnnDataType_t dataType; - cudnnTensorFormat_t format; - int nbDims; - int filterDimA[3]; - checkCUDNN(cudnnGetFilterNdDescriptor(linLayerMatDesc, 3, &dataType, - &format, &nbDims, filterDimA)); - std::cout<<"Wgs Dims: "<cudnnHandle, rnnDesc, - i, srcTensorDesc, paramDesc, paramsSpace, - j, linLayerBiasDesc, (void **)&linLayerBias)); - - if(linLayerMat == nullptr) { - FatalError("LSTM No bias in hidden layer"); - } - - checkCUDNN(cudnnGetFilterNdDescriptor(linLayerBiasDesc, 3, &dataType, - &format, &nbDims, filterDimA)); - std::cout<<"bias Dims: "<dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dy_vec[i], + net->dataType, 3, dimA, strideA)); } + // apply tensordesc + x_desc_vec_ = x_vec; + y_desc_vec_ = y_vec; + dx_desc_vec_ = dx_vec; + dy_desc_vec_ = dy_vec; + // set the state tensors + dimA[0] = numLayers * (bidirectional ? 2 : 1); + dimA[1] = batchSize; + dimA[2] = stateSize; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; + checkCUDNN(cudnnCreateTensorDescriptor(&hx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&cx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&hy_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&cy_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dhx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dcx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dhy_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dcy_desc_)); + checkCUDNN(cudnnSetTensorNdDescriptor(hx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(cx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(hy_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(cy_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dhx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dcx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dhy_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dcy_desc_, net->dataType, 3, dimA, strideA)); + // allocate dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; + checkCuda( cudaMalloc(&hx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&hy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCUDNN( cudnnCreateTensorDescriptor(&hiddenStateTensorDesc)); - checkCUDNN( cudnnSetTensor4dDescriptor(hiddenStateTensorDesc, - net->tensorFormat, net->dataType, 2*n, c, h, w) ); - checkCuda( cudaMalloc(&hiddenStateData, 2*input_dim.tot()*sizeof(dnnType)) ); - checkCUDNN( cudnnCreateTensorDescriptor(&cellStateTensorDesc)); - checkCUDNN( cudnnSetTensor4dDescriptor(cellStateTensorDesc, - net->tensorFormat, net->dataType, 2*n, c, h, w) ); - checkCuda( cudaMalloc(&cellStateData, 2*input_dim.tot()*sizeof(dnnType)) ); + + // Create Dropout descriptors // TODO: ??? IS IT NECESSARY ??? + float dropoutprob = 0.1f; // random val ???? + checkCUDNN(cudnnCreateDropoutDescriptor(&dropoutDesc)); + checkCUDNN(cudnnDropoutGetStatesSize(net->cudnnHandle, &dropout_byte_)); + dropout_size_ = dropout_byte_ / sizeof(dnnType); + checkCuda( cudaMalloc(&dropout_states_, dropout_byte_) ); + uint64_t seed_ = 17 + rand() % 4096; // NOLINT(runtime/threadsafe_fn) + checkCUDNN(cudnnSetDropoutDescriptor(dropoutDesc, + net->cudnnHandle, dropoutprob, dropout_states_, dropout_byte_, seed_)); + + + // RNN descriptors + checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc)); + + checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, + rnnDesc, stateSize, numLayers, dropoutDesc, + cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, + cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, + cudnnRNNMode_t::CUDNN_LSTM, + cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, + net->dataType)); + + + // Get temp space sizes + checkCUDNN(cudnnGetRNNWorkspaceSize(net->cudnnHandle, + rnnDesc, seqLen, x_desc_vec_.data(), &workspace_byte_)); + workspace_size_ = workspace_byte_ / sizeof(dnnType); + checkCuda( cudaMalloc(&work_space_, workspace_byte_) ); + + + // Check that number of params are correct + size_t cudnn_param_size; + checkCUDNN(cudnnGetRNNParamsSize(net->cudnnHandle, + rnnDesc,x_desc_vec_[0], &cudnn_param_size, net->dataType)); + int cudnn_params = cudnn_param_size/sizeof(dnnType); + std::cout<<"LSTM params size: "<dataType, net->tensorFormat, 3, dim_w)); + checkCUDNN(cudnnSetFilterNdDescriptor(dw_desc_, + net->dataType, net->tensorFormat, 3, dim_w)); + // allocate params dnnType *w_ptr, *dw_ptr; + checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&dw_ptr, cudnn_params*sizeof(dnnType)) ); output_dim = input_dim; - output_dim.c = hiddensize*2; - checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc, - net->tensorFormat, net->dataType, output_dim.n, output_dim.c, output_dim.h, output_dim.w) ); - - - + output_dim.c = stateSize*2; //allocate data for infer result checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); @@ -123,19 +155,29 @@ LSTM::~LSTM() { } dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { + std::cout<<"LSTM infer\n"; - checkCUDNN(cudnnRNNForwardInference( - net->cudnnHandle, rnnDesc, 1, - &srcTensorDesc, srcData, - hiddenStateTensorDesc, hiddenStateData, - cellStateTensorDesc, cellStateData, - paramDesc, paramsSpace, - &dstTensorDesc, dstData, - hiddenStateTensorDesc, hiddenStateData, - cellStateTensorDesc, cellStateData, - workSpace, ws_sizeInBytes - )); + checkCUDNN(cudnnRNNForwardInference(net->cudnnHandle, + rnnDesc, + seqLen, + x_desc_vec_.data(), // input array of desc + srcData, // input pointer + hx_desc_, // initial hidden state desc + hx_ptr, // initial hidden state pointer + cx_desc_, // initial cell state desc + cx_ptr, // initial cell state pointer + w_desc_, // weights desc + w_ptr, // weights pointer + y_desc_vec_.data(), // output desc + dstData, // output pointer + hy_desc_, // final hidden state desc + hy_ptr, // final hidden state pointer + cy_desc_, // final cell state desc + cy_ptr, // final cell state pointer + work_space_, // workspace pointer + workspace_byte_)); // workspace size + dim = output_dim; return dstData; } -- 2.52.0 From 03d39d991c5932aa4c94f7b86fc8c64e9d48238b Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 23:04:29 +0100 Subject: [PATCH 054/482] LSTM to be tested --- include/tkDNN/Layer.h | 40 ++++--- src/LSTM.cpp | 246 ++++++++++++++++++++++++------------------ 2 files changed, 170 insertions(+), 116 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index f15c781..c538fc9 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -207,9 +207,11 @@ protected: /** Bidirectional LSTM layer + + implementation info: https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp - - numlayers = 1 # hardcoded as 1 + https://github.com/Jeffery-Song/mxnet-test/blob/aab666faad44011f7a67b527b5f6c960367d0422/src/operator/cudnn_rnn-inl.h + https://stackoverflow.com/a/38737941 PARAMS (numlayers*2): layer0: @@ -221,7 +223,9 @@ protected: ( HIDDEN, ? ) ??? ( HIDDEN * 8 ) ??? - output shape: ( 2*HIDDEN, INH, INW ) + OUTPUT shape: + (N, C, 1, W) ---> LSTM(HIDDEN, returnSeq=True) ---> (N, 2*HIDDEN, 1, W) # W is seqLength + (N, C, 1, W) ---> LSTM(HIDDEN, returnSeq=False) ---> (N, 2*HIDDEN, 1, 1) */ class LSTM : public Layer { @@ -232,20 +236,28 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - int kernelH, kernelW, strideH, strideW, paddingH, paddingW; + const bool bidirectional = 1; /**> is the net bidir */ + int stateSize = 0; /**> number of hidden states */ + int seqLen = 0; /**> number of timesteps */ + int numLayers = 1; /**> number of internal layers */ protected: - cudnnFilterDescriptor_t paramDesc; - cudnnTensorDescriptor_t hiddenStateTensorDesc, cellStateTensorDesc; - cudnnRNNDescriptor_t rnnDesc; - cudnnRNNDataDescriptor_t rnnDataDesc; - cudnnDropoutDescriptor_t dropDesc; - cudnnRNNAlgo_t algo; + cudnnRNNDescriptor_t rnnDesc; + cudnnDropoutDescriptor_t dropoutDesc; + dnnType *dropout_states_, *work_space_; - dnnType *hiddenStateData, *cellStateData; - dnnType *paramsSpace; - void* workSpace; - size_t ws_sizeInBytes; + size_t workspace_byte_, reserve_space_byte_, dropout_byte_; + int workspace_size_, dropout_size_; + + std::vector x_desc_vec_, y_desc_vec_, dx_desc_vec_, dy_desc_vec_; + cudnnTensorDescriptor_t hx_desc_, cx_desc_; + cudnnTensorDescriptor_t hy_desc_, cy_desc_; + cudnnTensorDescriptor_t dhx_desc_, dcx_desc_; + cudnnTensorDescriptor_t dhy_desc_, dcy_desc_; + dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; + + cudnnFilterDescriptor_t w_desc_, dw_desc_; + dnnType *w_ptr, *dw_ptr; }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 3176819..be12cec 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -7,111 +7,143 @@ namespace tk { namespace dnn { LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : Layer(net) { - checkCUDNN( cudnnCreateFilterDescriptor(¶mDesc)); - checkCUDNN( cudnnCreateRNNDescriptor(&rnnDesc) ); - checkCUDNN( cudnnCreateRNNDataDescriptor(&rnnDataDesc) ); - checkCUDNN( cudnnCreateDropoutDescriptor(&dropDesc)); + int batchSize = input_dim.n; + int inputSize = input_dim.c; + seqLen = input_dim.w; + stateSize = hiddensize; - int n = input_dim.n; - int c = input_dim.c; - int h = input_dim.h; - int w = input_dim.w; - checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, - net->tensorFormat, net->dataType, n, 1, h, w) ); + std::cout<<"LSTM seqLen: "<cudnnHandle, rnnDesc, hiddensize, numlayers, dropDesc, - cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, - cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, - cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType) ); + // init Tensor Descriptors + std::vector x_vec(seqLen); + std::vector y_vec(seqLen); + std::vector dx_vec(seqLen); + std::vector dy_vec(seqLen); - // find dimension of params - size_t params_size = 0; - checkCUDNN( cudnnGetRNNParamsSize(net->cudnnHandle, rnnDesc, srcTensorDesc, ¶ms_size, net->dataType) ); - std::cout<<"Params size bytes: "<dataType, net->tensorFormat, 3, dimW)); - checkCuda( cudaMalloc(¶msSpace, params_size) ); + checkCUDNN(cudnnSetTensorNdDescriptor(x_vec[i], + net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dx_vec[i], + net->dataType, 3, dimA, strideA)); + dimA[0] = batchSize; + dimA[1] = bidirectional ? stateSize*2 : stateSize; + dimA[2] = 1; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; - - int numlinearlayers = 8; - - for(int i=0; icudnnHandle, rnnDesc, - i, srcTensorDesc, paramDesc, paramsSpace, - j, linLayerMatDesc, (void **)&linLayerMat)); - - if(linLayerMat == nullptr) { - FatalError("LSTM No weights in hidden layer"); - } - - cudnnDataType_t dataType; - cudnnTensorFormat_t format; - int nbDims; - int filterDimA[3]; - checkCUDNN(cudnnGetFilterNdDescriptor(linLayerMatDesc, 3, &dataType, - &format, &nbDims, filterDimA)); - std::cout<<"Wgs Dims: "<cudnnHandle, rnnDesc, - i, srcTensorDesc, paramDesc, paramsSpace, - j, linLayerBiasDesc, (void **)&linLayerBias)); - - if(linLayerMat == nullptr) { - FatalError("LSTM No bias in hidden layer"); - } - - checkCUDNN(cudnnGetFilterNdDescriptor(linLayerBiasDesc, 3, &dataType, - &format, &nbDims, filterDimA)); - std::cout<<"bias Dims: "<dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dy_vec[i], + net->dataType, 3, dimA, strideA)); } + // apply tensordesc + x_desc_vec_ = x_vec; + y_desc_vec_ = y_vec; + dx_desc_vec_ = dx_vec; + dy_desc_vec_ = dy_vec; + // set the state tensors + dimA[0] = numLayers * (bidirectional ? 2 : 1); + dimA[1] = batchSize; + dimA[2] = stateSize; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; + checkCUDNN(cudnnCreateTensorDescriptor(&hx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&cx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&hy_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&cy_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dhx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dcx_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dhy_desc_)); + checkCUDNN(cudnnCreateTensorDescriptor(&dcy_desc_)); + checkCUDNN(cudnnSetTensorNdDescriptor(hx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(cx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(hy_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(cy_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dhx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dcx_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dhy_desc_, net->dataType, 3, dimA, strideA)); + checkCUDNN(cudnnSetTensorNdDescriptor(dcy_desc_, net->dataType, 3, dimA, strideA)); + // allocate dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; + checkCuda( cudaMalloc(&hx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&hy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCUDNN( cudnnCreateTensorDescriptor(&hiddenStateTensorDesc)); - checkCUDNN( cudnnSetTensor4dDescriptor(hiddenStateTensorDesc, - net->tensorFormat, net->dataType, 2*n, c, h, w) ); - checkCuda( cudaMalloc(&hiddenStateData, 2*input_dim.tot()*sizeof(dnnType)) ); - checkCUDNN( cudnnCreateTensorDescriptor(&cellStateTensorDesc)); - checkCUDNN( cudnnSetTensor4dDescriptor(cellStateTensorDesc, - net->tensorFormat, net->dataType, 2*n, c, h, w) ); - checkCuda( cudaMalloc(&cellStateData, 2*input_dim.tot()*sizeof(dnnType)) ); + + // Create Dropout descriptors // TODO: ??? IS IT NECESSARY ??? + float dropoutprob = 0.1f; // random val ???? + checkCUDNN(cudnnCreateDropoutDescriptor(&dropoutDesc)); + checkCUDNN(cudnnDropoutGetStatesSize(net->cudnnHandle, &dropout_byte_)); + dropout_size_ = dropout_byte_ / sizeof(dnnType); + checkCuda( cudaMalloc(&dropout_states_, dropout_byte_) ); + uint64_t seed_ = 17 + rand() % 4096; // NOLINT(runtime/threadsafe_fn) + checkCUDNN(cudnnSetDropoutDescriptor(dropoutDesc, + net->cudnnHandle, dropoutprob, dropout_states_, dropout_byte_, seed_)); + + + // RNN descriptors + checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc)); + + checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, + rnnDesc, stateSize, numLayers, dropoutDesc, + cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, + cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, + cudnnRNNMode_t::CUDNN_LSTM, + cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, + net->dataType)); + + + // Get temp space sizes + checkCUDNN(cudnnGetRNNWorkspaceSize(net->cudnnHandle, + rnnDesc, seqLen, x_desc_vec_.data(), &workspace_byte_)); + workspace_size_ = workspace_byte_ / sizeof(dnnType); + checkCuda( cudaMalloc(&work_space_, workspace_byte_) ); + + + // Check that number of params are correct + size_t cudnn_param_size; + checkCUDNN(cudnnGetRNNParamsSize(net->cudnnHandle, + rnnDesc,x_desc_vec_[0], &cudnn_param_size, net->dataType)); + int cudnn_params = cudnn_param_size/sizeof(dnnType); + std::cout<<"LSTM params size: "<dataType, net->tensorFormat, 3, dim_w)); + checkCUDNN(cudnnSetFilterNdDescriptor(dw_desc_, + net->dataType, net->tensorFormat, 3, dim_w)); + // allocate params dnnType *w_ptr, *dw_ptr; + checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&dw_ptr, cudnn_params*sizeof(dnnType)) ); output_dim = input_dim; - output_dim.c = hiddensize*2; - checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc, - net->tensorFormat, net->dataType, output_dim.n, output_dim.c, output_dim.h, output_dim.w) ); - - - + output_dim.c = stateSize*2; //allocate data for infer result checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); @@ -123,19 +155,29 @@ LSTM::~LSTM() { } dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { + std::cout<<"LSTM infer\n"; - checkCUDNN(cudnnRNNForwardInference( - net->cudnnHandle, rnnDesc, 1, - &srcTensorDesc, srcData, - hiddenStateTensorDesc, hiddenStateData, - cellStateTensorDesc, cellStateData, - paramDesc, paramsSpace, - &dstTensorDesc, dstData, - hiddenStateTensorDesc, hiddenStateData, - cellStateTensorDesc, cellStateData, - workSpace, ws_sizeInBytes - )); + checkCUDNN(cudnnRNNForwardInference(net->cudnnHandle, + rnnDesc, + seqLen, + x_desc_vec_.data(), // input array of desc + srcData, // input pointer + hx_desc_, // initial hidden state desc + hx_ptr, // initial hidden state pointer + cx_desc_, // initial cell state desc + cx_ptr, // initial cell state pointer + w_desc_, // weights desc + w_ptr, // weights pointer + y_desc_vec_.data(), // output desc + dstData, // output pointer + hy_desc_, // final hidden state desc + hy_ptr, // final hidden state pointer + cy_desc_, // final cell state desc + cy_ptr, // final cell state pointer + work_space_, // workspace pointer + workspace_byte_)); // workspace size + dim = output_dim; return dstData; } -- 2.52.0 From 0329318ca333e62dfea22d6d05383e341606549a Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 23:10:48 +0100 Subject: [PATCH 055/482] removed unused var --- include/tkDNN/Layer.h | 10 ++++------ src/LSTM.cpp | 42 +++++++++++++----------------------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index c538fc9..596fbec 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -246,18 +246,16 @@ protected: cudnnDropoutDescriptor_t dropoutDesc; dnnType *dropout_states_, *work_space_; - size_t workspace_byte_, reserve_space_byte_, dropout_byte_; + size_t workspace_byte_, dropout_byte_; int workspace_size_, dropout_size_; - std::vector x_desc_vec_, y_desc_vec_, dx_desc_vec_, dy_desc_vec_; + std::vector x_desc_vec_, y_desc_vec_; cudnnTensorDescriptor_t hx_desc_, cx_desc_; cudnnTensorDescriptor_t hy_desc_, cy_desc_; - cudnnTensorDescriptor_t dhx_desc_, dcx_desc_; - cudnnTensorDescriptor_t dhy_desc_, dcy_desc_; dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; - cudnnFilterDescriptor_t w_desc_, dw_desc_; - dnnType *w_ptr, *dw_ptr; + cudnnFilterDescriptor_t w_desc_; + dnnType *w_ptr; }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index be12cec..46a0593 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -17,16 +17,12 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : // init Tensor Descriptors std::vector x_vec(seqLen); std::vector y_vec(seqLen); - std::vector dx_vec(seqLen); - std::vector dy_vec(seqLen); int dimA[3]; int strideA[3]; for (int i = 0; i < seqLen; i++) { checkCUDNN(cudnnCreateTensorDescriptor(&x_vec[i])); checkCUDNN(cudnnCreateTensorDescriptor(&y_vec[i])); - checkCUDNN(cudnnCreateTensorDescriptor(&dx_vec[i])); - checkCUDNN(cudnnCreateTensorDescriptor(&dy_vec[i])); dimA[0] = batchSize; dimA[1] = inputSize; @@ -36,29 +32,21 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : strideA[0] = dimA[2] * dimA[1]; strideA[1] = dimA[2]; strideA[2] = 1; - checkCUDNN(cudnnSetTensorNdDescriptor(x_vec[i], net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dx_vec[i], - net->dataType, 3, dimA, strideA)); + dimA[0] = batchSize; dimA[1] = bidirectional ? stateSize*2 : stateSize; dimA[2] = 1; strideA[0] = dimA[2] * dimA[1]; strideA[1] = dimA[2]; strideA[2] = 1; - checkCUDNN(cudnnSetTensorNdDescriptor(y_vec[i], net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dy_vec[i], - net->dataType, 3, dimA, strideA)); } - // apply tensordesc x_desc_vec_ = x_vec; y_desc_vec_ = y_vec; - dx_desc_vec_ = dx_vec; - dy_desc_vec_ = dy_vec; // set the state tensors @@ -72,18 +60,10 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : checkCUDNN(cudnnCreateTensorDescriptor(&cx_desc_)); checkCUDNN(cudnnCreateTensorDescriptor(&hy_desc_)); checkCUDNN(cudnnCreateTensorDescriptor(&cy_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dhx_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dcx_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dhy_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dcy_desc_)); checkCUDNN(cudnnSetTensorNdDescriptor(hx_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(cx_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(hy_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(cy_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dhx_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dcx_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dhy_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dcy_desc_, net->dataType, 3, dimA, strideA)); // allocate dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; checkCuda( cudaMalloc(&hx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); checkCuda( cudaMalloc(&cx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); @@ -130,28 +110,32 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : // Set param descriptors checkCUDNN(cudnnCreateFilterDescriptor(&w_desc_)); - checkCUDNN(cudnnCreateFilterDescriptor(&dw_desc_)); int dim_w[3] = {1, 1, 1}; dim_w[0] = cudnn_params; checkCUDNN(cudnnSetFilterNdDescriptor(w_desc_, net->dataType, net->tensorFormat, 3, dim_w)); - checkCUDNN(cudnnSetFilterNdDescriptor(dw_desc_, - net->dataType, net->tensorFormat, 3, dim_w)); - // allocate params dnnType *w_ptr, *dw_ptr; + // allocate params dnnType *w_ptr; checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&dw_ptr, cudnn_params*sizeof(dnnType)) ); - + // set output dim output_dim = input_dim; output_dim.c = stateSize*2; - + //allocate data for infer result checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); } LSTM::~LSTM() { + checkCuda(cudaFree(hx_ptr)); + checkCuda(cudaFree(cx_ptr)); + checkCuda(cudaFree(hy_ptr)); + checkCuda(cudaFree(cy_ptr)); + checkCuda(cudaFree(w_ptr )); - checkCuda( cudaFree(dstData) ); + checkCuda(cudaFree(work_space_ )); + checkCuda(cudaFree(dropout_states_)); + + checkCuda(cudaFree(dstData)); } dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { -- 2.52.0 From c1c2173e4d9ea4f59d95ee069fa9e0fc1f58f31b Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 23:10:48 +0100 Subject: [PATCH 056/482] removed unused var --- include/tkDNN/Layer.h | 10 ++++------ src/LSTM.cpp | 42 +++++++++++++----------------------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index c538fc9..596fbec 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -246,18 +246,16 @@ protected: cudnnDropoutDescriptor_t dropoutDesc; dnnType *dropout_states_, *work_space_; - size_t workspace_byte_, reserve_space_byte_, dropout_byte_; + size_t workspace_byte_, dropout_byte_; int workspace_size_, dropout_size_; - std::vector x_desc_vec_, y_desc_vec_, dx_desc_vec_, dy_desc_vec_; + std::vector x_desc_vec_, y_desc_vec_; cudnnTensorDescriptor_t hx_desc_, cx_desc_; cudnnTensorDescriptor_t hy_desc_, cy_desc_; - cudnnTensorDescriptor_t dhx_desc_, dcx_desc_; - cudnnTensorDescriptor_t dhy_desc_, dcy_desc_; dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; - cudnnFilterDescriptor_t w_desc_, dw_desc_; - dnnType *w_ptr, *dw_ptr; + cudnnFilterDescriptor_t w_desc_; + dnnType *w_ptr; }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index be12cec..46a0593 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -17,16 +17,12 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : // init Tensor Descriptors std::vector x_vec(seqLen); std::vector y_vec(seqLen); - std::vector dx_vec(seqLen); - std::vector dy_vec(seqLen); int dimA[3]; int strideA[3]; for (int i = 0; i < seqLen; i++) { checkCUDNN(cudnnCreateTensorDescriptor(&x_vec[i])); checkCUDNN(cudnnCreateTensorDescriptor(&y_vec[i])); - checkCUDNN(cudnnCreateTensorDescriptor(&dx_vec[i])); - checkCUDNN(cudnnCreateTensorDescriptor(&dy_vec[i])); dimA[0] = batchSize; dimA[1] = inputSize; @@ -36,29 +32,21 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : strideA[0] = dimA[2] * dimA[1]; strideA[1] = dimA[2]; strideA[2] = 1; - checkCUDNN(cudnnSetTensorNdDescriptor(x_vec[i], net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dx_vec[i], - net->dataType, 3, dimA, strideA)); + dimA[0] = batchSize; dimA[1] = bidirectional ? stateSize*2 : stateSize; dimA[2] = 1; strideA[0] = dimA[2] * dimA[1]; strideA[1] = dimA[2]; strideA[2] = 1; - checkCUDNN(cudnnSetTensorNdDescriptor(y_vec[i], net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dy_vec[i], - net->dataType, 3, dimA, strideA)); } - // apply tensordesc x_desc_vec_ = x_vec; y_desc_vec_ = y_vec; - dx_desc_vec_ = dx_vec; - dy_desc_vec_ = dy_vec; // set the state tensors @@ -72,18 +60,10 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : checkCUDNN(cudnnCreateTensorDescriptor(&cx_desc_)); checkCUDNN(cudnnCreateTensorDescriptor(&hy_desc_)); checkCUDNN(cudnnCreateTensorDescriptor(&cy_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dhx_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dcx_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dhy_desc_)); - checkCUDNN(cudnnCreateTensorDescriptor(&dcy_desc_)); checkCUDNN(cudnnSetTensorNdDescriptor(hx_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(cx_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(hy_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(cy_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dhx_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dcx_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dhy_desc_, net->dataType, 3, dimA, strideA)); - checkCUDNN(cudnnSetTensorNdDescriptor(dcy_desc_, net->dataType, 3, dimA, strideA)); // allocate dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; checkCuda( cudaMalloc(&hx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); checkCuda( cudaMalloc(&cx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); @@ -130,28 +110,32 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : // Set param descriptors checkCUDNN(cudnnCreateFilterDescriptor(&w_desc_)); - checkCUDNN(cudnnCreateFilterDescriptor(&dw_desc_)); int dim_w[3] = {1, 1, 1}; dim_w[0] = cudnn_params; checkCUDNN(cudnnSetFilterNdDescriptor(w_desc_, net->dataType, net->tensorFormat, 3, dim_w)); - checkCUDNN(cudnnSetFilterNdDescriptor(dw_desc_, - net->dataType, net->tensorFormat, 3, dim_w)); - // allocate params dnnType *w_ptr, *dw_ptr; + // allocate params dnnType *w_ptr; checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&dw_ptr, cudnn_params*sizeof(dnnType)) ); - + // set output dim output_dim = input_dim; output_dim.c = stateSize*2; - + //allocate data for infer result checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); } LSTM::~LSTM() { + checkCuda(cudaFree(hx_ptr)); + checkCuda(cudaFree(cx_ptr)); + checkCuda(cudaFree(hy_ptr)); + checkCuda(cudaFree(cy_ptr)); + checkCuda(cudaFree(w_ptr )); - checkCuda( cudaFree(dstData) ); + checkCuda(cudaFree(work_space_ )); + checkCuda(cudaFree(dropout_states_)); + + checkCuda(cudaFree(dstData)); } dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { -- 2.52.0 From f5706d8523714382b5e6c0e821bff36d595d358f Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 23:21:28 +0100 Subject: [PATCH 057/482] lstm return seq --- include/tkDNN/Layer.h | 3 ++- src/LSTM.cpp | 16 ++++++++++++---- tests/imuodom/imuodom.cpp | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 596fbec..240d9a2 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -230,13 +230,14 @@ protected: class LSTM : public Layer { public: - LSTM(Network *net, int hiddensize, std::string fname_weights); + LSTM(Network *net, int hiddensize, bool returnSeq, std::string fname_weights); virtual ~LSTM(); virtual layerType_t getLayerType() { return LAYER_LSTM; }; virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); const bool bidirectional = 1; /**> is the net bidir */ + bool returnSeq = false; /**> if false return only the result of last timestep */ int stateSize = 0; /**> number of hidden states */ int seqLen = 0; /**> number of timesteps */ int numLayers = 1; /**> number of internal layers */ diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 46a0593..c31bccf 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -4,9 +4,10 @@ namespace tk { namespace dnn { -LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : +LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weights) : Layer(net) { + this->returnSeq = returnSeq; int batchSize = input_dim.n; int inputSize = input_dim.c; seqLen = input_dim.w; @@ -117,12 +118,19 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : // allocate params dnnType *w_ptr; checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); + + + //allocate data for infer result + int dstDim = input_dim.n * stateSize*2 * input_dim.h * input_dim.w; + checkCuda( cudaMalloc(&dstData, dstDim*sizeof(dnnType)) ); + // set output dim output_dim = input_dim; output_dim.c = stateSize*2; - - //allocate data for infer result - checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); + if(!returnSeq) { + output_dim.h = 1; + output_dim.w = 1; + } } LSTM::~LSTM() { diff --git a/tests/imuodom/imuodom.cpp b/tests/imuodom/imuodom.cpp index 14146b2..6fc3fba 100644 --- a/tests/imuodom/imuodom.cpp +++ b/tests/imuodom/imuodom.cpp @@ -48,7 +48,8 @@ int main() { tk::dnn::Layer *concat_l[3] = { &x0_2, &x1_2, &x2_2 }; tk::dnn::Route concat (&net, concat_l, 3); - tk::dnn::LSTM lstm0(&net, 128, "ciao"); + tk::dnn::LSTM lstm0(&net, 128, true, "ciao"); + tk::dnn::LSTM lstm1(&net, 128, false, "ciao"); net.print(); -- 2.52.0 From 4fa5d2c231900e8ee9bfbab908b2ac97add974db Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Thu, 13 Feb 2020 23:21:28 +0100 Subject: [PATCH 058/482] lstm return seq --- include/tkDNN/Layer.h | 3 ++- src/LSTM.cpp | 16 ++++++++++++---- tests/imuodom/imuodom.cpp | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 596fbec..240d9a2 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -230,13 +230,14 @@ protected: class LSTM : public Layer { public: - LSTM(Network *net, int hiddensize, std::string fname_weights); + LSTM(Network *net, int hiddensize, bool returnSeq, std::string fname_weights); virtual ~LSTM(); virtual layerType_t getLayerType() { return LAYER_LSTM; }; virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); const bool bidirectional = 1; /**> is the net bidir */ + bool returnSeq = false; /**> if false return only the result of last timestep */ int stateSize = 0; /**> number of hidden states */ int seqLen = 0; /**> number of timesteps */ int numLayers = 1; /**> number of internal layers */ diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 46a0593..c31bccf 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -4,9 +4,10 @@ namespace tk { namespace dnn { -LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : +LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weights) : Layer(net) { + this->returnSeq = returnSeq; int batchSize = input_dim.n; int inputSize = input_dim.c; seqLen = input_dim.w; @@ -117,12 +118,19 @@ LSTM::LSTM( Network *net, int hiddensize, std::string fname_weights) : // allocate params dnnType *w_ptr; checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); + + + //allocate data for infer result + int dstDim = input_dim.n * stateSize*2 * input_dim.h * input_dim.w; + checkCuda( cudaMalloc(&dstData, dstDim*sizeof(dnnType)) ); + // set output dim output_dim = input_dim; output_dim.c = stateSize*2; - - //allocate data for infer result - checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); + if(!returnSeq) { + output_dim.h = 1; + output_dim.w = 1; + } } LSTM::~LSTM() { diff --git a/tests/imuodom/imuodom.cpp b/tests/imuodom/imuodom.cpp index 14146b2..6fc3fba 100644 --- a/tests/imuodom/imuodom.cpp +++ b/tests/imuodom/imuodom.cpp @@ -48,7 +48,8 @@ int main() { tk::dnn::Layer *concat_l[3] = { &x0_2, &x1_2, &x2_2 }; tk::dnn::Route concat (&net, concat_l, 3); - tk::dnn::LSTM lstm0(&net, 128, "ciao"); + tk::dnn::LSTM lstm0(&net, 128, true, "ciao"); + tk::dnn::LSTM lstm1(&net, 128, false, "ciao"); net.print(); -- 2.52.0 From 200e9466f648b42239fef7d37247c77b082fa723 Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 14 Feb 2020 18:44:13 +0100 Subject: [PATCH 059/482] Move pre-processing on GPU Signed-off-by: Davide Sapienza --- include/sorting.h | 19 ++- include/tkDNN/CenternetDetection.h | 23 ++- src/CenternetDetection.cpp | 245 ++++++++++++----------------- src/sorting.cu | 35 ++--- 4 files changed, 152 insertions(+), 170 deletions(-) diff --git a/include/sorting.h b/include/sorting.h index 9c968f3..23fd34b 100644 --- a/include/sorting.h +++ b/include/sorting.h @@ -6,14 +6,31 @@ #include #include +#include +#include +#include +#include "opencv2/opencv.hpp" #include "tkdnn.h" +struct threshold : public thrust::binary_function +{ + __host__ __device__ + float operator()(float x, float y) { + double toll = 1e-6; + if(fabsf(x-y)>toll) + return 0.0f; + else + return x; + } +}; + void sort(dnnType *src_begin, dnnType *src_end, int *idsrc); void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs); void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes); -void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out); +void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev); +void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op); void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys); void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out); diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index e39644f..7fd1c17 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -15,6 +15,7 @@ #include #include #include +#include "opencv2/opencv.hpp" #include "tkdnn.h" #include "sorting.h" @@ -29,20 +30,24 @@ class CenternetDetection { private: tk::dnn::NetworkRT *netRT = nullptr; - dnnType *input_h, *input, *input_d; + dnnType *input_d; int ndets = 0; // tk::dnn::Yolo::detection *dets = nullptr; cv::Mat imageF; - cv::Mat bgr[3]; + cv::cuda::GpuMat imageF1_d, imageF2_d; + cv::cuda::GpuMat bgr[3]; + // std::vector< cv::cuda::GpuMat > bgr; // variable to test cnet on dog pictures tk::dnn::dataDim_t dim; tk::dnn::dataDim_t dim2; - cv::Size sz; + cv::Size sz, sz_old; const char *input_bin = "../tests/resnet101_cnet/debug/input.bin"; + cv::cuda::Stream stream; + struct threshold op; // pre-process tk::dnn::dataDim_t dim_hm; tk::dnn::dataDim_t dim_wh; @@ -66,10 +71,15 @@ class CenternetDetection { float *target_coords; - cv::Vec mean; - cv::Vec stddev; + float *mean_d; + float *stddev_d; + + float *d_ptrs; + cv::Mat src; - cv::Mat dst; + cv::Mat dst; + cv::Mat dst2; + cv::Mat trans, trans2; //processing float toll = 0.000001; int K = 100; @@ -108,7 +118,6 @@ class CenternetDetection { * @return Success of the initialization */ bool init(std::string tensor_path); - void testdog(); cv::Mat draw(cv::Mat &frame); void update(cv::Mat &frame); diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index c707c6c..7840db7 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -37,10 +37,11 @@ bool CenternetDetection::init(std::string tensor_path) { coco_class_name = std::vector(coco_class_name_, std::end( coco_class_name_ )); src = cv::Mat(cv::Size(2,3), CV_32F); dst = cv::Mat(cv::Size(2,3), CV_32F); + dst2 = cv::Mat(cv::Size(2,3), CV_32F); + trans = cv::Mat(cv::Size(3,2), CV_32F); + trans2 = cv::Mat(cv::Size(3,2), CV_32F); // dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - checkCuda(cudaMallocHost(&input_h, sizeof(dnnType)*netRT->input_dim.tot())); - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); // dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1); @@ -94,61 +95,31 @@ bool CenternetDetection::init(std::string tensor_path) { checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) ); - mean << 0.408, 0.447, 0.47; - stddev << 0.289, 0.274, 0.278; + checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) ); + checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) ); + float mean[3] = {0.408, 0.447, 0.47}; + float stddev[3] = {0.289, 0.274, 0.278}; + + checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); + + checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) ); + // mean << 0.408, 0.447, 0.47; + // stddev << 0.289, 0.274, 0.278; // Alloc array used in the kernel checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); // checkCuda( cudaFree(src_out) ); // checkCuda( cudaFree(ids_out) ); - -} - -void CenternetDetection::testdog() { - - readBinaryFile(input_bin, dim.tot(), &input_h, &input_d); - - // -------- transofrm compose - cv::Mat imageORIG = cv::imread("../../dog.jpg"); - imageORIG.convertTo(imageF, CV_32FC3, 1/255.0); - sz = imageF.size(); - std::cout<<"image: "<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME threshold: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; // ----------- nms end // ----------- topk @@ -329,7 +308,7 @@ void CenternetDetection::update(cv::Mat &imageORIG) { ids_d); checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME sort: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME sort: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; topk(rt_out[0], ids_d, K, scores_d, @@ -337,14 +316,14 @@ void CenternetDetection::update(cv::Mat &imageORIG) { checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME topk: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME topk: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) ); topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME topk x y clses 2: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME topk x y clses 2: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); @@ -358,7 +337,7 @@ void CenternetDetection::update(cv::Mat &imageORIG) { // checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out); @@ -370,35 +349,13 @@ void CenternetDetection::update(cv::Mat &imageORIG) { checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME bboxes: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME bboxes: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; // ---------------------------------- post-process ----------------------------------------- // --------- ctdet_post_process // --------- transform_preds - src.at(0,0)=c[0]; - src.at(0,1)=c[1]; - src.at(1,0)=c[0]; - src.at(1,1)=c[1] + s[0] * -0.5; - dst.at(0,0)=width * 0.5; - dst.at(0,1)=width * 0.5; - dst.at(1,0)=width * 0.5; - dst.at(1,1)=width * 0.5 + width * -0.5; - - src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); - src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); - dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); - dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); - - - cv::Mat trans2(cv::Size(3,2), CV_32F); - trans2 = cv::getAffineTransform( dst, src ); - - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - cv::Mat new_pt1(cv::Size(1,2), CV_32F); cv::Mat new_pt2(cv::Size(1,2), CV_32F); @@ -422,7 +379,7 @@ void CenternetDetection::update(cv::Mat &imageORIG) { target_coords[i*4+2] = new_pt2.at(0,0); target_coords[i*4+3] = new_pt2.at(0,1); } - + detected.clear(); for(int i = 0; i(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME detections: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; std::cout<<"TOTAL: \n"; diff --git a/src/sorting.cu b/src/sorting.cu index 3ba5ac3..3c0f137 100644 --- a/src/sorting.cu +++ b/src/sorting.cu @@ -16,10 +16,7 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs) { checkCuda( cudaMemcpy(topk_scores, (float *)src_begin, K*sizeof(float), cudaMemcpyDeviceToDevice) ); - checkCuda( cudaMemcpy(topk_inds, idsrc, K*sizeof(int), cudaMemcpyDeviceToDevice) ); - // topk_ys_[i*K +count] = (int)(ids2[j] / width); - // topk_xs_[i*K +count] = (int)(ids2[j] % width); - + checkCuda( cudaMemcpy(topk_inds, idsrc, K*sizeof(int), cudaMemcpyDeviceToDevice) ); } __global__ @@ -28,7 +25,6 @@ void sortAndTopK_kernel(dnnType *src_begin, int *idsrc, float *topk_scores, int thrust::sort_by_key(thrust::device, src_begin + i * size, src_begin + i * size + size, idsrc + i * size, thrust::greater()); thrust::copy_n(thrust::device, src_begin + i * size, K, topk_scores + i * K); - // thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K ); thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K ); } @@ -41,20 +37,23 @@ void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int } -struct threshold : public thrust::binary_function -{ - __host__ __device__ - float operator()(float x, float y) { - double toll = 1e-6; - if(fabsf(x-y)>toll) - return 0.0f; - else - return x; - } -}; +__global__ +void normalize_kernel(float *bgr, const int dim, const float *mean, const float *stddev){ + int i = blockDim.x*blockIdx.x + threadIdx.x; + int j = blockIdx.y; + bgr[j*(dim)+i] = bgr[j*(dim)+i] - mean[j]; + bgr[j*(dim)+i] = bgr[j*(dim)+i] / stddev[j]; + +} -void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out){ - struct threshold op; +void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev) +{ + int num_thread = 256; + dim3 dimBlock(h*w/num_thread, ch); + normalize_kernel<<>>(bgr, h*w, mean, stddev); +} + +void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){ thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op); } -- 2.52.0 From 2c1df5619f4362f3922649d4430054739e81c0ed Mon Sep 17 00:00:00 2001 From: Davide Sapienza Date: Fri, 14 Feb 2020 18:44:13 +0100 Subject: [PATCH 060/482] Move pre-processing on GPU Signed-off-by: Davide Sapienza --- include/sorting.h | 19 ++- include/tkDNN/CenternetDetection.h | 23 ++- src/CenternetDetection.cpp | 245 ++++++++++++----------------- src/sorting.cu | 35 ++--- 4 files changed, 152 insertions(+), 170 deletions(-) diff --git a/include/sorting.h b/include/sorting.h index 9c968f3..23fd34b 100644 --- a/include/sorting.h +++ b/include/sorting.h @@ -6,14 +6,31 @@ #include #include +#include +#include +#include +#include "opencv2/opencv.hpp" #include "tkdnn.h" +struct threshold : public thrust::binary_function +{ + __host__ __device__ + float operator()(float x, float y) { + double toll = 1e-6; + if(fabsf(x-y)>toll) + return 0.0f; + else + return x; + } +}; + void sort(dnnType *src_begin, dnnType *src_end, int *idsrc); void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs); void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes); -void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out); +void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev); +void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op); void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys); void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out); diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index e39644f..7fd1c17 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -15,6 +15,7 @@ #include #include #include +#include "opencv2/opencv.hpp" #include "tkdnn.h" #include "sorting.h" @@ -29,20 +30,24 @@ class CenternetDetection { private: tk::dnn::NetworkRT *netRT = nullptr; - dnnType *input_h, *input, *input_d; + dnnType *input_d; int ndets = 0; // tk::dnn::Yolo::detection *dets = nullptr; cv::Mat imageF; - cv::Mat bgr[3]; + cv::cuda::GpuMat imageF1_d, imageF2_d; + cv::cuda::GpuMat bgr[3]; + // std::vector< cv::cuda::GpuMat > bgr; // variable to test cnet on dog pictures tk::dnn::dataDim_t dim; tk::dnn::dataDim_t dim2; - cv::Size sz; + cv::Size sz, sz_old; const char *input_bin = "../tests/resnet101_cnet/debug/input.bin"; + cv::cuda::Stream stream; + struct threshold op; // pre-process tk::dnn::dataDim_t dim_hm; tk::dnn::dataDim_t dim_wh; @@ -66,10 +71,15 @@ class CenternetDetection { float *target_coords; - cv::Vec mean; - cv::Vec stddev; + float *mean_d; + float *stddev_d; + + float *d_ptrs; + cv::Mat src; - cv::Mat dst; + cv::Mat dst; + cv::Mat dst2; + cv::Mat trans, trans2; //processing float toll = 0.000001; int K = 100; @@ -108,7 +118,6 @@ class CenternetDetection { * @return Success of the initialization */ bool init(std::string tensor_path); - void testdog(); cv::Mat draw(cv::Mat &frame); void update(cv::Mat &frame); diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index c707c6c..7840db7 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -37,10 +37,11 @@ bool CenternetDetection::init(std::string tensor_path) { coco_class_name = std::vector(coco_class_name_, std::end( coco_class_name_ )); src = cv::Mat(cv::Size(2,3), CV_32F); dst = cv::Mat(cv::Size(2,3), CV_32F); + dst2 = cv::Mat(cv::Size(2,3), CV_32F); + trans = cv::Mat(cv::Size(3,2), CV_32F); + trans2 = cv::Mat(cv::Size(3,2), CV_32F); // dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); - checkCuda(cudaMallocHost(&input_h, sizeof(dnnType)*netRT->input_dim.tot())); - checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot())); checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot())); // dim_hm = tk::dnn::dataDim_t(1, 80, 56, 56, 1); @@ -94,61 +95,31 @@ bool CenternetDetection::init(std::string tensor_path) { checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) ); - mean << 0.408, 0.447, 0.47; - stddev << 0.289, 0.274, 0.278; + checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) ); + checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) ); + float mean[3] = {0.408, 0.447, 0.47}; + float stddev[3] = {0.289, 0.274, 0.278}; + + checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice)); + checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice)); + + checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) ); + // mean << 0.408, 0.447, 0.47; + // stddev << 0.289, 0.274, 0.278; // Alloc array used in the kernel checkCuda( cudaMalloc(&src_out, K *sizeof(float)) ); checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) ); // checkCuda( cudaFree(src_out) ); // checkCuda( cudaFree(ids_out) ); - -} - -void CenternetDetection::testdog() { - - readBinaryFile(input_bin, dim.tot(), &input_h, &input_d); - - // -------- transofrm compose - cv::Mat imageORIG = cv::imread("../../dog.jpg"); - imageORIG.convertTo(imageF, CV_32FC3, 1/255.0); - sz = imageF.size(); - std::cout<<"image: "<(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME threshold: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; // ----------- nms end // ----------- topk @@ -329,7 +308,7 @@ void CenternetDetection::update(cv::Mat &imageORIG) { ids_d); checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME sort: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME sort: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; topk(rt_out[0], ids_d, K, scores_d, @@ -337,14 +316,14 @@ void CenternetDetection::update(cv::Mat &imageORIG) { checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME topk: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME topk: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) ); topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME topk x y clses 2: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME topk x y clses 2: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) ); @@ -358,7 +337,7 @@ void CenternetDetection::update(cv::Mat &imageORIG) { // checkCuda( cudaDeviceSynchronize() ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME add offset: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out); @@ -370,35 +349,13 @@ void CenternetDetection::update(cv::Mat &imageORIG) { checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); end_t = std::chrono::steady_clock::now(); - std::cout << " TIME bboxes: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME bboxes: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; // ---------------------------------- post-process ----------------------------------------- // --------- ctdet_post_process // --------- transform_preds - src.at(0,0)=c[0]; - src.at(0,1)=c[1]; - src.at(1,0)=c[0]; - src.at(1,1)=c[1] + s[0] * -0.5; - dst.at(0,0)=width * 0.5; - dst.at(0,1)=width * 0.5; - dst.at(1,0)=width * 0.5; - dst.at(1,1)=width * 0.5 + width * -0.5; - - src.at(2,0)=src.at(1,0) + (-src.at(0,1)+src.at(1,1) ); - src.at(2,1)=src.at(1,1) + (src.at(0,0)-src.at(1,0) ); - dst.at(2,0)=dst.at(1,0) + (-dst.at(0,1)+dst.at(1,1) ); - dst.at(2,1)=dst.at(1,1) + (dst.at(0,0)-dst.at(1,0) ); - - - cv::Mat trans2(cv::Size(3,2), CV_32F); - trans2 = cv::getAffineTransform( dst, src ); - - end_t = std::chrono::steady_clock::now(); - std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast(end_t - step_t).count() << " ms" << std::endl; - step_t = end_t; - cv::Mat new_pt1(cv::Size(1,2), CV_32F); cv::Mat new_pt2(cv::Size(1,2), CV_32F); @@ -422,7 +379,7 @@ void CenternetDetection::update(cv::Mat &imageORIG) { target_coords[i*4+2] = new_pt2.at(0,0); target_coords[i*4+3] = new_pt2.at(0,1); } - + detected.clear(); for(int i = 0; i(end_t - step_t).count() << " ms" << std::endl; + std::cout << " TIME detections: " << std::chrono::duration_cast(end_t - step_t).count() << " us" << std::endl; step_t = end_t; std::cout<<"TOTAL: \n"; diff --git a/src/sorting.cu b/src/sorting.cu index 3ba5ac3..3c0f137 100644 --- a/src/sorting.cu +++ b/src/sorting.cu @@ -16,10 +16,7 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs) { checkCuda( cudaMemcpy(topk_scores, (float *)src_begin, K*sizeof(float), cudaMemcpyDeviceToDevice) ); - checkCuda( cudaMemcpy(topk_inds, idsrc, K*sizeof(int), cudaMemcpyDeviceToDevice) ); - // topk_ys_[i*K +count] = (int)(ids2[j] / width); - // topk_xs_[i*K +count] = (int)(ids2[j] % width); - + checkCuda( cudaMemcpy(topk_inds, idsrc, K*sizeof(int), cudaMemcpyDeviceToDevice) ); } __global__ @@ -28,7 +25,6 @@ void sortAndTopK_kernel(dnnType *src_begin, int *idsrc, float *topk_scores, int thrust::sort_by_key(thrust::device, src_begin + i * size, src_begin + i * size + size, idsrc + i * size, thrust::greater()); thrust::copy_n(thrust::device, src_begin + i * size, K, topk_scores + i * K); - // thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K ); thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K ); } @@ -41,20 +37,23 @@ void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int } -struct threshold : public thrust::binary_function -{ - __host__ __device__ - float operator()(float x, float y) { - double toll = 1e-6; - if(fabsf(x-y)>toll) - return 0.0f; - else - return x; - } -}; +__global__ +void normalize_kernel(float *bgr, const int dim, const float *mean, const float *stddev){ + int i = blockDim.x*blockIdx.x + threadIdx.x; + int j = blockIdx.y; + bgr[j*(dim)+i] = bgr[j*(dim)+i] - mean[j]; + bgr[j*(dim)+i] = bgr[j*(dim)+i] / stddev[j]; + +} -void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out){ - struct threshold op; +void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev) +{ + int num_thread = 256; + dim3 dimBlock(h*w/num_thread, ch); + normalize_kernel<<>>(bgr, h*w, mean, stddev); +} + +void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){ thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op); } -- 2.52.0 From 8736a8c3da603fc6b340564d91359975f0519532 Mon Sep 17 00:00:00 2001 From: Francesco Gatti Date: Sat, 15 Feb 2020 20:37:08 +0100 Subject: [PATCH 061/482] LSTM params --- include/tkDNN/Layer.h | 5 ++- src/LSTM.cpp | 94 +++++++++++++++++++++++++++++++++------ tests/imuodom/imuodom.cpp | 20 ++++++--- tests/imuodom/infer.py | 1 + tests/weights_exporter.py | 16 +++++-- 5 files changed, 111 insertions(+), 25 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 240d9a2..319df6a 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -212,6 +212,7 @@ protected: https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp https://github.com/Jeffery-Song/mxnet-test/blob/aab666faad44011f7a67b527b5f6c960367d0422/src/operator/cudnn_rnn-inl.h https://stackoverflow.com/a/38737941 + https://colah.github.io/posts/2015-08-Understanding-LSTMs/ PARAMS (numlayers*2): layer0: @@ -236,7 +237,7 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - const bool bidirectional = 1; /**> is the net bidir */ + const bool bidirectional = false; /**> is the net bidir */ bool returnSeq = false; /**> if false return only the result of last timestep */ int stateSize = 0; /**> number of hidden states */ int seqLen = 0; /**> number of timesteps */ @@ -254,9 +255,11 @@ protected: cudnnTensorDescriptor_t hx_desc_, cx_desc_; cudnnTensorDescriptor_t hy_desc_, cy_desc_; dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; + int stateDataDim; cudnnFilterDescriptor_t w_desc_; dnnType *w_ptr; + dnnType *w_h; }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index c31bccf..ce2a9f5 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -66,10 +66,12 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnSetTensorNdDescriptor(hy_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(cy_desc_, net->dataType, 3, dimA, strideA)); // allocate dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; - checkCuda( cudaMalloc(&hx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&cx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&hy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&cy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + stateDataDim = dimA[0]*dimA[1]*dimA[2]; + checkCuda( cudaMalloc(&hx_ptr, stateDataDim*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cx_ptr, stateDataDim*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&hy_ptr, stateDataDim*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cy_ptr, stateDataDim*sizeof(dnnType)) ); + // Create Dropout descriptors // TODO: ??? IS IT NECESSARY ??? @@ -89,7 +91,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, - cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, + (bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), cudnnRNNMode_t::CUDNN_LSTM, cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType)); @@ -115,22 +117,81 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig dim_w[0] = cudnn_params; checkCUDNN(cudnnSetFilterNdDescriptor(w_desc_, net->dataType, net->tensorFormat, 3, dim_w)); - // allocate params dnnType *w_ptr; - checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); - + // load params + readBinaryFile(fname_weights, cudnn_params, &w_h, &w_ptr); //allocate data for infer result - int dstDim = input_dim.n * stateSize*2 * input_dim.h * input_dim.w; + int dstDim = input_dim.n * stateSize*(bidirectional ? 2 : 1) * input_dim.h * input_dim.w; checkCuda( cudaMalloc(&dstData, dstDim*sizeof(dnnType)) ); // set output dim output_dim = input_dim; - output_dim.c = stateSize*2; + output_dim.c = stateSize*(bidirectional ? 2 : 1); if(!returnSeq) { output_dim.h = 1; output_dim.w = 1; } + + + + + // Query weight layout + cudnnFilterDescriptor_t m_desc; + checkCUDNN(cudnnCreateFilterDescriptor(&m_desc)); + dnnType *p; + int n = 8; // lstm layers + + printCenteredTitle("WEIGHTS", '=', 20); + for (int i = 0; i < numLayers*(bidirectional?2:1); ++i) { + for (int j = 0; j < n; ++j) { + + checkCUDNN(cudnnGetRNNLinLayerMatrixParams(net->cudnnHandle, rnnDesc, + i, x_desc_vec_[0], w_desc_, 0, j, m_desc, (void**)&p)); + + std::cout << "ptr: " << ((int64_t)(p - NULL))/sizeof(dnnType)<<"\n"; + + cudnnDataType_t t; + cudnnTensorFormat_t f; + int ndim = 5; + int dims[5] = {0, 0, 0, 0, 0}; + checkCUDNN(cudnnGetFilterNdDescriptor(m_desc, ndim, &t, &f, &ndim, &dims[0])); + std::cout << "(layer, linlayer): " << i << " " << j << "\n"; + + int tot = 1; + for (int i = 0; i < ndim; ++i) { + std::cout << dims[i] << " "; + tot *= dims[i]; + } + std::cout<<"\t-> "< Date: Sat, 15 Feb 2020 20:37:08 +0100 Subject: [PATCH 062/482] LSTM params --- include/tkDNN/Layer.h | 5 ++- src/LSTM.cpp | 94 +++++++++++++++++++++++++++++++++------ tests/imuodom/imuodom.cpp | 20 ++++++--- tests/imuodom/infer.py | 1 + tests/weights_exporter.py | 16 +++++-- 5 files changed, 111 insertions(+), 25 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 240d9a2..319df6a 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -212,6 +212,7 @@ protected: https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp https://github.com/Jeffery-Song/mxnet-test/blob/aab666faad44011f7a67b527b5f6c960367d0422/src/operator/cudnn_rnn-inl.h https://stackoverflow.com/a/38737941 + https://colah.github.io/posts/2015-08-Understanding-LSTMs/ PARAMS (numlayers*2): layer0: @@ -236,7 +237,7 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - const bool bidirectional = 1; /**> is the net bidir */ + const bool bidirectional = false; /**> is the net bidir */ bool returnSeq = false; /**> if false return only the result of last timestep */ int stateSize = 0; /**> number of hidden states */ int seqLen = 0; /**> number of timesteps */ @@ -254,9 +255,11 @@ protected: cudnnTensorDescriptor_t hx_desc_, cx_desc_; cudnnTensorDescriptor_t hy_desc_, cy_desc_; dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; + int stateDataDim; cudnnFilterDescriptor_t w_desc_; dnnType *w_ptr; + dnnType *w_h; }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index c31bccf..ce2a9f5 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -66,10 +66,12 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnSetTensorNdDescriptor(hy_desc_, net->dataType, 3, dimA, strideA)); checkCUDNN(cudnnSetTensorNdDescriptor(cy_desc_, net->dataType, 3, dimA, strideA)); // allocate dnnType *hx_ptr, *cx_ptr, *hy_ptr, *cy_ptr; - checkCuda( cudaMalloc(&hx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&cx_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&hy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); - checkCuda( cudaMalloc(&cy_ptr, dimA[0]*dimA[1]*dimA[2]*sizeof(dnnType)) ); + stateDataDim = dimA[0]*dimA[1]*dimA[2]; + checkCuda( cudaMalloc(&hx_ptr, stateDataDim*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cx_ptr, stateDataDim*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&hy_ptr, stateDataDim*sizeof(dnnType)) ); + checkCuda( cudaMalloc(&cy_ptr, stateDataDim*sizeof(dnnType)) ); + // Create Dropout descriptors // TODO: ??? IS IT NECESSARY ??? @@ -89,7 +91,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, - cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL, + (bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), cudnnRNNMode_t::CUDNN_LSTM, cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType)); @@ -115,22 +117,81 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig dim_w[0] = cudnn_params; checkCUDNN(cudnnSetFilterNdDescriptor(w_desc_, net->dataType, net->tensorFormat, 3, dim_w)); - // allocate params dnnType *w_ptr; - checkCuda( cudaMalloc(&w_ptr, cudnn_params*sizeof(dnnType)) ); - + // load params + readBinaryFile(fname_weights, cudnn_params, &w_h, &w_ptr); //allocate data for infer result - int dstDim = input_dim.n * stateSize*2 * input_dim.h * input_dim.w; + int dstDim = input_dim.n * stateSize*(bidirectional ? 2 : 1) * input_dim.h * input_dim.w; checkCuda( cudaMalloc(&dstData, dstDim*sizeof(dnnType)) ); // set output dim output_dim = input_dim; - output_dim.c = stateSize*2; + output_dim.c = stateSize*(bidirectional ? 2 : 1); if(!returnSeq) { output_dim.h = 1; output_dim.w = 1; } + + + + + // Query weight layout + cudnnFilterDescriptor_t m_desc; + checkCUDNN(cudnnCreateFilterDescriptor(&m_desc)); + dnnType *p; + int n = 8; // lstm layers + + printCenteredTitle("WEIGHTS", '=', 20); + for (int i = 0; i < numLayers*(bidirectional?2:1); ++i) { + for (int j = 0; j < n; ++j) { + + checkCUDNN(cudnnGetRNNLinLayerMatrixParams(net->cudnnHandle, rnnDesc, + i, x_desc_vec_[0], w_desc_, 0, j, m_desc, (void**)&p)); + + std::cout << "ptr: " << ((int64_t)(p - NULL))/sizeof(dnnType)<<"\n"; + + cudnnDataType_t t; + cudnnTensorFormat_t f; + int ndim = 5; + int dims[5] = {0, 0, 0, 0, 0}; + checkCUDNN(cudnnGetFilterNdDescriptor(m_desc, ndim, &t, &f, &ndim, &dims[0])); + std::cout << "(layer, linlayer): " << i << " " << j << "\n"; + + int tot = 1; + for (int i = 0; i < ndim; ++i) { + std::cout << dims[i] << " "; + tot *= dims[i]; + } + std::cout<<"\t-> "< Date: Sun, 16 Feb 2020 16:28:39 +0100 Subject: [PATCH 063/482] works but it need cleaning --- include/tkDNN/Layer.h | 3 +- include/tkDNN/utils.h | 2 +- src/LSTM.cpp | 163 ++++++++++++++++++++++++++++------- src/utils.cpp | 4 +- tests/imuodom/imuodom.cpp | 26 +++--- tests/imuodom/infer.py | 25 +++--- tests/simple/test_model.py | 15 ++-- tests/simple/test_simple.cpp | 8 +- tests/weights_exporter.py | 44 ++++++++-- 9 files changed, 219 insertions(+), 71 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 319df6a..f12027f 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -237,7 +237,7 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - const bool bidirectional = false; /**> is the net bidir */ + const bool bidirectional = true; /**> is the net bidir */ bool returnSeq = false; /**> if false return only the result of last timestep */ int stateSize = 0; /**> number of hidden states */ int seqLen = 0; /**> number of timesteps */ @@ -260,6 +260,7 @@ protected: cudnnFilterDescriptor_t w_desc_; dnnType *w_ptr; dnnType *w_h; + dnnType *wf_ptr, *wb_ptr; // params pointer forward and backward layer }; diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index dc34a31..3fa9d34 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -91,7 +91,7 @@ void printCenteredTitle(const char *title, char fill, int dim); bool fileExist(const char *fname); void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0); -int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true); +int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10); void printDeviceVector(int size, dnnType* vec_d, bool device = true); void resize(int size, dnnType **data); diff --git a/src/LSTM.cpp b/src/LSTM.cpp index ce2a9f5..00d8f76 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -37,7 +37,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig net->dataType, 3, dimA, strideA)); dimA[0] = batchSize; - dimA[1] = bidirectional ? stateSize*2 : stateSize; + dimA[1] = stateSize; dimA[2] = 1; strideA[0] = dimA[2] * dimA[1]; strideA[1] = dimA[2]; @@ -51,7 +51,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig // set the state tensors - dimA[0] = numLayers * (bidirectional ? 2 : 1); + dimA[0] = numLayers; dimA[1] = batchSize; dimA[2] = stateSize; strideA[0] = dimA[2] * dimA[1]; @@ -91,7 +91,8 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, - (bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), + //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), + cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType)); @@ -119,23 +120,26 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig net->dataType, net->tensorFormat, 3, dim_w)); // load params - readBinaryFile(fname_weights, cudnn_params, &w_h, &w_ptr); - - //allocate data for infer result - int dstDim = input_dim.n * stateSize*(bidirectional ? 2 : 1) * input_dim.h * input_dim.w; - checkCuda( cudaMalloc(&dstData, dstDim*sizeof(dnnType)) ); + readBinaryFile(fname_weights, cudnn_params*2, &w_h, &w_ptr); + // set forward and backward params + wf_ptr = w_ptr; + wb_ptr = w_ptr + cudnn_params; + std::cout<<"wf: "<cublasHandle, srcData, trans, dim.c, dim.h*dim.w*dim.l); + srcData = trans; + + // reposition in invered order + dnnType *srcBack; + checkCuda( cudaMalloc(&srcBack, dim.tot()*sizeof(dnnType))); + for(int i=0; icudnnHandle, + rnnDesc, + seqLen, // number of time steps (nT) + x_desc_vec_.data(), // input array of desc (nT*nC_in) + srcData, // input pointer + hx_desc_, // initial hidden state desc + hx_ptr, // initial hidden state pointer + cx_desc_, // initial cell state desc + cx_ptr, // initial cell state pointer + w_desc_, // weights desc + wf_ptr, // weights pointer + y_desc_vec_.data(), // output desc (nT*nC_out) + dstF, // output pointer + hy_desc_, // final hidden state desc + hy_ptr, // final hidden state pointer + cy_desc_, // final cell state desc + cy_ptr, // final cell state pointer + work_space_, // workspace pointer + workspace_byte_)); // workspace size + } + std::cout<<"OUTPUT F:\n"; + printDeviceVector(singleOutput.tot(), dstF); + + std::cout<<"INPUT:\n"; + printDeviceVector(input_dim.tot(), srcBack); + + // backward + { + // reset states + checkCuda( cudaMemset(hx_ptr, 0, stateDataDim*sizeof(float)) ); + checkCuda( cudaMemset(cx_ptr, 0, stateDataDim*sizeof(float)) ); + + checkCUDNN(cudnnRNNForwardInference(net->cudnnHandle, + rnnDesc, + seqLen, // number of time steps (nT) + x_desc_vec_.data(), // input array of desc (nT*nC_in) + srcBack, // input pointer + hx_desc_, // initial hidden state desc + hx_ptr, // initial hidden state pointer + cx_desc_, // initial cell state desc + cx_ptr, // initial cell state pointer + w_desc_, // weights desc + wb_ptr, // weights pointer + y_desc_vec_.data(), // output desc (nT*nC_out) + dstB, // output pointer + hy_desc_, // final hidden state desc + hy_ptr, // final hidden state pointer + cy_desc_, // final cell state desc + cy_ptr, // final cell state pointer + work_space_, // workspace pointer + workspace_byte_)); // workspace size + } + + + // reposition in invered order + dnnType *dstBack; + checkCuda( cudaMalloc(&dstBack, singleOutput.tot()*sizeof(dnnType))); + for(int i=0; icublasHandle, dstF, trans, + singleOutput.h*singleOutput.w*singleOutput.l, singleOutput.c); + // backward transpose + matrixTranspose(net->cublasHandle, dstB, trans + singleOutput.tot(), + singleOutput.h*singleOutput.w*singleOutput.l, singleOutput.c); + dstData = trans; + } else { + // copy last of forward + checkCuda( cudaMemcpy(trans, dstF + singleOutput.tot() - singleOutput.c, singleOutput.c*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + // copy first of backward + checkCuda( cudaMemcpy(trans + singleOutput.c, dstB, singleOutput.c*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + dstData = trans; + } - checkCUDNN(cudnnRNNForwardInference(net->cudnnHandle, - rnnDesc, - seqLen, // number of time steps (nT) - x_desc_vec_.data(), // input array of desc (nT*nC_in) - srcData, // input pointer - hx_desc_, // initial hidden state desc - hx_ptr, // initial hidden state pointer - cx_desc_, // initial cell state desc - cx_ptr, // initial cell state pointer - w_desc_, // weights desc - w_ptr, // weights pointer - y_desc_vec_.data(), // output desc (nT*nC_out) - dstData, // output pointer - hy_desc_, // final hidden state desc - hy_ptr, // final hidden state pointer - cy_desc_, // final cell state desc - cy_ptr, // final cell state pointer - work_space_, // workspace pointer - workspace_byte_)); // workspace size dim = output_dim; return dstData; diff --git a/src/utils.cpp b/src/utils.cpp index 444318d..6789e8c 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -68,7 +68,7 @@ void printDeviceVector(int size, dnnType* vec_d, bool device) delete [] vec; } -int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device) { +int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit) { dnnType *data_h, *correct_h; const float eps = 0.02f; @@ -92,7 +92,7 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device) { diffs += 1; if(diffs == 1) std::cout<<"\n"; - if(diffs < 10) + if(diffs < limit) std::cout<<" | [ "< Date: Sun, 16 Feb 2020 16:28:39 +0100 Subject: [PATCH 064/482] works but it need cleaning --- include/tkDNN/Layer.h | 3 +- include/tkDNN/utils.h | 2 +- src/LSTM.cpp | 163 ++++++++++++++++++++++++++++------- src/utils.cpp | 4 +- tests/imuodom/imuodom.cpp | 26 +++--- tests/imuodom/infer.py | 25 +++--- tests/simple/test_model.py | 15 ++-- tests/simple/test_simple.cpp | 8 +- tests/weights_exporter.py | 44 ++++++++-- 9 files changed, 219 insertions(+), 71 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 319df6a..f12027f 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -237,7 +237,7 @@ public: virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); - const bool bidirectional = false; /**> is the net bidir */ + const bool bidirectional = true; /**> is the net bidir */ bool returnSeq = false; /**> if false return only the result of last timestep */ int stateSize = 0; /**> number of hidden states */ int seqLen = 0; /**> number of timesteps */ @@ -260,6 +260,7 @@ protected: cudnnFilterDescriptor_t w_desc_; dnnType *w_ptr; dnnType *w_h; + dnnType *wf_ptr, *wb_ptr; // params pointer forward and backward layer }; diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index dc34a31..3fa9d34 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -91,7 +91,7 @@ void printCenteredTitle(const char *title, char fill, int dim); bool fileExist(const char *fname); void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0); -int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true); +int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10); void printDeviceVector(int size, dnnType* vec_d, bool device = true); void resize(int size, dnnType **data); diff --git a/src/LSTM.cpp b/src/LSTM.cpp index ce2a9f5..00d8f76 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -37,7 +37,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig net->dataType, 3, dimA, strideA)); dimA[0] = batchSize; - dimA[1] = bidirectional ? stateSize*2 : stateSize; + dimA[1] = stateSize; dimA[2] = 1; strideA[0] = dimA[2] * dimA[1]; strideA[1] = dimA[2]; @@ -51,7 +51,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig // set the state tensors - dimA[0] = numLayers * (bidirectional ? 2 : 1); + dimA[0] = numLayers; dimA[1] = batchSize; dimA[2] = stateSize; strideA[0] = dimA[2] * dimA[1]; @@ -91,7 +91,8 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle, rnnDesc, stateSize, numLayers, dropoutDesc, cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT, - (bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), + //(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL), + cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL, cudnnRNNMode_t::CUDNN_LSTM, cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD, net->dataType)); @@ -119,23 +120,26 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig net->dataType, net->tensorFormat, 3, dim_w)); // load params - readBinaryFile(fname_weights, cudnn_params, &w_h, &w_ptr); - - //allocate data for infer result - int dstDim = input_dim.n * stateSize*(bidirectional ? 2 : 1) * input_dim.h * input_dim.w; - checkCuda( cudaMalloc(&dstData, dstDim*sizeof(dnnType)) ); + readBinaryFile(fname_weights, cudnn_params*2, &w_h, &w_ptr); + // set forward and backward params + wf_ptr = w_ptr; + wb_ptr = w_ptr + cudnn_params; + std::cout<<"wf: "<cublasHandle, srcData, trans, dim.c, dim.h*dim.w*dim.l); + srcData = trans; + + // reposition in invered order + dnnType *srcBack; + checkCuda( cudaMalloc(&srcBack, dim.tot()*sizeof(dnnType))); + for(int i=0; icudnnHandle, + rnnDesc, + seqLen, // number of time steps (nT) + x_desc_vec_.data(), // input array of desc (nT*nC_in) + srcData, // input pointer + hx_desc_, // initial hidden state desc + hx_ptr, // initial hidden state pointer + cx_desc_, // initial cell state desc + cx_ptr, // initial cell state pointer + w_desc_, // weights desc + wf_ptr, // weights pointer + y_desc_vec_.data(), // output desc (nT*nC_out) + dstF, // output pointer + hy_desc_, // final hidden state desc + hy_ptr, // final hidden state pointer + cy_desc_, // final cell state desc + cy_ptr, // final cell state pointer + work_space_, // workspace pointer + workspace_byte_)); // workspace size + } + std::cout<<"OUTPUT F:\n"; + printDeviceVector(singleOutput.tot(), dstF); + + std::cout<<"INPUT:\n"; + printDeviceVector(input_dim.tot(), srcBack); + + // backward + { + // reset states + checkCuda( cudaMemset(hx_ptr, 0, stateDataDim*sizeof(float)) ); + checkCuda( cudaMemset(cx_ptr, 0, stateDataDim*sizeof(float)) ); + + checkCUDNN(cudnnRNNForwardInference(net->cudnnHandle, + rnnDesc, + seqLen, // number of time steps (nT) + x_desc_vec_.data(), // input array of desc (nT*nC_in) + srcBack, // input pointer + hx_desc_, // initial hidden state desc + hx_ptr, // initial hidden state pointer + cx_desc_, // initial cell state desc + cx_ptr, // initial cell state pointer + w_desc_, // weights desc + wb_ptr, // weights pointer + y_desc_vec_.data(), // output desc (nT*nC_out) + dstB, // output pointer + hy_desc_, // final hidden state desc + hy_ptr, // final hidden state pointer + cy_desc_, // final cell state desc + cy_ptr, // final cell state pointer + work_space_, // workspace pointer + workspace_byte_)); // workspace size + } + + + // reposition in invered order + dnnType *dstBack; + checkCuda( cudaMalloc(&dstBack, singleOutput.tot()*sizeof(dnnType))); + for(int i=0; icublasHandle, dstF, trans, + singleOutput.h*singleOutput.w*singleOutput.l, singleOutput.c); + // backward transpose + matrixTranspose(net->cublasHandle, dstB, trans + singleOutput.tot(), + singleOutput.h*singleOutput.w*singleOutput.l, singleOutput.c); + dstData = trans; + } else { + // copy last of forward + checkCuda( cudaMemcpy(trans, dstF + singleOutput.tot() - singleOutput.c, singleOutput.c*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + // copy first of backward + checkCuda( cudaMemcpy(trans + singleOutput.c, dstB, singleOutput.c*sizeof(dnnType), cudaMemcpyDeviceToDevice)); + dstData = trans; + } - checkCUDNN(cudnnRNNForwardInference(net->cudnnHandle, - rnnDesc, - seqLen, // number of time steps (nT) - x_desc_vec_.data(), // input array of desc (nT*nC_in) - srcData, // input pointer - hx_desc_, // initial hidden state desc - hx_ptr, // initial hidden state pointer - cx_desc_, // initial cell state desc - cx_ptr, // initial cell state pointer - w_desc_, // weights desc - w_ptr, // weights pointer - y_desc_vec_.data(), // output desc (nT*nC_out) - dstData, // output pointer - hy_desc_, // final hidden state desc - hy_ptr, // final hidden state pointer - cy_desc_, // final cell state desc - cy_ptr, // final cell state pointer - work_space_, // workspace pointer - workspace_byte_)); // workspace size dim = output_dim; return dstData; diff --git a/src/utils.cpp b/src/utils.cpp index 444318d..6789e8c 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -68,7 +68,7 @@ void printDeviceVector(int size, dnnType* vec_d, bool device) delete [] vec; } -int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device) { +int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit) { dnnType *data_h, *correct_h; const float eps = 0.02f; @@ -92,7 +92,7 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device) { diffs += 1; if(diffs == 1) std::cout<<"\n"; - if(diffs < 10) + if(diffs < limit) std::cout<<" | [ "< Date: Sun, 16 Feb 2020 17:08:19 +0100 Subject: [PATCH 065/482] structure ok, result wrong --- include/tkDNN/Layer.h | 9 +++- src/LSTM.cpp | 103 ++++++++++++++++---------------------- tests/imuodom/imuodom.cpp | 4 +- 3 files changed, 55 insertions(+), 61 deletions(-) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index f12027f..7e827d8 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -207,7 +207,9 @@ protected: /** Bidirectional LSTM layer - + ONLY BIDIRECTIONAL (TODO: more configurable) + currently implemented as 2 inferences: forward and backward (TODO: only 1 cudnn inference) + implementation info: https://github.com/jiangnanhugo/seq2seq_cuda/blob/e4dbdcfa0517c972bfd4beea9f11a5233954093c/src/rnn.cpp https://github.com/Jeffery-Song/mxnet-test/blob/aab666faad44011f7a67b527b5f6c960367d0422/src/operator/cudnn_rnn-inl.h @@ -261,6 +263,11 @@ protected: dnnType *w_ptr; dnnType *w_h; dnnType *wf_ptr, *wb_ptr; // params pointer forward and backward layer + + // used during inference + dataDim_t one_output_dim; // output dim of as single inference + dnnType *srcF, *srcB; // input of single inference + dnnType *dstF, *dstB_NR, *dstB; // output of single inference, dstB_NR = dstB not reversed }; diff --git a/src/LSTM.cpp b/src/LSTM.cpp index 00d8f76..44502fa 100644 --- a/src/LSTM.cpp +++ b/src/LSTM.cpp @@ -13,8 +13,6 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig seqLen = input_dim.w; stateSize = hiddensize; - std::cout<<"LSTM seqLen: "< x_vec(seqLen); std::vector y_vec(seqLen); @@ -110,7 +108,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig checkCUDNN(cudnnGetRNNParamsSize(net->cudnnHandle, rnnDesc,x_desc_vec_[0], &cudnn_param_size, net->dataType)); int cudnn_params = cudnn_param_size/sizeof(dnnType); - std::cout<<"LSTM params size: "<cudnnHandle, rnnDesc, i, x_desc_vec_[0], w_desc_, 0, j, m_desc, (void**)&p)); @@ -209,37 +217,27 @@ LSTM::~LSTM() { checkCuda(cudaFree(work_space_ )); checkCuda(cudaFree(dropout_states_)); + checkCuda(cudaFree(srcF)); + checkCuda(cudaFree(srcB)); + checkCuda(cudaFree(dstF)); + checkCuda(cudaFree(dstB_NR)); + checkCuda(cudaFree(dstB)); checkCuda(cudaFree(dstData)); } dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) { - std::cout<<"LSTM infer\n"; + // transpose input + matrixTranspose(net->cublasHandle, srcData, srcF, dim.c, dim.h*dim.w*dim.l); - dnnType *trans; - checkCuda( cudaMalloc(&trans, dim.tot()*sizeof(dnnType))); - matrixTranspose(net->cublasHandle, srcData, trans, dim.c, dim.h*dim.w*dim.l); - srcData = trans; - - // reposition in invered order - dnnType *srcBack; - checkCuda( cudaMalloc(&srcBack, dim.tot()*sizeof(dnnType))); + // build srcB as reversed srcF for(int i=0; i