From 19f12d9c1567662661df76e2e8c761cfc2579aa7 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Mon, 15 Nov 2021 00:08:37 +0530 Subject: [PATCH 01/13] rds_slam -> tensorrt8 port for rds --- include/tkDNN/SegmentationNN.h | 241 +++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index 403bc28..fff27a8 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -18,6 +18,7 @@ #include "tkdnn.h" #include "NetworkViz.h" #include "kernelsThrust.h" +#define SLAM_MODE namespace tk { namespace dnn { @@ -99,6 +100,62 @@ class SegmentationNN { * * @param bi batch index */ + + #ifdef SLAM_MODE + cv::Mat postprocess(const int bi=0,bool apply_colormap=true){ + cv::Mat maskMatrix; + dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + dataDim_t odim = netRT->output_dim; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + + + dataDim_t vdim = odim; + vdim.c = 1; + dnnType *dataTemp = nullptr; + if(isCudaPointer(tmpOutData_h)) + { + dataTemp = new dnnType[vdim.tot()]; + checkCuda(cudaMemcpy(dataTemp,tmpOutData_h,vdim.tot()*sizeof(dnnType),cudaMemcpyDeviceToHost)); + } + else + { + dataTemp = tmpOutData_h; + } + for(int i =0;iinput_dim.h, netRT->input_dim.w, 0, classes, classes); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, dataTemp); + colored_fp32.convertTo(colored, CV_8UC1); + } + + int max_dim = (originalSize[bi].width > originalSize[bi].height) ? originalSize[bi].width : originalSize[bi].height; + resize(colored, colored, cv::Size(max_dim, max_dim)); + int top, bottom, left, right; + computeBorders(originalSize[bi].width, originalSize[bi].height, top, bottom, left, right); + cv::Rect roi(left,top,originalSize[bi].width, originalSize[bi].height); + cv::Mat or_size (colored, roi); + segmented[bi] = or_size; + + if(isCudaPointer(tmpOutData_h)) + { + delete [] dataTemp; + } + + return maskMatrix; + + } + #elif + void postprocess(const int bi=0, bool appy_colormap = true) { dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; @@ -128,6 +185,7 @@ class SegmentationNN { cv::Mat or_size (colored, roi); segmented[bi] = or_size; }; + #endif public: int classes = 0; @@ -237,6 +295,184 @@ class SegmentationNN { } } + #ifdef SLAM_MODE + cv::Mat updateOriginal(cv::Mat frame,bool apply_colormap=true){ + std::vector splitted_frames; + cv::Mat maskMatrix; + int H, W, net_H, net_W; + int top = 0, bottom = 0, left = 0, right = 0; + std::vector> pos; + + { + TKDNN_TSTART + cv::Size original_size = frame.size(); + + frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); + H = frame.rows; + W = frame.cols; + net_H = netRT->input_dim.h; + net_W = netRT->input_dim.w; + + cv::Mat frame_cropped; + + if( H <= net_H && W <= net_W ){ // smaller size wrt network + top = (net_H - H)/2; + bottom = net_H - H - top ; + left = (net_W - W)/2; + right = net_W - W - left ; + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + splitted_frames.push_back(frame_cropped); + } + else{ //bigger size wrt network + + + if(H < net_H || W < net_W){ + if(H < net_H){ + top = (net_H - H)/2; + bottom = net_H - H - top ; + } + else{ + left = (net_W - W)/2; + right = net_W - W - left ; + } + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); + } + + for(int x=0; x+net_W<=W ;){ + for(int y=0; y+net_H <=H ; ){ + cv::Rect roi(x, y, net_W, net_H); + cv::Mat image_roi = frame(roi); + splitted_frames.push_back(image_roi); + pos.push_back(std::make_pair(x,y)); + + y += net_H; + if(y == H) + break; + if(y + net_H > H) y = H - net_H; + } + x += net_W; + if(x == W) + break; + if(x + net_W > W) x = W - net_W; + } + } + + tk::dnn::dataDim_t idim = netRT->input_dim; + + if(splitted_frames.size()> nBatches) + FatalError(std::to_string(splitted_frames.size()) + " min batches required"); + + for(int bi=0; bistream)); + normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d); + } + TKDNN_TSTOP + stats_pre.push_back(t_ns); + } + + tk::dnn::dataDim_t dim = netRT->input_dim; + dim.n = splitted_frames.size(); + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + dataDim_t odim = netRT->output_dim; + + std::vector out_img; + std::vector out_mask; + + { + TKDNN_TSTART + + for(int bi=0; bibuffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + dataDim_t vdim = odim; + vdim.c = 1; + dnnType *dataTemp = nullptr; + if(isCudaPointer(tmpOutData_h)) + { + dataTemp = new dnnType[vdim.tot()]; + checkCuda(cudaMemcpy(dataTemp,tmpOutData_h,vdim.tot()*sizeof(dnnType),cudaMemcpyDeviceToHost)); + } + else + { + dataTemp = tmpOutData_h; + } + + cv::Mat colored; + for(int i=0;iinput_dim.h, netRT->input_dim.w, 0, classes, classes); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); + colored_fp32.convertTo(colored, CV_8UC1); + } + out_img.push_back(colored); + if(isCudaPointer(tmpOutData_h)) + { + delete [] dataTemp; + } + } + + cv::Mat tempMask(frame.size(), out_mask[0].type()); + cv::Mat seg(frame.size(), out_img[0].type()); + if(out_img.size() == 1) + { + cv::Rect roi(left, top, W, H); + seg = out_img[0](roi); + tempMask = out_mask[0](roi); + } + else{ + int bi=0; + + if(top == 0 && left == 0){ + + for(int i=0; i splitted_frames; @@ -385,6 +621,11 @@ class SegmentationNN { stats_post.push_back(t_ns); } } + #endif + + + + /** * Method to draw boundixg boxes and labels on a frame. From 7298dcfb2f4db8c98dbc47a0282d6b99256d8f49 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Thu, 6 Jan 2022 16:30:11 +0000 Subject: [PATCH 02/13] Added monodepth2.cpp --- CMakeLists.txt | 4 + tests/monodepth2/monodepth2.cpp | 275 ++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 tests/monodepth2/monodepth2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7cc9e33..6a775ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -202,6 +202,10 @@ target_link_libraries(test_shelfnet_berkeley tkDNN) add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp) target_link_libraries(test_shelfnet_mapillary tkDNN) +# MONODEPTH2 +add_executable(test_monodepth2 tests/monodepth2/monodepth2.cpp) +target_link_libraries(test_monodepth2 tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/tests/monodepth2/monodepth2.cpp b/tests/monodepth2/monodepth2.cpp new file mode 100644 index 0000000..0aa0f2a --- /dev/null +++ b/tests/monodepth2/monodepth2.cpp @@ -0,0 +1,275 @@ +#include +#include +#include +#include +#include + +const char* encoder_conv1_bin = "monodepth2/layers/encoder/encoder-conv1.bin"; +const char* encoder_layer1_bin[] = { + "monodepth2/layers/encoder/encoder-layer1-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer1-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer1-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer1-1-conv2.bin", +}; + +const char* encoder_layer2_bin[] = { + "monodepth2/layers/encoder/encoder-layer2-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer2-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer2-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer2-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer2-1-conv2.bin" +}; + +const char* encoder_layer3_bin[]={ + "monodepth2/layers/encoder/encoder-layer3-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer3-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer3-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer3-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer3-1-conv2.bin" +}; + +const char* encoder_layer4_bin[] = { + "monodepth2/layers/encoder/encoder-layer4-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer4-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer4-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer4-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer4-1-conv2.bin" +}; + +const char *encoder_fc_bin = "monodepth2/layers/encoder/encoder-fc.bin"; + +const char* decoder_layer_bin[] = { + "monodepth2/layers/depth_decoder/decoder-0-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-1-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-2-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-3-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-4-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-5-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-6-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-7-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-8-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-9-conv-conv.bin" +}; + +const char* decoder_dispconv_layer_bin[] = { + "monodepth2/layers/depth_decoder/decoder-10-conv.bin", + "monodepth2/layers/depth_decoder/decoder-11-conv.bin", + "monodepth2/layers/depth_decoder/decoder-12-conv.bin", + "monodepth2/layers/depth_decoder/decoder-13-conv.bin" +}; + +const char* output_bin[] = { + "monodepth2/debug/outputs/output-disp-0.bin", + "monodepth2/debug/outputs/output-disp-1.bin", + "monodepth2/debug/outputs/output-disp-2.bin", + "monodepth2/debug/outputs/output-disp-3.bin" +}; + +const char* input_monodepth2_bin[] = {"monodepth2/debug/input.bin","monodepth2/debug/input2.bin"}; + + +int main(){ + + tk::dnn::dataDim_t dim(1,3,192,640,1); + tk::dnn::Network net(dim); + std::vector features; + new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin, true,false,1, true); + tk::dnn::Layer *encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + features.push_back(encoder_relu_1); + tk::dnn::Layer *last = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); + + //layer 1 + for(int i=0;i<4;i=i+2){ + new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[i], true,false,1, true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[i+1],true,false,1,true); + new tk::dnn::Shortcut(&net,last); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + } + features.push_back(last); + + //layer2 + new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true,false,1,true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer *bn2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true,false,1, true); + new tk::dnn::Route(&net,&last,1); + new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2], true,false,1, true); + new tk::dnn::Shortcut(&net,bn2); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true,false,1, true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true,false,1, true); + new tk::dnn::Shortcut(&net,last); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + features.push_back(last); + + + //layer3 + new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true,false,1, true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + bn2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true,false,1, true); + new tk::dnn::Route(&net,&last,1); + new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2], true,false,1,true); + new tk::dnn::Shortcut(&net,bn2); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true,false,1, true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true,false,1, true); + new tk::dnn::Shortcut(&net,last); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + features.push_back(last); + + + //layer4 + new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true,false,1, true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + bn2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true,false,1, true); + new tk::dnn::Route(&net,&last,1); + new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2], true,false,1, true); + new tk::dnn::Shortcut(&net,bn2); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true,false,1, true); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true,false,1, true); + new tk::dnn::Shortcut(&net,last); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + features.push_back(last); + + std::vector depth_conv_features; + + //decoders + + new tk::dnn::Shortcut(&net,features[4]); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0], false,false,1, false); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer *upsample_layer_1 = new tk::dnn::Upsample(&net, 2); + tk::dnn::Layer *layer_1[2] = {features[3],upsample_layer_1}; + new tk::dnn::Route(&net,layer_1,2); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1], false,false,1,false); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2], false,false,1,false); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer *upsample_layer_2 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer *layer_2[2] = {features[2],upsample_layer_2}; + new tk::dnn::Route(&net,layer_2,2); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3], false,false,1,false); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + depth_conv_features.push_back(last); + + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4],false,false,1,false); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* upsample_layer_3 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer *layer_3[2] = {features[1],upsample_layer_3}; + new tk::dnn::Route(&net,layer_3,2); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5], false,false,1, false); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + depth_conv_features.push_back(last); + + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6], false,false,1, false); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* upsample_layer_4 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer *layer_4[2] = {features[0],upsample_layer_4}; + new tk::dnn::Route(&net,layer_4,2); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7], false,false,1,false); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + depth_conv_features.push_back(last); + + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8], false,false,1,false); + new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + new tk::dnn::Upsample(&net,2); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9], false, false,1, false); + last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + depth_conv_features.push_back(last); + + + new tk::dnn::Route(&net,&depth_conv_features[3],1); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0], false, false,1, false); + tk::dnn::Layer *disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp0->setFinal(); + + new tk::dnn::Route(&net,&depth_conv_features[2],1); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[1],false,false,1,false); + tk::dnn::Layer *disp1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp1->setFinal(); + + + new tk::dnn::Route(&net,&depth_conv_features[1],1); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[2], false,false,1, false); + tk::dnn::Layer *disp2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp2->setFinal(); + + new tk::dnn::Route(&net,&depth_conv_features[0],1); + new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[3], false,false,1, false); + tk::dnn::Layer *disp3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp3->setFinal(); + + + + dnnType *data; + dnnType *input_H; + readBinaryFile(input_monodepth2_bin[1],dim.tot(),&input_H,&data); + std::cout<<"INPUT DIMENSIONS : "<output_dim.print(); + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0;i<4;i++){ + printCenteredTitle((std::string("MONODEPTH2 CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i]; + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + + + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; +} \ No newline at end of file From f189efcbb1e3f8b3522b3e3d8c8844433c5af1e8 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Fri, 7 Jan 2022 05:41:26 +0000 Subject: [PATCH 03/13] Reflection Padding native plugin fix ,forgot to added input_dim.c in the plugin creator --- CMakeLists.txt | 5 +++-- include/tkDNN/pluginsRT/ReflectionPadding.h | 2 ++ src/NetworkRT.cpp | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a775ed..cadaae7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,12 +85,14 @@ endif() find_package(CUDNN REQUIRED) include_directories(${CUDNN_INCLUDE_DIR}) +find_package(yaml-cpp REQUIRED) + # compile file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu" "src/pluginsRT/*.cpp") cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS}) cuda_add_library(kernels SHARED ${tkdnn_CUSRC}) -target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES} ${CUDA_LIBRARIES} ${CUDNN_LIBRARIES}) +target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES} ${CUDA_LIBRARIES} ${CUDNN_LIBRARIES} yaml-cpp) @@ -120,7 +122,6 @@ endif() # endif() # gives problems in cross-compiling, probably malformed cmake config -find_package(yaml-cpp REQUIRED) #------------------------------------------------------------------------------- # Build Libraries diff --git a/include/tkDNN/pluginsRT/ReflectionPadding.h b/include/tkDNN/pluginsRT/ReflectionPadding.h index 894ed98..7b13710 100644 --- a/include/tkDNN/pluginsRT/ReflectionPadding.h +++ b/include/tkDNN/pluginsRT/ReflectionPadding.h @@ -94,6 +94,8 @@ namespace nvinfer1{ static std::vector mPluginAttributes; std::string mPluginNamespace; }; + + REGISTER_TENSORRT_PLUGIN(ReflectionPaddingRTPluginCreator); }; #endif diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 6192006..549b010 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -484,13 +484,15 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ mPluginAttributes.emplace_back(PluginField("outputH",&l->output_dim.h,PluginFieldType::kINT32,1)); mPluginAttributes.emplace_back(PluginField("outputW",&l->output_dim.w,PluginFieldType::kINT32,1)); mPluginAttributes.emplace_back(PluginField("n",&l->input_dim.n,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("c",&l->input_dim.c,PluginFieldType::kINT32,1)); mFC.nbFields = mPluginAttributes.size(); mFC.fields = mPluginAttributes.data(); auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC); - } auto *lRT = networkRT->addPluginV2(&input, 1, *plugin); checkNULL(lRT); return lRT; + } + #endif } From 3b58fbcb823f161262ddf15ec74d58a498666c90 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Fri, 7 Jan 2022 13:05:09 +0530 Subject: [PATCH 04/13] Added ConstantPadding plugin for TensorRT < 8.2 --- include/tkDNN/NetworkRT.h | 2 + include/tkDNN/pluginsRT/ConstantPaddingRT.h | 109 +++++++++++ src/NetworkRT.cpp | 55 ++++-- src/pluginsRT/ConstantPaddingRT.cpp | 201 ++++++++++++++++++++ 4 files changed, 350 insertions(+), 17 deletions(-) create mode 100644 include/tkDNN/pluginsRT/ConstantPaddingRT.h create mode 100644 src/pluginsRT/ConstantPaddingRT.cpp diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 571d127..b859074 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -23,6 +23,8 @@ #include #include #include +#include +#include diff --git a/include/tkDNN/pluginsRT/ConstantPaddingRT.h b/include/tkDNN/pluginsRT/ConstantPaddingRT.h new file mode 100644 index 0000000..15f4c0d --- /dev/null +++ b/include/tkDNN/pluginsRT/ConstantPaddingRT.h @@ -0,0 +1,109 @@ +// +// Created by perseusdg on 1/7/22. +// + +#ifndef _CONSTANTPADDINGRT_PLUGIN_H +#define _CONSTANTPADDINGRT_PLUGIN_H + +#include +#include +#include +#include +#include + +namespace nvinfer1{ + class ConstantPaddingRT : public IPluginV2Ext { + public: + ConstantPaddingRT(int32_t padH,int32_t padW,int32_t n,int32_t c,int32_t i_h,int32_t i_w,int32_t o_h,int32_t o_w,float constant); + + ConstantPaddingRT(const void *data,size_t length); + + ~ConstantPaddingRT(); + + int getNbOutputs() const NOEXCEPT override; + + Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ; + + int initialize() NOEXCEPT override ; + + void terminate() NOEXCEPT override ; + + size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ; + + +#if NV_TENSORRT_MAJOR > 7 + int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ; +#elif NV_TENSORRT_MAJOR <= 7 + int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override; +#endif + + size_t getSerializationSize() const NOEXCEPT override ; + + void serialize(void *buffer) const NOEXCEPT override ; + + void destroy() NOEXCEPT override ; + + const char *getPluginType() const NOEXCEPT override ; + + const char *getPluginVersion() const NOEXCEPT override; + + const char *getPluginNamespace() const NOEXCEPT override ; + + void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ; + + IPluginV2Ext *clone() const NOEXCEPT override ; + + DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override; + + void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override; + + bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override; + + bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override; + + void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, + int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes, + bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat, + int32_t maxBatchSize) NOEXCEPT override; + + void detachFromContext() NOEXCEPT override; + + bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override; + + int32_t i_h,i_w,o_h,o_w,n,c,padH,padW; + float constant; + private: + std::string mPluginNamespace; + + }; + + class ConstantPaddingRTPluginCreator : public IPluginCreator { + public: + ConstantPaddingRTPluginCreator(); + + void setPluginNamespace(const char* pluginNamespace) NOEXCEPT override; + + const char *getPluginNamespace() const NOEXCEPT override; + + IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ; + + IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ; + + const char *getPluginName() const NOEXCEPT override ; + + const char *getPluginVersion() const NOEXCEPT override; + + const PluginFieldCollection *getFieldNames() NOEXCEPT override ; + + private: + static PluginFieldCollection mFC; + static std::vector mPluginAttributes; + std::string mPluginNamespace; + + }; + + REGISTER_TENSORRT_PLUGIN(ConstantPaddingRTPluginCreator); +}; + + +#endif //TKDNN_CONSTANTPADDINGRT_H diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 549b010..3f086bd 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -474,24 +474,45 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ #else //todo add PADDING_MODE_CONSTANT AND PADDING_MODE_ZERO for tensorrt versions < 8.2 if(l->padding_mode == PADDING_MODE_REFLECTION){ - auto creator = getPluginRegistry()->getPluginCreator("ReflectionPaddingRT_tkDNN","1"); - std::vector mPluginAttributes; - PluginFieldCollection mFC{}; - mPluginAttributes.emplace_back(PluginField("padH",&l->paddingH,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("padW",&l->paddingW,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("inputH",&l->input_dim.h,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("inputW",&l->input_dim.w,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("outputH",&l->output_dim.h,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("outputW",&l->output_dim.w,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("n",&l->input_dim.n,PluginFieldType::kINT32,1)); - mPluginAttributes.emplace_back(PluginField("c",&l->input_dim.c,PluginFieldType::kINT32,1)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); - auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC); - auto *lRT = networkRT->addPluginV2(&input, 1, *plugin); - checkNULL(lRT); - return lRT; + auto creator = getPluginRegistry()->getPluginCreator("ReflectionPaddingRT_tkDNN","1"); + std::vector mPluginAttributes; + PluginFieldCollection mFC{}; + mPluginAttributes.emplace_back(PluginField("padH",&l->paddingH,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("padW",&l->paddingW,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("inputH",&l->input_dim.h,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("inputW",&l->input_dim.w,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("outputH",&l->output_dim.h,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("outputW",&l->output_dim.w,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("n",&l->input_dim.n,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("c",&l->input_dim.c,PluginFieldType::kINT32,1)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); + auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC); + auto *lRT = networkRT->addPluginV2(&input, 1, *plugin); + checkNULL(lRT); + return lRT; + }else if(l->padding_mode == PADDING_MODE_CONSTANT || l->padding_mode == PADDING_MODE_ZERO){ + auto creator = getPluginRegistry()->getPluginCreator("ConstantPaddingRT_tkDNN","1"); + std::vector mPluginAttributes; + PluginFieldCollection mFC{}; + mPluginAttributes.emplace_back(PluginField("padH",&l->paddingH,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("padW",&l->paddingW,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("inputH",&l->input_dim.h,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("inputW",&l->input_dim.w,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("outputH",&l->output_dim.h,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("outputW",&l->output_dim.w,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("n",&l->input_dim.n,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("c",&l->input_dim.c,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("constant",&l->constant,PluginFieldType::kFLOAT32,1)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); + auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC); + auto *lRT = networkRT->addPluginV2(&input,1,*plugin); + checkNULL(lRT); + return lRT; } + + return nullptr; #endif } diff --git a/src/pluginsRT/ConstantPaddingRT.cpp b/src/pluginsRT/ConstantPaddingRT.cpp new file mode 100644 index 0000000..ac37e9d --- /dev/null +++ b/src/pluginsRT/ConstantPaddingRT.cpp @@ -0,0 +1,201 @@ +#include + +using namespace nvinfer1; + +std::vector ConstantPaddingRTPluginCreator::mPluginAttributes; +PluginFieldCollection ConstantPaddingRTPluginCreator::mFC{}; + +static const char* CONSTANTPADDINGRT_PLUGIN_VERSION{"1"}; +static const char* CONSTANTPADDINGRT_PLUGIN_NAME{"ConstantPaddingRT_tkDNN"}; + +ConstantPaddingRT::ConstantPaddingRT(int32_t padH, int32_t padW, int32_t n, int32_t c, int32_t i_h, int32_t i_w, + int32_t o_h, int32_t o_w, float constant) { + this->padH = padH; + this->padW = padW; + this->n = n; + this->c = c; + this->i_h = i_h; + this->i_w = i_w; + this->o_h = o_h; + this->o_w = o_w; + this->constant = constant; + +} + +ConstantPaddingRT::ConstantPaddingRT(const void *data, size_t length) { + const char* buf = reinterpret_cast(data),*bufcheck=buf; + padH = readBUF(buf); + padW = readBUF(buf); + i_h = readBUF(buf); + i_w = readBUF(buf); + o_h = readBUF(buf); + o_w = readBUF(buf); + n = readBUF(buf); + c = readBUF(buf); + constant = readBUF(buf); + assert(buf = bufcheck + length); +} + +ConstantPaddingRT::~ConstantPaddingRT() {} + +int ConstantPaddingRT::getNbOutputs() const NOEXCEPT{ + return 1; +} + +Dims ConstantPaddingRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT { + return Dims3{c,o_h,o_w}; +} + +int ConstantPaddingRT::initialize() NOEXCEPT { + return 0; +} + +void ConstantPaddingRT::terminate() NOEXCEPT { + +} + +size_t ConstantPaddingRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT { + return 0; +} + +#if NV_TENSORRT_MAJOR > 7 +int ConstantPaddingRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT { + dnnType* srcData = (dnnType*)reinterpret_cast(inputs[0]); + dnnType* dstData = reinterpret_cast(outputs[0]); + constant_pad2d_forward(srcData,dstData,i_h,i_w,o_h,o_w,c,n,padH,padW,constant,stream); + return 0; +} +#elif NV_TENSORRT_MAJOR <= 7 + int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) { + dnnType* srcData = (dnnType*)reinterpret_cast(inputs[0]); + dnnType* dstData = reinterpret_cast(outputs[0]); + constant_pad2d_forward(srcData,dstData,i_h,i_w,o_h,o_w,c,n,padH,padW,constant,stream); + return 0; +} +#endif + +size_t ConstantPaddingRT::getSerializationSize() const NOEXCEPT { + return (8*sizeof(int32_t) + 1*sizeof(float)); +} + +void ConstantPaddingRT::serialize(void *buffer) const NOEXCEPT { + char *buf = reinterpret_cast(buffer),*a=buf; + writeBUF(buf,padH); + writeBUF(buf,padW); + writeBUF(buf,i_h); + writeBUF(buf,i_w); + writeBUF(buf,o_h); + writeBUF(buf,o_w); + writeBUF(buf,n); + writeBUF(buf,c); + writeBUF(buf,constant); +} + +void ConstantPaddingRT::destroy() NOEXCEPT { + delete this; +} + +const char* ConstantPaddingRT::getPluginType() const NOEXCEPT { + return CONSTANTPADDINGRT_PLUGIN_NAME; +} + +const char* ConstantPaddingRT::getPluginVersion() const NOEXCEPT { + return CONSTANTPADDINGRT_PLUGIN_VERSION; +} + +const char* ConstantPaddingRT::getPluginNamespace() const NOEXCEPT { + return mPluginNamespace.c_str(); +} + +void ConstantPaddingRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT { + mPluginNamespace = pluginNamespace; +} + +IPluginV2Ext *ConstantPaddingRT::clone() const NOEXCEPT { + auto *p = new ConstantPaddingRT(padH,padW,n,c,i_h,i_w,o_h,o_w,constant); + p->setPluginNamespace(mPluginNamespace.c_str()); + return p; +} + +DataType ConstantPaddingRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, + int nbInputs) const NOEXCEPT { + return DataType::kFLOAT; +} + +void ConstantPaddingRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext, + IGpuAllocator *gpuAllocator) NOEXCEPT { + +} + +bool ConstantPaddingRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, + int nbInputs) const NOEXCEPT { + return false; +} + +bool ConstantPaddingRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT { + return false; +} + +void ConstantPaddingRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, + int32_t nbOutputs, const DataType *inputTypes, const DataType *outputTypes, + const bool *inputIsBroadcast, const bool *outputIsBroadcast, + PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT { + +} + +void ConstantPaddingRT::detachFromContext() NOEXCEPT { + +} + +bool ConstantPaddingRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT { + return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR); +} + +ConstantPaddingRTPluginCreator::ConstantPaddingRTPluginCreator() { + mPluginAttributes.clear(); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +void ConstantPaddingRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT { + mPluginNamespace = pluginNamespace; +} + +const char *ConstantPaddingRTPluginCreator::getPluginNamespace() const NOEXCEPT { + return mPluginNamespace.c_str(); +} + +IPluginV2Ext *ConstantPaddingRTPluginCreator::deserializePlugin(const char *name, const void *serialData, + size_t serialLength) NOEXCEPT { + auto *pluginObj = new ConstantPaddingRT(serialData,serialLength); + pluginObj->setPluginNamespace(mPluginNamespace.c_str()); + return pluginObj; +} + +IPluginV2Ext *ConstantPaddingRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT { + const PluginField *fields = fc->fields; + int padH = *(static_cast(fields[0].data)); + int padW = *(static_cast(fields[1].data)); + int inputH = *(static_cast(fields[2].data)); + int inputW = *(static_cast(fields[3].data)); + int outputH = *(static_cast(fields[4].data)); + int outputW = *(static_cast(fields[5].data)); + int n = *(static_cast(fields[6].data)); + int c = *(static_cast(fields[7].data)); + float constant = *(static_cast(fields[8].data)); + auto *pluginObj = new ConstantPaddingRT(padH,padW,n,c,inputH,inputW,outputH,outputW,constant); + pluginObj->setPluginNamespace(mPluginNamespace.c_str()); + return pluginObj; +} + +const char *ConstantPaddingRTPluginCreator::getPluginName() const NOEXCEPT { + return CONSTANTPADDINGRT_PLUGIN_NAME; +} + +const char *ConstantPaddingRTPluginCreator::getPluginVersion() const NOEXCEPT { + return CONSTANTPADDINGRT_PLUGIN_VERSION; +} + +const PluginFieldCollection *ConstantPaddingRTPluginCreator::getFieldNames() NOEXCEPT { + return &mFC; +} From cabebc95a238d7fcbd27d0bca565b3fedf5bcd4e Mon Sep 17 00:00:00 2001 From: perseusdg Date: Sun, 9 Jan 2022 13:45:44 +0530 Subject: [PATCH 05/13] added individual layer names in monodepth2 --- tests/monodepth2/monodepth2.cpp | 249 +++++++++++++++----------------- 1 file changed, 115 insertions(+), 134 deletions(-) diff --git a/tests/monodepth2/monodepth2.cpp b/tests/monodepth2/monodepth2.cpp index 0aa0f2a..c763baf 100644 --- a/tests/monodepth2/monodepth2.cpp +++ b/tests/monodepth2/monodepth2.cpp @@ -72,156 +72,135 @@ int main(){ tk::dnn::dataDim_t dim(1,3,192,640,1); tk::dnn::Network net(dim); - std::vector features; - new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin, true,false,1, true); - tk::dnn::Layer *encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - features.push_back(encoder_relu_1); - tk::dnn::Layer *last = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); + tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin,true,false,1,true); + tk::dnn::Layer* encoder_relu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_maxpool = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); - //layer 1 - for(int i=0;i<4;i=i+2){ - new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[i], true,false,1, true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[i+1],true,false,1,true); - new tk::dnn::Shortcut(&net,last); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - } - features.push_back(last); + //layer-1 + tk::dnn::Layer* encoder_layer_1_0_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_0_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_0_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_maxpool); + tk::dnn::Layer* encoder_relu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_1_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_relu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_1_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_1_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_relu_2); + tk::dnn::Layer* encoder_relu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - //layer2 - new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true,false,1,true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer *bn2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true,false,1, true); - new tk::dnn::Route(&net,&last,1); - new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2], true,false,1, true); - new tk::dnn::Shortcut(&net,bn2); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true,false,1, true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true,false,1, true); - new tk::dnn::Shortcut(&net,last); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - features.push_back(last); + //layer-2 + tk::dnn::Layer* encoder_layer_2_0_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_relu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_0_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_route = new tk::dnn::Route(&net,&encoder_relu_4,1); + tk::dnn::Layer* encoder_layer_2_0_downsample_convbn = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_2_0_convbn_2); + tk::dnn::Layer* encoder_relu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_1_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_relu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_1_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_6); + tk::dnn::Layer* encoder_relu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + //layer-3 + tk::dnn::Layer* encoder_layer_3_0_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_relu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_0_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_route = new tk::dnn::Route(&net,&encoder_relu_8,1); + tk::dnn::Layer* encoder_layer_3_0_downsample_convbn = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_3_0_convbn_2); + tk::dnn::Layer* encoder_relu_10 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_1_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_relu_11 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_1_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_10); + tk::dnn::Layer* encoder_relu_12 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - //layer3 - new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true,false,1, true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - bn2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true,false,1, true); - new tk::dnn::Route(&net,&last,1); - new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2], true,false,1,true); - new tk::dnn::Shortcut(&net,bn2); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true,false,1, true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true,false,1, true); - new tk::dnn::Shortcut(&net,last); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - features.push_back(last); - - - //layer4 - new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true,false,1, true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - bn2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true,false,1, true); - new tk::dnn::Route(&net,&last,1); - new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2], true,false,1, true); - new tk::dnn::Shortcut(&net,bn2); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true,false,1, true); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true,false,1, true); - new tk::dnn::Shortcut(&net,last); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - features.push_back(last); + //layer-4 + tk::dnn::Layer* encoder_layer_4_0_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_relu_13 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_0_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_route = new tk::dnn::Route(&net,&encoder_relu_12,1); + tk::dnn::Layer* encoder_layer_4_0_downsample_convbn = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_4_0_convbn_2); + tk::dnn::Layer* encoder_relu_14 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_1_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_relu_15 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_1_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_14); + tk::dnn::Layer* encoder_relu_16 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - std::vector depth_conv_features; - - //decoders - - new tk::dnn::Shortcut(&net,features[4]); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0], false,false,1, false); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer *upsample_layer_1 = new tk::dnn::Upsample(&net, 2); - tk::dnn::Layer *layer_1[2] = {features[3],upsample_layer_1}; - new tk::dnn::Route(&net,layer_1,2); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1], false,false,1,false); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2], false,false,1,false); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer *upsample_layer_2 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer *layer_2[2] = {features[2],upsample_layer_2}; - new tk::dnn::Route(&net,layer_2,2); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3], false,false,1,false); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - depth_conv_features.push_back(last); - - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4],false,false,1,false); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* upsample_layer_3 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer *layer_3[2] = {features[1],upsample_layer_3}; - new tk::dnn::Route(&net,layer_3,2); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5], false,false,1, false); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - depth_conv_features.push_back(last); - - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6], false,false,1, false); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* upsample_layer_4 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer *layer_4[2] = {features[0],upsample_layer_4}; - new tk::dnn::Route(&net,layer_4,2); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7], false,false,1,false); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - depth_conv_features.push_back(last); - - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8], false,false,1,false); - new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - new tk::dnn::Upsample(&net,2); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9], false, false,1, false); - last = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - depth_conv_features.push_back(last); - - - new tk::dnn::Route(&net,&depth_conv_features[3],1); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0], false, false,1, false); - tk::dnn::Layer *disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + //decoder + tk::dnn::Layer* decoder_reflection_padding_2d = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_4_0 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0]); + tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer[2] = {decoder_upsampling_2d,encoder_relu_12}; + tk::dnn::Layer* decoder_concatenate = new tk::dnn::Route(&net,concatenate_layer,2); + tk::dnn::Layer* decoder_reflection_padding_2d_1 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_4_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1]); + tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_2 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_3_0 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2]); + tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_1 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_1[2] = {decoder_upsampling_2d_1,encoder_relu_8}; + tk::dnn::Layer* decoder_concatenate_layer_1 = new tk::dnn::Route{&net,concatenate_layer_1,2}; + tk::dnn::Layer* decoder_reflection_padding_2d_3 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_3_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3]); + tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_5 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_2_0 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4]); + tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_2 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_2[2] = {decoder_upsampling_2d_2,encoder_relu_4}; + tk::dnn::Layer* decoder_concatenate_layer_2 = new tk::dnn::Route(&net,concatenate_layer_2,2); + tk::dnn::Layer* decoder_reflection_padding_2d_6 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_2_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5]); + tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_8 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_1_0 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6]); + tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_3 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_3[2] = {decoder_upsampling_2d_3,encoder_relu}; + tk::dnn::Layer* decoder_concatenate_layer_3 = new tk::dnn::Route(&net,concatenate_layer_3,2); + tk::dnn::Layer* decoder_reflection_padding_2d_9 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_1_1 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7]); + tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_11 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_0_0 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8]); + tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_4 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* decoder_reflection_padding_2d_12 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_0_1 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9]); + tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_13 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_0 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0]); + tk::dnn::Layer* disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); disp0->setFinal(); - new tk::dnn::Route(&net,&depth_conv_features[2],1); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[1],false,false,1,false); - tk::dnn::Layer *disp1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Layer* route_elu_7 = new tk::dnn::Route(&net,&decoder_elu_7,1); + tk::dnn::Layer* decoder_reflection_padding_2d_10 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_1 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[1]); + tk::dnn::Layer* disp1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); disp1->setFinal(); - - new tk::dnn::Route(&net,&depth_conv_features[1],1); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[2], false,false,1, false); - tk::dnn::Layer *disp2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Layer* route_elu_5 = new tk::dnn::Route(&net,&decoder_elu_5,1); + tk::dnn::Layer* decoder_reflection_padding_2d_7 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_2 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[2]); + tk::dnn::Layer* disp2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); disp2->setFinal(); - new tk::dnn::Route(&net,&depth_conv_features[0],1); - new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[3], false,false,1, false); - tk::dnn::Layer *disp3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + tk::dnn::Layer* route_elu_3 = new tk::dnn::Route(&net,&decoder_elu_3,1); + tk::dnn::Layer* decoder_reflection_padding_2d_4 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_3 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[3]); + tk::dnn::Layer* disp3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); disp3->setFinal(); - dnnType *data; dnnType *input_H; - readBinaryFile(input_monodepth2_bin[1],dim.tot(),&input_H,&data); + readBinaryFile(input_monodepth2_bin[0],dim.tot(),&input_H,&data); std::cout<<"INPUT DIMENSIONS : "< Date: Sun, 9 Jan 2022 23:03:26 +0530 Subject: [PATCH 06/13] Added a seperate layer file for batchnorm,to support an independent batchnorm class in order to pass necessary parameters read from the bin file. --- include/tkDNN/Layer.h | 59 +++++++++++++++++++++++++++++ src/LayerBNWgs.cpp | 88 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 src/LayerBNWgs.cpp diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 662eb6a..28a8817 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -180,6 +180,65 @@ public: }; +class LayerBNWgs : public Layer { +public: + LayerBNWgs(Network* net, int input, int output, std::string fname_weights); + ~LayerBNWgs(); + + int inputs, outputs; + std::string weights_path; + + dnnType* bias_h, * bias_d; + dnnType* power_h = nullptr; + dnnType* scales_h = nullptr, * scales_d = nullptr; + dnnType* mean_h = nullptr, * mean_d = nullptr; + dnnType* variance_h = nullptr, * variance_d = nullptr; + + __half* bias16_h = nullptr, * bias16_d = nullptr; + __half* power16_h = nullptr, * power16_d = nullptr; + __half* scales16_h = nullptr, * scales16_d = nullptr; + __half* mean16_h = nullptr, * mean16_d = nullptr; + __half* variance16_h = nullptr, * variance16_d = nullptr; + + + void releaseHost(bool release32 = true, bool release16 = true) { + if (release32) { + if (bias_h != nullptr) { delete[] bias_h; bias_h = nullptr; } + if (scales_h != nullptr) { delete[] scales_h; scales_h = nullptr; } + if (mean_h != nullptr) { delete[] mean_h; mean_h = nullptr; } + if (variance_h != nullptr) { delete[] variance_h; variance_h = nullptr; } + if (power_h != nullptr) { delete[] power_h; power_h = nullptr; } + } + if (net->fp16 && release16) { + if (bias16_h != nullptr) { delete[] bias16_h; bias16_h = nullptr; } + if (scales16_h != nullptr) { delete[] scales16_h; scales16_h = nullptr; } + if (mean16_h != nullptr) { delete[] mean16_h; mean16_h = nullptr; } + if (variance16_h != nullptr) { delete[] variance16_h; variance16_h = nullptr; } + if (power16_h != nullptr) { delete[] power16_h; power16_h = nullptr; } + + } + } + + void releaseDevice(bool release32 = true, bool release16 = true) { + if (release32) { + if (bias_d != nullptr) { cudaFree(bias_d); bias_d = nullptr; } + if (scales_d != nullptr) { cudaFree(scales_d); scales_d = nullptr; } + if (mean_d != nullptr) { cudaFree(mean_d); mean_d = nullptr; } + if (variance_d != nullptr) { cudaFree(variance_d); variance_d = nullptr; } + } + if (net->fp16 && release16) { + if (bias16_d != nullptr) { cudaFree(bias16_d); bias16_d = nullptr; } + if (scales16_d != nullptr) { cudaFree(scales16_d); scales16_d = nullptr; } + if (mean16_d != nullptr) { cudaFree(mean16_d); mean16_d = nullptr; } + if (variance16_d != nullptr) { cudaFree(variance16_d); variance16_d = nullptr; } + if (power16_d != nullptr) { cudaFree(power16_d); power16_d = nullptr; } + } + } + + + +}; + /** Input layer (it doesn't need weights) */ diff --git a/src/LayerBNWgs.cpp b/src/LayerBNWgs.cpp new file mode 100644 index 0000000..7414ef3 --- /dev/null +++ b/src/LayerBNWgs.cpp @@ -0,0 +1,88 @@ +#include +#include + +#include "Layer.h" +#include "kernels.h" + +namespace tk { namespace dnn { + LayerBNWgs::LayerBNWgs(Network* net, int input, int output, std::string fname_weights) : Layer(net) { + this->inputs = inputs; + this->outputs = output; + this->weights_path = fname_weights; + + std::cout << "Reading BatchNorm O = " << outputs << std::endl; + int seek = 0; + readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek); + seek += outputs; + readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek); + seek += outputs; + readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek); + seek += outputs; + readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek); + seek += outputs; + + float eps = TKDNN_BN_MIN_EPSILON; + + power_h = new dnnType[outputs]; + for (int i = 0; i < outputs; i++) power_h[i] = 1.0f; + + for (int i = 0; i < outputs; i++) + mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]); + + for (int i = 0; i < outputs; i++) + variance_h[i] = 1.0f / sqrt(eps + variance_h[i]); + + if (!net->fp16) + return; + + int b_size = outputs; + bias16_h = new __half[b_size]; + cudaMalloc(&bias16_d, b_size * sizeof(__half)); + float2half(bias_d, bias16_d, b_size); + cudaMemcpy(bias16_h, bias16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); + + power16_h = new __half[b_size]; + mean16_h = new __half[b_size]; + variance16_h = new __half[b_size]; + scales16_h = new __half[b_size]; + + cudaMalloc(&power16_d, b_size * sizeof(__half)); + cudaMalloc(&mean16_d, b_size * sizeof(__half)); + cudaMalloc(&variance16_d, b_size * sizeof(__half)); + cudaMalloc(&scales16_d, b_size * sizeof(__half)); + + //temporary buffers + float* tmp_d; + cudaMalloc(&tmp_d, b_size * sizeof(float)); + + //init power array of ones + cudaMemcpy(tmp_d, power_h, b_size * sizeof(float), cudaMemcpyHostToDevice); + float2half(tmp_d, power16_d, b_size); + cudaMemcpy(power16_h, power16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); + + //mean array + cudaMemcpy(tmp_d, mean_h, b_size * sizeof(float), cudaMemcpyHostToDevice); + float2half(tmp_d, mean16_d, b_size); + cudaMemcpy(mean16_h, mean16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); + + //convert variance + + cudaMemcpy(tmp_d, variance_h, b_size * sizeof(float), cudaMemcpyHostToDevice); + float2half(tmp_d, variance16_d, b_size); + cudaMemcpy(variance16_h, variance16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); + + //convert scales + float2half(scales_d, scales16_d, b_size); + cudaMemcpy(scales16_h, scales16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); + + cudaFree(tmp_d); + + + } + + LayerBNWgs::~LayerBNWgs() { + releaseHost(); + releaseDevice(); + } + +} } \ No newline at end of file From bcf0c4eab3d63a2a01ea73b898ee032c98f400c0 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Mon, 10 Jan 2022 18:43:12 +0530 Subject: [PATCH 07/13] Added BatchNorm Layer (Testing still needs to be done) --- include/tkDNN/Layer.h | 23 ++++++++++++++- src/BatchNorm.cpp | 67 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/BatchNorm.cpp diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 28a8817..62d5a3e 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -32,7 +32,8 @@ enum layerType_t { LAYER_UPSAMPLE, LAYER_REGION, LAYER_YOLO, - LAYER_PADDING + LAYER_PADDING, + LAYER_BATCHNORM }; #define TKDNN_BN_MIN_EPSILON 1e-5 @@ -89,6 +90,7 @@ public: case LAYER_REGION: return "Region"; case LAYER_YOLO: return "Yolo"; case LAYER_PADDING: return "Padding"; + case LAYER_BATCHNORM: return "BatchNorm"; default: return "unknown"; } } @@ -604,6 +606,25 @@ public: }; + +class BatchNorm : public LayerBNWgs { +public: + BatchNorm(Network *net,int output,std::string fname_weights); + virtual ~BatchNorm(); + virtual layerType_t getLayerType(){return LAYER_BATCHNORM;}; + virtual dnnType* infer(dataDim_t& dim,dnnType* srcData); + std::string weights_bin; +protected: + cudnnFilterDescriptor_t filterDesc; + cudnnConvolutionFwdAlgoPerf_t algo; + cudnnConvolutionBwdDataAlgoPerf_t bwAlgo; + cudnnTensorDescriptor_t biasTensorDesc; + + void initCUDNN(); + void inferCUDNN(dnnType* srcData); + void* workSpace; + size_t ws_sizeInBytes; +}; /** Softmax layer */ diff --git a/src/BatchNorm.cpp b/src/BatchNorm.cpp new file mode 100644 index 0000000..b406011 --- /dev/null +++ b/src/BatchNorm.cpp @@ -0,0 +1,67 @@ +#include + +#include "Layer.h" + +namespace tk { namespace dnn { + void BatchNorm::initCUDNN(){ + cudnnTensorDescriptor_t srcTensor = srcTensorDesc; + cudnnTensorDescriptor_t dstTensor = dstTensorDesc; + dataDim_t idim,odim; + idim = input_dim; + odim = output_dim; + + checkCUDNN( cudnnSetTensor4dDescriptor(srcTensor, + net->tensorFormat, net->dataType, idim.n, idim.c, idim.h, idim.w) ); + + checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) ); + + checkCUDNN( cudnnSetTensor4dDescriptor(dstTensor, + net->tensorFormat, net->dataType, odim.n, odim.c, odim.h, odim.w) ); + + checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc, + net->tensorFormat, net->dataType, + 1, output_dim.c, 1, 1) ); + + + } + + void BatchNorm::inferCUDNN(float *srcData){ + dnnType alpha = dnnType(1); + dnnType beta = dnnType(0); + + alpha = dnnType(1); + beta = dnnType(1); + checkCUDNN( cudnnAddTensor(net->cudnnHandle, + &alpha, biasTensorDesc, bias_d, + &beta, dstTensorDesc, dstData) ); + alpha = dnnType(1); + beta = dnnType(0); + checkCUDNN( cudnnBatchNormalizationForwardInference(net->cudnnHandle, + CUDNN_BATCHNORM_SPATIAL, &alpha, &beta, + dstTensorDesc, dstData, dstTensorDesc, + dstData, biasTensorDesc, //same tensor descriptor as bias + scales_d, bias_d, mean_d, variance_d, + TKDNN_BN_MIN_EPSILON) ); + } + + BatchNorm::BatchNorm(Network *net,int output,std::string fname_weights) : + LayerBNWgs(net,net->getOutputDim().c,output,fname_weights){ + output_dim = input_dim; + initCUDNN(); + checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); + + } + + dnnType* BatchNorm::infer(dataDim_t &dim,dnnType* srcData){ + inferCUDNN(srcData); + + dim = output_dim; + return dstData; + } + + BatchNorm::~BatchNorm(){ + checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) ); + checkCuda( cudaFree(dstData) ); + } + +}} \ No newline at end of file From 061bc79a69e8aa4960c12bd137169269c0fc8779 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Mon, 10 Jan 2022 22:40:31 +0530 Subject: [PATCH 08/13] Added BatchNorm to NetworkRT (conver_layer) --- include/tkDNN/NetworkRT.h | 1 + src/NetworkRT.cpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index b859074..10394bc 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -98,6 +98,7 @@ public: nvinfer1::IResizeLayer* convert_layer(nvinfer1::ITensor *input, Upsample *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input,Padding *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor* input,BatchNorm *l); #if NV_TENSORRT_MAJOR > 5 && NV_TENSORRT_MAJOR < 8 bool serialize(const char *filename); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 3f086bd..b83de23 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -277,6 +277,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (DeformConv2d*) l); if(type == LAYER_PADDING) return convert_layer(input, (Padding*) l); + if(type == LAYER_BATCHNORM) + return convert_layer(input,(BatchNorm*) l); std::cout<getLayerName()<<"\n"; FatalError("Layer not implemented in tensorRT"); @@ -407,6 +409,38 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { return lRT; } +ILayer* NetworkRT::convert_layer(ITensor *input,BatchNorm *l){ + void *bias_b, *power_b, *mean_b, *variance_b, *scales_b; + if(dtRT == DataType::kHALF) { + bias_b = l->bias16_h; + power_b = l->power16_h; + mean_b = l->mean16_h; + variance_b = l->variance16_h; + scales_b = l->scales16_h; + } else { + bias_b = l->bias_h; + power_b = l->power_h; + mean_b = l->mean_h; + variance_b = l->variance_h; + scales_b = l->scales_h; + } + Weights power{dtRT, power_b, l->outputs}; + Weights shift{dtRT, mean_b, l->outputs}; + Weights scale{dtRT, variance_b, l->outputs}; + + IScaleLayer *lRT = networkRT->addScale(*input, ScaleMode::kCHANNEL, + shift, scale, power); + checkNULL(lRT); + Weights shift2{dtRT, bias_b, l->outputs}; + Weights scale2{dtRT, scales_b, l->outputs}; + IScaleLayer *lRT2 = networkRT->addScale(*lRT->getOutput(0), ScaleMode::kCHANNEL, + shift2, scale2, power); + checkNULL(lRT2); + + return lRT2; + +} + ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { // std::cout<<"convert Pooling\n"; From 19e41a8b992a005cb019f6061291e47e97900b36 Mon Sep 17 00:00:00 2001 From: perseusdg Date: Tue, 11 Jan 2022 12:38:36 +0530 Subject: [PATCH 09/13] monodepth2 new format (test) conv + independent batch norm --- CMakeLists.txt | 3 + tests/monodepth2/monodepth2_new_format.cpp | 306 +++++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 tests/monodepth2/monodepth2_new_format.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cadaae7..8a6d616 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -207,6 +207,9 @@ target_link_libraries(test_shelfnet_mapillary tkDNN) add_executable(test_monodepth2 tests/monodepth2/monodepth2.cpp) target_link_libraries(test_monodepth2 tkDNN) +add_executable(test_monodepth2_new_format tests/monodepth2/monodepth2_new_format.cpp) +target_link_libraries(test_monodepth2_new_format tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/tests/monodepth2/monodepth2_new_format.cpp b/tests/monodepth2/monodepth2_new_format.cpp new file mode 100644 index 0000000..313978c --- /dev/null +++ b/tests/monodepth2/monodepth2_new_format.cpp @@ -0,0 +1,306 @@ +#include +#include +#include +#include +#include + +const char* encoder_conv1_bin = "monodepth2/layers/encoder/encoder-conv1.bin"; +const char* encoder_bn1_bin = "monodepth2/layers/encoder/encoder-bn1.bin"; + +const char* encoder_layer1_conv_bin[] = { + "monodepth2/layers/encoder/encoder-layer1-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer1-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer1-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer1-1-conv2.bin", +}; + +const char* encoder_layer1_bn_bin[] = { + "monodepth2/layers/encoder/encoder-layer1-0-bn1.bin", + "monodepth2/layers/encoder/encoder-layer1-0-bn2.bin", + "monodepth2/layers/encoder/encoder-layer1-1-bn1.bin", + "monodepth2/layers/encoder/encoder-layer1-1-bn2.bin", +}; + + + +const char* encoder_layer2_conv_bin[] = { + "monodepth2/layers/encoder/encoder-layer2-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer2-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer2-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer2-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer2-1-conv2.bin" +}; + +const char* encoder_layer2_bn_bin[] = { + "monodepth2/layers/encoder/encoder-layer2-0-bn1.bin", + "monodepth2/layers/encoder/encoder-layer2-0-bn2.bin", + "monodepth2/layers/encoder/encoder-layer2-0-downsample-1.bin", + "monodepth2/layers/encoder/encoder-layer2-1-bn1.bin", + "monodepth2/layers/encoder/encoder-layer2-1-bn2.bin" +}; + +const char* encoder_layer3_conv_bin[]={ + "monodepth2/layers/encoder/encoder-layer3-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer3-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer3-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer3-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer3-1-conv2.bin" +}; + +const char* encoder_layer3_bn_bin[]={ + "monodepth2/layers/encoder/encoder-layer3-0-bn1.bin", + "monodepth2/layers/encoder/encoder-layer3-0-bn2.bin", + "monodepth2/layers/encoder/encoder-layer3-0-downsample-1.bin", + "monodepth2/layers/encoder/encoder-layer3-1-bn1.bin", + "monodepth2/layers/encoder/encoder-layer3-1-bn2.bin" +}; + +const char* encoder_layer4_conv_bin[] = { + "monodepth2/layers/encoder/encoder-layer4-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer4-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer4-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer4-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer4-1-conv2.bin" +}; + +const char* encoder_layer4_bn_bin[] = { + "monodepth2/layers/encoder/encoder-layer4-0-bn1.bin", + "monodepth2/layers/encoder/encoder-layer4-0-bn2.bin", + "monodepth2/layers/encoder/encoder-layer4-0-downsample-1.bin", + "monodepth2/layers/encoder/encoder-layer4-1-bn1.bin", + "monodepth2/layers/encoder/encoder-layer4-1-bn2.bin" +}; + +const char* decoder_layer_bin[] = { + "monodepth2/layers/depth_decoder/decoder-0-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-1-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-2-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-3-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-4-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-5-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-6-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-7-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-8-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-9-conv-conv.bin" +}; + +const char* decoder_dispconv_layer_bin[] = { + "monodepth2/layers/depth_decoder/decoder-10-conv.bin", + "monodepth2/layers/depth_decoder/decoder-11-conv.bin", + "monodepth2/layers/depth_decoder/decoder-12-conv.bin", + "monodepth2/layers/depth_decoder/decoder-13-conv.bin" +}; + +const char* output_bin[] = { + "monodepth2/debug/outputs/output-disp-0.bin", + "monodepth2/debug/outputs/output-disp-1.bin", + "monodepth2/debug/outputs/output-disp-2.bin", + "monodepth2/debug/outputs/output-disp-3.bin" +}; + +const char* input_monodepth2_bin[] = {"monodepth2/debug/input.bin","monodepth2/debug/input2.bin"}; + +int main(){ + tk::dnn::dataDim_t dim(1,3,192,640,1); + tk::dnn::Network net(dim); + tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin); + tk::dnn::Layer* encoder_bn = new tk::dnn::BatchNorm(&net,64,encoder_bn1_bin); + tk::dnn::Layer* encoder_relu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_maxpool = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); + + //layer-1 + tk::dnn::Layer* encoder_layer_1_0_conv_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[0]); + tk::dnn::Layer* encoder_layer_1_0_bn_1 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[0]); + tk::dnn::Layer* encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_0_conv_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[1]); + tk::dnn::Layer* encoder_layer_1_0_bn_2 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[1]); + tk::dnn::Layer* encoder_layer_1_0_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_maxpool); + tk::dnn::Layer* encoder_relu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_1_conv_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[2]); + tk::dnn::Layer* encoder_layer_1_1_bn_1 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[2]); + tk::dnn::Layer* encoder_relu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_1_conv_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[3]); + tk::dnn::Layer* encoder_layer_1_1_bn_2 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[3]); + tk::dnn::Layer* encoder_layer_1_1_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_relu_2); + tk::dnn::Layer* encoder_relu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //layer-2 + tk::dnn::Layer* encoder_layer_2_0_conv_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_conv_bin[0]); + tk::dnn::Layer* encoder_layer_2_0_bn_1 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[0]); + tk::dnn::Layer* encoder_relu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_0_conv_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_conv_bin[1]); + tk::dnn::Layer* encoder_layer_2_0_bn_2 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[1]); + tk::dnn::Layer* encoder_layer_2_0_route = new tk::dnn::Route(&net,&encoder_relu_4,1); + tk::dnn::Layer* encoder_layer_2_0_downsample_conv = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_conv_bin[2]); + tk::dnn::Layer* encoder_layer_2_0_downsample_bn = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[2]); + tk::dnn::Layer* encoder_layer_2_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_2_0_downsample_bn); + tk::dnn::Layer* encoder_relu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_1_conv_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_conv_bin[3]); + tk::dnn::Layer* encoder_layer_2_1_bn_1 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[3]); + tk::dnn::Layer* encoder_relu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_1_conv_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_conv_bin[4]); + tk::dnn::Layer* encoder_layer_2_1_bn_2 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[4]); + tk::dnn::Layer* encoder_layer_2_1_shortcut = new tk::dnn::Shortcut(&net,encoder_relu_6); + tk::dnn::Layer* encoder_relu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //layer-3 + tk::dnn::Layer* encoder_layer_3_0_conv_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_conv_bin[0]); + tk::dnn::Layer* encoder_layer_3_0_bn_1 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[0]); + tk::dnn::Layer* encoder_relu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_0_conv_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_conv_bin[1]); + tk::dnn::Layer* encoder_layer_3_0_bn_2 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[1]); + tk::dnn::Layer* encoder_layer_3_0_route = new tk::dnn::Route(&net,&encoder_relu_8,1); + tk::dnn::Layer* encoder_layer_3_0_downsample_conv = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_conv_bin[2]); + tk::dnn::Layer* encoder_layer_3_0_downsample_bn = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[2]); + tk::dnn::Layer* encoder_layer_3_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_3_0_bn_2); + tk::dnn::Layer* encoder_relu_10 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_1_conv_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_conv_bin[3]); + tk::dnn::Layer* encoder_layer_3_1_bn_1 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[3]); + tk::dnn::Layer* encoder_relu_11 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_1_conv_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_conv_bin[4]); + tk::dnn::Layer* encoder_layer_3_1_bn_2 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[4]); + tk::dnn::Layer* encoder_layer_3_1_shortcut = new tk::dnn::Shortcut(&net,encoder_relu_10); + tk::dnn::Layer* encoder_relu_12 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //layer-4 + tk::dnn::Layer* encoder_layer_4_0_conv_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_conv_bin[0]); + tk::dnn::Layer* encoder_layer_4_0_bn_1 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[0]); + tk::dnn::Layer* encoder_relu_13 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_0_conv_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_conv_bin[1]); + tk::dnn::Layer* encoder_layer_4_0_bn_2 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[1]); + tk::dnn::Layer* encoder_layer_4_0_route = new tk::dnn::Route(&net,&encoder_relu_12,1); + tk::dnn::Layer* encoder_layer_4_0_downsample_conv = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_conv_bin[2]); + tk::dnn::Layer* encoder_layer_4_0_downsample_bn = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[2]); + tk::dnn::Layer* encoder_layer_4_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_4_0_bn_2); + tk::dnn::Layer* encoder_relu_14 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_1_conv_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_conv_bin[3]); + tk::dnn::Layer* encoder_layer_4_1_bn_1 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[3]); + tk::dnn::Layer* encoder_relu_15 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_1_conv_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_conv_bin[4]); + tk::dnn::Layer* encoder_layer_4_1_bn_2 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[4]); + tk::dnn::Layer* encoder_layer_4_1_shortcut = new tk::dnn::Shortcut(&net,encoder_relu_14); + tk::dnn::Layer* encoder_relu_16 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + + //decoder + tk::dnn::Layer* decoder_reflection_padding_2d = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_4_0 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0]); + tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer[2] = {decoder_upsampling_2d,encoder_relu_12}; + tk::dnn::Layer* decoder_concatenate = new tk::dnn::Route(&net,concatenate_layer,2); + tk::dnn::Layer* decoder_reflection_padding_2d_1 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_4_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1]); + tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_2 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_3_0 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2]); + tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_1 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_1[2] = {decoder_upsampling_2d_1,encoder_relu_8}; + tk::dnn::Layer* decoder_concatenate_layer_1 = new tk::dnn::Route{&net,concatenate_layer_1,2}; + tk::dnn::Layer* decoder_reflection_padding_2d_3 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_3_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3]); + tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_5 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_2_0 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4]); + tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_2 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_2[2] = {decoder_upsampling_2d_2,encoder_relu_4}; + tk::dnn::Layer* decoder_concatenate_layer_2 = new tk::dnn::Route(&net,concatenate_layer_2,2); + tk::dnn::Layer* decoder_reflection_padding_2d_6 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_2_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5]); + tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_8 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_1_0 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6]); + tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_3 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_3[2] = {decoder_upsampling_2d_3,encoder_relu}; + tk::dnn::Layer* decoder_concatenate_layer_3 = new tk::dnn::Route(&net,concatenate_layer_3,2); + tk::dnn::Layer* decoder_reflection_padding_2d_9 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_1_1 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7]); + tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_11 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_0_0 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8]); + tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_4 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* decoder_reflection_padding_2d_12 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_0_1 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9]); + tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_13 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_0 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0]); + tk::dnn::Layer* disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp0->setFinal(); + + tk::dnn::Layer* route_elu_7 = new tk::dnn::Route(&net,&decoder_elu_7,1); + tk::dnn::Layer* decoder_reflection_padding_2d_10 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_1 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[1]); + tk::dnn::Layer* disp1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp1->setFinal(); + + tk::dnn::Layer* route_elu_5 = new tk::dnn::Route(&net,&decoder_elu_5,1); + tk::dnn::Layer* decoder_reflection_padding_2d_7 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_2 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[2]); + tk::dnn::Layer* disp2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp2->setFinal(); + + tk::dnn::Layer* route_elu_3 = new tk::dnn::Route(&net,&decoder_elu_3,1); + tk::dnn::Layer* decoder_reflection_padding_2d_4 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_3 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[3]); + tk::dnn::Layer* disp3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp3->setFinal(); + + dnnType *data; + dnnType *input_H; + readBinaryFile(input_monodepth2_bin[1],dim.tot(),&input_H,&data); + std::cout<<"INPUT DIMENSIONS : "<output_dim.print(); + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0;i<4;i++){ + printCenteredTitle((std::string("MONODEPTH2 CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[i]; + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + } + + + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; + +} \ No newline at end of file From 907df27e07ce2da961ec6729983f144f305619e1 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 18 Jan 2022 21:34:16 -0800 Subject: [PATCH 10/13] Fix monodepth2, add demoDepth: - Fix monodepth2 network, now works with both cuDNN and tensorRT - Substitute cuDNN ELU with tkDNN one - add DepthNN class - add demoDepth demo, now only works with monodepth2 net Signed-off-by: Micaela Verucchi Francesco Gatti --- CMakeLists.txt | 3 + demo/demo/demoDepth.cpp | 103 +++++++++++++++++++ include/tkDNN/DepthNN.h | 174 ++++++++++++++++++++++++++++++++ include/tkDNN/Layer.h | 1 - include/tkDNN/NetworkRT.h | 1 + include/tkDNN/NetworkViz.h | 2 +- src/Activation.cpp | 3 + src/NetworkRT.cpp | 76 ++++++++++++++ src/NetworkViz.cpp | 2 +- tests/monodepth2/monodepth2.cpp | 70 +++++++------ 10 files changed, 401 insertions(+), 34 deletions(-) create mode 100644 demo/demo/demoDepth.cpp create mode 100644 include/tkDNN/DepthNN.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a6d616..a109c36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,6 +229,9 @@ target_link_libraries(demoTracker tkDNN) add_executable(seg_demo demo/demo/seg_demo.cpp) target_link_libraries(seg_demo tkDNN) +add_executable(demoDepth demo/demo/demoDepth.cpp) +target_link_libraries(demoDepth tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demoDepth.cpp b/demo/demo/demoDepth.cpp new file mode 100644 index 0000000..eda04c6 --- /dev/null +++ b/demo/demo/demoDepth.cpp @@ -0,0 +1,103 @@ +#include +#include +#include /* srand, rand */ +//#include +#include + +#include "tkDNN/DepthNN.h" + +bool gRun; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + signal(SIGINT, sig_handler); + + std::string net = "monodepth2_fp32.rt"; + if(argc > 1) + net = argv[1]; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) + input = argv[2]; + bool show = true; + if(argc > 3) + show = atoi(argv[3]); + bool save = true; + if(argc > 4) + save = atoi(argv[4]); + + std::cout <<"Net settings - net: "<< net + <<"\n"; + std::cout <<"Demo settings - input: "<< input + <<", show: "<< show + <<", save: "<< save<<"\n\n"; + + tk::dnn::DepthNN depthNN; + + // create depth network + int n_batch = 1; + depthNN.init(net, n_batch); + + // open video stream + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(save) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + if(show) + cv::namedWindow("depth", cv::WINDOW_NORMAL); + + cv::Mat frame; + std::vector batch_frame; + std::vector batch_dnn_input; + + // start detection loop + gRun = true; + while(gRun) { + batch_dnn_input.clear(); + batch_frame.clear(); + + //read frame + cap >> frame; + if(!frame.data) + break; + batch_frame.push_back(frame); + batch_dnn_input.push_back(frame.clone()); + + //inference + depthNN.update(batch_dnn_input, 1); + if(show){ + cv::imshow("depth", depthNN.depthMats[0]); + cv::waitKey(1); + + } + } + + std::cout<<"detection end\n"; + + double mean = 0; + std::cout< +#include +#include +#ifdef __linux__ +#include +#endif + +#include + +#include +#include +#include + +#include "tkDNN/utils.h" +#include "tkDNN/tkdnn.h" + +#include "NetworkViz.h" + + +namespace tk { namespace dnn { + +class DepthNN { + + public: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_h; + dnnType *input_d; + float* depth_h; + + int nBatches = 1; + + cv::Mat bgr[3]; + cv::Mat imagePreproc; + + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector> depths; + std::vector depthMats; + + DepthNN() {}; + ~DepthNN(){}; + + /** + * Method used to initialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file of the NN. + * @param n_batches maximum number of batches to use in inference + * @return true if everything is correct, false otherwise. + */ + void init(const std::string& tensor_path, const int n_batches=1){ + //create net + + std::cout<<(tensor_path).c_str()<<"\n"; + nBatches = n_batches; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); + + //allocate memory for NN input + checkCuda(cudaMallocHost(&input_h, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + + //allocate memory for NN output + depthMats.resize(nBatches); + depths.resize(nBatches); + for(int i=0; i< depths.size();++i) + depths[i].resize(netRT->buffersDIM[1].tot()); + + depth_h = (float *)malloc(netRT->buffersDIM[1].tot() * sizeof(float)); + + } + + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + void preprocess(cv::Mat &frame, const int bi=0) { + //resize image, remove mean, divide by std + cv::Mat frame_nomean; + resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); + frame.convertTo(frame_nomean, CV_32FC3); + frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 255.0, 0); + + //copy image into tensor and copy it into GPU + cv::split(imagePreproc, bgr); + for (int i = 0; i < netRT->input_dim.c; i++){ + int idx = i * imagePreproc.rows * imagePreproc.cols; + int ch = netRT->input_dim.c-1 -i; + memcpy((void *)&input_h[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType)); + } + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input_h + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + } + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + void postprocess(const int bi=0) { + + dnnType *rt_out[1]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + checkCuda(cudaMemcpy(depth_h, rt_out[0], netRT->buffersDIM[1].tot()* sizeof(float), cudaMemcpyDeviceToHost)); + memcpy(&depths[bi][0], &depth_h[0], netRT->buffersDIM[1].tot()* sizeof(float)); + + // cv::Mat d(netRT->buffersDIM[1].h, netRT->buffersDIM[1].w, CV_8UC1, depth_h); + // depthMats[bi] = d.clone(); + + cv::Mat depth_mat = vizData2Mat(rt_out[0], netRT->buffersDIM[1], netRT->buffersDIM[1].h, netRT->buffersDIM[1].w); + // cv::Mat depth_mat = vizData2Mat((dnnType *)netRT->buffersRT[0], netRT->buffersDIM[0], netRT->buffersDIM[0].h, netRT->buffersDIM[0].w); + depthMats[bi] = depth_mat.clone(); + + } + + /** + * This method performs the inference of the NN. + * + * @param frames frames to build the embedding from. + * @param cur_batches number of batches to use in inference + */ + void update(std::vector& frames, const int cur_batches=1){ + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT feature extraction ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + { + TKDNN_TSTART + for(int bi=0; bi 5 && NV_TENSORRT_MAJOR < 8 bool serialize(const char *filename); diff --git a/include/tkDNN/NetworkViz.h b/include/tkDNN/NetworkViz.h index 2cf8009..ffdf361 100644 --- a/include/tkDNN/NetworkViz.h +++ b/include/tkDNN/NetworkViz.h @@ -6,7 +6,7 @@ namespace tk { namespace dnn { cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19); -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19); +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=0); cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000); }} diff --git a/src/Activation.cpp b/src/Activation.cpp index 0b113a7..947b019 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -56,6 +56,9 @@ dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { else if(act_mode == ACTIVATION_LOGISTIC) { activationLOGISTICForward(srcData, dstData, dim.tot()); + } else if(act_mode == ACTIVATION_ELU) { + activationELUForward(srcData, dstData, dim.tot()); + } else { dnnType alpha = dnnType(1); dnnType beta = dnnType(0); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index b83de23..9b5f14c 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -279,6 +279,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (Padding*) l); if(type == LAYER_BATCHNORM) return convert_layer(input,(BatchNorm*) l); + if(type == LAYER_MULADD) + return convert_layer(input,(MulAdd*) l); std::cout<getLayerName()<<"\n"; FatalError("Layer not implemented in tensorRT"); @@ -441,6 +443,80 @@ ILayer* NetworkRT::convert_layer(ITensor *input,BatchNorm *l){ } +ILayer* NetworkRT::convert_layer(ITensor *input,MulAdd *l){ + + void *power_b, *shift_b, *scales_b; + int size = l->input_dim.tot(); + + power_b = new dnnType[size]; + shift_b = new dnnType[size]; + scales_b = new dnnType[size]; + + for(int i=0; iadd; + ((dnnType*) scales_b)[i] = l->mul; + } + + if(dtRT == DataType::kHALF) { + + __half *power16_h = nullptr, *power16_d = nullptr; + __half *scales16_h = nullptr, *scales16_d = nullptr; + __half *shift16_h = nullptr, *shift16_d = nullptr; + + dnnType * power_d = nullptr; + dnnType * scales_d = nullptr; + dnnType * shift_d = nullptr; + + cudaMalloc(&power_d, size*sizeof(dnnType)); + cudaMemcpy(power_d, power_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + cudaMalloc(&shift_d, size*sizeof(dnnType)); + cudaMemcpy(shift_d, shift_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + cudaMalloc(&scales_d, size*sizeof(dnnType)); + cudaMemcpy(scales_d, scales_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + //convert to fp16 + power16_h = new __half[size]; + cudaMalloc(&power16_d, size*sizeof(__half)); + float2half(power_d, power16_d, size); + cudaMemcpy(power16_h, power16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + shift16_h = new __half[size]; + cudaMalloc(&shift16_d, size*sizeof(__half)); + float2half(shift_d, shift16_d, size); + cudaMemcpy(shift16_h, shift16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + scales16_h = new __half[size]; + cudaMalloc(&scales16_d, size*sizeof(__half)); + float2half(scales_d, scales16_d, size); + cudaMemcpy(scales16_h, scales16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + power_b = power16_h; + shift_b = shift16_h; + scales_b = scales16_h; + + + cudaFree(power16_d); + cudaFree(shift16_d); + cudaFree(scales16_d); + + cudaFree(power_d); + cudaFree(shift_d); + cudaFree(scales_d); + } + + Weights power{dtRT, power_b, size}; + Weights shift{dtRT, shift_b, size}; + Weights scale{dtRT, scales_b, size}; + IScaleLayer *lRT = networkRT->addScale(*input, ScaleMode::kELEMENTWISE, + shift, scale, power); + checkNULL(lRT); + return lRT; +} + + ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { // std::cout<<"convert Pooling\n"; diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp index b6c95d9..8dbb20e 100644 --- a/src/NetworkViz.cpp +++ b/src/NetworkViz.cpp @@ -383,7 +383,7 @@ cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) { default: // expand your range to 0..255. Similar to histEq(); map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); - applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET); + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_PARULA); } return falseColorsMap; } diff --git a/tests/monodepth2/monodepth2.cpp b/tests/monodepth2/monodepth2.cpp index c763baf..4e0767a 100644 --- a/tests/monodepth2/monodepth2.cpp +++ b/tests/monodepth2/monodepth2.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "tkDNN/NetworkViz.h" const char* encoder_conv1_bin = "monodepth2/layers/encoder/encoder-conv1.bin"; const char* encoder_layer1_bin[] = { @@ -72,108 +73,111 @@ int main(){ tk::dnn::dataDim_t dim(1,3,192,640,1); tk::dnn::Network net(dim); - tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin,true,false,1,true); + + tk::dnn::Layer* muladd_sub = new tk::dnn::MulAdd(&net, 1.0f, -0.45f); + tk::dnn::Layer* muladd_mul = new tk::dnn::MulAdd(&net, 1.0f / 0.225f, 0.0f); + tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin,true); tk::dnn::Layer* encoder_relu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); tk::dnn::Layer* encoder_maxpool = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); //layer-1 - tk::dnn::Layer* encoder_layer_1_0_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_0_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[0],true); tk::dnn::Layer* encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_0_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_0_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[1],true); tk::dnn::Layer* encoder_layer_1_0_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_maxpool); tk::dnn::Layer* encoder_relu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_1_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_1_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[2],true); tk::dnn::Layer* encoder_relu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_1_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_1_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[3],true); tk::dnn::Layer* encoder_layer_1_1_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_relu_2); tk::dnn::Layer* encoder_relu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //layer-2 - tk::dnn::Layer* encoder_layer_2_0_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true); tk::dnn::Layer* encoder_relu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_0_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true); tk::dnn::Layer* encoder_layer_2_0_route = new tk::dnn::Route(&net,&encoder_relu_4,1); - tk::dnn::Layer* encoder_layer_2_0_downsample_convbn = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_downsample_convbn = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2],true); tk::dnn::Layer* encoder_layer_2_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_2_0_convbn_2); tk::dnn::Layer* encoder_relu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_1_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_1_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true); tk::dnn::Layer* encoder_relu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_1_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_1_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true); tk::dnn::Layer* encoder_layer_2_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_6); tk::dnn::Layer* encoder_relu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //layer-3 - tk::dnn::Layer* encoder_layer_3_0_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true); tk::dnn::Layer* encoder_relu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_0_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true); tk::dnn::Layer* encoder_layer_3_0_route = new tk::dnn::Route(&net,&encoder_relu_8,1); - tk::dnn::Layer* encoder_layer_3_0_downsample_convbn = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_downsample_convbn = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2],true); tk::dnn::Layer* encoder_layer_3_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_3_0_convbn_2); tk::dnn::Layer* encoder_relu_10 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_1_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_1_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true); tk::dnn::Layer* encoder_relu_11 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_1_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_1_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true); tk::dnn::Layer* encoder_layer_3_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_10); tk::dnn::Layer* encoder_relu_12 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //layer-4 - tk::dnn::Layer* encoder_layer_4_0_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true); tk::dnn::Layer* encoder_relu_13 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_0_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true); tk::dnn::Layer* encoder_layer_4_0_route = new tk::dnn::Route(&net,&encoder_relu_12,1); - tk::dnn::Layer* encoder_layer_4_0_downsample_convbn = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_downsample_convbn = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2],true); tk::dnn::Layer* encoder_layer_4_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_4_0_convbn_2); tk::dnn::Layer* encoder_relu_14 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_1_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_1_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true); tk::dnn::Layer* encoder_relu_15 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_1_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_1_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true); tk::dnn::Layer* encoder_layer_4_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_14); tk::dnn::Layer* encoder_relu_16 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //decoder tk::dnn::Layer* decoder_reflection_padding_2d = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_4_0 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0]); - tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer[2] = {decoder_upsampling_2d,encoder_relu_12}; tk::dnn::Layer* decoder_concatenate = new tk::dnn::Route(&net,concatenate_layer,2); tk::dnn::Layer* decoder_reflection_padding_2d_1 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_4_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1]); - tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_2 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_3_0 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2]); - tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_1 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer_1[2] = {decoder_upsampling_2d_1,encoder_relu_8}; tk::dnn::Layer* decoder_concatenate_layer_1 = new tk::dnn::Route{&net,concatenate_layer_1,2}; tk::dnn::Layer* decoder_reflection_padding_2d_3 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_3_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3]); - tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_5 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_2_0 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4]); - tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_2 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer_2[2] = {decoder_upsampling_2d_2,encoder_relu_4}; tk::dnn::Layer* decoder_concatenate_layer_2 = new tk::dnn::Route(&net,concatenate_layer_2,2); tk::dnn::Layer* decoder_reflection_padding_2d_6 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_2_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5]); - tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_8 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_1_0 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6]); - tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_3 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer_3[2] = {decoder_upsampling_2d_3,encoder_relu}; tk::dnn::Layer* decoder_concatenate_layer_3 = new tk::dnn::Route(&net,concatenate_layer_3,2); tk::dnn::Layer* decoder_reflection_padding_2d_9 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_1_1 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7]); - tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_11 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_0_0 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8]); - tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_4 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* decoder_reflection_padding_2d_12 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_0_1 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9]); - tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_13 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_dispconv_0 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0]); tk::dnn::Layer* disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); @@ -240,13 +244,17 @@ int main(){ dnnType *cudnn_out, *rt_out; cudnn_out = outs[i]->dstData; - rt_out = (dnnType *)netRT.buffersRT[i]; + rt_out = (dnnType *)netRT.buffersRT[1+i]; std::cout<<"CUDNN vs correct"; ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; std::cout<<"TRT vs correct"; ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; std::cout<<"CUDNN vs TRT "; ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + + cv::Mat depth_mat = vizData2Mat(outs[i]->dstData, outs[i]->output_dim, outs[i]->output_dim.h, outs[i]->output_dim.w); + cv::imshow("depth", depth_mat); + cv::waitKey(0); } From decd73d29889859110da3ed7a32a0a4b532605f2 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Wed, 19 Jan 2022 10:22:06 -0800 Subject: [PATCH 11/13] Fix saving result video in demoDepth, add automatic download for monodepth2 weights Signed-off-by: Micaela Verucchi --- demo/demo/demoDepth.cpp | 9 ++++++--- include/tkDNN/DepthNN.h | 6 ++++++ tests/monodepth2/monodepth2.cpp | 6 ++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/demo/demo/demoDepth.cpp b/demo/demo/demoDepth.cpp index eda04c6..41f5dbf 100644 --- a/demo/demo/demoDepth.cpp +++ b/demo/demo/demoDepth.cpp @@ -55,9 +55,9 @@ int main(int argc, char *argv[]) { cv::VideoWriter resultVideo; if(save) { - int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); - int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); - resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + int w = depthNN.output_w; + int h = depthNN.output_h; + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','J','P','G'), 30, cv::Size(w, h)); } if(show) @@ -87,6 +87,9 @@ int main(int argc, char *argv[]) { cv::waitKey(1); } + + if(save) + resultVideo << depthNN.depthMats[0]; } std::cout<<"detection end\n"; diff --git a/include/tkDNN/DepthNN.h b/include/tkDNN/DepthNN.h index 0d89ac0..4d34239 100644 --- a/include/tkDNN/DepthNN.h +++ b/include/tkDNN/DepthNN.h @@ -30,6 +30,9 @@ class DepthNN { dnnType *input_d; float* depth_h; + int output_w; + int output_h; + int nBatches = 1; cv::Mat bgr[3]; @@ -68,6 +71,9 @@ class DepthNN { depths[i].resize(netRT->buffersDIM[1].tot()); depth_h = (float *)malloc(netRT->buffersDIM[1].tot() * sizeof(float)); + + output_h = netRT->buffersDIM[1].h; + output_w = netRT->buffersDIM[1].w; } diff --git a/tests/monodepth2/monodepth2.cpp b/tests/monodepth2/monodepth2.cpp index 4e0767a..8618539 100644 --- a/tests/monodepth2/monodepth2.cpp +++ b/tests/monodepth2/monodepth2.cpp @@ -66,11 +66,13 @@ const char* output_bin[] = { "monodepth2/debug/outputs/output-disp-3.bin" }; -const char* input_monodepth2_bin[] = {"monodepth2/debug/input.bin","monodepth2/debug/input2.bin"}; +const char* input_bin = "monodepth2/debug/input.bin"; int main(){ + downloadWeightsifDoNotExist(input_bin, "monodepth2", "https://cloud.hipert.unimore.it/s/iYw9QwgP6CsqxLR/download"); + tk::dnn::dataDim_t dim(1,3,192,640,1); tk::dnn::Network net(dim); @@ -204,7 +206,7 @@ int main(){ dnnType *data; dnnType *input_H; - readBinaryFile(input_monodepth2_bin[0],dim.tot(),&input_H,&data); + readBinaryFile(input_bin, dim.tot(),&input_H,&data); std::cout<<"INPUT DIMENSIONS : "< Date: Thu, 20 Jan 2022 00:52:22 +0530 Subject: [PATCH 12/13] -Replaced ISliceLayer based paddings(reflection,constant and zero) with the IPluginV2 version for TensorRT >= 8.2.0 -Fixed demo build issue on Windows Signed-off-by: perseusdg --- demo/demo/demo.cpp | 6 +++--- src/NetworkRT.cpp | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 6de7214..f46ca80 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -46,9 +46,9 @@ int main(int argc, char *argv[]) { std::string cfgPath = YAMLgetConf(conf,"cfg_input", "../tests/darknet/cfg/yolo4tiny.cfg"); std::string namePath = YAMLgetConf(conf,"name_input","../tests/darknet/names/coco.names"); #elif _WIN32 - std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); - std::string cfgPath = YAMLgetConf(conf,"cfg_win_input","..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg"); - std::string namePath = YAMLgetConf(conf,"name_win_input","..\\..\\..\\tests\\darknet\\names\\coco.names"); + std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); + std::string cfgPath = YAMLgetConf(conf,"cfg_win_input","..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg"); + std::string namePath = YAMLgetConf(conf,"name_win_input","..\\..\\..\\tests\\darknet\\names\\coco.names"); #endif if(!fileExist(input.c_str())) FatalError("The given input video does not exist."); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 9b5f14c..71e45ed 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -571,7 +571,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ float(NV_TENSORRT_MINOR)/10 + float(NV_TENSORRT_PATCH)/100; -#if ((NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR >= 2) || NV_TENSORRT_MAJOR > 8) +/*#if ((NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR >= 2) || NV_TENSORRT_MAJOR > 8) auto *lRT = networkRT->addSlice(*input,Dims3{0,0,0},Dims3{l->output_dim.c,l->output_dim.h,l->output_dim.w},Dims3{0,0,0}); if(l->padding_mode == PADDING_MODE_REFLECTION){ lRT->setMode(SliceMode::kREFLECT); @@ -581,8 +581,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ } checkNULL(lRT); return lRT; -#else - //todo add PADDING_MODE_CONSTANT AND PADDING_MODE_ZERO for tensorrt versions < 8.2 +#else*/ + //todo use ISliceLayer for padding,currently using ISliceLayer for reflection padding generates an error with monodepth2 if(l->padding_mode == PADDING_MODE_REFLECTION){ auto creator = getPluginRegistry()->getPluginCreator("ReflectionPaddingRT_tkDNN","1"); std::vector mPluginAttributes; @@ -623,8 +623,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ } return nullptr; - -#endif + } ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { From 3e86671c5073414a43079f2ce5002d709eba1e7b Mon Sep 17 00:00:00 2001 From: Harshvardhan Chandirasekar <43143075+perseusdg@users.noreply.github.com> Date: Mon, 24 Jan 2022 21:41:59 +0530 Subject: [PATCH 13/13] Moved batchnorm and test_monodepth2_new_format layer to dev Signed-off-by: perseusdg --- CMakeLists.txt | 3 - include/tkDNN/Layer.h | 78 ------ src/BatchNorm.cpp | 67 ----- src/LayerBNWgs.cpp | 88 ------ tests/monodepth2/monodepth2_new_format.cpp | 306 --------------------- 5 files changed, 542 deletions(-) delete mode 100644 src/BatchNorm.cpp delete mode 100644 src/LayerBNWgs.cpp delete mode 100644 tests/monodepth2/monodepth2_new_format.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a109c36..7392fd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -207,9 +207,6 @@ target_link_libraries(test_shelfnet_mapillary tkDNN) add_executable(test_monodepth2 tests/monodepth2/monodepth2.cpp) target_link_libraries(test_monodepth2 tkDNN) -add_executable(test_monodepth2_new_format tests/monodepth2/monodepth2_new_format.cpp) -target_link_libraries(test_monodepth2_new_format tkDNN) - # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index 1e22216..daa27e5 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -33,7 +33,6 @@ enum layerType_t { LAYER_REGION, LAYER_YOLO, LAYER_PADDING, - LAYER_BATCHNORM }; #define TKDNN_BN_MIN_EPSILON 1e-5 @@ -90,7 +89,6 @@ public: case LAYER_REGION: return "Region"; case LAYER_YOLO: return "Yolo"; case LAYER_PADDING: return "Padding"; - case LAYER_BATCHNORM: return "BatchNorm"; default: return "unknown"; } } @@ -182,65 +180,6 @@ public: }; -class LayerBNWgs : public Layer { -public: - LayerBNWgs(Network* net, int input, int output, std::string fname_weights); - ~LayerBNWgs(); - - int inputs, outputs; - std::string weights_path; - - dnnType* bias_h, * bias_d; - dnnType* power_h = nullptr; - dnnType* scales_h = nullptr, * scales_d = nullptr; - dnnType* mean_h = nullptr, * mean_d = nullptr; - dnnType* variance_h = nullptr, * variance_d = nullptr; - - __half* bias16_h = nullptr, * bias16_d = nullptr; - __half* power16_h = nullptr, * power16_d = nullptr; - __half* scales16_h = nullptr, * scales16_d = nullptr; - __half* mean16_h = nullptr, * mean16_d = nullptr; - __half* variance16_h = nullptr, * variance16_d = nullptr; - - - void releaseHost(bool release32 = true, bool release16 = true) { - if (release32) { - if (bias_h != nullptr) { delete[] bias_h; bias_h = nullptr; } - if (scales_h != nullptr) { delete[] scales_h; scales_h = nullptr; } - if (mean_h != nullptr) { delete[] mean_h; mean_h = nullptr; } - if (variance_h != nullptr) { delete[] variance_h; variance_h = nullptr; } - if (power_h != nullptr) { delete[] power_h; power_h = nullptr; } - } - if (net->fp16 && release16) { - if (bias16_h != nullptr) { delete[] bias16_h; bias16_h = nullptr; } - if (scales16_h != nullptr) { delete[] scales16_h; scales16_h = nullptr; } - if (mean16_h != nullptr) { delete[] mean16_h; mean16_h = nullptr; } - if (variance16_h != nullptr) { delete[] variance16_h; variance16_h = nullptr; } - if (power16_h != nullptr) { delete[] power16_h; power16_h = nullptr; } - - } - } - - void releaseDevice(bool release32 = true, bool release16 = true) { - if (release32) { - if (bias_d != nullptr) { cudaFree(bias_d); bias_d = nullptr; } - if (scales_d != nullptr) { cudaFree(scales_d); scales_d = nullptr; } - if (mean_d != nullptr) { cudaFree(mean_d); mean_d = nullptr; } - if (variance_d != nullptr) { cudaFree(variance_d); variance_d = nullptr; } - } - if (net->fp16 && release16) { - if (bias16_d != nullptr) { cudaFree(bias16_d); bias16_d = nullptr; } - if (scales16_d != nullptr) { cudaFree(scales16_d); scales16_d = nullptr; } - if (mean16_d != nullptr) { cudaFree(mean16_d); mean16_d = nullptr; } - if (variance16_d != nullptr) { cudaFree(variance16_d); variance16_d = nullptr; } - if (power16_d != nullptr) { cudaFree(power16_d); power16_d = nullptr; } - } - } - - - -}; - /** Input layer (it doesn't need weights) */ @@ -606,24 +545,7 @@ public: }; -class BatchNorm : public LayerBNWgs { -public: - BatchNorm(Network *net,int output,std::string fname_weights); - virtual ~BatchNorm(); - virtual layerType_t getLayerType(){return LAYER_BATCHNORM;}; - virtual dnnType* infer(dataDim_t& dim,dnnType* srcData); - std::string weights_bin; -protected: - cudnnFilterDescriptor_t filterDesc; - cudnnConvolutionFwdAlgoPerf_t algo; - cudnnConvolutionBwdDataAlgoPerf_t bwAlgo; - cudnnTensorDescriptor_t biasTensorDesc; - void initCUDNN(); - void inferCUDNN(dnnType* srcData); - void* workSpace; - size_t ws_sizeInBytes; -}; /** Softmax layer */ diff --git a/src/BatchNorm.cpp b/src/BatchNorm.cpp deleted file mode 100644 index b406011..0000000 --- a/src/BatchNorm.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include - -#include "Layer.h" - -namespace tk { namespace dnn { - void BatchNorm::initCUDNN(){ - cudnnTensorDescriptor_t srcTensor = srcTensorDesc; - cudnnTensorDescriptor_t dstTensor = dstTensorDesc; - dataDim_t idim,odim; - idim = input_dim; - odim = output_dim; - - checkCUDNN( cudnnSetTensor4dDescriptor(srcTensor, - net->tensorFormat, net->dataType, idim.n, idim.c, idim.h, idim.w) ); - - checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) ); - - checkCUDNN( cudnnSetTensor4dDescriptor(dstTensor, - net->tensorFormat, net->dataType, odim.n, odim.c, odim.h, odim.w) ); - - checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc, - net->tensorFormat, net->dataType, - 1, output_dim.c, 1, 1) ); - - - } - - void BatchNorm::inferCUDNN(float *srcData){ - dnnType alpha = dnnType(1); - dnnType beta = dnnType(0); - - alpha = dnnType(1); - beta = dnnType(1); - checkCUDNN( cudnnAddTensor(net->cudnnHandle, - &alpha, biasTensorDesc, bias_d, - &beta, dstTensorDesc, dstData) ); - alpha = dnnType(1); - beta = dnnType(0); - checkCUDNN( cudnnBatchNormalizationForwardInference(net->cudnnHandle, - CUDNN_BATCHNORM_SPATIAL, &alpha, &beta, - dstTensorDesc, dstData, dstTensorDesc, - dstData, biasTensorDesc, //same tensor descriptor as bias - scales_d, bias_d, mean_d, variance_d, - TKDNN_BN_MIN_EPSILON) ); - } - - BatchNorm::BatchNorm(Network *net,int output,std::string fname_weights) : - LayerBNWgs(net,net->getOutputDim().c,output,fname_weights){ - output_dim = input_dim; - initCUDNN(); - checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); - - } - - dnnType* BatchNorm::infer(dataDim_t &dim,dnnType* srcData){ - inferCUDNN(srcData); - - dim = output_dim; - return dstData; - } - - BatchNorm::~BatchNorm(){ - checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) ); - checkCuda( cudaFree(dstData) ); - } - -}} \ No newline at end of file diff --git a/src/LayerBNWgs.cpp b/src/LayerBNWgs.cpp deleted file mode 100644 index 7414ef3..0000000 --- a/src/LayerBNWgs.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include - -#include "Layer.h" -#include "kernels.h" - -namespace tk { namespace dnn { - LayerBNWgs::LayerBNWgs(Network* net, int input, int output, std::string fname_weights) : Layer(net) { - this->inputs = inputs; - this->outputs = output; - this->weights_path = fname_weights; - - std::cout << "Reading BatchNorm O = " << outputs << std::endl; - int seek = 0; - readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek); - seek += outputs; - readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek); - seek += outputs; - readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek); - seek += outputs; - readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek); - seek += outputs; - - float eps = TKDNN_BN_MIN_EPSILON; - - power_h = new dnnType[outputs]; - for (int i = 0; i < outputs; i++) power_h[i] = 1.0f; - - for (int i = 0; i < outputs; i++) - mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]); - - for (int i = 0; i < outputs; i++) - variance_h[i] = 1.0f / sqrt(eps + variance_h[i]); - - if (!net->fp16) - return; - - int b_size = outputs; - bias16_h = new __half[b_size]; - cudaMalloc(&bias16_d, b_size * sizeof(__half)); - float2half(bias_d, bias16_d, b_size); - cudaMemcpy(bias16_h, bias16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); - - power16_h = new __half[b_size]; - mean16_h = new __half[b_size]; - variance16_h = new __half[b_size]; - scales16_h = new __half[b_size]; - - cudaMalloc(&power16_d, b_size * sizeof(__half)); - cudaMalloc(&mean16_d, b_size * sizeof(__half)); - cudaMalloc(&variance16_d, b_size * sizeof(__half)); - cudaMalloc(&scales16_d, b_size * sizeof(__half)); - - //temporary buffers - float* tmp_d; - cudaMalloc(&tmp_d, b_size * sizeof(float)); - - //init power array of ones - cudaMemcpy(tmp_d, power_h, b_size * sizeof(float), cudaMemcpyHostToDevice); - float2half(tmp_d, power16_d, b_size); - cudaMemcpy(power16_h, power16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); - - //mean array - cudaMemcpy(tmp_d, mean_h, b_size * sizeof(float), cudaMemcpyHostToDevice); - float2half(tmp_d, mean16_d, b_size); - cudaMemcpy(mean16_h, mean16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); - - //convert variance - - cudaMemcpy(tmp_d, variance_h, b_size * sizeof(float), cudaMemcpyHostToDevice); - float2half(tmp_d, variance16_d, b_size); - cudaMemcpy(variance16_h, variance16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); - - //convert scales - float2half(scales_d, scales16_d, b_size); - cudaMemcpy(scales16_h, scales16_d, b_size * sizeof(__half), cudaMemcpyDeviceToHost); - - cudaFree(tmp_d); - - - } - - LayerBNWgs::~LayerBNWgs() { - releaseHost(); - releaseDevice(); - } - -} } \ No newline at end of file diff --git a/tests/monodepth2/monodepth2_new_format.cpp b/tests/monodepth2/monodepth2_new_format.cpp deleted file mode 100644 index 313978c..0000000 --- a/tests/monodepth2/monodepth2_new_format.cpp +++ /dev/null @@ -1,306 +0,0 @@ -#include -#include -#include -#include -#include - -const char* encoder_conv1_bin = "monodepth2/layers/encoder/encoder-conv1.bin"; -const char* encoder_bn1_bin = "monodepth2/layers/encoder/encoder-bn1.bin"; - -const char* encoder_layer1_conv_bin[] = { - "monodepth2/layers/encoder/encoder-layer1-0-conv1.bin", - "monodepth2/layers/encoder/encoder-layer1-0-conv2.bin", - "monodepth2/layers/encoder/encoder-layer1-1-conv1.bin", - "monodepth2/layers/encoder/encoder-layer1-1-conv2.bin", -}; - -const char* encoder_layer1_bn_bin[] = { - "monodepth2/layers/encoder/encoder-layer1-0-bn1.bin", - "monodepth2/layers/encoder/encoder-layer1-0-bn2.bin", - "monodepth2/layers/encoder/encoder-layer1-1-bn1.bin", - "monodepth2/layers/encoder/encoder-layer1-1-bn2.bin", -}; - - - -const char* encoder_layer2_conv_bin[] = { - "monodepth2/layers/encoder/encoder-layer2-0-conv1.bin", - "monodepth2/layers/encoder/encoder-layer2-0-conv2.bin", - "monodepth2/layers/encoder/encoder-layer2-0-downsample-0.bin", - "monodepth2/layers/encoder/encoder-layer2-1-conv1.bin", - "monodepth2/layers/encoder/encoder-layer2-1-conv2.bin" -}; - -const char* encoder_layer2_bn_bin[] = { - "monodepth2/layers/encoder/encoder-layer2-0-bn1.bin", - "monodepth2/layers/encoder/encoder-layer2-0-bn2.bin", - "monodepth2/layers/encoder/encoder-layer2-0-downsample-1.bin", - "monodepth2/layers/encoder/encoder-layer2-1-bn1.bin", - "monodepth2/layers/encoder/encoder-layer2-1-bn2.bin" -}; - -const char* encoder_layer3_conv_bin[]={ - "monodepth2/layers/encoder/encoder-layer3-0-conv1.bin", - "monodepth2/layers/encoder/encoder-layer3-0-conv2.bin", - "monodepth2/layers/encoder/encoder-layer3-0-downsample-0.bin", - "monodepth2/layers/encoder/encoder-layer3-1-conv1.bin", - "monodepth2/layers/encoder/encoder-layer3-1-conv2.bin" -}; - -const char* encoder_layer3_bn_bin[]={ - "monodepth2/layers/encoder/encoder-layer3-0-bn1.bin", - "monodepth2/layers/encoder/encoder-layer3-0-bn2.bin", - "monodepth2/layers/encoder/encoder-layer3-0-downsample-1.bin", - "monodepth2/layers/encoder/encoder-layer3-1-bn1.bin", - "monodepth2/layers/encoder/encoder-layer3-1-bn2.bin" -}; - -const char* encoder_layer4_conv_bin[] = { - "monodepth2/layers/encoder/encoder-layer4-0-conv1.bin", - "monodepth2/layers/encoder/encoder-layer4-0-conv2.bin", - "monodepth2/layers/encoder/encoder-layer4-0-downsample-0.bin", - "monodepth2/layers/encoder/encoder-layer4-1-conv1.bin", - "monodepth2/layers/encoder/encoder-layer4-1-conv2.bin" -}; - -const char* encoder_layer4_bn_bin[] = { - "monodepth2/layers/encoder/encoder-layer4-0-bn1.bin", - "monodepth2/layers/encoder/encoder-layer4-0-bn2.bin", - "monodepth2/layers/encoder/encoder-layer4-0-downsample-1.bin", - "monodepth2/layers/encoder/encoder-layer4-1-bn1.bin", - "monodepth2/layers/encoder/encoder-layer4-1-bn2.bin" -}; - -const char* decoder_layer_bin[] = { - "monodepth2/layers/depth_decoder/decoder-0-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-1-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-2-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-3-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-4-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-5-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-6-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-7-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-8-conv-conv.bin", - "monodepth2/layers/depth_decoder/decoder-9-conv-conv.bin" -}; - -const char* decoder_dispconv_layer_bin[] = { - "monodepth2/layers/depth_decoder/decoder-10-conv.bin", - "monodepth2/layers/depth_decoder/decoder-11-conv.bin", - "monodepth2/layers/depth_decoder/decoder-12-conv.bin", - "monodepth2/layers/depth_decoder/decoder-13-conv.bin" -}; - -const char* output_bin[] = { - "monodepth2/debug/outputs/output-disp-0.bin", - "monodepth2/debug/outputs/output-disp-1.bin", - "monodepth2/debug/outputs/output-disp-2.bin", - "monodepth2/debug/outputs/output-disp-3.bin" -}; - -const char* input_monodepth2_bin[] = {"monodepth2/debug/input.bin","monodepth2/debug/input2.bin"}; - -int main(){ - tk::dnn::dataDim_t dim(1,3,192,640,1); - tk::dnn::Network net(dim); - tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin); - tk::dnn::Layer* encoder_bn = new tk::dnn::BatchNorm(&net,64,encoder_bn1_bin); - tk::dnn::Layer* encoder_relu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_maxpool = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); - - //layer-1 - tk::dnn::Layer* encoder_layer_1_0_conv_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[0]); - tk::dnn::Layer* encoder_layer_1_0_bn_1 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[0]); - tk::dnn::Layer* encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_0_conv_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[1]); - tk::dnn::Layer* encoder_layer_1_0_bn_2 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[1]); - tk::dnn::Layer* encoder_layer_1_0_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_maxpool); - tk::dnn::Layer* encoder_relu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_1_conv_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[2]); - tk::dnn::Layer* encoder_layer_1_1_bn_1 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[2]); - tk::dnn::Layer* encoder_relu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_1_conv_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_conv_bin[3]); - tk::dnn::Layer* encoder_layer_1_1_bn_2 = new tk::dnn::BatchNorm(&net,64,encoder_layer1_bn_bin[3]); - tk::dnn::Layer* encoder_layer_1_1_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_relu_2); - tk::dnn::Layer* encoder_relu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - - //layer-2 - tk::dnn::Layer* encoder_layer_2_0_conv_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_conv_bin[0]); - tk::dnn::Layer* encoder_layer_2_0_bn_1 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[0]); - tk::dnn::Layer* encoder_relu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_0_conv_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_conv_bin[1]); - tk::dnn::Layer* encoder_layer_2_0_bn_2 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[1]); - tk::dnn::Layer* encoder_layer_2_0_route = new tk::dnn::Route(&net,&encoder_relu_4,1); - tk::dnn::Layer* encoder_layer_2_0_downsample_conv = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_conv_bin[2]); - tk::dnn::Layer* encoder_layer_2_0_downsample_bn = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[2]); - tk::dnn::Layer* encoder_layer_2_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_2_0_downsample_bn); - tk::dnn::Layer* encoder_relu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_1_conv_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_conv_bin[3]); - tk::dnn::Layer* encoder_layer_2_1_bn_1 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[3]); - tk::dnn::Layer* encoder_relu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_1_conv_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_conv_bin[4]); - tk::dnn::Layer* encoder_layer_2_1_bn_2 = new tk::dnn::BatchNorm(&net,128,encoder_layer2_bn_bin[4]); - tk::dnn::Layer* encoder_layer_2_1_shortcut = new tk::dnn::Shortcut(&net,encoder_relu_6); - tk::dnn::Layer* encoder_relu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - - //layer-3 - tk::dnn::Layer* encoder_layer_3_0_conv_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_conv_bin[0]); - tk::dnn::Layer* encoder_layer_3_0_bn_1 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[0]); - tk::dnn::Layer* encoder_relu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_0_conv_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_conv_bin[1]); - tk::dnn::Layer* encoder_layer_3_0_bn_2 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[1]); - tk::dnn::Layer* encoder_layer_3_0_route = new tk::dnn::Route(&net,&encoder_relu_8,1); - tk::dnn::Layer* encoder_layer_3_0_downsample_conv = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_conv_bin[2]); - tk::dnn::Layer* encoder_layer_3_0_downsample_bn = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[2]); - tk::dnn::Layer* encoder_layer_3_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_3_0_bn_2); - tk::dnn::Layer* encoder_relu_10 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_1_conv_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_conv_bin[3]); - tk::dnn::Layer* encoder_layer_3_1_bn_1 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[3]); - tk::dnn::Layer* encoder_relu_11 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_1_conv_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_conv_bin[4]); - tk::dnn::Layer* encoder_layer_3_1_bn_2 = new tk::dnn::BatchNorm(&net,256,encoder_layer3_bn_bin[4]); - tk::dnn::Layer* encoder_layer_3_1_shortcut = new tk::dnn::Shortcut(&net,encoder_relu_10); - tk::dnn::Layer* encoder_relu_12 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - - //layer-4 - tk::dnn::Layer* encoder_layer_4_0_conv_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_conv_bin[0]); - tk::dnn::Layer* encoder_layer_4_0_bn_1 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[0]); - tk::dnn::Layer* encoder_relu_13 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_0_conv_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_conv_bin[1]); - tk::dnn::Layer* encoder_layer_4_0_bn_2 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[1]); - tk::dnn::Layer* encoder_layer_4_0_route = new tk::dnn::Route(&net,&encoder_relu_12,1); - tk::dnn::Layer* encoder_layer_4_0_downsample_conv = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_conv_bin[2]); - tk::dnn::Layer* encoder_layer_4_0_downsample_bn = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[2]); - tk::dnn::Layer* encoder_layer_4_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_4_0_bn_2); - tk::dnn::Layer* encoder_relu_14 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_1_conv_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_conv_bin[3]); - tk::dnn::Layer* encoder_layer_4_1_bn_1 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[3]); - tk::dnn::Layer* encoder_relu_15 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_1_conv_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_conv_bin[4]); - tk::dnn::Layer* encoder_layer_4_1_bn_2 = new tk::dnn::BatchNorm(&net,512,encoder_layer4_bn_bin[4]); - tk::dnn::Layer* encoder_layer_4_1_shortcut = new tk::dnn::Shortcut(&net,encoder_relu_14); - tk::dnn::Layer* encoder_relu_16 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - - - //decoder - tk::dnn::Layer* decoder_reflection_padding_2d = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_4_0 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0]); - tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_upsampling_2d = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer* concatenate_layer[2] = {decoder_upsampling_2d,encoder_relu_12}; - tk::dnn::Layer* decoder_concatenate = new tk::dnn::Route(&net,concatenate_layer,2); - tk::dnn::Layer* decoder_reflection_padding_2d_1 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_4_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1]); - tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_reflection_padding_2d_2 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_3_0 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2]); - tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_upsampling_2d_1 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer* concatenate_layer_1[2] = {decoder_upsampling_2d_1,encoder_relu_8}; - tk::dnn::Layer* decoder_concatenate_layer_1 = new tk::dnn::Route{&net,concatenate_layer_1,2}; - tk::dnn::Layer* decoder_reflection_padding_2d_3 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_3_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3]); - tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_reflection_padding_2d_5 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_2_0 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4]); - tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_upsampling_2d_2 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer* concatenate_layer_2[2] = {decoder_upsampling_2d_2,encoder_relu_4}; - tk::dnn::Layer* decoder_concatenate_layer_2 = new tk::dnn::Route(&net,concatenate_layer_2,2); - tk::dnn::Layer* decoder_reflection_padding_2d_6 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_2_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5]); - tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_reflection_padding_2d_8 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_1_0 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6]); - tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_upsampling_2d_3 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer* concatenate_layer_3[2] = {decoder_upsampling_2d_3,encoder_relu}; - tk::dnn::Layer* decoder_concatenate_layer_3 = new tk::dnn::Route(&net,concatenate_layer_3,2); - tk::dnn::Layer* decoder_reflection_padding_2d_9 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_1_1 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7]); - tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_reflection_padding_2d_11 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_0_0 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8]); - tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_upsampling_2d_4 = new tk::dnn::Upsample(&net,2); - tk::dnn::Layer* decoder_reflection_padding_2d_12 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_upconv_0_1 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9]); - tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); - tk::dnn::Layer* decoder_reflection_padding_2d_13 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_dispconv_0 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0]); - tk::dnn::Layer* disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); - disp0->setFinal(); - - tk::dnn::Layer* route_elu_7 = new tk::dnn::Route(&net,&decoder_elu_7,1); - tk::dnn::Layer* decoder_reflection_padding_2d_10 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_dispconv_1 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[1]); - tk::dnn::Layer* disp1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); - disp1->setFinal(); - - tk::dnn::Layer* route_elu_5 = new tk::dnn::Route(&net,&decoder_elu_5,1); - tk::dnn::Layer* decoder_reflection_padding_2d_7 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_dispconv_2 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[2]); - tk::dnn::Layer* disp2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); - disp2->setFinal(); - - tk::dnn::Layer* route_elu_3 = new tk::dnn::Route(&net,&decoder_elu_3,1); - tk::dnn::Layer* decoder_reflection_padding_2d_4 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); - tk::dnn::Layer* decoder_dispconv_3 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[3]); - tk::dnn::Layer* disp3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); - disp3->setFinal(); - - dnnType *data; - dnnType *input_H; - readBinaryFile(input_monodepth2_bin[1],dim.tot(),&input_H,&data); - std::cout<<"INPUT DIMENSIONS : "<output_dim.print(); - int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; - for(int i=0;i<4;i++){ - printCenteredTitle((std::string("MONODEPTH2 CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); - outs[i]->output_dim.print(); - - dnnType *out, *out_h; - int odim = outs[i]->output_dim.tot(); - readBinaryFile(output_bin[i], odim, &out_h, &out); - - dnnType *cudnn_out, *rt_out; - cudnn_out = outs[i]->dstData; - rt_out = (dnnType *)netRT.buffersRT[i]; - std::cout<<"CUDNN vs correct"; - ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; - std::cout<<"TRT vs correct"; - ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; - std::cout<<"CUDNN vs TRT "; - ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; - } - - - return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; - -} \ No newline at end of file