From 907df27e07ce2da961ec6729983f144f305619e1 Mon Sep 17 00:00:00 2001 From: Micaela Verucchi Date: Tue, 18 Jan 2022 21:34:16 -0800 Subject: [PATCH] Fix monodepth2, add demoDepth: - Fix monodepth2 network, now works with both cuDNN and tensorRT - Substitute cuDNN ELU with tkDNN one - add DepthNN class - add demoDepth demo, now only works with monodepth2 net Signed-off-by: Micaela Verucchi Francesco Gatti --- CMakeLists.txt | 3 + demo/demo/demoDepth.cpp | 103 +++++++++++++++++++ include/tkDNN/DepthNN.h | 174 ++++++++++++++++++++++++++++++++ include/tkDNN/Layer.h | 1 - include/tkDNN/NetworkRT.h | 1 + include/tkDNN/NetworkViz.h | 2 +- src/Activation.cpp | 3 + src/NetworkRT.cpp | 76 ++++++++++++++ src/NetworkViz.cpp | 2 +- tests/monodepth2/monodepth2.cpp | 70 +++++++------ 10 files changed, 401 insertions(+), 34 deletions(-) create mode 100644 demo/demo/demoDepth.cpp create mode 100644 include/tkDNN/DepthNN.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a6d616..a109c36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,6 +229,9 @@ target_link_libraries(demoTracker tkDNN) add_executable(seg_demo demo/demo/seg_demo.cpp) target_link_libraries(seg_demo tkDNN) +add_executable(demoDepth demo/demo/demoDepth.cpp) +target_link_libraries(demoDepth tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demoDepth.cpp b/demo/demo/demoDepth.cpp new file mode 100644 index 0000000..eda04c6 --- /dev/null +++ b/demo/demo/demoDepth.cpp @@ -0,0 +1,103 @@ +#include +#include +#include /* srand, rand */ +//#include +#include + +#include "tkDNN/DepthNN.h" + +bool gRun; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + signal(SIGINT, sig_handler); + + std::string net = "monodepth2_fp32.rt"; + if(argc > 1) + net = argv[1]; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) + input = argv[2]; + bool show = true; + if(argc > 3) + show = atoi(argv[3]); + bool save = true; + if(argc > 4) + save = atoi(argv[4]); + + std::cout <<"Net settings - net: "<< net + <<"\n"; + std::cout <<"Demo settings - input: "<< input + <<", show: "<< show + <<", save: "<< save<<"\n\n"; + + tk::dnn::DepthNN depthNN; + + // create depth network + int n_batch = 1; + depthNN.init(net, n_batch); + + // open video stream + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(save) { + int w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + if(show) + cv::namedWindow("depth", cv::WINDOW_NORMAL); + + cv::Mat frame; + std::vector batch_frame; + std::vector batch_dnn_input; + + // start detection loop + gRun = true; + while(gRun) { + batch_dnn_input.clear(); + batch_frame.clear(); + + //read frame + cap >> frame; + if(!frame.data) + break; + batch_frame.push_back(frame); + batch_dnn_input.push_back(frame.clone()); + + //inference + depthNN.update(batch_dnn_input, 1); + if(show){ + cv::imshow("depth", depthNN.depthMats[0]); + cv::waitKey(1); + + } + } + + std::cout<<"detection end\n"; + + double mean = 0; + std::cout< +#include +#include +#ifdef __linux__ +#include +#endif + +#include + +#include +#include +#include + +#include "tkDNN/utils.h" +#include "tkDNN/tkdnn.h" + +#include "NetworkViz.h" + + +namespace tk { namespace dnn { + +class DepthNN { + + public: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_h; + dnnType *input_d; + float* depth_h; + + int nBatches = 1; + + cv::Mat bgr[3]; + cv::Mat imagePreproc; + + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector> depths; + std::vector depthMats; + + DepthNN() {}; + ~DepthNN(){}; + + /** + * Method used to initialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file of the NN. + * @param n_batches maximum number of batches to use in inference + * @return true if everything is correct, false otherwise. + */ + void init(const std::string& tensor_path, const int n_batches=1){ + //create net + + std::cout<<(tensor_path).c_str()<<"\n"; + nBatches = n_batches; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); + + //allocate memory for NN input + checkCuda(cudaMallocHost(&input_h, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + + //allocate memory for NN output + depthMats.resize(nBatches); + depths.resize(nBatches); + for(int i=0; i< depths.size();++i) + depths[i].resize(netRT->buffersDIM[1].tot()); + + depth_h = (float *)malloc(netRT->buffersDIM[1].tot() * sizeof(float)); + + } + + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + void preprocess(cv::Mat &frame, const int bi=0) { + //resize image, remove mean, divide by std + cv::Mat frame_nomean; + resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); + frame.convertTo(frame_nomean, CV_32FC3); + frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 255.0, 0); + + //copy image into tensor and copy it into GPU + cv::split(imagePreproc, bgr); + for (int i = 0; i < netRT->input_dim.c; i++){ + int idx = i * imagePreproc.rows * imagePreproc.cols; + int ch = netRT->input_dim.c-1 -i; + memcpy((void *)&input_h[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType)); + } + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input_h + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + } + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + void postprocess(const int bi=0) { + + dnnType *rt_out[1]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + checkCuda(cudaMemcpy(depth_h, rt_out[0], netRT->buffersDIM[1].tot()* sizeof(float), cudaMemcpyDeviceToHost)); + memcpy(&depths[bi][0], &depth_h[0], netRT->buffersDIM[1].tot()* sizeof(float)); + + // cv::Mat d(netRT->buffersDIM[1].h, netRT->buffersDIM[1].w, CV_8UC1, depth_h); + // depthMats[bi] = d.clone(); + + cv::Mat depth_mat = vizData2Mat(rt_out[0], netRT->buffersDIM[1], netRT->buffersDIM[1].h, netRT->buffersDIM[1].w); + // cv::Mat depth_mat = vizData2Mat((dnnType *)netRT->buffersRT[0], netRT->buffersDIM[0], netRT->buffersDIM[0].h, netRT->buffersDIM[0].w); + depthMats[bi] = depth_mat.clone(); + + } + + /** + * This method performs the inference of the NN. + * + * @param frames frames to build the embedding from. + * @param cur_batches number of batches to use in inference + */ + void update(std::vector& frames, const int cur_batches=1){ + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT feature extraction ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + { + TKDNN_TSTART + for(int bi=0; bi 5 && NV_TENSORRT_MAJOR < 8 bool serialize(const char *filename); diff --git a/include/tkDNN/NetworkViz.h b/include/tkDNN/NetworkViz.h index 2cf8009..ffdf361 100644 --- a/include/tkDNN/NetworkViz.h +++ b/include/tkDNN/NetworkViz.h @@ -6,7 +6,7 @@ namespace tk { namespace dnn { cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19); -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19); +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=0); cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000); }} diff --git a/src/Activation.cpp b/src/Activation.cpp index 0b113a7..947b019 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -56,6 +56,9 @@ dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { else if(act_mode == ACTIVATION_LOGISTIC) { activationLOGISTICForward(srcData, dstData, dim.tot()); + } else if(act_mode == ACTIVATION_ELU) { + activationELUForward(srcData, dstData, dim.tot()); + } else { dnnType alpha = dnnType(1); dnnType beta = dnnType(0); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index b83de23..9b5f14c 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -279,6 +279,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (Padding*) l); if(type == LAYER_BATCHNORM) return convert_layer(input,(BatchNorm*) l); + if(type == LAYER_MULADD) + return convert_layer(input,(MulAdd*) l); std::cout<getLayerName()<<"\n"; FatalError("Layer not implemented in tensorRT"); @@ -441,6 +443,80 @@ ILayer* NetworkRT::convert_layer(ITensor *input,BatchNorm *l){ } +ILayer* NetworkRT::convert_layer(ITensor *input,MulAdd *l){ + + void *power_b, *shift_b, *scales_b; + int size = l->input_dim.tot(); + + power_b = new dnnType[size]; + shift_b = new dnnType[size]; + scales_b = new dnnType[size]; + + for(int i=0; iadd; + ((dnnType*) scales_b)[i] = l->mul; + } + + if(dtRT == DataType::kHALF) { + + __half *power16_h = nullptr, *power16_d = nullptr; + __half *scales16_h = nullptr, *scales16_d = nullptr; + __half *shift16_h = nullptr, *shift16_d = nullptr; + + dnnType * power_d = nullptr; + dnnType * scales_d = nullptr; + dnnType * shift_d = nullptr; + + cudaMalloc(&power_d, size*sizeof(dnnType)); + cudaMemcpy(power_d, power_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + cudaMalloc(&shift_d, size*sizeof(dnnType)); + cudaMemcpy(shift_d, shift_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + cudaMalloc(&scales_d, size*sizeof(dnnType)); + cudaMemcpy(scales_d, scales_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + //convert to fp16 + power16_h = new __half[size]; + cudaMalloc(&power16_d, size*sizeof(__half)); + float2half(power_d, power16_d, size); + cudaMemcpy(power16_h, power16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + shift16_h = new __half[size]; + cudaMalloc(&shift16_d, size*sizeof(__half)); + float2half(shift_d, shift16_d, size); + cudaMemcpy(shift16_h, shift16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + scales16_h = new __half[size]; + cudaMalloc(&scales16_d, size*sizeof(__half)); + float2half(scales_d, scales16_d, size); + cudaMemcpy(scales16_h, scales16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + power_b = power16_h; + shift_b = shift16_h; + scales_b = scales16_h; + + + cudaFree(power16_d); + cudaFree(shift16_d); + cudaFree(scales16_d); + + cudaFree(power_d); + cudaFree(shift_d); + cudaFree(scales_d); + } + + Weights power{dtRT, power_b, size}; + Weights shift{dtRT, shift_b, size}; + Weights scale{dtRT, scales_b, size}; + IScaleLayer *lRT = networkRT->addScale(*input, ScaleMode::kELEMENTWISE, + shift, scale, power); + checkNULL(lRT); + return lRT; +} + + ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { // std::cout<<"convert Pooling\n"; diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp index b6c95d9..8dbb20e 100644 --- a/src/NetworkViz.cpp +++ b/src/NetworkViz.cpp @@ -383,7 +383,7 @@ cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) { default: // expand your range to 0..255. Similar to histEq(); map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); - applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET); + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_PARULA); } return falseColorsMap; } diff --git a/tests/monodepth2/monodepth2.cpp b/tests/monodepth2/monodepth2.cpp index c763baf..4e0767a 100644 --- a/tests/monodepth2/monodepth2.cpp +++ b/tests/monodepth2/monodepth2.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "tkDNN/NetworkViz.h" const char* encoder_conv1_bin = "monodepth2/layers/encoder/encoder-conv1.bin"; const char* encoder_layer1_bin[] = { @@ -72,108 +73,111 @@ int main(){ tk::dnn::dataDim_t dim(1,3,192,640,1); tk::dnn::Network net(dim); - tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin,true,false,1,true); + + tk::dnn::Layer* muladd_sub = new tk::dnn::MulAdd(&net, 1.0f, -0.45f); + tk::dnn::Layer* muladd_mul = new tk::dnn::MulAdd(&net, 1.0f / 0.225f, 0.0f); + tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin,true); tk::dnn::Layer* encoder_relu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); tk::dnn::Layer* encoder_maxpool = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); //layer-1 - tk::dnn::Layer* encoder_layer_1_0_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_0_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[0],true); tk::dnn::Layer* encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_0_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_0_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[1],true); tk::dnn::Layer* encoder_layer_1_0_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_maxpool); tk::dnn::Layer* encoder_relu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_1_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_1_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[2],true); tk::dnn::Layer* encoder_relu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_1_1_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_1_1_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[3],true); tk::dnn::Layer* encoder_layer_1_1_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_relu_2); tk::dnn::Layer* encoder_relu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //layer-2 - tk::dnn::Layer* encoder_layer_2_0_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true); tk::dnn::Layer* encoder_relu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_0_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true); tk::dnn::Layer* encoder_layer_2_0_route = new tk::dnn::Route(&net,&encoder_relu_4,1); - tk::dnn::Layer* encoder_layer_2_0_downsample_convbn = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_0_downsample_convbn = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2],true); tk::dnn::Layer* encoder_layer_2_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_2_0_convbn_2); tk::dnn::Layer* encoder_relu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_1_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_1_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true); tk::dnn::Layer* encoder_relu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_2_1_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_2_1_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true); tk::dnn::Layer* encoder_layer_2_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_6); tk::dnn::Layer* encoder_relu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //layer-3 - tk::dnn::Layer* encoder_layer_3_0_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true); tk::dnn::Layer* encoder_relu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_0_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true); tk::dnn::Layer* encoder_layer_3_0_route = new tk::dnn::Route(&net,&encoder_relu_8,1); - tk::dnn::Layer* encoder_layer_3_0_downsample_convbn = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_0_downsample_convbn = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2],true); tk::dnn::Layer* encoder_layer_3_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_3_0_convbn_2); tk::dnn::Layer* encoder_relu_10 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_1_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_1_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true); tk::dnn::Layer* encoder_relu_11 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_3_1_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_3_1_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true); tk::dnn::Layer* encoder_layer_3_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_10); tk::dnn::Layer* encoder_relu_12 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //layer-4 - tk::dnn::Layer* encoder_layer_4_0_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true); tk::dnn::Layer* encoder_relu_13 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_0_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true); tk::dnn::Layer* encoder_layer_4_0_route = new tk::dnn::Route(&net,&encoder_relu_12,1); - tk::dnn::Layer* encoder_layer_4_0_downsample_convbn = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_0_downsample_convbn = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2],true); tk::dnn::Layer* encoder_layer_4_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_4_0_convbn_2); tk::dnn::Layer* encoder_relu_14 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_1_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_1_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true); tk::dnn::Layer* encoder_relu_15 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); - tk::dnn::Layer* encoder_layer_4_1_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true,false,1,true); + tk::dnn::Layer* encoder_layer_4_1_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true); tk::dnn::Layer* encoder_layer_4_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_14); tk::dnn::Layer* encoder_relu_16 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); //decoder tk::dnn::Layer* decoder_reflection_padding_2d = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_4_0 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0]); - tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer[2] = {decoder_upsampling_2d,encoder_relu_12}; tk::dnn::Layer* decoder_concatenate = new tk::dnn::Route(&net,concatenate_layer,2); tk::dnn::Layer* decoder_reflection_padding_2d_1 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_4_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1]); - tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_2 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_3_0 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2]); - tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_1 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer_1[2] = {decoder_upsampling_2d_1,encoder_relu_8}; tk::dnn::Layer* decoder_concatenate_layer_1 = new tk::dnn::Route{&net,concatenate_layer_1,2}; tk::dnn::Layer* decoder_reflection_padding_2d_3 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_3_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3]); - tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_5 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_2_0 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4]); - tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_2 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer_2[2] = {decoder_upsampling_2d_2,encoder_relu_4}; tk::dnn::Layer* decoder_concatenate_layer_2 = new tk::dnn::Route(&net,concatenate_layer_2,2); tk::dnn::Layer* decoder_reflection_padding_2d_6 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_2_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5]); - tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_8 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_1_0 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6]); - tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_3 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* concatenate_layer_3[2] = {decoder_upsampling_2d_3,encoder_relu}; tk::dnn::Layer* decoder_concatenate_layer_3 = new tk::dnn::Route(&net,concatenate_layer_3,2); tk::dnn::Layer* decoder_reflection_padding_2d_9 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_1_1 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7]); - tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_11 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_0_0 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8]); - tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_upsampling_2d_4 = new tk::dnn::Upsample(&net,2); tk::dnn::Layer* decoder_reflection_padding_2d_12 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_upconv_0_1 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9]); - tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_ELU); + tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); tk::dnn::Layer* decoder_reflection_padding_2d_13 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); tk::dnn::Layer* decoder_dispconv_0 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0]); tk::dnn::Layer* disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); @@ -240,13 +244,17 @@ int main(){ dnnType *cudnn_out, *rt_out; cudnn_out = outs[i]->dstData; - rt_out = (dnnType *)netRT.buffersRT[i]; + rt_out = (dnnType *)netRT.buffersRT[1+i]; std::cout<<"CUDNN vs correct"; ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; std::cout<<"TRT vs correct"; ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; std::cout<<"CUDNN vs TRT "; ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + + cv::Mat depth_mat = vizData2Mat(outs[i]->dstData, outs[i]->output_dim, outs[i]->output_dim.h, outs[i]->output_dim.w); + cv::imshow("depth", depth_mat); + cv::waitKey(0); }