diff --git a/CMakeLists.txt b/CMakeLists.txt index d919b46..7392fd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,6 +203,10 @@ target_link_libraries(test_shelfnet_berkeley tkDNN) add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp) target_link_libraries(test_shelfnet_mapillary tkDNN) +# MONODEPTH2 +add_executable(test_monodepth2 tests/monodepth2/monodepth2.cpp) +target_link_libraries(test_monodepth2 tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) @@ -222,6 +226,9 @@ target_link_libraries(demoTracker tkDNN) add_executable(seg_demo demo/demo/seg_demo.cpp) target_link_libraries(seg_demo tkDNN) +add_executable(demoDepth demo/demo/demoDepth.cpp) +target_link_libraries(demoDepth tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index 6de7214..f46ca80 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -46,9 +46,9 @@ int main(int argc, char *argv[]) { std::string cfgPath = YAMLgetConf(conf,"cfg_input", "../tests/darknet/cfg/yolo4tiny.cfg"); std::string namePath = YAMLgetConf(conf,"name_input","../tests/darknet/names/coco.names"); #elif _WIN32 - std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); - std::string cfgPath = YAMLgetConf(conf,"cfg_win_input","..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg"); - std::string namePath = YAMLgetConf(conf,"name_win_input","..\\..\\..\\tests\\darknet\\names\\coco.names"); + std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); + std::string cfgPath = YAMLgetConf(conf,"cfg_win_input","..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg"); + std::string namePath = YAMLgetConf(conf,"name_win_input","..\\..\\..\\tests\\darknet\\names\\coco.names"); #endif if(!fileExist(input.c_str())) FatalError("The given input video does not exist."); diff --git a/demo/demo/demoDepth.cpp b/demo/demo/demoDepth.cpp new file mode 100644 index 0000000..41f5dbf --- /dev/null +++ b/demo/demo/demoDepth.cpp @@ -0,0 +1,106 @@ +#include +#include +#include /* srand, rand */ +//#include +#include + +#include "tkDNN/DepthNN.h" + +bool gRun; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +int main(int argc, char *argv[]) { + + signal(SIGINT, sig_handler); + + std::string net = "monodepth2_fp32.rt"; + if(argc > 1) + net = argv[1]; + #ifdef __linux__ + std::string input = "../demo/yolo_test.mp4"; + #elif _WIN32 + std::string input = "..\\..\\..\\demo\\yolo_test.mp4"; + #endif + if(argc > 2) + input = argv[2]; + bool show = true; + if(argc > 3) + show = atoi(argv[3]); + bool save = true; + if(argc > 4) + save = atoi(argv[4]); + + std::cout <<"Net settings - net: "<< net + <<"\n"; + std::cout <<"Demo settings - input: "<< input + <<", show: "<< show + <<", save: "<< save<<"\n\n"; + + tk::dnn::DepthNN depthNN; + + // create depth network + int n_batch = 1; + depthNN.init(net, n_batch); + + // open video stream + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(save) { + int w = depthNN.output_w; + int h = depthNN.output_h; + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','J','P','G'), 30, cv::Size(w, h)); + } + + if(show) + cv::namedWindow("depth", cv::WINDOW_NORMAL); + + cv::Mat frame; + std::vector batch_frame; + std::vector batch_dnn_input; + + // start detection loop + gRun = true; + while(gRun) { + batch_dnn_input.clear(); + batch_frame.clear(); + + //read frame + cap >> frame; + if(!frame.data) + break; + batch_frame.push_back(frame); + batch_dnn_input.push_back(frame.clone()); + + //inference + depthNN.update(batch_dnn_input, 1); + if(show){ + cv::imshow("depth", depthNN.depthMats[0]); + cv::waitKey(1); + + } + + if(save) + resultVideo << depthNN.depthMats[0]; + } + + std::cout<<"detection end\n"; + + double mean = 0; + std::cout< +#include +#include +#ifdef __linux__ +#include +#endif + +#include + +#include +#include +#include + +#include "tkDNN/utils.h" +#include "tkDNN/tkdnn.h" + +#include "NetworkViz.h" + + +namespace tk { namespace dnn { + +class DepthNN { + + public: + tk::dnn::NetworkRT *netRT = nullptr; + dnnType *input_h; + dnnType *input_d; + float* depth_h; + + int output_w; + int output_h; + + int nBatches = 1; + + cv::Mat bgr[3]; + cv::Mat imagePreproc; + + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector> depths; + std::vector depthMats; + + DepthNN() {}; + ~DepthNN(){}; + + /** + * Method used to initialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file of the NN. + * @param n_batches maximum number of batches to use in inference + * @return true if everything is correct, false otherwise. + */ + void init(const std::string& tensor_path, const int n_batches=1){ + //create net + + std::cout<<(tensor_path).c_str()<<"\n"; + nBatches = n_batches; + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); + + //allocate memory for NN input + checkCuda(cudaMallocHost(&input_h, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + + //allocate memory for NN output + depthMats.resize(nBatches); + depths.resize(nBatches); + for(int i=0; i< depths.size();++i) + depths[i].resize(netRT->buffersDIM[1].tot()); + + depth_h = (float *)malloc(netRT->buffersDIM[1].tot() * sizeof(float)); + + output_h = netRT->buffersDIM[1].h; + output_w = netRT->buffersDIM[1].w; + + } + + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + void preprocess(cv::Mat &frame, const int bi=0) { + //resize image, remove mean, divide by std + cv::Mat frame_nomean; + resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); + frame.convertTo(frame_nomean, CV_32FC3); + frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 255.0, 0); + + //copy image into tensor and copy it into GPU + cv::split(imagePreproc, bgr); + for (int i = 0; i < netRT->input_dim.c; i++){ + int idx = i * imagePreproc.rows * imagePreproc.cols; + int ch = netRT->input_dim.c-1 -i; + memcpy((void *)&input_h[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType)); + } + checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input_h + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + } + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + void postprocess(const int bi=0) { + + dnnType *rt_out[1]; + rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + checkCuda(cudaMemcpy(depth_h, rt_out[0], netRT->buffersDIM[1].tot()* sizeof(float), cudaMemcpyDeviceToHost)); + memcpy(&depths[bi][0], &depth_h[0], netRT->buffersDIM[1].tot()* sizeof(float)); + + // cv::Mat d(netRT->buffersDIM[1].h, netRT->buffersDIM[1].w, CV_8UC1, depth_h); + // depthMats[bi] = d.clone(); + + cv::Mat depth_mat = vizData2Mat(rt_out[0], netRT->buffersDIM[1], netRT->buffersDIM[1].h, netRT->buffersDIM[1].w); + // cv::Mat depth_mat = vizData2Mat((dnnType *)netRT->buffersRT[0], netRT->buffersDIM[0], netRT->buffersDIM[0].h, netRT->buffersDIM[0].w); + depthMats[bi] = depth_mat.clone(); + + } + + /** + * This method performs the inference of the NN. + * + * @param frames frames to build the embedding from. + * @param cur_batches number of batches to use in inference + */ + void update(std::vector& frames, const int cur_batches=1){ + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT feature extraction ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + { + TKDNN_TSTART + for(int bi=0; bi 5 && NV_TENSORRT_MAJOR < 8 bool serialize(const char *filename); diff --git a/include/tkDNN/NetworkViz.h b/include/tkDNN/NetworkViz.h index 2cf8009..ffdf361 100644 --- a/include/tkDNN/NetworkViz.h +++ b/include/tkDNN/NetworkViz.h @@ -6,7 +6,7 @@ namespace tk { namespace dnn { cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19); -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19); +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=0); cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000); }} diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index 403bc28..fff27a8 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -18,6 +18,7 @@ #include "tkdnn.h" #include "NetworkViz.h" #include "kernelsThrust.h" +#define SLAM_MODE namespace tk { namespace dnn { @@ -99,6 +100,62 @@ class SegmentationNN { * * @param bi batch index */ + + #ifdef SLAM_MODE + cv::Mat postprocess(const int bi=0,bool apply_colormap=true){ + cv::Mat maskMatrix; + dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + dataDim_t odim = netRT->output_dim; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + + + dataDim_t vdim = odim; + vdim.c = 1; + dnnType *dataTemp = nullptr; + if(isCudaPointer(tmpOutData_h)) + { + dataTemp = new dnnType[vdim.tot()]; + checkCuda(cudaMemcpy(dataTemp,tmpOutData_h,vdim.tot()*sizeof(dnnType),cudaMemcpyDeviceToHost)); + } + else + { + dataTemp = tmpOutData_h; + } + for(int i =0;iinput_dim.h, netRT->input_dim.w, 0, classes, classes); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, dataTemp); + colored_fp32.convertTo(colored, CV_8UC1); + } + + int max_dim = (originalSize[bi].width > originalSize[bi].height) ? originalSize[bi].width : originalSize[bi].height; + resize(colored, colored, cv::Size(max_dim, max_dim)); + int top, bottom, left, right; + computeBorders(originalSize[bi].width, originalSize[bi].height, top, bottom, left, right); + cv::Rect roi(left,top,originalSize[bi].width, originalSize[bi].height); + cv::Mat or_size (colored, roi); + segmented[bi] = or_size; + + if(isCudaPointer(tmpOutData_h)) + { + delete [] dataTemp; + } + + return maskMatrix; + + } + #elif + void postprocess(const int bi=0, bool appy_colormap = true) { dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; @@ -128,6 +185,7 @@ class SegmentationNN { cv::Mat or_size (colored, roi); segmented[bi] = or_size; }; + #endif public: int classes = 0; @@ -237,6 +295,184 @@ class SegmentationNN { } } + #ifdef SLAM_MODE + cv::Mat updateOriginal(cv::Mat frame,bool apply_colormap=true){ + std::vector splitted_frames; + cv::Mat maskMatrix; + int H, W, net_H, net_W; + int top = 0, bottom = 0, left = 0, right = 0; + std::vector> pos; + + { + TKDNN_TSTART + cv::Size original_size = frame.size(); + + frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); + H = frame.rows; + W = frame.cols; + net_H = netRT->input_dim.h; + net_W = netRT->input_dim.w; + + cv::Mat frame_cropped; + + if( H <= net_H && W <= net_W ){ // smaller size wrt network + top = (net_H - H)/2; + bottom = net_H - H - top ; + left = (net_W - W)/2; + right = net_W - W - left ; + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + splitted_frames.push_back(frame_cropped); + } + else{ //bigger size wrt network + + + if(H < net_H || W < net_W){ + if(H < net_H){ + top = (net_H - H)/2; + bottom = net_H - H - top ; + } + else{ + left = (net_W - W)/2; + right = net_W - W - left ; + } + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); + } + + for(int x=0; x+net_W<=W ;){ + for(int y=0; y+net_H <=H ; ){ + cv::Rect roi(x, y, net_W, net_H); + cv::Mat image_roi = frame(roi); + splitted_frames.push_back(image_roi); + pos.push_back(std::make_pair(x,y)); + + y += net_H; + if(y == H) + break; + if(y + net_H > H) y = H - net_H; + } + x += net_W; + if(x == W) + break; + if(x + net_W > W) x = W - net_W; + } + } + + tk::dnn::dataDim_t idim = netRT->input_dim; + + if(splitted_frames.size()> nBatches) + FatalError(std::to_string(splitted_frames.size()) + " min batches required"); + + for(int bi=0; bistream)); + normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d); + } + TKDNN_TSTOP + stats_pre.push_back(t_ns); + } + + tk::dnn::dataDim_t dim = netRT->input_dim; + dim.n = splitted_frames.size(); + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + dataDim_t odim = netRT->output_dim; + + std::vector out_img; + std::vector out_mask; + + { + TKDNN_TSTART + + for(int bi=0; bibuffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + dataDim_t vdim = odim; + vdim.c = 1; + dnnType *dataTemp = nullptr; + if(isCudaPointer(tmpOutData_h)) + { + dataTemp = new dnnType[vdim.tot()]; + checkCuda(cudaMemcpy(dataTemp,tmpOutData_h,vdim.tot()*sizeof(dnnType),cudaMemcpyDeviceToHost)); + } + else + { + dataTemp = tmpOutData_h; + } + + cv::Mat colored; + for(int i=0;iinput_dim.h, netRT->input_dim.w, 0, classes, classes); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); + colored_fp32.convertTo(colored, CV_8UC1); + } + out_img.push_back(colored); + if(isCudaPointer(tmpOutData_h)) + { + delete [] dataTemp; + } + } + + cv::Mat tempMask(frame.size(), out_mask[0].type()); + cv::Mat seg(frame.size(), out_img[0].type()); + if(out_img.size() == 1) + { + cv::Rect roi(left, top, W, H); + seg = out_img[0](roi); + tempMask = out_mask[0](roi); + } + else{ + int bi=0; + + if(top == 0 && left == 0){ + + for(int i=0; i splitted_frames; @@ -385,6 +621,11 @@ class SegmentationNN { stats_post.push_back(t_ns); } } + #endif + + + + /** * Method to draw boundixg boxes and labels on a frame. diff --git a/src/Activation.cpp b/src/Activation.cpp index 0b113a7..947b019 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -56,6 +56,9 @@ dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { else if(act_mode == ACTIVATION_LOGISTIC) { activationLOGISTICForward(srcData, dstData, dim.tot()); + } else if(act_mode == ACTIVATION_ELU) { + activationELUForward(srcData, dstData, dim.tot()); + } else { dnnType alpha = dnnType(1); dnnType beta = dnnType(0); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index 3f086bd..71e45ed 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -277,6 +277,10 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (DeformConv2d*) l); if(type == LAYER_PADDING) return convert_layer(input, (Padding*) l); + if(type == LAYER_BATCHNORM) + return convert_layer(input,(BatchNorm*) l); + if(type == LAYER_MULADD) + return convert_layer(input,(MulAdd*) l); std::cout<getLayerName()<<"\n"; FatalError("Layer not implemented in tensorRT"); @@ -407,6 +411,112 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) { return lRT; } +ILayer* NetworkRT::convert_layer(ITensor *input,BatchNorm *l){ + void *bias_b, *power_b, *mean_b, *variance_b, *scales_b; + if(dtRT == DataType::kHALF) { + bias_b = l->bias16_h; + power_b = l->power16_h; + mean_b = l->mean16_h; + variance_b = l->variance16_h; + scales_b = l->scales16_h; + } else { + bias_b = l->bias_h; + power_b = l->power_h; + mean_b = l->mean_h; + variance_b = l->variance_h; + scales_b = l->scales_h; + } + Weights power{dtRT, power_b, l->outputs}; + Weights shift{dtRT, mean_b, l->outputs}; + Weights scale{dtRT, variance_b, l->outputs}; + + IScaleLayer *lRT = networkRT->addScale(*input, ScaleMode::kCHANNEL, + shift, scale, power); + checkNULL(lRT); + Weights shift2{dtRT, bias_b, l->outputs}; + Weights scale2{dtRT, scales_b, l->outputs}; + IScaleLayer *lRT2 = networkRT->addScale(*lRT->getOutput(0), ScaleMode::kCHANNEL, + shift2, scale2, power); + checkNULL(lRT2); + + return lRT2; + +} + +ILayer* NetworkRT::convert_layer(ITensor *input,MulAdd *l){ + + void *power_b, *shift_b, *scales_b; + int size = l->input_dim.tot(); + + power_b = new dnnType[size]; + shift_b = new dnnType[size]; + scales_b = new dnnType[size]; + + for(int i=0; iadd; + ((dnnType*) scales_b)[i] = l->mul; + } + + if(dtRT == DataType::kHALF) { + + __half *power16_h = nullptr, *power16_d = nullptr; + __half *scales16_h = nullptr, *scales16_d = nullptr; + __half *shift16_h = nullptr, *shift16_d = nullptr; + + dnnType * power_d = nullptr; + dnnType * scales_d = nullptr; + dnnType * shift_d = nullptr; + + cudaMalloc(&power_d, size*sizeof(dnnType)); + cudaMemcpy(power_d, power_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + cudaMalloc(&shift_d, size*sizeof(dnnType)); + cudaMemcpy(shift_d, shift_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + cudaMalloc(&scales_d, size*sizeof(dnnType)); + cudaMemcpy(scales_d, scales_b, size*sizeof(dnnType), cudaMemcpyHostToDevice); + + //convert to fp16 + power16_h = new __half[size]; + cudaMalloc(&power16_d, size*sizeof(__half)); + float2half(power_d, power16_d, size); + cudaMemcpy(power16_h, power16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + shift16_h = new __half[size]; + cudaMalloc(&shift16_d, size*sizeof(__half)); + float2half(shift_d, shift16_d, size); + cudaMemcpy(shift16_h, shift16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + scales16_h = new __half[size]; + cudaMalloc(&scales16_d, size*sizeof(__half)); + float2half(scales_d, scales16_d, size); + cudaMemcpy(scales16_h, scales16_d, size*sizeof(__half), cudaMemcpyDeviceToHost); + + power_b = power16_h; + shift_b = shift16_h; + scales_b = scales16_h; + + + cudaFree(power16_d); + cudaFree(shift16_d); + cudaFree(scales16_d); + + cudaFree(power_d); + cudaFree(shift_d); + cudaFree(scales_d); + } + + Weights power{dtRT, power_b, size}; + Weights shift{dtRT, shift_b, size}; + Weights scale{dtRT, scales_b, size}; + IScaleLayer *lRT = networkRT->addScale(*input, ScaleMode::kELEMENTWISE, + shift, scale, power); + checkNULL(lRT); + return lRT; +} + + ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) { // std::cout<<"convert Pooling\n"; @@ -461,7 +571,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ float(NV_TENSORRT_MINOR)/10 + float(NV_TENSORRT_PATCH)/100; -#if ((NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR >= 2) || NV_TENSORRT_MAJOR > 8) +/*#if ((NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR >= 2) || NV_TENSORRT_MAJOR > 8) auto *lRT = networkRT->addSlice(*input,Dims3{0,0,0},Dims3{l->output_dim.c,l->output_dim.h,l->output_dim.w},Dims3{0,0,0}); if(l->padding_mode == PADDING_MODE_REFLECTION){ lRT->setMode(SliceMode::kREFLECT); @@ -471,8 +581,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ } checkNULL(lRT); return lRT; -#else - //todo add PADDING_MODE_CONSTANT AND PADDING_MODE_ZERO for tensorrt versions < 8.2 +#else*/ + //todo use ISliceLayer for padding,currently using ISliceLayer for reflection padding generates an error with monodepth2 if(l->padding_mode == PADDING_MODE_REFLECTION){ auto creator = getPluginRegistry()->getPluginCreator("ReflectionPaddingRT_tkDNN","1"); std::vector mPluginAttributes; @@ -513,8 +623,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input,Padding *l){ } return nullptr; - -#endif + } ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp index b6c95d9..8dbb20e 100644 --- a/src/NetworkViz.cpp +++ b/src/NetworkViz.cpp @@ -383,7 +383,7 @@ cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) { default: // expand your range to 0..255. Similar to histEq(); map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); - applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET); + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_PARULA); } return falseColorsMap; } diff --git a/tests/monodepth2/monodepth2.cpp b/tests/monodepth2/monodepth2.cpp new file mode 100644 index 0000000..8618539 --- /dev/null +++ b/tests/monodepth2/monodepth2.cpp @@ -0,0 +1,266 @@ +#include +#include +#include +#include +#include +#include "tkDNN/NetworkViz.h" + +const char* encoder_conv1_bin = "monodepth2/layers/encoder/encoder-conv1.bin"; +const char* encoder_layer1_bin[] = { + "monodepth2/layers/encoder/encoder-layer1-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer1-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer1-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer1-1-conv2.bin", +}; + +const char* encoder_layer2_bin[] = { + "monodepth2/layers/encoder/encoder-layer2-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer2-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer2-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer2-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer2-1-conv2.bin" +}; + +const char* encoder_layer3_bin[]={ + "monodepth2/layers/encoder/encoder-layer3-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer3-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer3-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer3-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer3-1-conv2.bin" +}; + +const char* encoder_layer4_bin[] = { + "monodepth2/layers/encoder/encoder-layer4-0-conv1.bin", + "monodepth2/layers/encoder/encoder-layer4-0-conv2.bin", + "monodepth2/layers/encoder/encoder-layer4-0-downsample-0.bin", + "monodepth2/layers/encoder/encoder-layer4-1-conv1.bin", + "monodepth2/layers/encoder/encoder-layer4-1-conv2.bin" +}; + +const char *encoder_fc_bin = "monodepth2/layers/encoder/encoder-fc.bin"; + +const char* decoder_layer_bin[] = { + "monodepth2/layers/depth_decoder/decoder-0-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-1-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-2-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-3-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-4-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-5-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-6-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-7-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-8-conv-conv.bin", + "monodepth2/layers/depth_decoder/decoder-9-conv-conv.bin" +}; + +const char* decoder_dispconv_layer_bin[] = { + "monodepth2/layers/depth_decoder/decoder-10-conv.bin", + "monodepth2/layers/depth_decoder/decoder-11-conv.bin", + "monodepth2/layers/depth_decoder/decoder-12-conv.bin", + "monodepth2/layers/depth_decoder/decoder-13-conv.bin" +}; + +const char* output_bin[] = { + "monodepth2/debug/outputs/output-disp-0.bin", + "monodepth2/debug/outputs/output-disp-1.bin", + "monodepth2/debug/outputs/output-disp-2.bin", + "monodepth2/debug/outputs/output-disp-3.bin" +}; + +const char* input_bin = "monodepth2/debug/input.bin"; + + +int main(){ + + downloadWeightsifDoNotExist(input_bin, "monodepth2", "https://cloud.hipert.unimore.it/s/iYw9QwgP6CsqxLR/download"); + + tk::dnn::dataDim_t dim(1,3,192,640,1); + tk::dnn::Network net(dim); + + tk::dnn::Layer* muladd_sub = new tk::dnn::MulAdd(&net, 1.0f, -0.45f); + tk::dnn::Layer* muladd_mul = new tk::dnn::MulAdd(&net, 1.0f / 0.225f, 0.0f); + tk::dnn::Layer* encoder_conv = new tk::dnn::Conv2d(&net,64,7,7,2,2,3,3,encoder_conv1_bin,true); + tk::dnn::Layer* encoder_relu = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_maxpool = new tk::dnn::Pooling(&net,3,3,2,2,1,1,tk::dnn::POOLING_MAX); + + //layer-1 + tk::dnn::Layer* encoder_layer_1_0_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[0],true); + tk::dnn::Layer* encoder_relu_1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_0_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[1],true); + tk::dnn::Layer* encoder_layer_1_0_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_maxpool); + tk::dnn::Layer* encoder_relu_2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_1_convbn_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[2],true); + tk::dnn::Layer* encoder_relu_3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_1_1_convbn_2 = new tk::dnn::Conv2d(&net,64,3,3,1,1,1,1,encoder_layer1_bin[3],true); + tk::dnn::Layer* encoder_layer_1_1_shortcut_1 = new tk::dnn::Shortcut(&net,encoder_relu_2); + tk::dnn::Layer* encoder_relu_4 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //layer-2 + tk::dnn::Layer* encoder_layer_2_0_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,2,2,1,1,encoder_layer2_bin[0],true); + tk::dnn::Layer* encoder_relu_5 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_0_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[1],true); + tk::dnn::Layer* encoder_layer_2_0_route = new tk::dnn::Route(&net,&encoder_relu_4,1); + tk::dnn::Layer* encoder_layer_2_0_downsample_convbn = new tk::dnn::Conv2d(&net,128,1,1,2,2,0,0,encoder_layer2_bin[2],true); + tk::dnn::Layer* encoder_layer_2_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_2_0_convbn_2); + tk::dnn::Layer* encoder_relu_6 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_1_convbn_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[3],true); + tk::dnn::Layer* encoder_relu_7 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_2_1_convbn_2 = new tk::dnn::Conv2d(&net,128,3,3,1,1,1,1,encoder_layer2_bin[4],true); + tk::dnn::Layer* encoder_layer_2_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_6); + tk::dnn::Layer* encoder_relu_8 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //layer-3 + tk::dnn::Layer* encoder_layer_3_0_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,2,2,1,1,encoder_layer3_bin[0],true); + tk::dnn::Layer* encoder_relu_9 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_0_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[1],true); + tk::dnn::Layer* encoder_layer_3_0_route = new tk::dnn::Route(&net,&encoder_relu_8,1); + tk::dnn::Layer* encoder_layer_3_0_downsample_convbn = new tk::dnn::Conv2d(&net,256,1,1,2,2,0,0,encoder_layer3_bin[2],true); + tk::dnn::Layer* encoder_layer_3_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_3_0_convbn_2); + tk::dnn::Layer* encoder_relu_10 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_1_convbn_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[3],true); + tk::dnn::Layer* encoder_relu_11 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_3_1_convbn_2 = new tk::dnn::Conv2d(&net,256,3,3,1,1,1,1,encoder_layer3_bin[4],true); + tk::dnn::Layer* encoder_layer_3_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_10); + tk::dnn::Layer* encoder_relu_12 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //layer-4 + tk::dnn::Layer* encoder_layer_4_0_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,2,2,1,1,encoder_layer4_bin[0],true); + tk::dnn::Layer* encoder_relu_13 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_0_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[1],true); + tk::dnn::Layer* encoder_layer_4_0_route = new tk::dnn::Route(&net,&encoder_relu_12,1); + tk::dnn::Layer* encoder_layer_4_0_downsample_convbn = new tk::dnn::Conv2d(&net,512,1,1,2,2,0,0,encoder_layer4_bin[2],true); + tk::dnn::Layer* encoder_layer_4_0_shortcut = new tk::dnn::Shortcut(&net,encoder_layer_4_0_convbn_2); + tk::dnn::Layer* encoder_relu_14 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_1_convbn_1 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[3],true); + tk::dnn::Layer* encoder_relu_15 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + tk::dnn::Layer* encoder_layer_4_1_convbn_2 = new tk::dnn::Conv2d(&net,512,3,3,1,1,1,1,encoder_layer4_bin[4],true); + tk::dnn::Layer* encoder_layer_4_1shortcut = new tk::dnn::Shortcut(&net,encoder_relu_14); + tk::dnn::Layer* encoder_relu_16 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_RELU); + + //decoder + tk::dnn::Layer* decoder_reflection_padding_2d = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_4_0 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[0]); + tk::dnn::Layer* decoder_elu = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer[2] = {decoder_upsampling_2d,encoder_relu_12}; + tk::dnn::Layer* decoder_concatenate = new tk::dnn::Route(&net,concatenate_layer,2); + tk::dnn::Layer* decoder_reflection_padding_2d_1 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_4_1 = new tk::dnn::Conv2d(&net,256,3,3,1,1,0,0,decoder_layer_bin[1]); + tk::dnn::Layer* decoder_elu_1 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_2 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_3_0 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[2]); + tk::dnn::Layer* decoder_elu_2 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_1 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_1[2] = {decoder_upsampling_2d_1,encoder_relu_8}; + tk::dnn::Layer* decoder_concatenate_layer_1 = new tk::dnn::Route{&net,concatenate_layer_1,2}; + tk::dnn::Layer* decoder_reflection_padding_2d_3 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_3_1 = new tk::dnn::Conv2d(&net,128,3,3,1,1,0,0,decoder_layer_bin[3]); + tk::dnn::Layer* decoder_elu_3 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_5 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_2_0 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[4]); + tk::dnn::Layer* decoder_elu_4 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_2 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_2[2] = {decoder_upsampling_2d_2,encoder_relu_4}; + tk::dnn::Layer* decoder_concatenate_layer_2 = new tk::dnn::Route(&net,concatenate_layer_2,2); + tk::dnn::Layer* decoder_reflection_padding_2d_6 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_2_1 = new tk::dnn::Conv2d(&net,64,3,3,1,1,0,0,decoder_layer_bin[5]); + tk::dnn::Layer* decoder_elu_5 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_8 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_1_0 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[6]); + tk::dnn::Layer* decoder_elu_6 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_3 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* concatenate_layer_3[2] = {decoder_upsampling_2d_3,encoder_relu}; + tk::dnn::Layer* decoder_concatenate_layer_3 = new tk::dnn::Route(&net,concatenate_layer_3,2); + tk::dnn::Layer* decoder_reflection_padding_2d_9 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_1_1 = new tk::dnn::Conv2d(&net,32,3,3,1,1,0,0,decoder_layer_bin[7]); + tk::dnn::Layer* decoder_elu_7 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_11 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_0_0 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[8]); + tk::dnn::Layer* decoder_elu_8 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_upsampling_2d_4 = new tk::dnn::Upsample(&net,2); + tk::dnn::Layer* decoder_reflection_padding_2d_12 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_upconv_0_1 = new tk::dnn::Conv2d(&net,16,3,3,1,1,0,0,decoder_layer_bin[9]); + tk::dnn::Layer* decoder_elu_9 = new tk::dnn::Activation(&net,tk::dnn::ACTIVATION_ELU); + tk::dnn::Layer* decoder_reflection_padding_2d_13 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_0 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[0]); + tk::dnn::Layer* disp0 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp0->setFinal(); + + tk::dnn::Layer* route_elu_7 = new tk::dnn::Route(&net,&decoder_elu_7,1); + tk::dnn::Layer* decoder_reflection_padding_2d_10 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_1 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[1]); + tk::dnn::Layer* disp1 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp1->setFinal(); + + tk::dnn::Layer* route_elu_5 = new tk::dnn::Route(&net,&decoder_elu_5,1); + tk::dnn::Layer* decoder_reflection_padding_2d_7 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_2 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[2]); + tk::dnn::Layer* disp2 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp2->setFinal(); + + tk::dnn::Layer* route_elu_3 = new tk::dnn::Route(&net,&decoder_elu_3,1); + tk::dnn::Layer* decoder_reflection_padding_2d_4 = new tk::dnn::Padding(&net,1,1,tk::dnn::PADDING_MODE_REFLECTION); + tk::dnn::Layer* decoder_dispconv_3 = new tk::dnn::Conv2d(&net,1,3,3,1,1,0,0,decoder_dispconv_layer_bin[3]); + tk::dnn::Layer* disp3 = new tk::dnn::Activation(&net,CUDNN_ACTIVATION_SIGMOID); + disp3->setFinal(); + + + dnnType *data; + dnnType *input_H; + readBinaryFile(input_bin, dim.tot(),&input_H,&data); + std::cout<<"INPUT DIMENSIONS : "<output_dim.print(); + int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; + for(int i=0;i<4;i++){ + printCenteredTitle((std::string("MONODEPTH2 CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30); + outs[i]->output_dim.print(); + + dnnType *out, *out_h; + int odim = outs[i]->output_dim.tot(); + readBinaryFile(output_bin[i], odim, &out_h, &out); + + dnnType *cudnn_out, *rt_out; + cudnn_out = outs[i]->dstData; + rt_out = (dnnType *)netRT.buffersRT[1+i]; + std::cout<<"CUDNN vs correct"; + ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN; + std::cout<<"TRT vs correct"; + ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT; + std::cout<<"CUDNN vs TRT "; + ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT; + + cv::Mat depth_mat = vizData2Mat(outs[i]->dstData, outs[i]->output_dim, outs[i]->output_dim.h, outs[i]->output_dim.w); + cv::imshow("depth", depth_mat); + cv::waitKey(0); + } + + + return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt; + + +} \ No newline at end of file