diff --git a/CMakeLists.txt b/CMakeLists.txt index f03c98c..5692979 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,6 +119,16 @@ target_link_libraries(test_resnet101_cnet tkDNN) add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp) target_link_libraries(test_dla34_cnet tkDNN) +# SHELFNET +add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp) +target_link_libraries(test_shelfnet tkDNN) + +add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp) +target_link_libraries(test_shelfnet_berkeley tkDNN) + +add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp) +target_link_libraries(test_shelfnet_mapillary tkDNN) + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) @@ -129,6 +139,9 @@ target_link_libraries(map_demo tkDNN) add_executable(demo demo/demo/demo.cpp) target_link_libraries(demo tkDNN) +add_executable(seg_demo demo/demo/seg_demo.cpp) +target_link_libraries(seg_demo tkDNN) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/README.md b/README.md index 2325eb5..7dcd080 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ If you use tkDNN in your research, please cite the [following paper](https://iee } ``` +### What's new (20 July 2021) +- [x] Support to sematic segmentation [README](docs/README_seg.md) +- [ ] Support to TensorRT8 (WIP) + ## FPS Results Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); @@ -89,16 +93,21 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001 - [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows) - - - ## Dependencies -This branch works on every NVIDIA GPU that supports the dependencies: -* CUDA 10.0 -* CUDNN 7.603 -* TENSORRT 6.01 -* OPENCV 3.4 -* yaml-cpp 0.5.2 (sudo apt install libyaml-cpp-dev) +This branch works on every NVIDIA GPU that supports the following (latest tested) dependencies: +* CUDA 11.0 (or >= 10) +* cuDNN 8.0.4 (or >= 7.3) +* TensorRT 7.2.0 (or >=5) +* OpenCV 4.5.2 (or >=4) +* cmake 3.21 (or >= 3.15) +* yaml-cpp 0.5.2 +* eigen3 3.3.4 +* curl 7.58 + +``` +sudo apt install libyaml-cpp-dev curl libeigen3-dev + +``` ## About OpenCV To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh```. It will download and compile OpenCV in Download folder. diff --git a/demo/demo/seg_demo.cpp b/demo/demo/seg_demo.cpp new file mode 100644 index 0000000..2b491d8 --- /dev/null +++ b/demo/demo/seg_demo.cpp @@ -0,0 +1,148 @@ +#include +#include +#include /* srand, rand */ +#include +#include + +#include "SegmentationNN.h" + +bool gRun; +bool SAVE_RESULT = true; + +void sig_handler(int signo) { + std::cout<<"request gateway stop\n"; + gRun = false; +} + +void writePred(const std::string& images_names, const std::string& gt_folder, const std::string& out_folder, tk::dnn::SegmentationNN& segNN, int& width, int& height, bool show=false){ + std::ifstream all_gt(images_names); + std::string filename; + cv::Mat frame; + for (; std::getline(all_gt, filename); ) { + std::cout< 1) + net = argv[1]; + std::string input = "../demo/yolo_test.mp4"; + if(argc > 2) + input = argv[2]; + int n_batch = 1; + if(argc > 3) + n_batch = atoi(argv[3]); + int n_classes = 19; + if(argc > 4) + n_classes = atoi(argv[4]); + bool resize = false; + if(argc > 5) + resize = atoi(argv[5]); + int baseline_resize = 1024; + if(argc > 6) + baseline_resize = atoi(argv[6]); + bool show = true; + if(argc > 7) + show = atoi(argv[7]); + bool write_pred = false; + if(argc > 8) + write_pred = atoi(argv[8]); + + if(resize && (baseline_resize < 0 || baseline_resize > 5000)) + FatalError("Problem with baseline resize") + if(n_batch < 1 || n_batch > 64) + FatalError("Batch dim not supported"); + + //net initialization + tk::dnn::SegmentationNN segNN; + segNN.init(net, n_classes, n_batch); + + int height = 0, width = 0; + int basewidth=baseline_resize, hsize; + + if(write_pred){ + std::string gt_folder = "../demo/CityScapes_val/images/"; + std::string images_names = "../demo/CityScapes_val/all_images.txt"; + std::string out_folder = "seg/"; + + writePred(images_names, gt_folder, out_folder, segNN, width, height, show); + } + else{ + if(!show) + SAVE_RESULT = true; + + gRun = true; + + cv::VideoCapture cap(input); + if(!cap.isOpened()) + gRun = false; + else + std::cout<<"camera started\n"; + + cv::VideoWriter resultVideo; + if(SAVE_RESULT) { + int w,h; + if(resize){ + w = basewidth; + h = int((float(cap.get(cv::CAP_PROP_FRAME_HEIGHT))*float(basewidth/float(cap.get(cv::CAP_PROP_FRAME_WIDTH))))); + } + else{ + w = cap.get(cv::CAP_PROP_FRAME_WIDTH); + h = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + } + resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); + } + + cv::Mat frame; + while(gRun) { + cap >> frame; + if(!frame.data) + break; + + if(resize){ + hsize = int((float(frame.rows)*float(basewidth/float(frame.cols)))); + cv::resize(frame, frame, cv::Size(basewidth, hsize)); + } + + height = frame.rows; + width = frame.cols; + + //inference + segNN.updateOriginal(frame, true); + if(show) + segNN.draw(); + + if(SAVE_RESULT) + resultVideo << segNN.segmented[0]; + } + } + + std::cout<<"segmentation end\n"; + double mean = 0, mean_pre = 0, mean_post = 0; + + std::cout< than 1 if you want to run inference on images bigger than 1024 +./test_shelfnet # run the yolo test (is slow) +./demo shelfnet_fp32.rt ../demo/yolo_test.mp4 1 19 +``` +In general the demo program takes the following parameters: +``` +./seg_demo +``` +where +* `````` is the rt file generated by a test +* ```<``` is the path to a video file or a camera input +* `````` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network). +* ``````is the number of classes the network is trained on +* `````` if set to 0 the demo will not resize the input frames, but use it as it is, otherwise it will resize it. +* `````` is `````` is set to 1, then the input frames will be proportionally resized using `````` as width baseline. +* `````` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1) +* `````` if set to 0 (default) the demo will run, otherwise the evaluation of a dataset will run and the output of the segmentation will be saved. Attention: this is under development and paths are embedded, so change them in the code in advance. + +NB) By default it is used FP32 inference +NB) The batching is not used to work on more streams, rather to work on more tiles of the same image. Shelfnet never resized the input image, therefore for images greater than 1024x1024 tiles of 1024x1024 are given in input to the network in batch. + +![gif](output.gif "Results on yolo_test.mp4") + +For other demo videos refer to [this playlist](https://www.youtube.com/playlist?list=PLv0nEQYDD45y5EdSiywwCGPBmJVUzIWwe). + + +## Existing tests and supported networks + +| Test Name | Network | Dataset | N Classes | Input size | Weights | +| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ | +| shelfnet | ShelfNet18_realtime1 | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) | +| shelfnet_berkeley | ShelfNet18_realtime1 | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) | + +1. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019. + + +## FPS Results + +Inference FPS of shelfnet with tkDNN, average of 1200 images on: + * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5); + * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 ); + +| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS | +| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | +| RTX 2080Ti | shelfnet 1024x1024 (B=1) | pre | 6.11863 | 163.435 | 5.81465 | 171.979 | 5.88699 | 169.866 | +| RTX 2080Ti | shelfnet 1024x1024 (B=1) | inf | 11.5464 | 86.6074 | 7.35396 | 135.981 | 6.37623 | 156.832 | +| RTX 2080Ti | shelfnet 1024x1024 (B=1) | post | 4.09058 | 244.464 | 3.91961 | 255.128 | 4.07343 | 245.493 | +| RTX 2080Ti | shelfnet 1024x1024 (B=1) | tot | 21.7556 | 45.9652 | 17.0882 | 58.5199 | 16.3366 | 61.2121 | +| RTX 2080Ti | shelfnet 2048x2048 (B=4) | pre | 25.435 | 39.3158 | 25.2953 | 39.5331 | 25.9303 | 38.565 | +| RTX 2080Ti | shelfnet 2048x2048 (B=4) | inf | 36.5015 | 27.3961 | 17.0534 | 58.6395 | 15.6061 | 64.0773 | +| RTX 2080Ti | shelfnet 2048x2048 (B=4) | post | 17.3917 | 57.4985 | 17.1649 | 58.2583 | 17.5539 | 56.9675 | +| RTX 2080Ti | shelfnet 2048x2048 (B=4) | tot | 79.3283 | 12.6058 | 59.5136 | 16.8029 | 59.0903 | 16.9233 | +| AGX Xavier | shelfnet 1024x1024 (B=1) | pre | 8.0174 | 124.729 | 7.5117 | 133.126 | 7.47333 | 133.809 | +| AGX Xavier | shelfnet 1024x1024 (B=1) | inf | 72.4173 | 13.8089 | 37.505 | 26.6631 | 31.3286 | 31.9197 | +| AGX Xavier | shelfnet 1024x1024 (B=1) | post | 8.89958 | 112.365 | 8.83576 | 113.176 | 9.42655 | 106.083 | +| AGX Xavier | shelfnet 1024x1024 (B=1) | tot | 89.3342 | 11.1939 | 53.8525 | 18.5692 | 48.2285 | 20.7346 | +| AGX Xavier | shelfnet 2048x2048 (B=4) | pre | 47.1454 | 21.211 | 21.6475 | 46.1947 | 21.4201 | 46.6851 | +| AGX Xavier | shelfnet 2048x2048 (B=4) | inf | 266.537 | 3.75183 | 128.321 | 7.79293 | 107.621 | 9.29185 | +| AGX Xavier | shelfnet 2048x2048 (B=4) | post | 44.0711 | 22.6906 | 40.1732 | 24.8922 | 39.873 | 25.0796 | +| AGX Xavier | shelfnet 2048x2048 (B=4) | tot | 357.753 | 2.79522 | 190.142 | 5.25922 | 168.914 | 5.92016 | + + +## Known issues + +When creating the rt file all the checks returns errors. It is due to a different resize function and handling of the original ShelfNet outputs. +However, the network is supposed to work. diff --git a/docs/output.gif b/docs/output.gif new file mode 100644 index 0000000..64a9396 Binary files /dev/null and b/docs/output.gif differ diff --git a/include/tkDNN/Layer.h b/include/tkDNN/Layer.h index e097372..91c33ec 100644 --- a/include/tkDNN/Layer.h +++ b/include/tkDNN/Layer.h @@ -22,6 +22,7 @@ enum layerType_t { LAYER_ACTIVATION_LOGISTIC, LAYER_FLATTEN, LAYER_RESHAPE, + LAYER_RESIZE, LAYER_MULADD, LAYER_POOLING, LAYER_SOFTMAX, @@ -72,6 +73,7 @@ public: case LAYER_ACTIVATION_LOGISTIC: return "ActivationLogistic"; case LAYER_FLATTEN: return "Flatten"; case LAYER_RESHAPE: return "Reshape"; + case LAYER_RESIZE: return "Resize"; case LAYER_MULADD: return "MulAdd"; case LAYER_POOLING: return "Pooling"; case LAYER_SOFTMAX: return "Softmax"; @@ -226,8 +228,9 @@ class Activation : public Layer { public: int act_mode; float ceiling; + float slope; - Activation(Network *net, int act_mode, const float ceiling=0.0); + Activation(Network *net, int act_mode, const float ceiling=0.0, const float slope=0.1); virtual ~Activation(); virtual layerType_t getLayerType() { if(act_mode == CUDNN_ACTIVATION_CLIPPED_RELU) @@ -432,6 +435,23 @@ public: }; +enum ResizeMode_t { NEAREST= 0, + LINEAR= 1}; + +/** + Resize layer +*/ +class Resize : public Layer { + +public: + Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed=false, ResizeMode_t mode=NEAREST); + virtual ~Resize(); + virtual layerType_t getLayerType() { return LAYER_RESIZE; }; + + virtual dnnType* infer(dataDim_t &dim, dnnType* srcData); + + ResizeMode_t mode; +}; /** MulAdd layer @@ -552,7 +572,7 @@ public: class Shortcut : public Layer { public: - Shortcut(Network *net, Layer *backLayer); + Shortcut(Network *net, Layer *backLayer, bool mul=false); virtual ~Shortcut(); virtual layerType_t getLayerType() { return LAYER_SHORTCUT; }; @@ -560,6 +580,7 @@ public: public: Layer *backLayer; + bool mul = false; }; /** diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index 29cacf5..9892a24 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -108,6 +108,7 @@ public: nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l); + nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l); nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l); diff --git a/include/tkDNN/NetworkViz.h b/include/tkDNN/NetworkViz.h index c8b1bea..2cf8009 100644 --- a/include/tkDNN/NetworkViz.h +++ b/include/tkDNN/NetworkViz.h @@ -5,8 +5,8 @@ namespace tk { namespace dnn { -cv::Mat vizFloat2colorMap(cv::Mat map); -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim); +cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19); +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19); cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000); }} diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h new file mode 100644 index 0000000..b691cbc --- /dev/null +++ b/include/tkDNN/SegmentationNN.h @@ -0,0 +1,403 @@ +#ifndef SEGMENTATIONNN_H +#define SEGMENTATIONNN_H + +#include +#include +#include +#include +#include +#include "utils.h" + +#include +#include +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" +#include "kernelsThrust.h" + +namespace tk { namespace dnn { + +class SegmentationNN { + + protected: + tk::dnn::NetworkRT *netRT = nullptr; + int nBatches = 1; + + std::vector originalSize; + cv::Mat bgr[3]; + dnnType *input; + dnnType *input_d; + float* confidences_h; + + float * tmpInputData_d; + float *tmpOutData_d; + float *tmpOutData_h; + + float *mean_d, *stddev_d; + + cublasHandle_t cublasHandle; + + void computeBorders(const int or_width, const int or_height, int& top, int& bottom, int& left, int&right){ + top = 0; + bottom = 0; + left = 0; + right = 0; + + if(or_height != or_width){ + if(or_height < or_width){ + top = (or_width - or_height)/2; + bottom = or_width - top - or_height; + } + else{ + left = (or_height - or_width)/2; + right = or_height - left - or_width; + } + } + } + + /** + * This method preprocess the image, before feeding it to the NN. + * + * @param frame original frame to adapt for inference. + * @param bi batch index + */ + void preprocess(cv::Mat &frame, const int bi=0) { + originalSize[bi] = frame.size(); + + frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); + int H = frame.rows; + int W = frame.cols; + cv::Mat frame_cropped; + + int top, bottom, left, right; + computeBorders(W, H, top, bottom, left, right); + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + + tk::dnn::dataDim_t idim = netRT->input_dim; + + resize(frame_cropped, frame_cropped, cv::Size(idim.w, idim.h)); + + cv::split(frame_cropped, bgr); + for (int i = 0; i < idim.c; i++){ + int idx = i * frame_cropped.rows * frame_cropped.cols; + int ch = idim.c-1 -i; + memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType)); + } + + checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream)); + + normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d); + } + + /** + * This method postprocess the output of the NN to obtain the correct + * boundig boxes. + * + * @param bi batch index + */ + void postprocess(const int bi=0, bool appy_colormap = true) { + dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + dataDim_t odim = netRT->output_dim; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + dataDim_t vdim = odim; + vdim.c = 1; + + cv::Mat colored; + + if(appy_colormap) + colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); + colored_fp32.convertTo(colored, CV_8UC1); + } + + int max_dim = (originalSize[bi].width > originalSize[bi].height) ? originalSize[bi].width : originalSize[bi].height; + resize(colored, colored, cv::Size(max_dim, max_dim)); + int top, bottom, left, right; + computeBorders(originalSize[bi].width, originalSize[bi].height, top, bottom, left, right); + cv::Rect roi(left,top,originalSize[bi].width, originalSize[bi].height); + cv::Mat or_size (colored, roi); + segmented[bi] = or_size; + }; + + public: + int classes = 0; + std::vector stats; /*keeps track of inference times (ms)*/ + std::vector stats_pre; + std::vector stats_post; + std::vector classesNames; + std::vector segmented; + + SegmentationNN() { + checkERROR( cublasCreate(&cublasHandle) ); + }; + ~SegmentationNN(){ + checkERROR( cublasDestroy(cublasHandle) ); + }; + + /** + * Method used to inialize the class, allocate memory and compute + * needed data. + * + * @param tensor_path path to the rt file og the NN. + * @param n_classes number of classes for the given dataset. + * @param n_batches maximum number of batches to use in inference + * @return true if everything is correct, false otherwise. + */ + bool init(const std::string& tensor_path, const int n_classes=19, const int n_batches=1){ + std::cout<<(tensor_path).c_str()<<"\n"; + if(!fileExist(tensor_path.c_str())) + FatalError("This file do not exists" + tensor_path ); + + netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); + classes = n_classes; + nBatches = n_batches; + + checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches)); + + dataDim_t odim = netRT->output_dim; + + checkCuda(cudaMallocHost(&confidences_h, sizeof(float) * odim.tot())); + checkCuda(cudaMalloc(&tmpInputData_d, sizeof(float) * odim.tot())); + checkCuda(cudaMalloc(&tmpOutData_d, sizeof(float) * odim.w*odim.h)); + checkCuda(cudaMallocHost(&tmpOutData_h, sizeof(float) * odim.w*odim.h)); + + segmented.resize(nBatches); + originalSize.resize(nBatches); + + std::vector mean = {0.485, 0.456, 0.406}; + std::vector stddev = {0.229, 0.224, 0.225}; + + checkCuda(cudaMalloc(&mean_d, sizeof(float) * mean.size())); + checkCuda(cudaMalloc(&stddev_d, sizeof(float) * stddev.size())); + + checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); + checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); + + } + + /** + * This method performs the whole detection of the NN. + * + * @param frames frames to run detection on. + * @param cur_batches number of batches to use in inference + * @param save_times if set to true, preprocess, inference and postprocess times + * are saved on a csv file, otherwise not. + * @param times pointer to the output stream where to write times + * @param mAP set to true only if all the probabilities for a bounding + * box are needed, as in some cases for the mAP calculation + */ + void update(std::vector& frames, const int cur_batches=1, bool apply_colormap=true){ + if(cur_batches > nBatches) + FatalError("A batch size greater than nBatches cannot be used"); + + originalSize.clear(); + if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); + { + TKDNN_TSTART + for(int bi=0; biinput_dim; + dim.n = cur_batches; + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + { + TKDNN_TSTART + for(int bi=0; bi splitted_frames; + int H, W, net_H, net_W; + int top = 0, bottom = 0, left = 0, right = 0; + std::vector> pos; + + { + TKDNN_TSTART + cv::Size original_size = frame.size(); + + frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0); + H = frame.rows; + W = frame.cols; + net_H = netRT->input_dim.h; + net_W = netRT->input_dim.w; + + cv::Mat frame_cropped; + + if( H <= net_H && W <= net_W ){ // smaller size wrt network + top = (net_H - H)/2; + bottom = net_H - H - top ; + left = (net_W - W)/2; + right = net_W - W - left ; + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) ); + splitted_frames.push_back(frame_cropped); + } + else{ //bigger size wrt network + + + if(H < net_H || W < net_W){ + if(H < net_H){ + top = (net_H - H)/2; + bottom = net_H - H - top ; + } + else{ + left = (net_W - W)/2; + right = net_W - W - left ; + } + cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); + } + + for(int x=0; x+net_W<=W ;){ + for(int y=0; y+net_H <=H ; ){ + cv::Rect roi(x, y, net_W, net_H); + cv::Mat image_roi = frame(roi); + splitted_frames.push_back(image_roi); + pos.push_back(std::make_pair(x,y)); + + y += net_H; + if(y == H) + break; + if(y + net_H > H) y = H - net_H; + } + x += net_W; + if(x == W) + break; + if(x + net_W > W) x = W - net_W; + } + } + + tk::dnn::dataDim_t idim = netRT->input_dim; + + if(splitted_frames.size()> nBatches) + FatalError(std::to_string(splitted_frames.size()) + " min batches required"); + + for(int bi=0; bistream)); + normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d); + } + TKDNN_TSTOP + stats_pre.push_back(t_ns); + } + + tk::dnn::dataDim_t dim = netRT->input_dim; + dim.n = splitted_frames.size(); + { + if(TKDNN_VERBOSE) dim.print(); + TKDNN_TSTART + netRT->infer(dim, input_d); + TKDNN_TSTOP + if(TKDNN_VERBOSE) dim.print(); + stats.push_back(t_ns); + } + + dataDim_t odim = netRT->output_dim; + + std::vector out_img; + + { + TKDNN_TSTART + + for(int bi=0; bibuffersRT[1]+ netRT->buffersDIM[1].tot()*bi; + + matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h); + maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w); + checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost)); + + dataDim_t vdim = odim; + vdim.c = 1; + + cv::Mat colored; + + if(apply_colormap) + colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes); + else{ + cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h); + colored_fp32.convertTo(colored, CV_8UC1); + } + out_img.push_back(colored); + } + + + cv::Mat seg(frame.size(), out_img[0].type()); + if(out_img.size() == 1) + { + cv::Rect roi(left, top, W, H); + seg = out_img[0](roi); + } + else{ + int bi=0; + + if(top == 0 && left == 0){ + + for(int i=0; i #include #include #include @@ -9,6 +10,8 @@ #include #include #include +#include + #include "tkdnn.h" @@ -36,4 +39,6 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_be void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out); +void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w); + #endif //KERNELSTHRUST_H \ No newline at end of file diff --git a/include/tkDNN/pluginsRT/ActivationLeakyRT.h b/include/tkDNN/pluginsRT/ActivationLeakyRT.h index 9e26b2b..330ed37 100644 --- a/include/tkDNN/pluginsRT/ActivationLeakyRT.h +++ b/include/tkDNN/pluginsRT/ActivationLeakyRT.h @@ -4,9 +4,8 @@ class ActivationLeakyRT : public IPlugin { public: - ActivationLeakyRT() { - - + ActivationLeakyRT(float s) { + slope = s; } ~ActivationLeakyRT(){ @@ -42,13 +41,13 @@ public: virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override { activationLEAKYForward((dnnType*)reinterpret_cast(inputs[0]), - reinterpret_cast(outputs[0]), batchSize*size, stream); + reinterpret_cast(outputs[0]), batchSize*size, slope, stream); return 0; } virtual size_t getSerializationSize() override { - return 1*sizeof(int); + return 1*sizeof(int) + 1*sizeof(float); } virtual void serialize(void* buffer) override { @@ -58,4 +57,5 @@ public: } int size; + float slope; }; diff --git a/include/tkDNN/pluginsRT/ShortcutRT.h b/include/tkDNN/pluginsRT/ShortcutRT.h index 17f050f..04091ac 100644 --- a/include/tkDNN/pluginsRT/ShortcutRT.h +++ b/include/tkDNN/pluginsRT/ShortcutRT.h @@ -4,10 +4,11 @@ class ShortcutRT : public IPlugin { public: - ShortcutRT(tk::dnn::dataDim_t bdim) { + ShortcutRT(tk::dnn::dataDim_t bdim, bool mul) { this->bc = bdim.c; this->bh = bdim.h; this->bw = bdim.w; + this->mul = mul; } ~ShortcutRT(){ @@ -47,15 +48,14 @@ public: dnnType *dstData = reinterpret_cast(outputs[0]); checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream)); - for(int b=0; b < batchSize; ++b) - shortcutForward(srcDataBack + b*bc*bh*bw, dstData + b*c*h*w, 1, c, h, w, 1, 1, bc, bh, bw, 1, stream); + shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream); return 0; } virtual size_t getSerializationSize() override { - return 6*sizeof(int); + return 6*sizeof(int) + sizeof(bool); } virtual void serialize(void* buffer) override { @@ -63,6 +63,7 @@ public: tk::dnn::writeBUF(buf, bc); tk::dnn::writeBUF(buf, bh); tk::dnn::writeBUF(buf, bw); + tk::dnn::writeBUF(buf, mul); tk::dnn::writeBUF(buf, c); tk::dnn::writeBUF(buf, h); tk::dnn::writeBUF(buf, w); @@ -72,4 +73,5 @@ public: int c, h, w; int bc, bh, bw; + bool mul; }; diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index eeef3c2..dd5d673 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -40,7 +40,7 @@ #define COL_PURPLEB "\033[1;35m" #define COL_CYANB "\033[1;36m" -#define TKDNN_VERBOSE 0 +#define TKDNN_VERBOSE 1 // Simple Timer #ifdef __linux__ diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index 6aab775..f0ae161 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -72,6 +72,8 @@ do # ./test_imuodom &>> $out_file # print_output $? imuodom + test_net shelfnet + test_net shelfnet_berkeley test_net yolo4 test_net yolo4-csp test_net yolo4x diff --git a/src/Activation.cpp b/src/Activation.cpp index a8642e7..0b113a7 100644 --- a/src/Activation.cpp +++ b/src/Activation.cpp @@ -5,11 +5,12 @@ namespace tk { namespace dnn { -Activation::Activation(Network *net, int act_mode, const float ceiling) : +Activation::Activation(Network *net, int act_mode, const float ceiling, const float slope) : Layer(net) { - this->act_mode = act_mode; - this->ceiling = ceiling; + this->act_mode = act_mode; + this->ceiling = ceiling; + this->slope = slope; checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) ); if(int(act_mode) < 100) { @@ -46,8 +47,7 @@ Activation::~Activation() { dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) { if(act_mode == ACTIVATION_LEAKY) { - activationLEAKYForward(srcData, dstData, dim.tot()); - + activationLEAKYForward(srcData, dstData, dim.tot(), this->slope); } else if(act_mode == ACTIVATION_MISH) { activationMishForward(srcData, dstData, dim.tot()); diff --git a/src/LayerWgs.cpp b/src/LayerWgs.cpp index a761327..ba7239a 100644 --- a/src/LayerWgs.cpp +++ b/src/LayerWgs.cpp @@ -26,10 +26,11 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs, } readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek); + seek += outputs; this->batchnorm = batchnorm; if(batchnorm) { - seek += outputs; + readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek); seek += outputs; readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek); diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index a95423d..6ac7235 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -237,6 +237,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) { return convert_layer(input, (Flatten*) l); if(type == LAYER_RESHAPE) return convert_layer(input, (Reshape*) l); + if(type == LAYER_RESIZE) + return convert_layer(input, (Resize*) l); if(type == LAYER_REORG) return convert_layer(input, (Reorg*) l); if(type == LAYER_REGION) @@ -390,13 +392,13 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) { #if NV_TENSORRT_MAJOR < 6 // plugin version - IPlugin *plugin = new ActivationLeakyRT(); + IPlugin *plugin = new ActivationLeakyRT(l->slope); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; #else IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU); - lRT->setAlpha(0.1); + lRT->setAlpha(l->slope); checkNULL(lRT); return lRT; #endif @@ -479,13 +481,23 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) { ILayer* NetworkRT::convert_layer(ITensor *input, Reshape *l) { // std::cout<<"convert Reshape\n"; - l->output_dim.print(); IPlugin *plugin = new ReshapeRT(l->output_dim); IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin); checkNULL(lRT); return lRT; } +ILayer* NetworkRT::convert_layer(ITensor *input, Resize *l) { + // std::cout<<"convert Resize\n"; + + IResizeLayer *lRT = networkRT->addResize(*input); //default is kNEAREST + checkNULL(lRT); + Dims d{}; + lRT->setResizeMode(ResizeMode(l->mode)); + lRT->setOutputDimensions(DimsCHW{l->output_dim.c, l->output_dim.h, l->output_dim.w}); + return lRT; +} + ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) { //std::cout<<"convert Reorg\n"; @@ -513,7 +525,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) { ITensor *back_tens = tensors[l->backLayer]; - if(l->backLayer->output_dim.c == l->output_dim.c) + if(l->backLayer->output_dim.c == l->output_dim.c && !l->mul) { IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM); checkNULL(lRT); @@ -522,7 +534,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) { else { // plugin version - IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim); + IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim, l->mul); ITensor **inputs = new ITensor*[2]; inputs[0] = input; inputs[1] = back_tens; @@ -651,7 +663,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa //std::cout<(buf)); a->size = readBUF(buf); assert(buf == bufCheck + serialLength); return a; @@ -710,7 +722,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa bdim.w = readBUF(buf); bdim.l = 1; - ShortcutRT *r = new ShortcutRT(bdim); + ShortcutRT *r = new ShortcutRT(bdim, readBUF(buf)); r->c = readBUF(buf); r->h = readBUF(buf); r->w = readBUF(buf); diff --git a/src/NetworkViz.cpp b/src/NetworkViz.cpp index 6ac274c..b6c95d9 100644 --- a/src/NetworkViz.cpp +++ b/src/NetworkViz.cpp @@ -6,23 +6,389 @@ namespace tk { namespace dnn { -cv::Mat vizFloat2colorMap(cv::Mat map) { +cv::Mat mapillary_15_map(cv::Mat adjMap){ + + // cv::imshow("test", adjMap); + // cv::waitKey(0); + cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1); + + //animal + M3.at(0)=165; + M2.at(0)=42; + M1.at(0)=45; + + //curb + M3.at(1)=196; + M2.at(1)=196; + M1.at(1)=196; + + //barrier + M3.at(2)=90; + M2.at(2)=120; + M1.at(2)=150; + + //road + M3.at(3)=128; + M2.at(3)=64; + M1.at(3)=128; + + //building + M3.at(4)=70; + M2.at(4)=70; + M1.at(4)=70; + + //person + M3.at(5)=220; + M2.at(5)=20; + M1.at(5)=60; + + //roadmark + M3.at(6)=255; + M2.at(6)=255; + M1.at(6)=255; + + //nature + M3.at(7)=107; + M2.at(7)=142; + M1.at(7)=35; + + //sky + M3.at(8)=70; + M2.at(8)=130; + M1.at(8)=180; + + //billboard + M3.at(9)=220; + M2.at(9)=220; + M1.at(9)=220; + + //pole + M3.at(10)=153; + M2.at(10)=153; + M1.at(10)=153; + + //traffic sign + M3.at(11)=128; + M2.at(11)=128; + M1.at(11)=128; + + //bike + M3.at(12)=119; + M2.at(12)=11; + M1.at(12)=32; + + //vehicle + M3.at(13)=0; + M2.at(13)=0; + M1.at(13)=142; + + //void + for(int i=14;i<256;i++) + { + M1.at(i)=0; + M2.at(i)=0; + M3.at(i)=0; + } + + cv::Mat r1,r2,r3; + + cv::LUT(adjMap,M1,r1); + cv::LUT(adjMap,M2,r2); + cv::LUT(adjMap,M3,r3); + + std::vector planes; + planes.push_back(r1); + planes.push_back(r2); + planes.push_back(r3); + + cv::Mat dst; + cv::merge(planes,dst); + return dst; + + +} + +cv::Mat berkeley_20_map(cv::Mat adjMap){ + + cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1); + + //road + M3.at(0)=128; + M2.at(0)=64; + M1.at(0)=128; + + //sidewalk + M3.at(1)=244; + M2.at(1)=35; + M1.at(1)=232; + + //building + M3.at(2)=70; + M2.at(2)=70; + M1.at(2)=70; + + //wall + M3.at(3)=102; + M2.at(3)=102; + M1.at(3)=156; + + //fence + M3.at(4)=90; + M2.at(4)=120; + M1.at(4)=150; + + //pole + M3.at(5)=153; + M2.at(5)=153; + M1.at(5)=153; + + //traffic light + M3.at(6)=250; + M2.at(6)=170; + M1.at(6)=30; + + //traffic sign + M3.at(7)=128; + M2.at(7)=128; + M1.at(7)=128; + + //nature + M3.at(8)=107; + M2.at(8)=142; + M1.at(8)=35; + + //ground + M3.at(9)=0; + M2.at(9)=192; + M1.at(9)=0; + + //sky + M3.at(10)=70; + M2.at(10)=130; + M1.at(10)=180; + + //person + M3.at(11)=220; + M2.at(11)=20; + M1.at(11)=60; + + //rider + M3.at(12)=255; + M2.at(12)=0; + M1.at(12)=100; + + //car + M3.at(13)=0; + M2.at(13)=0; + M1.at(13)=142; + + //truck + M3.at(14)=0; + M2.at(14)=0; + M1.at(14)=70; + + //bus + M3.at(15)=0; + M2.at(15)=60; + M1.at(15)=100; + + //train + M3.at(16)=0; + M2.at(16)=0; + M1.at(16)=192; + + //motorbike + M3.at(17)=0; + M2.at(17)=0; + M1.at(17)=230; + + //bike + M3.at(18)=119; + M2.at(18)=11; + M1.at(18)=32; + + //void + for(int i=19;i<256;i++) + { + M1.at(i)=0; + M2.at(i)=0; + M3.at(i)=0; + } + + cv::Mat r1,r2,r3; + + cv::LUT(adjMap,M1,r1); + cv::LUT(adjMap,M2,r2); + cv::LUT(adjMap,M3,r3); + + std::vector planes; + planes.push_back(r1); + planes.push_back(r2); + planes.push_back(r3); + + cv::Mat dst; + cv::merge(planes,dst); + return dst; + +} + +cv::Mat cityscapes_19_map(cv::Mat adjMap){ + + cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1); + + //road + M3.at(0)=128; + M2.at(0)=64; + M1.at(0)=128; + + //sidewalk + M3.at(1)=244; + M2.at(1)=35; + M1.at(1)=232; + + //building + M3.at(2)=70; + M2.at(2)=70; + M1.at(2)=70; + + //wall + M3.at(3)=102; + M2.at(3)=102; + M1.at(3)=156; + + //fence + M3.at(4)=190; + M2.at(4)=153; + M1.at(4)=153; + + //pole + M3.at(5)=153; + M2.at(5)=153; + M1.at(5)=153; + + //traffic light + M3.at(6)=250; + M2.at(6)=170; + M1.at(6)=30; + + //traffic sign + M3.at(7)=220; + M2.at(7)=220; + M1.at(7)=0; + + //vegetation + M3.at(8)=107; + M2.at(8)=142; + M1.at(8)=35; + + //terrain + M3.at(9)=152; + M2.at(9)=251; + M1.at(9)=152; + + //sky + M3.at(10)=70; + M2.at(10)=130; + M1.at(10)=180; + + //person + M3.at(11)=220; + M2.at(11)=20; + M1.at(11)=60; + + //rider + M3.at(12)=255; + M2.at(12)=0; + M1.at(12)=0; + + //car + M3.at(13)=0; + M2.at(13)=0; + M1.at(13)=142; + + //truck + M3.at(14)=0; + M2.at(14)=0; + M1.at(14)=70; + + //bus + M3.at(15)=0; + M2.at(15)=60; + M1.at(15)=100; + + //train + M3.at(16)=0; + M2.at(16)=80; + M1.at(16)=100; + + //motorcycle + M3.at(17)=0; + M2.at(17)=0; + M1.at(17)=230; + + //bicycle + M3.at(18)=119; + M2.at(18)=11; + M1.at(18)=32; + + //void + for(int i=19;i<256;i++) + { + M1.at(i)=0; + M2.at(i)=0; + M3.at(i)=0; + } + + cv::Mat r1,r2,r3; + + cv::LUT(adjMap,M1,r1); + cv::LUT(adjMap,M2,r2); + cv::LUT(adjMap,M3,r3); + + std::vector planes; + planes.push_back(r1); + planes.push_back(r2); + planes.push_back(r3); + + cv::Mat dst; + cv::merge(planes,dst); + return dst; + +} + + +cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) { + + if(min == 0 && max == 0) + cv::minMaxIdx(map, &min, &max); - double min; - double max; - cv::minMaxIdx(map, &min, &max); cv::Mat adjMap; - // expand your range to 0..255. Similar to histEq(); - map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); - //return adjMap; - - cv::Mat falseColorsMap; - applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_HOT); + + switch (classes) + { + case 15: + map.convertTo(adjMap,CV_8UC1); + falseColorsMap = mapillary_15_map(adjMap); + break; + case 20: + map.convertTo(adjMap,CV_8UC1); + falseColorsMap = berkeley_20_map(adjMap); + break; + case 19: + map.convertTo(adjMap,CV_8UC1); + falseColorsMap = cityscapes_19_map(adjMap); + break; + + default: + // expand your range to 0..255. Similar to histEq(); + map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); + applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET); + } return falseColorsMap; } -cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) { +cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min, double max, int classes) { dnnType *data = nullptr; // copy to CPU @@ -38,14 +404,13 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) { cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0)); for(int i=0; i= net->num_layers) FatalError("Could not viz layer\n"); - return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim); + return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim, imgdim); //cv::imwrite("viz/layer" + std::to_string(layer) + ".png", viz); //cv::imshow("layer", viz); diff --git a/src/Reshape.cpp b/src/Reshape.cpp index c1d814a..f43c4ee 100644 --- a/src/Reshape.cpp +++ b/src/Reshape.cpp @@ -15,6 +15,11 @@ Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) { output_dim.w = new_dim.w; output_dim.l = new_dim.l; + output_dim = new_dim; + + if(input_dim.tot() != output_dim.tot()) + FatalError("Reshape dimension mismatch"); + } Reshape::~Reshape() { diff --git a/src/Resize.cpp b/src/Resize.cpp new file mode 100644 index 0000000..897b823 --- /dev/null +++ b/src/Resize.cpp @@ -0,0 +1,39 @@ +#include + +#include "Layer.h" +#include "kernels.h" + +namespace tk { namespace dnn { + +Resize::Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed, ResizeMode_t mode) : Layer(net) { + + this->mode = mode; + if(fixed){ + output_dim.c = scale_c; + output_dim.h = scale_h; + output_dim.w = scale_w; + } + else{ + output_dim.c *= scale_c; + output_dim.h *= scale_h; + output_dim.w *= scale_w; + } + + checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); +} + +Resize::~Resize() { + + checkCuda( cudaFree(dstData) ); +} + +dnnType* Resize::infer(dataDim_t &dim, dnnType* srcData) { + + resizeForward(srcData, dstData, dim.n, dim.c, dim.h, dim.w, + output_dim.c, output_dim.h, output_dim.w); + dim = output_dim; + + return dstData; +} + +}} \ No newline at end of file diff --git a/src/Shortcut.cpp b/src/Shortcut.cpp index 2c7a4f4..b1053c8 100644 --- a/src/Shortcut.cpp +++ b/src/Shortcut.cpp @@ -5,15 +5,16 @@ namespace tk { namespace dnn { -Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) { +Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) { this->backLayer = backLayer; + this->mul = mul; checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) ); - if( /*backLayer->output_dim.c != input_dim.c ||*/ - backLayer->output_dim.w != input_dim.w || - backLayer->output_dim.h != input_dim.h ) - FatalError("Shortcut dim mismatch"); + if( ( backLayer->output_dim.c != input_dim.c && mul ) || + (( backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) && !mul ) ) + FatalError("Shortcut dim missmatch"); + } Shortcut::~Shortcut() { @@ -26,7 +27,7 @@ dnnType* Shortcut::infer(dataDim_t &dim, dnnType* srcData) { dataDim_t bdim = this->backLayer->output_dim; checkCuda(cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1); + shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1, mul); //update data dimensions dim = output_dim; diff --git a/src/kernels/activation_leaky.cu b/src/kernels/activation_leaky.cu index a9029ad..2ecb3af 100644 --- a/src/kernels/activation_leaky.cu +++ b/src/kernels/activation_leaky.cu @@ -1,7 +1,7 @@ #include "kernels.h" __global__ -void activation_leaky(dnnType *input, dnnType *output, int size) { +void activation_leaky(dnnType *input, dnnType *output, int size, float slope) { int i = blockDim.x*blockIdx.x + threadIdx.x; @@ -9,7 +9,7 @@ void activation_leaky(dnnType *input, dnnType *output, int size) { if (input[i]>0) output[i] = input[i]; else - output[i] = 0.1f*input[i]; + output[i] = slope*input[i]; } } @@ -17,12 +17,12 @@ void activation_leaky(dnnType *input, dnnType *output, int size) { /** ELU activation function */ -void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream) +void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, float slope, cudaStream_t stream) { int blocks = (size+255)/256; int threads = 256; - activation_leaky<<>>(srcData, dstData, size); + activation_leaky<<>>(srcData, dstData, size, slope); } diff --git a/src/kernels/postprocessing.cu b/src/kernels/postprocessing.cu index 3510200..53234a2 100644 --- a/src/kernels/postprocessing.cu +++ b/src/kernels/postprocessing.cu @@ -34,6 +34,25 @@ void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int sortAndTopK_kernel<<>>(src_begin, idsrc, topk_scores, topk_inds, topk_ys, topk_xs, size, K); } +__global__ +void maxElem_kernel(float *src_begin, float *dst_begin, const int n_classes, const int size){ + int i = blockDim.x*blockIdx.x + threadIdx.x; + if (i > size) + return; + + thrust::device_ptr dPbeg ( &src_begin[i*n_classes] ) ; + thrust::device_ptr dPend = dPbeg + n_classes; + thrust::device_ptr result = thrust::max_element(thrust::device,dPbeg, dPend); + + dst_begin[i] = result - dPbeg; +} + +void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w){ + int blocks = (h*w)/32+1; + int threads = 32; + maxElem_kernel<<>>(src_begin, dst_begin, c, h*w); +} + void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys){ thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), clses, thrust::divides()); thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), ids_begin, thrust::modulus()); diff --git a/src/kernels/resize.cu b/src/kernels/resize.cu index 169b853..6059ef5 100644 --- a/src/kernels/resize.cu +++ b/src/kernels/resize.cu @@ -1,46 +1,33 @@ #include "kernels.h" #include -#define MIN(a,b) (((a)<(b))?(a):(b)) -#define MAX(a,b) (((a)>(b))?(a):(b)) -__global__ void resize_kernel( int i_N,float *x, int i_w, int i_h, int i_c, +__global__ void resize_kernel( int size,float *x, int i_w, int i_h, int i_c, int o_w, int o_h, int o_c, int batch, float *out) { - int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; - if(i >= i_N) return; + int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; + if(id >= size) return; - int out_index = i; - int out_w = i%o_w; - i = i/o_w; - int out_h = i%o_h; - i = i/o_h; - int out_c = i%o_c; - i = i/o_c; + int i = id % o_w; + id /= o_w; + int j = id % o_h; + id /= o_h; + int k = id % o_c; + id /= o_c; + int b = id % batch; - //copying last column/last row as padding - int in_index = ((i*i_c + MIN(out_c,i_c-1))*i_h + MIN(out_h,i_h-1))*i_w + MIN(out_w, i_w-1); - out[out_index] = x[in_index]; + int out_index = i + o_w*(j + o_h*(k + o_c*b)); + int add_index = i/(o_w/i_w) + i_w*(j/(o_h/i_h) + i_h*(k + i_c*b)); + out[out_index] = x[add_index]; } void resizeForward( dnnType* srcData, dnnType* dstData, int n, int i_c, int i_h, int i_w, int o_c, int o_h, int o_w, cudaStream_t stream ) { - int i_size = n*i_c*i_h*i_w; int o_size = n*o_c*o_h*o_w; int blocks = (o_size+255)/256; int threads = 256; - if(i_c == o_c && i_h == o_h && i_w == o_w ) - { - checkCuda(cudaMemcpy(dstData, srcData, i_size*sizeof(dnnType), cudaMemcpyDeviceToDevice)); - } - else - { - checkCuda(cudaMemset(dstData, 0, o_size*sizeof(dnnType))); - resize_kernel<<>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData); - // printDeviceVector(i_size, srcData); - // printDeviceVector(o_size, dstData); - } + resize_kernel<<>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData); } diff --git a/src/kernels/shortcut.cu b/src/kernels/shortcut.cu index 83539a8..c2d99bd 100644 --- a/src/kernels/shortcut.cu +++ b/src/kernels/shortcut.cu @@ -21,27 +21,60 @@ __global__ void shortcut_kernel(int size, int minw, int minh, int minc, int stri //out[out_index] += add[add_index]; } +__global__ void shortcut_mul_kernel(int size, int minw, int minh, int minc, int sample, int batch, + int w1, int h1, int c1, dnnType *mul, + int w2, int h2, int c2, float s1, float s2, dnnType *out) +{ + int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; + if (id >= size) return; + int i = id % minw; + id /= minw; + int j = id % minh; + id /= minh; + int k = id % minc; + id /= minc; + int b = id % batch; + + int out_index = i*sample + w1*(j*sample + h1*(k + c1*b)); + out[out_index] = out[out_index] * mul[k + c2*b]; +} + void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1, int n2, int c2, int h2, int w2, int s2, - cudaStream_t stream) + bool mul, cudaStream_t stream) { assert(n1 == n2); int batch = n1; - int minw = (w1 < w2) ? w1 : w2; - int minh = (h1 < h2) ? h1 : h2; - int minc = (c1 < c2) ? c1 : c2; + if(!mul){ + int minw = (w1 < w2) ? w1 : w2; + int minh = (h1 < h2) ? h1 : h2; + int minc = (c1 < c2) ? c1 : c2; + int stride = w1/w2; + int sample = w2/w1; + assert(stride == h1/h2); + assert(sample == h2/h1); + if(stride < 1) stride = 1; + if(sample < 1) sample = 1; - int stride = w1/w2; - int sample = w2/w1; - assert(stride == h1/h2); - assert(sample == h2/h1); - if(stride < 1) stride = 1; - if(sample < 1) sample = 1; + int size = batch * minw * minh * minc; + int blocks = (size+255)/256; + int threads = 256; + + shortcut_kernel<<>>(size, minw, minh, minc, stride, sample, batch, + w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData); + } + else{ + int minw = w1; + int minh = h1; + int minc = c1; + int sample = 1; - int size = batch * minw * minh * minc; - int blocks = (size+255)/256; - int threads = 256; - shortcut_kernel<<>>(size, minw, minh, minc, stride, sample, batch, - w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData); + int size = batch * minw * minh * minc; + int blocks = (size+255)/256; + int threads = 256; + + shortcut_mul_kernel<<>>(size, minw, minh, minc, sample, batch, + w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData); + } } diff --git a/src/utils.cpp b/src/utils.cpp index fa6458f..c73d42e 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -111,6 +111,7 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int } int diffs = 0; for(int i=0; i eps) { diffs += 1; diff --git a/tests/shelfnet/shelfnet.cpp b/tests/shelfnet/shelfnet.cpp new file mode 100644 index 0000000..48cad04 --- /dev/null +++ b/tests/shelfnet/shelfnet.cpp @@ -0,0 +1,295 @@ +#include +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" + + +const char *input_bin = "shelfnet/debug/input.bin"; + +const char *backbone[] = { + "shelfnet/layers/backbone-conv1.bin", + "shelfnet/layers/backbone-layer1-0-conv1.bin", + "shelfnet/layers/backbone-layer1-0-conv2.bin", + "shelfnet/layers/backbone-layer1-1-conv1.bin", + "shelfnet/layers/backbone-layer1-1-conv2.bin", + "shelfnet/layers/backbone-layer2-0-conv1.bin", + "shelfnet/layers/backbone-layer2-0-conv2.bin", + "shelfnet/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet/layers/backbone-layer2-1-conv1.bin", + "shelfnet/layers/backbone-layer2-1-conv2.bin", + "shelfnet/layers/backbone-layer3-0-conv1.bin", + "shelfnet/layers/backbone-layer3-0-conv2.bin", + "shelfnet/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet/layers/backbone-layer3-1-conv1.bin", + "shelfnet/layers/backbone-layer3-1-conv2.bin", + "shelfnet/layers/backbone-layer4-0-conv1.bin", + "shelfnet/layers/backbone-layer4-0-conv2.bin", + "shelfnet/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet/layers/backbone-layer4-1-conv1.bin", + "shelfnet/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet/layers/conv_out-conv-conv.bin", + "shelfnet/layers/conv_out-conv_out.bin", + "shelfnet/layers/conv_out16-conv-conv.bin", + "shelfnet/layers/conv_out16-conv_out.bin", + "shelfnet/layers/conv_out32-conv-conv.bin", + "shelfnet/layers/conv_out32-conv_out.bin" + }; + +const char *decoder[] = { + "shelfnet/layers/decoder-bottom-conv1.bin", + "shelfnet/layers/decoder-bottom-conv12.bin", + "shelfnet/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet/layers/decoder-up_dense_list-1-conv.bin" + }; + + +const char *ladder[] = { + "shelfnet/layers/ladder-inconv-conv1.bin", + "shelfnet/layers/ladder-inconv-conv12.bin", + "shelfnet/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet/layers/ladder-down_conv_list-0.bin", + + "shelfnet/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet/layers/ladder-down_conv_list-1.bin", + + "shelfnet/layers/ladder-bottom-conv1.bin", + "shelfnet/layers/ladder-bottom-conv12.bin", + + + + "shelfnet/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet/layers/trans1-conv.bin", + "shelfnet/layers/trans2-conv.bin", + "shelfnet/layers/trans3-conv.bin"}; +int main() +{ + + downloadWeightsifDoNotExist(input_bin, "shelfnet", "https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download"); + + int classes = 19; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0, di = 0, li = 0, ci = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 19, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, 19, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); + // } + + new tk::dnn::Softmax(&net); + + const char *output_bin = "shelfnet/debug/softmax.bin"; + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + std::cout<<"Input:"< +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" + + +const char *input_bin = "shelfnet_mapillary/debug/input.bin"; + +const char *backbone[] = { + "shelfnet_mapillary/layers/backbone-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer1-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer1-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer1-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer1-1-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer2-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer2-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet_mapillary/layers/backbone-layer2-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer2-1-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer3-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer3-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet_mapillary/layers/backbone-layer3-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer3-1-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer4-0-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer4-0-conv2.bin", + "shelfnet_mapillary/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet_mapillary/layers/backbone-layer4-1-conv1.bin", + "shelfnet_mapillary/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet_mapillary/layers/conv_out-conv-conv.bin", + "shelfnet_mapillary/layers/conv_out-conv_out.bin", + "shelfnet_mapillary/layers/conv_out16-conv-conv.bin", + "shelfnet_mapillary/layers/conv_out16-conv_out.bin", + "shelfnet_mapillary/layers/conv_out32-conv-conv.bin", + "shelfnet_mapillary/layers/conv_out32-conv_out.bin" + }; + +const char *decoder[] = { + "shelfnet_mapillary/layers/decoder-bottom-conv1.bin", + "shelfnet_mapillary/layers/decoder-bottom-conv12.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet_mapillary/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet_mapillary/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet_mapillary/layers/decoder-up_dense_list-1-conv.bin" + }; + + +const char *ladder[] = { + "shelfnet_mapillary/layers/ladder-inconv-conv1.bin", + "shelfnet_mapillary/layers/ladder-inconv-conv12.bin", + "shelfnet_mapillary/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet_mapillary/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet_mapillary/layers/ladder-down_conv_list-0.bin", + + "shelfnet_mapillary/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet_mapillary/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet_mapillary/layers/ladder-down_conv_list-1.bin", + + "shelfnet_mapillary/layers/ladder-bottom-conv1.bin", + "shelfnet_mapillary/layers/ladder-bottom-conv12.bin", + + + + "shelfnet_mapillary/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet_mapillary/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet_mapillary/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet_mapillary/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet_mapillary/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet_mapillary/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet_mapillary/layers/trans1-conv.bin", + "shelfnet_mapillary/layers/trans2-conv.bin", + "shelfnet_mapillary/layers/trans3-conv.bin"}; +int main() +{ + + // downloadWeightsifDoNotExist(input_bin, "shelfnet_mapillary", ""); + // download the weights from here: https://cloud.hipert.unimore.it/f/652476 + + // Mapillary Vistas has originally 66 classes, but we reduced them to 15 to improve the results on the categories of our interest. + int classes = 15; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0, di = 0, li = 0, ci = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); + // } + + new tk::dnn::Softmax(&net); + + const char *output_bin = "shelfnet_mapillary/debug/softmax.bin"; + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + std::cout<<"Input:"<