diff --git a/CMakeLists.txt b/CMakeLists.txt index 1af836d..c29e987 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,10 @@ set(CMAKE_CXX_STANDARD 14) option(ENABLE_OPENCV_CUDA_CONTRIB "Enable OpenCV CUDA Contrib" OFF ) +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "default build" FORCE) +endif(NOT CMAKE_BUILD_TYPE) + find_package(CUDA 9.0 REQUIRED) if (CUDA_FOUND) set(OUTPUTFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cuda_script) # No suffix required @@ -78,8 +82,7 @@ endif() #------------------------------------------------------------------------------- # CUDA #------------------------------------------------------------------------------- - -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'") +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}" --compiler-options '-fPIC') find_package(CUDNN REQUIRED) @@ -203,12 +206,17 @@ target_link_libraries(test_shelfnet_berkeley tkDNN) add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp) target_link_libraries(test_shelfnet_mapillary tkDNN) +add_executable(test_shelfnet_coco tests/shelfnet/shelfnet_coco.cpp) +target_link_libraries(test_shelfnet_coco tkDNN) + # MONODEPTH2 add_executable(test_monodepth2_640 tests/monodepth2/monodepth2_640.cpp) target_link_libraries(test_monodepth2_640 tkDNN) add_executable(test_monodepth2_1024 tests/monodepth2/monodepth2_1024.cpp) target_link_libraries(test_monodepth2_1024 tkDNN) + + # DEMOS add_executable(test_rtinference tests/test_rtinference/rtinference.cpp) target_link_libraries(test_rtinference tkDNN) diff --git a/README.md b/README.md index af7c9c5..825c00b 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,11 @@ If you use tkDNN in your research, please cite the [following paper](https://iee - [x] Support 2D/3D Object Detection and Tracking [README](docs/README_2d3dtracking.md) #### 24 November 2021 - [x] Support to sematic segmentation on cuda 11 -- [x] Support to TensorRT8. +- [x] Support to TensorRT8. (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg)) +#### 30 March 2022 +- [x] Support to monocular depth esitmation [README](docs/README_depth.md) (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg)) + -TensorRT8 (and therefore Jetpack 4.6) is currently supported only on the branch tensort8 due to [performance issue with TensorRT8](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/tensorrt-8.html)). We will merge it to the master as soon as those issues are fixed (probably in future minor releases). ## FPS Results Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on @@ -117,8 +119,8 @@ git clone https://github.com/ceccocats/tkDNN cd tkDNN mkdir build cd build -cmake -DCMAKE_BUILD_TYPE=Release -G"Ninja" .. -ninja +cmake -DCMAKE_BUILD_TYPE=Release .. +make ``` ## Workflow @@ -138,6 +140,7 @@ For specific details on how to export weights see [HERE](./docs/exporting_weight For specific details on how to run: - 2D object detection demos, details on FP16, INT8 and batching see [HERE](./docs/demo.md). - segmentation demos see [HERE](./docs/README_seg.md). +- monocular depth estimation see [HERE](./docs/README_depth.md). - 2D/3D object detection and tracking demos see [HERE](./docs/README_2d3dtracking.md). - mAP demo to evaluate 2D object detectors see [HERE](./docs/mAP_demo.md). @@ -175,16 +178,17 @@ For specific details on how to run tkDNN on Windows 10/11 see [HERE](./docs/wind | yolo4_320 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 320x320 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | | yolo4_512 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | | yolo4_608 | Yolov4 8 | [COCO 2017](http://cocodataset.org/) | 80 | 608x608 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) | -| yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | +| yolo4_berkeley | Yolov4 8 | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 544x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) | | yolo4tiny | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | -| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | +| yolo4x | Yolov4x-mish 9 | [COCO 2017](http://cocodataset.org/) | 80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) | | yolo4tiny_512 | Yolov4 tiny 9 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) | -80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) | | yolo4x-cps | Scaled Yolov4 10 | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download) | | shelfnet | ShelfNet18_realtime11 | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) | | shelfnet_berkeley | ShelfNet18_realtime11 | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) | | dla34_cnet3d | Centernet3D (DLA34 backend)4 | [KITTI 2017](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d) | 1 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download) | | dla34_ctrack | CenterTrack (DLA34 backend)12 | [NuScenes 3D](https://www.nuscenes.org/) | 7 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download) | +| monodepth2 | Monodepth2 13 | [KITTI DEPTH](http://www.cvlibs.net/datasets/kitti/raw_data.php) | - | 640x192 | [weights-mono](https://cloud.hipert.unimore.it/s/iYw9QwgP6CsqxLR/download) | +| monodepth2 | Monodepth2 13 | [KITTI DEPTH](http://www.cvlibs.net/datasets/kitti/raw_data.php) | - | 640x192 | [weights-stereo](https://cloud.hipert.unimore.it/s/XmwbWNXDfqyQ4EL/download) | ## References @@ -201,3 +205,11 @@ For specific details on how to run tkDNN on Windows 10/11 see [HERE](./docs/wind 10. Wang, Chien-Yao, Alexey Bochkovskiy, and Hong-Yuan Mark Liao. "Scaled-YOLOv4: Scaling Cross Stage Partial Network." arXiv preprint arXiv:2011.08036 (2020). 11. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019. 12. Zhou, Xingyi, Vladlen Koltun, and Philipp Krähenbühl. "Tracking objects as points." European Conference on Computer Vision. Springer, Cham, 2020. +13. Godard, Clément, et al. "Digging into self-supervised monocular depth estimation." Proceedings of the IEEE/CVF International Conference on Computer Vision. 2019. + +## Contributors +The main contibutors, in chronological order, are: +- [Francesco Gatti](https://github.com/ceccocats), francesco.gatti@hipert.it +- [Micaela Verucchi](https://github.com/mive93), micaela.verucchi@unimore.it +- [Davide Sapienza](https://github.com/sapienzadavide), davide.sapienza@unimore.it +- [Harshvardhan Chandirasekar](https://github.com/perseusdg), f20180523@goa.bits-pilani.ac.in diff --git a/demo/demo/demo.cpp b/demo/demo/demo.cpp index f46ca80..b6f78d7 100644 --- a/demo/demo/demo.cpp +++ b/demo/demo/demo.cpp @@ -9,7 +9,6 @@ #include "Yolo3Detection.h" bool gRun; -bool SAVE_RESULT = false; void sig_handler(int signo) { std::cout<<"request gateway stop\n"; @@ -20,44 +19,37 @@ int main(int argc, char *argv[]) { signal(SIGINT, sig_handler); -#ifdef __linux__ - std::string config_file = "../demo/demoConfig.yaml"; -#elif _WIN32 - std::string config_file = "..\\..\\..\\demo\\demoConfig.yaml"; -#endif - - if(argc > 1){ - config_file = argv[1]; - } - - YAML::Node conf = YAMLloadConf(config_file); - if(!conf){ + // get config file path and read it + #ifdef __linux__ + std::string config_file = "../demo/demoConfig.yaml"; + #elif _WIN32 + std::string config_file = "..\\..\\..\\demo\\demoConfig.yaml"; + #endif + if(argc > 1) + config_file = argv[1]; + + YAML::Node conf = YAMLloadConf(config_file); + if(!conf) FatalError("Problem with config file"); - } - - std::string net = YAMLgetConf(conf,"net","yolo4tiny_fp32.rt"); - if(!fileExist(net.c_str())) { + // read settings from config file + std::string net = YAMLgetConf(conf, "net", "yolo4tiny_fp32.rt"); + if(!fileExist(net.c_str())) FatalError("The given network does not exist. Create the rt first."); - } -#ifdef __linux__ - std::string input = YAMLgetConf(conf, "input", "../demo/yolo_test.mp4"); - std::string cfgPath = YAMLgetConf(conf,"cfg_input", "../tests/darknet/cfg/yolo4tiny.cfg"); - std::string namePath = YAMLgetConf(conf,"name_input","../tests/darknet/names/coco.names"); -#elif _WIN32 - std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); - std::string cfgPath = YAMLgetConf(conf,"cfg_win_input","..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg"); - std::string namePath = YAMLgetConf(conf,"name_win_input","..\\..\\..\\tests\\darknet\\names\\coco.names"); -#endif - if(!fileExist(input.c_str())) - FatalError("The given input video does not exist."); + #ifdef __linux__ + std::string input = YAMLgetConf(conf, "input", "../demo/yolo_test.mp4"); + #elif _WIN32 + std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4"); + #endif + if(!fileExist(input.c_str())) + FatalError("The given input video does not exist."); char ntype = YAMLgetConf(conf, "ntype", 'y'); int n_classes = YAMLgetConf(conf, "n_classes", 80); int n_batch = YAMLgetConf(conf, "n_batch", 1); if(n_batch < 1 || n_batch > 64) - FatalError("Batch dim not supported"); + FatalError("Batch dim not supported"); float conf_thresh = YAMLgetConf(conf, "conf_thresh", 0.3); bool show = YAMLgetConf(conf, "show", true); bool save = YAMLgetConf(conf, "save", false); @@ -70,7 +62,8 @@ int main(int argc, char *argv[]) { std::cout <<"Demo settings - input: "<< input <<", show: "<< show <<", save: "<< save<<"\n\n"; - + + // create detection network tk::dnn::Yolo3Detection yolo; tk::dnn::CenternetDetection cnet; tk::dnn::MobilenetDetection mbnet; @@ -93,15 +86,9 @@ int main(int argc, char *argv[]) { FatalError("Network type not allowed (3rd parameter)\n"); } - if(ntype == 'c' || ntype == 'm'){ - cfgPath = ""; - namePath = ""; - } - - detNN->init(net,cfgPath,namePath,n_classes,n_batch,conf_thresh); - - gRun = true; + detNN->init(net,n_classes,n_batch,conf_thresh); + // open video stream cv::VideoCapture cap(input); if(!cap.isOpened()) gRun = false; @@ -115,13 +102,15 @@ int main(int argc, char *argv[]) { resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h)); } - cv::Mat frame; if(show) cv::namedWindow("detection", cv::WINDOW_NORMAL); + cv::Mat frame; std::vector batch_frame; std::vector batch_dnn_input; + // start detection loop + gRun = true; while(gRun) { batch_dnn_input.clear(); batch_frame.clear(); @@ -154,14 +143,13 @@ int main(int argc, char *argv[]) { } std::cout<<"detection end\n"; + double mean = 0; - std::cout<stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; - std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n"; + std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; + std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n"; for(int i=0; istats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size(); - std::cout<<"Avg: "< 2) ntype = argv[2][0]; if(argc > 3) - cfg_path = argv[3]; + labels_path = argv[3]; if(argc > 4) - name_path = argv[4]; + config_filename = argv[4]; if(argc > 5) - labels_path = argv[5]; + n_batches = atoi(argv[5]); if(argc > 6) - config_filename = argv[6]; - if(argc > 7) - n_batches = atoi(argv[7]); - if(argc > 8) - confidence_thresh = atof(argv[8]); + confidence_thresh = atof(argv[6]); std::cout<<"conf t: "<init(net,cfg_path,name_path,n_classes, 1, conf_thresh); + detNN->init(net,n_classes, 1, conf_thresh); //read images std::ifstream all_labels(labels_path); diff --git a/demo/demoConfig.yaml b/demo/demoConfig.yaml index f14ffaa..5785852 100644 --- a/demo/demoConfig.yaml +++ b/demo/demoConfig.yaml @@ -2,16 +2,8 @@ input : "../demo/yolo_test.mp4" win_input : "..\\..\\..\\demo\\yolo_test.mp4" -#cfg input -cfg_input : "../tests/darknet/cfg/yolo4tiny.cfg" -cfg_win_input : "..\\..\\..\\tests\\darknet\\cfg\\yolo4tiny.cfg" - -#name input -name_input : "../tests/darknet/names/coco.names" -name_win_input : "..\\..\\..\\tests\\darknet\\names\\coco.names" - # network config -net : "yolo4tiny_fp32.rt" +net : "yolo4_berkeley_fp32.rt" ntype : 'y' n_classes : 80 n_batch : 1 @@ -19,4 +11,4 @@ conf_thresh : 0.3 # demo config show : true -save : false \ No newline at end of file +save : false diff --git a/docs/README_depth.md b/docs/README_depth.md new file mode 100644 index 0000000..ac87bb6 --- /dev/null +++ b/docs/README_depth.md @@ -0,0 +1,54 @@ +# Monocular depth estimation with tkDNN + +Currently tkDNN supports only Monodepth2 as monocular depth esitmation network. + + +## Run the demo + +To run the depth estimation demo follow these steps (example with monodepth2): +``` +rm monodepth2_fp32.rt # be sure to delete(or move) old tensorRT files +./test_monodepth2 # run the yolo test (is slow) +./demoDepth monodepth2_fp32.rt ../demo/yolo_test.mp4 +``` +In general the demo program takes the following parameters: +``` +./demoDepth +``` +where +* `````` is the rt file generated by a test +* ```<``` is the path to a video file or a camera input +* `````` if set to 0 the demo will not show the visualization, it will otherwise (default=1) +* `````` if set to 1 the demo will save the video into result.mp4, it won't otherwise (default=1) + +NB) By default it is used FP32 inference + + +![demo](https://user-images.githubusercontent.com/11939259/160845358-0d6ab15d-c5f4-46ae-b9da-bfaf3903389d.gif "Results on yolo_test.mp4") + + + + diff --git a/docs/demo.md b/docs/demo.md index bf6b79d..5961acb 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -46,8 +46,6 @@ The config file is a yaml file with the following attributes: * ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed. * ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1) * ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1) -* ```cfg_input``` (for linux) \ ```cfg_win_input``` (for windows) is the location of the cfg path of the network for mobilenet and centernet networks use ```" "``` -* ```name_input``` (for linux) \ ```name_win_input``` (for windows) is the location of the name path of the network for mobilenet and centernet networks use ```" "``` N.B. By default it is used FP32 inference diff --git a/docs/exporting_weights.md b/docs/exporting_weights.md index 811431d..b4cb366 100644 --- a/docs/exporting_weights.md +++ b/docs/exporting_weights.md @@ -86,6 +86,18 @@ mkdir layer debug python export.py ``` +### 6)Export weights for monodepth2 +To get the weights needed to run Shelfnet tests use [this](https://github.com/perseusdg/monodepth2) fork of a Pytorch implementation of monodepth2 network. + +``` +git clone https://github.com/perseusdg/monodepth2 +cd monodepth2 +mkdir models # Download the official weights and put depth.pth and encorder.pth inside this new folder +conda env create --file monodepth.yaml +conda activate monodepth2 +python exporter.py # you will find the weights inside the tkDNN_bin folder +``` + ## Darknet Parser tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*: ``` diff --git a/include/tkDNN/CenternetDetection.h b/include/tkDNN/CenternetDetection.h index 07c80cd..3c8cfbb 100644 --- a/include/tkDNN/CenternetDetection.h +++ b/include/tkDNN/CenternetDetection.h @@ -73,7 +73,7 @@ public: CenternetDetection() {}; ~CenternetDetection() {}; - bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/DetectionNN.h b/include/tkDNN/DetectionNN.h index 3a757ef..a8c81f7 100644 --- a/include/tkDNN/DetectionNN.h +++ b/include/tkDNN/DetectionNN.h @@ -87,7 +87,7 @@ class DetectionNN { * @param n_batches maximum number of batches to use in inference * @return true if everything is correct, false otherwise. */ - virtual bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0; + virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0; /** * This method performs the whole detection of the NN. diff --git a/include/tkDNN/MobilenetDetection.h b/include/tkDNN/MobilenetDetection.h index ec35b20..58512d5 100644 --- a/include/tkDNN/MobilenetDetection.h +++ b/include/tkDNN/MobilenetDetection.h @@ -65,7 +65,7 @@ public: MobilenetDetection() {}; ~MobilenetDetection() {}; - bool init(const std::string& tensor_path, const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches=1, const float conf_thresh=0.3); + bool init(const std::string& tensor_path,const int n_classes, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/NetworkRT.h b/include/tkDNN/NetworkRT.h index a7e67c2..a422134 100644 --- a/include/tkDNN/NetworkRT.h +++ b/include/tkDNN/NetworkRT.h @@ -30,10 +30,6 @@ namespace tk { namespace dnn { -using namespace nvinfer1; - - - class NetworkRT { public: @@ -57,6 +53,7 @@ public: dnnType *output; cudaStream_t stream; + std::vector yolo_plugins; // yolo layers in network NetworkRT(Network *net, const char *name); virtual ~NetworkRT(); diff --git a/include/tkDNN/SegmentationNN.h b/include/tkDNN/SegmentationNN.h index 403bc28..c39323c 100644 --- a/include/tkDNN/SegmentationNN.h +++ b/include/tkDNN/SegmentationNN.h @@ -185,6 +185,7 @@ class SegmentationNN { checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream)); return true; + return true; } /** diff --git a/include/tkDNN/Yolo3Detection.h b/include/tkDNN/Yolo3Detection.h index 5a29d9c..b49915a 100644 --- a/include/tkDNN/Yolo3Detection.h +++ b/include/tkDNN/Yolo3Detection.h @@ -19,13 +19,12 @@ private: tk::dnn::Yolo* getYoloLayer(int n=0); cv::Mat bgr_h; - std::vector noYolos; public: Yolo3Detection() {}; ~Yolo3Detection() {}; - bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path,const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); + bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3); void preprocess(cv::Mat &frame, const int bi=0); void postprocess(const int bi=0,const bool mAP=false); }; diff --git a/include/tkDNN/pluginsRT/ReshapeRT.h b/include/tkDNN/pluginsRT/ReshapeRT.h index e56c79c..65d577d 100644 --- a/include/tkDNN/pluginsRT/ReshapeRT.h +++ b/include/tkDNN/pluginsRT/ReshapeRT.h @@ -5,7 +5,6 @@ #include #include #include -using namespace tk::dnn; namespace nvinfer1 { diff --git a/include/tkDNN/pluginsRT/YoloRT.h b/include/tkDNN/pluginsRT/YoloRT.h index 352a0cb..f0e5733 100644 --- a/include/tkDNN/pluginsRT/YoloRT.h +++ b/include/tkDNN/pluginsRT/YoloRT.h @@ -5,7 +5,6 @@ #include #include "../kernels.h" #include -#include #define YOLORT_CLASSNAME_W 256 @@ -80,8 +79,10 @@ namespace nvinfer1 { float nms_thresh; int nms_kind; int new_coords; - int NUM = 0; + std::vector classesNames; + std::vector mask; + std::vector bias; int entry_index(int batch, int location, int entry) { diff --git a/include/tkDNN/tkdnn.h b/include/tkDNN/tkdnn.h index 26aaa0b..b129e6e 100644 --- a/include/tkDNN/tkdnn.h +++ b/include/tkDNN/tkdnn.h @@ -5,4 +5,4 @@ #include "Layer.h" #include "NetworkRT.h" -#define TKDNN_VERSION 500 +#define TKDNN_VERSION 700 diff --git a/include/tkDNN/utils.h b/include/tkDNN/utils.h index a1a1f1c..607697a 100644 --- a/include/tkDNN/utils.h +++ b/include/tkDNN/utils.h @@ -23,13 +23,16 @@ #include #include +#include -#if NV_TENSORRT_MAJOR > 7 -#define NOEXCEPT noexcept -#else -#define NOEXCEPT +#ifndef NOEXCEPT + #if NV_TENSORRT_MAJOR > 7 + #define NOEXCEPT noexcept + #else + #define NOEXCEPT + #endif #endif @@ -176,5 +179,4 @@ inline T YAMLgetConf(YAML::Node conf, std::string key, T defaultVal) { return val; } - #endif //UTILS_H diff --git a/scripts/test_all_tests.sh b/scripts/test_all_tests.sh index 0da9b6b..e44f7e8 100644 --- a/scripts/test_all_tests.sh +++ b/scripts/test_all_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -cd build +#cd build RED='\033[1;31m' GREEN='\033[1;32m' @@ -29,24 +29,28 @@ function print_output { } +out_dir=results out_file=results.log -rm $out_file +rm -rf $out_dir/ +mkdir -p $out_dir function test_net { - ./test_$1 &>> $out_file + ./test_$1 &> $out_dir/$1_${TKDNN_MODE}_build_$out_file print_output $? $1 - ./test_rtinference $1*.rt $TKDNN_BATCHSIZE &>> $out_file + ./test_rtinference $1*.rt 1 &> $out_dir/$1_${TKDNN_MODE}_inference_batch1_$out_file + print_output $? "infer $1" + ./test_rtinference $1*.rt $TKDNN_BATCHSIZE &> $out_dir/$1_${TKDNN_MODE}_inference_batch${TKDNN_BATCHSIZE}_$out_file print_output $? "batched $1" } -modes=( 1 ) # only FP32 -# modes=( 1 2 ) # FP32 and FP16 +# modes=( 1 ) # only FP32 +modes=( 1 2 ) # FP32 and FP16 # modes=( 1 2 3 ) # FP32, FP16 and INT8 for i in "${modes[@]}" do - rm *rt + rm -f *rt if [ $i -eq 1 ] then export TKDNN_MODE=FP32 @@ -73,37 +77,37 @@ do # print_output $? imuodom test_net yolo4 - test_net yolo4_320 - test_net yolo4_320_coco2 - test_net yolo4_512 - test_net yolo4_608 - test_net yolo4-csp - test_net yolo4x - test_net yolo4_berkeley - test_net yolo4_berkeley_f1 - test_net yolo4tiny - test_net yolo4tiny_512 - test_net yolo3 - test_net yolo3_berkeley - test_net yolo3_coco4 - test_net yolo3_flir - test_net yolo3_512 - test_net yolo3tiny - test_net yolo3tiny_512 - test_net yolo2 - test_net yolo2_voc - #test_net yolo2tiny - test_net csresnext50-panet-spp - #test_net csresnext50-panet-spp_berkeley - test_net resnet101_cnet - test_net dla34_cnet - test_net dla34_cnet3d - test_net mobilenetv2ssd - test_net mobilenetv2ssd512 - test_net bdd-mobilenetv2ssd - test_net dla34_ctrack - test_net shelfnet - test_net shelfnet_berkeley + # test_net yolo4_320 + # test_net yolo4_320_coco2 + # test_net yolo4_512 + # test_net yolo4_608 + # test_net yolo4-csp + # test_net yolo4x + # test_net yolo4_berkeley + # test_net yolo4_berkeley_f1 + # test_net yolo4tiny + # test_net yolo4tiny_512 + # test_net yolo3 + # test_net yolo3_berkeley + # test_net yolo3_coco4 + # test_net yolo3_flir + # test_net yolo3_512 + # test_net yolo3tiny + # test_net yolo3tiny_512 + # test_net yolo2 + # test_net yolo2_voc + # test_net yolo2tiny + # test_net csresnext50-panet-spp + # test_net csresnext50-panet-spp_berkeley + # test_net resnet101_cnet + # test_net dla34_cnet + # test_net dla34_cnet3d + # test_net mobilenetv2ssd + # test_net mobilenetv2ssd512 + # test_net bdd-mobilenetv2ssd + # test_net dla34_ctrack + # test_net shelfnet + # test_net shelfnet_berkeley done -echo "If errors occured, check logfile $out_file" +echo "If errors occured, check logfiles in directory: $out_dir" diff --git a/src/CenternetDetection.cpp b/src/CenternetDetection.cpp index 133c682..c18e08c 100644 --- a/src/CenternetDetection.cpp +++ b/src/CenternetDetection.cpp @@ -3,7 +3,7 @@ namespace tk { namespace dnn { -bool CenternetDetection::init(const std::string& tensor_path, const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches, const float conf_thresh){ +bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() ); classes = n_classes; @@ -120,6 +120,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const std::string& dst2.at(2,1)=dst2.at(1,1) + (dst2.at(0,0)-dst2.at(1,0) ); return true; + return true; } diff --git a/src/MobilenetDetection.cpp b/src/MobilenetDetection.cpp index 1d3832d..13b836f 100644 --- a/src/MobilenetDetection.cpp +++ b/src/MobilenetDetection.cpp @@ -126,7 +126,7 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){ return iou; } -bool MobilenetDetection::init(const std::string& tensor_path, const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches, const float conf_thresh){ +bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){ std::cout<<(tensor_path).c_str()<<"\n"; netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str()); imageSize = netRT->input_dim.h; @@ -198,7 +198,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const std::string& "bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" , "apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" , "donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" , - "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , + "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , "cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" , "book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"}; classesNames = std::vector(classes_names_, std::end(classes_names_)); @@ -207,7 +207,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const std::string& else{ FatalError("Number of classes not supported for mobilenet"); } - return 1; + return true; } void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){ diff --git a/src/NetworkRT.cpp b/src/NetworkRT.cpp index c7427d1..26489bf 100644 --- a/src/NetworkRT.cpp +++ b/src/NetworkRT.cpp @@ -15,6 +15,9 @@ using namespace nvinfer1; +extern std::mutex gYoloPlugins_mutex; +extern std::vector gYoloPlugins; + // Logger for info/warning/errors class Logger : public ILogger { void log(Severity severity, const char* msg) NOEXCEPT override { @@ -826,6 +829,12 @@ IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Yolo *l) { mPluginAttributes.emplace_back(PluginField("nms_thresh",&l->nms_thresh,PluginFieldType::kFLOAT32,1)); mPluginAttributes.emplace_back(PluginField("nms_kins",&l->nsm_kind,PluginFieldType::kINT32,1)); mPluginAttributes.emplace_back(PluginField("new_coords",&l->new_coords,PluginFieldType::kINT32,1)); + mPluginAttributes.emplace_back(PluginField("mask",l->mask_h,PluginFieldType::kFLOAT32,l->n_masks)); + mPluginAttributes.emplace_back(PluginField("bias",l->bias_h,PluginFieldType::kFLOAT32,l->n_masks*2*l->num)); + for(int i=0; iclasses; i++) { + mPluginAttributes.emplace_back(PluginField("class_name",l->classesNames[i].data(),PluginFieldType::kCHAR,l->classesNames[i].size())); + } + mFC.nbFields = mPluginAttributes.size(); mFC.fields = mPluginAttributes.data(); auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC); @@ -1001,7 +1010,14 @@ bool NetworkRT::deserialize(const char *filename) { } runtimeRT = createInferRuntime(loggerRT); + + gYoloPlugins_mutex.lock(); + gYoloPlugins.clear(); engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size); + yolo_plugins = gYoloPlugins; + gYoloPlugins.clear(); + gYoloPlugins_mutex.unlock(); + std::cout< nms_thresh) + if (nsm_kind == GREEDY_NMS && yolo_box_iou(a, b) > thresh) dets[j].prob[k] = 0; - else if (nsm_kind == DIOU_NMS && yolo_box_diou(a, b, nms_thresh) > nms_thresh) + else if (nsm_kind == DIOU_NMS && yolo_box_diou(a, b, nms_thresh) > thresh) dets[j].prob[k] = 0; } } diff --git a/src/Yolo3Detection.cpp b/src/Yolo3Detection.cpp index bda206a..c7f226e 100644 --- a/src/Yolo3Detection.cpp +++ b/src/Yolo3Detection.cpp @@ -3,7 +3,7 @@ namespace tk { namespace dnn { - bool Yolo3Detection::init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches, const float conf_thresh) { + bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) { //convert network to tensorRT std::cout<<(tensor_path).c_str()<<"\n"; @@ -14,43 +14,27 @@ namespace tk { namespace dnn { tk::dnn::dataDim_t idim = netRT->input_dim; idim.n = nBatches; - std::vector yolosLine = noYolosLine(cfg_path); - noYolos = yolosLine; - int channels,height,width; - loadYoloInitInfo(channels,width,height,cfg_path); - - - - if(yolosLine.size() < 2 ) { + if(netRT->yolo_plugins.size() < 2 ) { FatalError("this is not yolo3"); } + for(int i=0; iyolo_plugins.size(); i++) { + nvinfer1::YoloRT *yRT = netRT->yolo_plugins[i]; + classes = yRT->classes; + num = yRT->num; + nMasks = yRT->n_masks; - for(int i=0; i maskTemp,anchorsTemp; - std::vector classNamesTemp; - int nms_kind,coords,numTemp; - float nmsthresh; - loadYoloInfo(cfg_path,yolosLine[i],maskTemp,anchorsTemp,numTemp,classes,nmsthresh,nms_kind,coords); - classNamesTemp = darknetReadNames(name_path); - num = numTemp/maskTemp.size(); - nMasks = maskTemp.size(); - dnnType* maskTempF; - dnnType* biasTempF; - maskTempF = maskTemp.data(); - biasTempF = anchorsTemp.data(); // make a yolo layer to interpret predictions yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias yolo[i]->mask_h = new dnnType[nMasks]; yolo[i]->bias_h = new dnnType[num*nMasks*2]; - memcpy(yolo[i]->mask_h, maskTempF, sizeof(dnnType)*nMasks); - memcpy(yolo[i]->bias_h, biasTempF, sizeof(dnnType)*num*nMasks*2); - auto dim = netRT->engineRT->getBindingDimensions(i+1); - yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, dim.d[0], dim.d[1], dim.d[2]); - yolo[i]->classesNames = classNamesTemp; - yolo[i]->nms_thresh = nmsthresh; - yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) nms_kind; - yolo[i]->new_coords = coords; + memcpy(yolo[i]->mask_h, yRT->mask.data(), sizeof(dnnType)*nMasks); + memcpy(yolo[i]->bias_h, yRT->bias.data(), sizeof(dnnType)*num*nMasks*2); + yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w); + yolo[i]->classesNames = yRT->classesNames; + yolo[i]->nms_thresh = yRT->nms_thresh; + yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) yRT->nms_kind; + yolo[i]->new_coords = yRT->new_coords; } dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes); @@ -112,12 +96,12 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ //get yolo outputs - if(noYolos.size() < 2){ + if(netRT->yolo_plugins.size() < 2){ FatalError("YOLOS WRONG!!"); } std::vector rt_out; //dnnType *rt_out[netRT->pluginFactory->n_yolos]; - for(int i=0; iyolo_plugins.size(); i++) rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi); float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w); @@ -125,7 +109,7 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){ // compute dets nDets = 0; - for(int i=0; iyolo_plugins.size(); i++) { yolo[i]->dstData = rt_out[i]; yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords); } diff --git a/src/pluginsRT/YoloRT.cpp b/src/pluginsRT/YoloRT.cpp index 55cb462..590acb3 100644 --- a/src/pluginsRT/YoloRT.cpp +++ b/src/pluginsRT/YoloRT.cpp @@ -1,8 +1,13 @@ #include #include +#include using namespace nvinfer1; +// used to retrive Yolo plugin during network deserialization +std::mutex gYoloPlugins_mutex; +std::vector gYoloPlugins; + std::vector YoloRTPluginCreator::mPluginAttributes; PluginFieldCollection YoloRTPluginCreator::mFC{}; @@ -22,6 +27,10 @@ YoloRT::YoloRT(int classes, int num, int c,int h,int w,int n_masks, float scale_ this->nms_thresh = nms_thresh; this->nms_kind = nms_kind; this->new_coords = new_coords; + + bias.clear(); + mask.clear(); + classesNames.clear(); } YoloRT::YoloRT(const void *data, size_t length) { @@ -36,7 +45,24 @@ YoloRT::YoloRT(const void *data, size_t length) { c = readBUF(buf); h = readBUF(buf); w = readBUF(buf); + + mask.resize(n_masks); + for(int i=0; i(buf); + bias.resize(n_masks*2*num); + for(int i=0; i(buf); + + // save classes names + classesNames.resize(classes); + for(int i=0; i(buf); + classesNames[i] = std::string(tmp); + } assert(buf == bufCheck + length); + gYoloPlugins.push_back(this); } YoloRT::~YoloRT() {} @@ -126,7 +152,7 @@ int32_t YoloRT::enqueue(int32_t batchSize, const void *const *inputs, void **out size_t YoloRT::getSerializationSize() const NOEXCEPT { - return 8 * sizeof(int) + 2 * sizeof(float) ; + return 8 * sizeof(int) + 2 * sizeof(float) + n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char); } bool YoloRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT { @@ -145,6 +171,19 @@ void YoloRT::serialize(void *buffer) const NOEXCEPT { writeBUF(buf, c); //std::cout << "C : " << c << std::endl; writeBUF(buf, h); //std::cout << "H : " << h << std::endl; writeBUF(buf, w); //std::cout << "C : " << c << std::endl; + for (int i = 0; i < n_masks; i++) + writeBUF(buf, mask[i]); //std::cout << "mask[i] : " << mask[i] << std::endl; + for (int i = 0; i < n_masks * 2 * num; i++) + writeBUF(buf, bias[i]); //std::cout << "bias[i] : " << bias[i] << std::endl; + + // save classes names + for(int i=0; imask = mask; + p->bias = bias; + p->classesNames = classesNames; p->setPluginNamespace(mPluginNamespace.c_str()); return p; } @@ -235,6 +277,17 @@ IPluginV2Ext *YoloRTPluginCreator::createPlugin(const char *name, const PluginFi int nms_kind = *(static_cast(fields[8].data)); int new_coords = *(static_cast(fields[9].data)); auto *pluginObj = new YoloRT(classes,num,c,h,w,n_masks,scaleXY,nmsThresh,nms_kind,new_coords); + + // fill additional data + pluginObj->mask.resize(fields[10].length*sizeof(float)); + memcpy(pluginObj->mask.data(), fields[10].data, fields[10].length*sizeof(float)); + pluginObj->bias.resize(fields[11].length*sizeof(float)); + memcpy(pluginObj->bias.data(), fields[11].data, fields[11].length*sizeof(float)); + pluginObj->classesNames.resize(classes); + for(int i=0; iclassesNames[i].resize(fields[12+i].length); + memcpy(&pluginObj->classesNames[i][0], fields[12+i].data, fields[12+i].length*sizeof(char)); + } return pluginObj; } diff --git a/tests/darknet/cfg/yolo4-csp_crowd.cfg b/tests/darknet/cfg/yolo4-csp_crowd.cfg new file mode 100644 index 0000000..4c50f4d --- /dev/null +++ b/tests/darknet/cfg/yolo4-csp_crowd.cfg @@ -0,0 +1,1279 @@ +[net] +# Testing +#batch=1 +#subdivisions=1 +# Training +batch=64 +subdivisions=16 +width=512 +height=512 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 8000 +policy=steps +steps=6400,7200 +scales=.1,.1 + +mosaic=1 + +letter_box=1 + +ema_alpha=0.9998 + +#optimized_memory=1 + +#23:104x104 54:52x52 85:26x26 104:13x13 for 416 + + + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -2 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +#[route] +#layers = -1,-7 + +#[convolutional] +#batch_normalize=1 +#filters=64 +#size=1 +#stride=1 +#pad=1 +#activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -13 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 79 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[upsample] +stride=2 + +[route] +layers = 48 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=128 +activation=mish + +[route] +layers = -1, -6 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=logistic + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=0 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=4.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1, -20 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=logistic + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=1.0 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=5 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1, -49 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=mish + +[route] +layers = -1,-6 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=mish + +[convolutional] +size=1 +stride=1 +pad=1 +filters=21 +activation=logistic + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=2 +num=9 +jitter=.1 +scale_x_y = 2.0 +objectness_smooth=1 +ignore_thresh = .7 +truth_thresh = 1 +#random=1 +resize=1.5 +iou_thresh=0.2 +iou_normalizer=0.05 +cls_normalizer=0.5 +obj_normalizer=0.4 +iou_loss=ciou +nms_kind=diounms +beta_nms=0.6 +new_coords=1 +max_delta=2 diff --git a/tests/darknet/names/crowdhuman.names b/tests/darknet/names/crowdhuman.names new file mode 100644 index 0000000..5ba1275 --- /dev/null +++ b/tests/darknet/names/crowdhuman.names @@ -0,0 +1,2 @@ +person +head \ No newline at end of file diff --git a/tests/darknet/yolo4-csp_crowd.cpp b/tests/darknet/yolo4-csp_crowd.cpp new file mode 100644 index 0000000..b8f0e6b --- /dev/null +++ b/tests/darknet/yolo4-csp_crowd.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "tkdnn.h" +#include "test.h" +#include "DarknetParser.h" + +int main() { + std::string bin_path = "yolo4-csp_crowd"; + std::vector input_bins = { + bin_path + "/layers/input.bin" + }; + std::vector output_bins = { + bin_path + "/debug/layer144_out.bin", + bin_path + "/debug/layer159_out.bin", + bin_path + "/debug/layer174_out.bin" + }; + std::string wgs_path = bin_path + "/layers"; + std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4-csp_crowd.cfg"; + std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/crowdhuman.names"; + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/RKWfWNmWXfJigsK/download"); + + // parse darknet network + tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); + net->print(); + + //convert network to tensorRT + tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str())); + + int ret = testInference(input_bins, output_bins, net, netRT); + net->releaseLayers(); + delete net; + delete netRT; + return ret; +} \ No newline at end of file diff --git a/tests/darknet/yolo4_berkeley_f1.cpp b/tests/darknet/yolo4_berkeley_f1.cpp index f6f4e62..7d830c3 100644 --- a/tests/darknet/yolo4_berkeley_f1.cpp +++ b/tests/darknet/yolo4_berkeley_f1.cpp @@ -17,7 +17,7 @@ int main() { std::string wgs_path = bin_path + "/layers"; std::string cfg_path = std::string(TKDNN_PATH) + "/tests/darknet/cfg/yolo4_berkeley.cfg"; std::string name_path = std::string(TKDNN_PATH) + "/tests/darknet/names/berkeley.names"; - downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/M7WJdGoGDaDACnN/download"); + downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/q9dwoqQ5YQqEi7s/download"); // parse darknet network tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path); diff --git a/tests/shelfnet/shelfnet_coco.cpp b/tests/shelfnet/shelfnet_coco.cpp new file mode 100644 index 0000000..276a09d --- /dev/null +++ b/tests/shelfnet/shelfnet_coco.cpp @@ -0,0 +1,295 @@ +#include +#include +#include + +#include "tkdnn.h" +#include "NetworkViz.h" + + +const char *input_bin = "shelfnet_coco/debug/input.bin"; + +const char *backbone[] = { + "shelfnet_coco/layers/backbone-conv1.bin", + "shelfnet_coco/layers/backbone-layer1-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer1-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer1-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer1-1-conv2.bin", + "shelfnet_coco/layers/backbone-layer2-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer2-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer2-0-downsample-0.bin", + "shelfnet_coco/layers/backbone-layer2-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer2-1-conv2.bin", + "shelfnet_coco/layers/backbone-layer3-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer3-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer3-0-downsample-0.bin", + "shelfnet_coco/layers/backbone-layer3-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer3-1-conv2.bin", + "shelfnet_coco/layers/backbone-layer4-0-conv1.bin", + "shelfnet_coco/layers/backbone-layer4-0-conv2.bin", + "shelfnet_coco/layers/backbone-layer4-0-downsample-0.bin", + "shelfnet_coco/layers/backbone-layer4-1-conv1.bin", + "shelfnet_coco/layers/backbone-layer4-1-conv2.bin"}; + +const char *conv_out[] = { + "shelfnet_coco/layers/conv_out-conv-conv.bin", + "shelfnet_coco/layers/conv_out-conv_out.bin", + "shelfnet_coco/layers/conv_out16-conv-conv.bin", + "shelfnet_coco/layers/conv_out16-conv_out.bin", + "shelfnet_coco/layers/conv_out32-conv-conv.bin", + "shelfnet_coco/layers/conv_out32-conv_out.bin" + }; + +const char *decoder[] = { + "shelfnet_coco/layers/decoder-bottom-conv1.bin", + "shelfnet_coco/layers/decoder-bottom-conv12.bin", + "shelfnet_coco/layers/decoder-up_conv_list-0-conv-conv.bin", + "shelfnet_coco/layers/decoder-up_conv_list-0-conv_atten.bin", + "shelfnet_coco/layers/decoder-up_dense_list-0-conv.bin", + "shelfnet_coco/layers/decoder-up_conv_list-1-conv-conv.bin", + "shelfnet_coco/layers/decoder-up_conv_list-1-conv_atten.bin", + "shelfnet_coco/layers/decoder-up_dense_list-1-conv.bin" + }; + + +const char *ladder[] = { + "shelfnet_coco/layers/ladder-inconv-conv1.bin", + "shelfnet_coco/layers/ladder-inconv-conv12.bin", + "shelfnet_coco/layers/ladder-down_module_list-0-conv1.bin", + "shelfnet_coco/layers/ladder-down_module_list-0-conv12.bin", + "shelfnet_coco/layers/ladder-down_conv_list-0.bin", + + "shelfnet_coco/layers/ladder-down_module_list-1-conv1.bin", + "shelfnet_coco/layers/ladder-down_module_list-1-conv12.bin", + "shelfnet_coco/layers/ladder-down_conv_list-1.bin", + + "shelfnet_coco/layers/ladder-bottom-conv1.bin", + "shelfnet_coco/layers/ladder-bottom-conv12.bin", + + + + "shelfnet_coco/layers/ladder-up_conv_list-0-conv-conv.bin", + "shelfnet_coco/layers/ladder-up_conv_list-0-conv_atten.bin", + "shelfnet_coco/layers/ladder-up_dense_list-0-conv.bin", + + + "shelfnet_coco/layers/ladder-up_conv_list-1-conv-conv.bin", + "shelfnet_coco/layers/ladder-up_conv_list-1-conv_atten.bin", + "shelfnet_coco/layers/ladder-up_dense_list-1-conv.bin"}; + +const char *trans[] = { + "shelfnet_coco/layers/trans1-conv.bin", + "shelfnet_coco/layers/trans2-conv.bin", + "shelfnet_coco/layers/trans3-conv.bin"}; +int main() +{ + + downloadWeightsifDoNotExist(input_bin, "shelfnet_coco", "https://cloud.hipert.unimore.it/s/KfQ9fGJQsgzNbiW/download"); + + int classes = 183; + + // Network layout + tk::dnn::dataDim_t dim(1, 3, 1024, 1024, 1); + tk::dnn::Network net(dim); + + int bi = 0, di = 0, li = 0, ci = 0; + new tk::dnn::Conv2d(&net, 64, 7, 7, 2, 2, 3, 3, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + tk::dnn::Layer* last = new tk::dnn::Pooling (&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX); + + + + for(int i=0; i<2; ++i){ + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, backbone[bi++], true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + } + + std::vector features; + for(int i=0;i<3;++i){ + int out_channel = pow(2,7+i); + std::cout< up_out; + //bottom + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, decoder[di++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + std::cout<output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, decoder[di++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, features[1-i]); + + //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, decoder[di++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + //LADDER + + std::vector down_out; + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + + for(int i=0; i<2;++i){ + int out_channel = pow(2,6+i); + tk::dnn::Layer* l_last = new tk::dnn::Shortcut(&net, up_out[2-i]); + + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, l_last); + l_last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + down_out.push_back(l_last); + + new tk::dnn::Conv2d (&net, out_channel*2, 3, 3, 2, 2, 1, 1, ladder[li++], false); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.0f); //should be ReLU + } + + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, 256, 3, 3, 1, 1, 1, 1, ladder[li++], true, false, 1, true); + new tk::dnn::Shortcut(&net, last); + last = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_RELU); + up_out.clear(); + up_out.push_back(last); + + for(int i=0; i<2; ++i){ + int out_channel = pow(2,7-i); + //up-conv + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + + new tk::dnn::Pooling(&net, last->output_dim.w, last->output_dim.h, last->output_dim.w, last->output_dim.h, 0, 0, tk::dnn::POOLING_AVERAGE); + new tk::dnn::Conv2d (&net, out_channel, 1, 1, 1, 1, 0, 0, ladder[li++], true); + + tk::dnn::Layer* act = new tk::dnn::Activation (&net, CUDNN_ACTIVATION_SIGMOID); + new tk::dnn::Route(&net, &last, 1); + new tk::dnn::Shortcut(&net, act, true); + + //interpolate + new tk::dnn::Resize(&net, 1,2,2); + new tk::dnn::Shortcut(&net, down_out[1-i]); + + // //up-dense + new tk::dnn::Conv2d (&net, out_channel, 3, 3, 1, 1, 1, 1, ladder[li++], true); + last = new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + up_out.push_back(last); + } + + + // for(int i=2;i>=0;--i){ + // new tk::dnn::Route(&net, &up_out[i], 1); + new tk::dnn::Conv2d (&net, 64, 3, 3, 1, 1, 1, 1, conv_out[ci++], true); + new tk::dnn::Activation (&net, tk::dnn::ACTIVATION_LEAKY, 0.0f, 0.01); + new tk::dnn::Conv2d (&net, classes, 3, 3, 1, 1, 1, 1, conv_out[ci++], false); + /*up_out[i] =*/ new tk::dnn::Resize(&net, classes, net.input_dim.h, net.input_dim.w, true, tk::dnn::ResizeMode_t::LINEAR); + // // } + + new tk::dnn::Softmax(&net); + + const char *output_bin = "shelfnet_coco/debug/softmax.bin"; + + // Load input + dnnType *data; + dnnType *input_h; + readBinaryFile(input_bin, dim.tot(), &input_h, &data); + std::cout<<"Input:"<