rtinference nstep

Tensorrt8

.gitignore

@@ -19,4 +19,6 @@ demo/BDD100K_val
 cmake-build-minsizerel/*
 scripts/COCO_val2017/*
 scripts/COCO_val2017.zip
-scripts/all_labels.txt
+scripts/all_labels.txt
+/cmake/cuda_script
+/cmake-build-debug/

CMakeLists.txt

@@ -1,15 +1,69 @@
 cmake_minimum_required(VERSION 3.15)
-
-project (tkDNN)
+project(tkDNN)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
-if(UNIX)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations")
-endif()
-if(WIN32)
 set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_FLAGS "/O2 /FS /EHsc")
+
+option(ENABLE_OPENCV_CUDA_CONTRIB "Enable OpenCV CUDA Contrib" OFF )
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "default build" FORCE)
+endif(NOT CMAKE_BUILD_TYPE)
+
+find_package(CUDA 9.0 REQUIRED)
+if (CUDA_FOUND)
+    set(OUTPUTFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cuda_script) # No suffix required
+    execute_process(COMMAND "rm ${OUTPUTFILE}")
+    set(CUDAFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/getCudaArch.cu)
+    execute_process(COMMAND ${CUDA_NVCC_EXECUTABLE} -lcuda ${CUDAFILE} -o ${OUTPUTFILE})
+    execute_process(COMMAND ${OUTPUTFILE}
+            RESULT_VARIABLE CUDA_RETURN_CODE
+            OUTPUT_VARIABLE ARCH)
+
+    if(${CUDA_RETURN_CODE} EQUAL 0)
+        set(CUDA_SUCCESS "TRUE")
+    else()
+        set(CUDA_SUCCESS "FALSE")
+    endif()
+
+    if (${CUDA_SUCCESS})
+        message(STATUS "CUDA Architecture: ${ARCH}")
+        message(STATUS "CUDA Version: ${CUDA_VERSION_STRING}")
+        message(STATUS "CUDA Path: ${CUDA_TOOLKIT_ROOT_DIR}")
+        message(STATUS "CUDA Libararies: ${CUDA_LIBRARIES}")
+        message(STATUS "CUDA Performance Primitives: ${CUDA_npp_LIBRARY}")
+        set(CUDA_NVCC_FLAGS "${ARCH}")
+    else()
+        message(WARNING ${ARCH})
+    endif()
+endif()
+
+SET(CUDA_SEPARABLE_COMPILATION ON)
+
+if(UNIX)
+    if(CMAKE_BUILD_TYPE MATCHES Release)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -fPIC -Wno-deprecated-declarations -Wno-unused-variable -O3")
+        set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
+    endif()
+
+    if(CMAKE_BUILD_TYPE MATCHES Debug)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -fPIC -Wno-deprecated-declarations -Wno-unused-variable -g3")
+        set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32 -G -g)
+    endif()
+endif()
+
+if(WIN32)
+    if(CMAKE_BUILD_TYPE MATCHES Release)
+        set(CMAKE_CXX_FLAGS "/O2 /FS /EHsc /MD")
+        set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
+    endif()
+
+    if(CMAKE_BUILD_TYPE MATCHES Debug)
+        set(CMAKE_CXX_FLAGS "/Od /FS /EHsc /MDd")
+        set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32 -G -g)
+    endif()
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif(WIN32)
+
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN)
 
 # project specific flags
@@ -18,7 +72,7 @@ if(DEBUG)
 endif()
 
 if(TKDNN_PATH)
-    message("SET TKDNN_PATH:"${TKDNN_PATH})
+    message("SET TKDNN_PATH:" ${TKDNN_PATH})
     add_definitions(-DTKDNN_PATH="${TKDNN_PATH}")
 else()
     add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}")
@@ -28,20 +82,21 @@ endif()
 #-------------------------------------------------------------------------------
 # CUDA
 #-------------------------------------------------------------------------------
-find_package(CUDA 9.0 REQUIRED)
-SET(CUDA_SEPARABLE_COMPILATION ON)
-#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'")
-set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
+set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}" --compiler-options '-fPIC')
+
 
 find_package(CUDNN REQUIRED)
 include_directories(${CUDNN_INCLUDE_DIR})
 
+find_package(yaml-cpp REQUIRED)
+
 
 # compile
-file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu")
+file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu" "src/pluginsRT/*.cpp")
 cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
 cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
-target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES})
+target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES} ${CUDA_LIBRARIES} ${CUDNN_LIBRARIES} yaml-cpp)
+
 
 
 #-------------------------------------------------------------------------------
@@ -53,12 +108,23 @@ include_directories(${EIGEN3_INCLUDE_DIR})
 
 find_package(OpenCV REQUIRED)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV")
+if(ENABLE_OPENCV_CUDA_CONTRIB)
+    if (OpenCV_FOUND)
+        find_package(OpenCV COMPONENTS cudawarping cudaarithm)
+        if(OpenCV_cudawarping_FOUND AND OpenCV_cudaarithm_FOUND)
+            add_compile_definitions(OPENCV_CUDACONTRIB)
+            message("OpenCV Cuda Contrib modules found")
+        else()
+            message("OpenCV Cuda Contrib modules not found")
+            set(ENABLE_OPENCV_CUDA_CONTRIB OFF)
+        endif()
+    endif()
+endif()
 # if(OpenCV_CUDA_VERSION)
 #     add_compile_definitions(OPENCV_CUDACONTRIB)
 # endif()
 
 # gives problems in cross-compiling, probably malformed cmake config
-find_package(yaml-cpp REQUIRED)
 
 #-------------------------------------------------------------------------------
 # Build Libraries
@@ -69,7 +135,7 @@ set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRAR
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES})
 add_library(tkDNN SHARED ${tkdnn_SRC})
-target_link_libraries(tkDNN ${tkdnn_LIBS})
+target_link_libraries(tkDNN ${tkdnn_LIBS} ${CUDA_CUBLAS_LIBRARIES})
 
 #static 
 #add_library(tkDNN_static STATIC ${tkdnn_SRC})
@@ -143,6 +209,14 @@ target_link_libraries(test_shelfnet_mapillary tkDNN)
 add_executable(test_shelfnet_coco tests/shelfnet/shelfnet_coco.cpp)
 target_link_libraries(test_shelfnet_coco tkDNN)
 
+# MONODEPTH2
+add_executable(test_monodepth2_640 tests/monodepth2/monodepth2_640.cpp)
+target_link_libraries(test_monodepth2_640 tkDNN)
+
+add_executable(test_monodepth2_1024 tests/monodepth2/monodepth2_1024.cpp)
+target_link_libraries(test_monodepth2_1024 tkDNN)
+
+
 # DEMOS
 add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
 target_link_libraries(test_rtinference tkDNN)
@@ -162,6 +236,9 @@ target_link_libraries(demoTracker tkDNN)
 add_executable(seg_demo demo/demo/seg_demo.cpp)
 target_link_libraries(seg_demo tkDNN)
 
+add_executable(demoDepth demo/demo/demoDepth.cpp)
+target_link_libraries(demoDepth tkDNN)
+
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
@@ -171,7 +248,7 @@ target_link_libraries(seg_demo tkDNN)
 #endif()
 message("install dir:" ${CMAKE_INSTALL_PREFIX})
 install(DIRECTORY include/ DESTINATION include/)
-install(TARGETS tkDNN kernels DESTINATION lib)
+install(TARGETS tkDNN  DESTINATION lib)
 install(TARGETS test_simple test_mnist test_mnistRT test_rtinference demo map_demo DESTINATION bin)
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory
         DESTINATION "share/tkDNN/cmake/" # target directory

README.md

@@ -23,9 +23,11 @@ If you use tkDNN in your research, please cite the [following paper](https://iee
 - [x] Support 2D/3D Object Detection and Tracking [README](docs/README_2d3dtracking.md)
 #### 24 November 2021
 - [x] Support to sematic segmentation on cuda 11
-- [x] Support to TensorRT8 (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg)). 
+- [x] Support to TensorRT8. (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg))
+#### 30 March 2022
+- [x] Support to monocular depth esitmation [README](docs/README_depth.md) (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg))
+
 
-TensorRT8 (and therefore Jetpack 4.6) is currently supported only on the branch tensorrt8 due to [performance issue with TensorRT8](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/tensorrt-8.html)). We will merge it to the master as soon as those issues are fixed (probably in future minor releases).
 
 ## FPS Results
 Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on 
@@ -80,17 +82,17 @@ Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001
   - [Workflow](#workflow)
   - [Exporting weights](#exporting-weights)
   - [Run the demos](#run-the-demos)
-  - [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental)
+  - [tkDNN on Windows 10 or Windows 11](#tkdnn-on-windows-10-or-windows-11)
   - [Existing tests and supported networks](#existing-tests-and-supported-networks)
   - [References](#references)
   
 
 ## Dependencies
 This branch works on every NVIDIA GPU that supports the following (latest tested) dependencies:
-* CUDA 11.0 (or >= 10) [the segmentation only works with CUDA 10 for now]
-* cuDNN 8.0.4 (or >= 7.3)
-* TensorRT 7.2.0 (or >=5)
-* OpenCV 4.5.2 (or >=4)
+* CUDA 11.3 (or >= 10.2) 
+* cuDNN 8.2.1 (or >= 8.0.4)
+* TensorRT 8.0.3 (or >=7.2)
+* OpenCV 4.5.4 (or >=4)
 * cmake 3.21 (or >= 3.15)
 * yaml-cpp 0.5.2
 * eigen3 3.3.4
@@ -106,16 +108,18 @@ To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh`
 ```
 bash scripts/install_OpenCV4.sh
 ```
-When using openCV not compiled with contrib, comment the definition of OPENCV_CUDACONTRIBCONTRIB in include/tkDNN/DetectionNN.h. When commented, the preprocessing of the networks is computed on the CPU, otherwise on the GPU. In the latter case some milliseconds are saved in the end-to-end latency. 
+If you have OpenCV compiled with cuda and contrib and want to use it with tkDNN pass  ```ENABLE_OPENCV_CUDA_CONTRIB=ON``` flag when compiling tkDBB
+. If the flag is not passed,the preprocessing of the networks is computed on the CPU, otherwise on the GPU. In the latter case some milliseconds are saved in the end-to-end latency. 
 
 ## How to compile this repo
-Build with cmake. If using Ubuntu 18.04 a new version of cmake is needed (3.15 or above). 
+Build with cmake. If using Ubuntu 18.04 a new version of cmake is needed (3.15 or above).
+On both linux and windows ,the ```CMAKE_BUILD_TYPE``` variable needs to be defined as either ```Release``` or ```Debug```.
 ```
 git clone https://github.com/ceccocats/tkDNN
 cd tkDNN
 mkdir build
 cd build
-cmake .. 
+cmake -DCMAKE_BUILD_TYPE=Release .. 
 make
 ```
 
@@ -136,14 +140,15 @@ For specific details on how to export weights see [HERE](./docs/exporting_weight
 For specific details on how to run:
 - 2D object detection demos, details on FP16, INT8 and batching see [HERE](./docs/demo.md).
 - segmentation demos see [HERE](./docs/README_seg.md).
+- monocular depth estimation see [HERE](./docs/README_depth.md).
 - 2D/3D object detection and tracking demos see [HERE](./docs/README_2d3dtracking.md).
 - mAP demo to evaluate 2D object detectors see [HERE](./docs/mAP_demo.md).
 
 ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
 
-## tkDNN on Windows 10 (experimental)
+## tkDNN on Windows 10 or Windows 11
 
-For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows.md).
+For specific details on how to run tkDNN on Windows 10/11 see [HERE](./docs/windows.md).
 
 ## Existing tests and supported networks
 
@@ -182,6 +187,8 @@ For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows
 | shelfnet_berkeley              | ShelfNet18_realtime<sup>11</sup>                           | [DeepDrive](https://bdd-data.berkeley.edu/)                          | 20        | 1024x1024       | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download)                                                                   |
 | dla34_cnet3d        | Centernet3D (DLA34 backend)<sup>4</sup>         | [KITTI 2017](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d)                          | 1        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download)     |
 | dla34_ctrack        | CenterTrack (DLA34 backend)<sup>12</sup>         | [NuScenes 3D](https://www.nuscenes.org/)                          | 7        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download)     |
+| monodepth2        | Monodepth2 <sup>13</sup>         | [KITTI DEPTH](http://www.cvlibs.net/datasets/kitti/raw_data.php)                          | -        | 640x192       | [weights-mono](https://cloud.hipert.unimore.it/s/iYw9QwgP6CsqxLR/download)     |
+| monodepth2        | Monodepth2 <sup>13</sup>         | [KITTI DEPTH](http://www.cvlibs.net/datasets/kitti/raw_data.php)                          | -        | 640x192       | [weights-stereo](https://cloud.hipert.unimore.it/s/XmwbWNXDfqyQ4EL/download)     |
 
 
 ## References
@@ -198,3 +205,11 @@ For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows
 10. Wang, Chien-Yao, Alexey Bochkovskiy, and Hong-Yuan Mark Liao. "Scaled-YOLOv4: Scaling Cross Stage Partial Network." arXiv preprint arXiv:2011.08036 (2020).
 11. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
 12. Zhou, Xingyi, Vladlen Koltun, and Philipp Krähenbühl. "Tracking objects as points." European Conference on Computer Vision. Springer, Cham, 2020.
+13. Godard, Clément, et al. "Digging into self-supervised monocular depth estimation." Proceedings of the IEEE/CVF International Conference on Computer Vision. 2019.
+
+## Contributors
+The main contibutors, in chronological order, are: 
+- [Francesco Gatti](https://github.com/ceccocats), francesco.gatti@hipert.it
+- [Micaela Verucchi](https://github.com/mive93), micaela.verucchi@unimore.it
+- [Davide Sapienza](https://github.com/sapienzadavide), davide.sapienza@unimore.it
+- [Harshvardhan Chandirasekar](https://github.com/perseusdg), f20180523@goa.bits-pilani.ac.in

cmake/getCudaArch.cu

@@ -0,0 +1,20 @@
+#include <stdio.h>
+
+int main(int argc, char **argv){
+    cudaDeviceProp dP;
+    float min_cc = 5.0;
+
+    int rc = cudaGetDeviceProperties(&dP, 0);
+    if(rc != cudaSuccess) {
+        cudaError_t error = cudaGetLastError();
+        printf("CUDA error: %s", cudaGetErrorString(error));
+        return rc; /* Failure */
+    }
+    if((dP.major+(dP.minor/10)) < min_cc) {
+        printf("Min Compute Capability of %2.1f required:  %d.%d found\n Not Building CUDA Code", min_cc, dP.major, dP.minor);
+        return 1; /* Failure */
+    } else {
+        printf("-arch=sm_%d%d", dP.major, dP.minor);
+        return 0; /* Success */
+    }
+}

demo/demo/demo.cpp

@@ -37,14 +37,14 @@ int main(int argc, char *argv[]) {
     if(!fileExist(net.c_str()))
         FatalError("The given network does not exist. Create the rt first.");
 
-    #ifdef __linux__ 
+    #ifdef __linux__
         std::string input = YAMLgetConf<std::string>(conf, "input", "../demo/yolo_test.mp4");
     #elif _WIN32
-        std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4");
+        std::string input = YAMLgetConf<std::string>(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4");
     #endif
-    if(!fileExist(input.c_str()))
+        if(!fileExist(input.c_str()))
         FatalError("The given input video does not exist.");
-    
+
     char ntype          = YAMLgetConf<char>(conf, "ntype", 'y');
     int n_classes       = YAMLgetConf<int>(conf, "n_classes", 80);
     int n_batch         = YAMLgetConf<int>(conf, "n_batch", 1);
@@ -66,7 +66,7 @@ int main(int argc, char *argv[]) {
     // create detection network
     tk::dnn::Yolo3Detection yolo;
     tk::dnn::CenternetDetection cnet;
-    tk::dnn::MobilenetDetection mbnet;  
+    tk::dnn::MobilenetDetection mbnet;
 
     tk::dnn::DetectionNN *detNN;  
 
@@ -86,7 +86,7 @@ int main(int argc, char *argv[]) {
         FatalError("Network type not allowed (3rd parameter)\n");
     }
 
-    detNN->init(net, n_classes, n_batch, conf_thresh);
+    detNN->init(net,n_classes,n_batch,conf_thresh);
 
     // open video stream
     cv::VideoCapture cap(input);
@@ -146,10 +146,10 @@ int main(int argc, char *argv[]) {
     
     double mean = 0; 
     std::cout<<COL_GREENB<<"\n\nTime stats:\n";
-    std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";    
-    std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";    
+    std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
+    std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";    
     for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
-    std::cout<<"Avg: "<<mean/n_batch<<" ms\t"<<1000/(mean/n_batch)<<" FPS\n"<<COL_END;   
+    std::cout<<"Avg: "<<mean<<" ms\t"<<1000/(mean)<<" FPS\n"<<COL_END;   
 
     return 0;
 }

demo/demo/demoDepth.cpp

@@ -0,0 +1,106 @@
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>     /* srand, rand */
+//#include <unistd.h>
+#include <mutex>
+
+#include "tkDNN/DepthNN.h"
+
+bool gRun;
+
+void sig_handler(int signo) {
+    std::cout<<"request gateway stop\n";
+    gRun = false;
+}
+
+int main(int argc, char *argv[]) {
+
+    signal(SIGINT, sig_handler);
+
+    std::string net = "monodepth2_fp32.rt";
+    if(argc > 1)
+        net = argv[1]; 
+    #ifdef __linux__ 
+        std::string input = "../demo/yolo_test.mp4";
+    #elif _WIN32
+        std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
+    #endif
+    if(argc > 2)
+        input = argv[2]; 
+    bool show = true;
+    if(argc > 3)
+        show = atoi(argv[3]); 
+    bool save = true;
+    if(argc > 4)
+        save = atoi(argv[4]); 
+
+    std::cout   <<"Net settings - net: "<< net
+                <<"\n"; 
+    std::cout   <<"Demo settings - input: "<< input
+                <<", show: "<< show
+                <<", save: "<< save<<"\n\n"; 
+    
+    tk::dnn::DepthNN depthNN;  
+
+    // create depth network 
+    int n_batch = 1;
+    depthNN.init(net, n_batch);
+
+    // open video stream
+    cv::VideoCapture cap(input);
+    if(!cap.isOpened())
+        gRun = false; 
+    else
+        std::cout<<"camera started\n";
+
+    cv::VideoWriter resultVideo;
+    if(save) {
+        int w = depthNN.output_w;
+        int h = depthNN.output_h;
+        resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
+    }
+
+    if(show)
+        cv::namedWindow("depth", cv::WINDOW_NORMAL);
+
+    cv::Mat frame;
+    std::vector<cv::Mat> batch_frame;
+    std::vector<cv::Mat> batch_dnn_input;
+
+    // start detection loop
+    gRun = true;
+    while(gRun) {
+        batch_dnn_input.clear();
+        batch_frame.clear();
+
+        //read frame
+        cap >> frame; 
+        if(!frame.data) 
+            break;
+        batch_frame.push_back(frame);
+        batch_dnn_input.push_back(frame.clone());
+    
+        //inference
+        depthNN.update(batch_dnn_input, 1);
+        if(show){
+            cv::imshow("depth", depthNN.depthMats[0]);
+            cv::waitKey(1);
+
+        }
+
+        if(save)
+            resultVideo << depthNN.depthMats[0];
+    }
+
+    std::cout<<"detection end\n";   
+    
+    double mean = 0; 
+    std::cout<<COL_GREENB<<"\n\nTime stats depth:\n";
+    std::cout<<"Min: "<<*std::min_element(depthNN.stats.begin(), depthNN.stats.end())<<" ms\n";    
+    std::cout<<"Max: "<<*std::max_element(depthNN.stats.begin(), depthNN.stats.end())<<" ms\n";    
+    for(int i=0; i<depthNN.stats.size(); i++) mean += depthNN.stats[i]; mean /= depthNN.stats.size();
+    std::cout<<"Avg: "<<mean<<" ms\t"<<1000/(mean)<<" FPS\n";   
+
+    return 0;
+}
+

demo/demo/map.cpp

@@ -32,7 +32,7 @@ int main(int argc, char *argv[])
 {
     char ntype = 'y';
     const char *config_filename = "../demo/config.yaml";
-    const char * net = "yolo3.rt";
+    const char * net = "yolo4tiny_fp32.rt";
     const char * labels_path = "../demo/COCO_val2017/all_labels.txt";
     int n_batches = 1;
     float confidence_thresh = 0.3;
@@ -45,7 +45,6 @@ int main(int argc, char *argv[])
     bool verbose;
     int classes, map_points, map_levels;
     float map_step, IoU_thresh, conf_thresh;
-
     double vm_total = 0, rss_total = 0;
     double vm, rss;
 
@@ -53,7 +52,7 @@ int main(int argc, char *argv[])
     if(argc > 1)
         net = argv[1]; 
     if(argc > 2)
-        ntype = argv[2][0];    
+        ntype = argv[2][0];
     if(argc > 3)
         labels_path = argv[3]; 
     if(argc > 4)
@@ -116,7 +115,7 @@ int main(int argc, char *argv[])
         default:
             FatalError("Network type not allowed (3rd parameter)\n");
     }
-    detNN->init(net, n_classes, 1, conf_thresh);
+    detNN->init(net,n_classes, 1, conf_thresh);
 
     //read images 
     std::ifstream all_labels(labels_path);

demo/demo/seg_demo.cpp

@@ -1,7 +1,9 @@
 #include <iostream>
 #include <signal.h>
 #include <stdlib.h>     /* srand, rand */
+#ifdef __linux__
 #include <unistd.h>
+#endif 
 #include <mutex>
 
 #include "SegmentationNN.h"

demo/demoConfig.yaml

@@ -3,7 +3,7 @@ input     : "../demo/yolo_test.mp4"
 win_input : "..\\..\\..\\demo\\yolo_test.mp4"
 
 # network config
-net         : "yolo4tiny_fp32.rt"
+net         : "yolo4_berkeley_fp32.rt"
 ntype       : 'y'
 n_classes   : 80
 n_batch     : 1
@@ -11,4 +11,4 @@ conf_thresh : 0.3
 
 # demo config
 show  : true
-save  : true
+save  : false

docker/Dockerfile.base

@@ -1,57 +1,140 @@
-FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04
-LABEL maintainer "Francesco Gatti"
+FROM nvidia/cudagl:11.3.1-devel-ubuntu20.04
 
-ADD nv-tensorrt-repo-ubuntu1804-cuda10.2-trt7.0.0.11-ga-20191216_1-1_amd64.deb /tmp/trt.deb
-RUN apt-get update && dpkg -i /tmp/trt.deb && rm /tmp/trt.deb && apt-get update
-RUN apt install -y libnvinfer7=7.0.0-1+cuda10.2 libnvinfer-dev=7.0.0-1+cuda10.2
-RUN DEBIAN_FRONTEND=noninteractive apt install -y git wget libeigen3-dev libyaml-cpp-dev
-RUN cd /tmp && \
-    wget https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh && \
-    chmod +x cmake-3.17.3-Linux-x86_64.sh && \
-    ./cmake-3.17.3-Linux-x86_64.sh --prefix=/usr/local --exclude-subdir --skip-license && \
-    rm ./cmake-3.17.3-Linux-x86_64.sh
+LABEL maintainer "TKDNN AUTHORS"
+LABEL Description="tkDNN+cudagl"
+LABEL com.tkdnn.nvidia.version="11.3.1"
 
-RUN echo "INSTALL OPENCV"
-RUN apt-get install -y build-essential \
-    unzip \
-    pkg-config \
-    libjpeg-dev \
-    libpng-dev \
-    libtiff-dev \
-    libavcodec-dev \
-    libavformat-dev \
-    libswscale-dev \
-    libv4l-dev \
-    libxvidcore-dev \
-    libx264-dev \
-    libgtk-3-dev \
-    libatlas-base-dev \
-    gfortran \
-    libgstreamer1.0-dev \
-    libgstreamer-plugins-base1.0-dev \
-    libdc1394-22-dev \
-    libavresample-dev
-RUN cd && wget https://github.com/opencv/opencv/archive/4.3.0.tar.gz && tar -xf 4.3.0.tar.gz && rm *.tar.gz
-RUN cd && wget https://github.com/opencv/opencv_contrib/archive/4.3.0.tar.gz && tar -xf 4.3.0.tar.gz && rm *.tar.gz
-RUN cd && \
-    cd opencv-4.3.0 && mkdir build && cd build && \
+ENV DEBIAN_FRONTEND noninteractive
+ENV CC gcc
+ENV CXX g++
+
+RUN apt-get update && apt-get install -y \
+libblkid-dev && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y \
+libcudnn8-dev=8.2.1.32-1+cuda11.3 \
+libcudnn8=8.2.1.32-1+cuda11.3 \
+libnvinfer-dev=8.0.3-1+cuda11.3 \
+libnvinfer8=8.0.3-1+cuda11.3 && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+libblkid-dev \
+locales \
+lsb-release \
+mesa-utils \
+git \
+nano \
+terminator \
+wget \
+curl \
+libssl-dev \
+htop \
+dbus-x11 \
+libqt5opengl5-dev \
+libgtk-3-dev \
+libvtk7-dev \
+libv4l-dev \
+tar \
+libgoogle-glog-dev \
+libgflags-dev \
+gfortran-9 \
+libtbb-dev \
+libgstreamer1.0-dev \
+libgstreamer-plugins-base1.0-dev \
+libdc1394-22-dev \
+libavresample-dev \
+libatlas-cpp-0.6-dev \
+python3-dev \
+gdb \
+python3-pip \
+unzip libtbb-dev && \
+apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+software-properties-common && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN apt-add-repository universe
+RUN apt-get update && apt-get install -y python3-pip  python3 openssh-server ssh pyqt5-dev sip-dev && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade virtualenv
+RUN pip3 install --upgrade paramiko
+RUN pip3 install --ignore-installed --upgrade numpy protobuf
+
+
+RUN cd ~ && mkdir build
+RUN cd ~/build && wget https://github.com/Kitware/CMake/releases/download/v3.21.4/cmake-3.21.4.tar.gz  && \
+tar -xvf cmake-3.21.4.tar.gz && cd cmake-3.21.4 && ./configure --prefix=/usr/local --qt-gui --parallel=12 && \
+make -j8 && make install
+
+RUN apt-get update && apt-get install -y automake autoconf pkg-config libevent-dev libncurses5-dev bison && \
+apt-get clean && rm -rf /var/lib/apt/lists/
+
+RUN git clone https://github.com/tmux/tmux.git && \
+cd tmux && git checkout tags/3.2 && ls -la && sh autogen.sh && ./configure && make -j8 && make install
+
+RUN apt-get update && apt-get install -y zsh && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true
+RUN chsh -s /usr/bin/zsh root
+RUN git clone https://github.com/sindresorhus/pure /root/.oh-my-zsh/custom/pure
+RUN ln -s /root/.oh-my-zsh/custom/pure/pure.zsh-theme /root/.oh-my-zsh/custom/
+RUN ln -s /root/.oh-my-zsh/custom/pure/async.zsh /root/.oh-my-zsh/custom/
+RUN sed -i -e 's/robbyrussell/refined/g' /root/.zshrc
+RUN sed -i '/plugins=(/c\plugins=(git git-flow adb pyenv tmux)' /root/.zshrc
+
+RUN mkdir -p /root/.config/terminator/
+COPY assets/terminator_config /root/.config/terminator/config
+
+RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
+    echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf && \
+    echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/nvidia.conf
+
+
+ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/lib:/usr/lib/x86_64-linux-gnu:/usr/local/lib:${LD_LIBRARY_PATH}
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility,graphics
+
+
+
+RUN cd ~/build && wget https://github.com/opencv/opencv/archive/4.5.4.tar.gz && tar -xf 4.5.4.tar.gz && rm 4.5.4.tar.gz
+RUN cd ~/build && wget https://github.com/opencv/opencv_contrib/archive/4.5.4.tar.gz && tar -xf 4.5.4.tar.gz && rm 4.5.4.tar.gz
+RUN cd ~/build && \
+    cd opencv-4.5.4 && mkdir build && cd build && \
     cmake -D CMAKE_BUILD_TYPE=RELEASE \
     -D CMAKE_INSTALL_PREFIX=/usr/local \
     -D INSTALL_PYTHON_EXAMPLES=OFF \
     -D INSTALL_C_EXAMPLES=OFF \
-    -D OPENCV_EXTRA_MODULES_PATH='~/opencv_contrib-4.3.0/modules' \
+    -D OPENCV_EXTRA_MODULES_PATH='~/build/opencv_contrib-4.5.4/modules' \
     -D BUILD_EXAMPLES=OFF \
+    -D BUILD_TESTS=OFF \
+    -D BUILD_PERF_TESTS=OFF \
+    -D BUILD_DOCS=OFF \
     -D WITH_CUDA=ON \
+    -D WITH_OPENGL=ON \
+    -D WITH_NVCUVID=ON \
     -D CUDA_ARCH_BIN=7.2 \
-    -D CUDA_ARCH_PTX="" \
+    -D CUDA_ARCH_PTX=7.2 \
     -D ENABLE_FAST_MATH=ON \
     -D CUDA_FAST_MATH=ON \
     -D WITH_CUBLAS=ON \
+    -D WITH_CUDNN=ON \
+    -D WITH_OPENMP=ON \
+    -D WITH_NONFREE=ON \
     -D WITH_LIBV4L=ON \
     -D WITH_GSTREAMER=ON \
     -D WITH_GSTREAMER_0_10=OFF \
     -D WITH_TBB=ON \
-    ../ && make -j12 && make install
-RUN apt clean
+    ../ && make -j12 && make install && ldconfig
 
+RUN cd ~ && rm -rf build
 
+RUN cd ~ && mkdir Development && cd Development && \
+git clone https://github.com/ceccocats/tkDNN.git && cd tkDNN && \
+mkdir build && cd build && \
+cmake -DCMAKE_BUILD_TYPE=Release .. && \
+make -j6
+
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY assets/entrypoint_setup.sh /
+ENTRYPOINT ["/entrypoint_setup.sh"]
+CMD ["terminator"]

docker/README.md

@@ -9,13 +9,10 @@ docker build -t tkdnn:build -f Dockerfile .
 # make nvidia docker working
 # follow this guide: https://github.com/NVIDIA/nvidia-docker
 
-# dowload tensorrt
-# from: https://developer.nvidia.com/compute/machine-learning/tensorrt/secure/7.0/7.0.0.11/local_repo/nv-tensorrt-repo-ubuntu1804-cuda10.2-trt7.0.0.11-ga-20191216_1-1_amd64.deb
-
 # build image
 docker build -t ceccocats/tkdnn:latest -f Dockerfile.base .
 
 # run image
-docker run -ti --gpus all --rm ceccocats/tkdnn:latest bash
+./docker_launch.sh
 ```
 

docker/assets/entrypoint_setup.sh

@@ -0,0 +1,123 @@
+#! /bin/bash
+
+CMD=
+
+# Functions
+# TOOD: Check if we can use: getent passwd $USER to extract all variables
+# TODO: Check for valid inputs, cause now it will go through even with bad inputs
+check_envs () {
+    DOCKER_CUSTOM_USER_OK=true;
+    if [ -z ${DOCKER_USER_NAME+x} ]; then 
+        DOCKER_CUSTOM_USER_OK=false;
+        return;
+    fi
+    
+    if [ -z ${DOCKER_USER_ID+x} ]; then 
+        DOCKER_CUSTOM_USER_OK=false;
+        return;
+    else
+        if ! [ -z "${DOCKER_USER_ID##[0-9]*}" ]; then 
+            echo -e "\033[1;33mWarning: User-ID should be a number. Falling back to defaults.\033[0m"
+            DOCKER_CUSTOM_USER_OK=false;
+            return;
+        fi
+    fi
+    
+    if [ -z ${DOCKER_USER_GROUP_NAME+x} ]; then 
+        DOCKER_CUSTOM_USER_OK=false;
+        return;
+    fi
+
+    if [ -z ${DOCKER_USER_GROUP_ID+x} ]; then 
+        DOCKER_CUSTOM_USER_OK=false;
+        return;
+    else
+        if ! [ -z "${DOCKER_USER_GROUP_ID##[0-9]*}" ]; then 
+            echo -e "\033[1;33mWarning: Group-ID should be a number. Falling back to defaults.\033[0m"
+            DOCKER_CUSTOM_USER_OK=false;
+            return;
+        fi
+    fi
+}
+
+setup_env_user () {
+    USER=$1
+    USER_ID=$2
+    GROUP=$3
+    GROUP_ID=$4
+
+    ## Create user
+    useradd -m $USER
+
+    ## Copy zsh/sh configs
+    cp /root/.profile /home/$USER/
+    cp /root/.bashrc /home/$USER/
+    cp /root/.zshrc /home/$USER/
+    ## Copy terminator configs
+    mkdir -p /home/$USER/.config/terminator
+    cp /root/.config/terminator/config /home/$USER/.config/terminator/config
+    cp /root/.config/terminator/background.png /home/$USER/.config/terminator/background.png
+    cp -rf /root/.oh-my-zsh /home/$USER/
+    cp -rf /root/tkDNN /home/$USER/
+    rm -rf /home/$USER/.oh-my-zsh/custom/pure.zsh-theme /home/$USER/.oh-my-zsh/custom/async.zsh
+    ln -s /home/$USER/.oh-my-zsh/custom/pure/pure.zsh-theme /home/$USER/.oh-my-zsh/custom/
+    ln -s /home/$USER/.oh-my-zsh/custom/pure/async.zsh /home/$USER/.oh-my-zsh/custom/
+    sed -i -e 's@ZSH=\"/root@ZSH=\"/home/$USER@g' /home/$USER/.zshrc
+    # Copy SSH keys & fix owner
+    if [ -d "/root/.ssh" ]; then
+        cp -rf /root/.ssh /home/$USER/
+        chown -R $USER:$GROUP /home/$USER/.ssh
+    fi
+    
+    ## Fix owner
+    chown $USER:$GROUP /home/$USER
+    chown -R $USER:$GROUP /home/$USER/.config
+    chown $USER:$GROUP /home/$USER/.profile
+    chown $USER:$GROUP /home/$USER/.bashrc
+    chown $USER:$GROUP /home/$USER/.zshrc
+    chown -R $USER:$GROUP /home/$USER/.oh-my-zsh
+    chown -R $USER:$GROUP /home/$USER/tkDNN
+
+    ## This a trick to keep the evnironmental variables of root which is important!
+    echo "if ! [ \"$DOCKER_USER_NAME\" = \"$(id -un)\" ]; then" >> /root/.bashrc
+    echo "    cd /home/$DOCKER_USER_NAME" >> /root/.bashrc
+    echo "    su $DOCKER_USER_NAME" >> /root/.bashrc
+    echo "fi" >> /root/.bashrc
+
+    echo "if ! [ \"$DOCKER_USER_NAME\" = \"$(id -un)\" ]; then" >> /root/.zshrc
+    echo "    cd /home/$DOCKER_USER_NAME" >> /root/.zshrc
+    echo "    su $DOCKER_USER_NAME" >> /root/.zshrc
+    echo "fi" >> /root/.zshrc
+
+    ## Setup Password-file
+    PASSWDCONTENTS=$(grep -v "^${USER}:" /etc/passwd)
+    GROUPCONTENTS=$(grep -v -e "^${GROUP}:" -e "^docker:" /etc/group)
+
+    (echo "${PASSWDCONTENTS}" && echo "${USER}:x:$USER_ID:$GROUP_ID::/home/$USER:/bin/bash") > /etc/passwd
+    (echo "${GROUPCONTENTS}" && echo "${GROUP}:x:${GROUP_ID}:") > /etc/group
+    (if test -f /etc/sudoers ; then echo "${USER}  ALL=(ALL)   NOPASSWD: ALL" >> /etc/sudoers ; fi)
+}
+
+
+# ---Main---
+
+# Create new user
+## Check Inputs
+check_envs
+
+## Determine user & Setup Environment
+if [ $DOCKER_CUSTOM_USER_OK == true ]; then
+    echo "  -->DOCKER_USER Input is set to '$DOCKER_USER_NAME:$DOCKER_USER_ID:$DOCKER_USER_GROUP_NAME:$DOCKER_USER_GROUP_ID'";
+    echo -e "\033[0;32mSetting up environment for user=$DOCKER_USER_NAME\033[0m"
+    setup_env_user $DOCKER_USER_NAME $DOCKER_USER_ID $DOCKER_USER_GROUP_NAME $DOCKER_USER_GROUP_ID
+else
+    echo "  -->DOCKER_USER* variables not set. Using 'root'.";
+    echo -e "\033[0;32mSetting up environment for user=root\033[0m"
+    DOCKER_USER_NAME="root"
+fi
+
+# Change shell to zsh
+chsh -s /usr/bin/zsh $DOCKER_USER_NAME
+
+# Run CMD from Docker
+"$@"

docker/assets/terminator_config

@@ -0,0 +1,18 @@
+[global_config]
+  title_transmit_bg_color = "#2e3436"
+[keybindings]
+[layouts]
+  [[default]]
+    [[[child1]]]
+      parent = window0
+      type = Terminal
+    [[[window0]]]
+      parent = ""
+      type = Window
+[plugins]
+[profiles]
+  [[default]]
+    background_color = "#282828"
+    cursor_color = "#aaaaaa"
+    foreground_color = "#f3f3f3"
+    palette = "#000000:#aa0000:#00aa00:#c4a000:#3465a4:#75507b:#06989a:#d3d7cf:#88807c:#f15d22:#73c48f:#ffce51:#48b9c7:#ad7fa8:#34e2e2:#eeeeec"

docker/docker_launch.sh

@@ -0,0 +1,9 @@
+xhost local:root
+docker run --rm -it --runtime=nvidia --privileged --net=host --cap-add sys_ptrace -d --ipc=host \
+-v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY \
+-v $HOME/.Xauthority:/home/$(id -un)/.Xauthority -e XAUTHORITY=/home/$(id -un)/.Xauthority \
+-e DOCKER_USER_NAME=$(id -un) \
+-e DOCKER_USER_ID=$(id -u) \
+-e DOCKER_USER_GROUP_NAME=$(id -gn) \
+-e DOCKER_USER_GROUP_ID=$(id -g) \
+-v $HOME/.ssh:/home/$(id -un)/.ssh  ceccocats/tkdnn

docs/README_depth.md

@@ -0,0 +1,54 @@
+# Monocular depth estimation with tkDNN
+
+Currently tkDNN supports only Monodepth2 as monocular depth esitmation network.
+
+
+## Run the demo
+
+To run the depth estimation demo follow these steps (example with monodepth2):
+```
+rm monodepth2_fp32.rt        # be sure to delete(or move) old tensorRT files
+./test_monodepth2            # run the yolo test (is slow)
+./demoDepth monodepth2_fp32.rt ../demo/yolo_test.mp4 
+```
+In general the demo program takes the following parameters:
+```
+./demoDepth <network-rt-file> <path-to-video> <show-flag> <save-flag>
+```
+where
+*  ```<network-rt-file>``` is the rt file generated by a test
+*  ```<<path-to-video>``` is the path to a video file or a camera input  
+*  ```<show-flag>``` if set to 0 the demo will not show the visualization, it will otherwise (default=1)
+*  ```<save-flag>``` if set to 1 the demo will save the video into result.mp4, it won't otherwise (default=1)
+
+NB) By default it is used FP32 inference
+
+
+![demo](https://user-images.githubusercontent.com/11939259/160845358-0d6ab15d-c5f4-46ae-b9da-bfaf3903389d.gif "Results on yolo_test.mp4")  
+
+
+<!-- ## FPS Results
+
+Inference FPS of shelfnet with tkDNN, average of 1200 images on:
+  * RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
+  * Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
+
+| Platform   | Test                     | Phase   | FP32, ms  | FP32, FPS | FP16, ms  |	FP16, FPS  | INT8, ms |	INT8, FPS | 
+| :------:   | :-----:                  | :-----: | :-----:   | :-----:   | :-----:   |	:-----:    | :-----:  |	:-----:   | 
+| RTX 2080Ti | shelfnet 1024x1024 (B=1) | pre     | 6.11863   |  163.435  |   5.81465 |  171.979   |  5.88699 |   169.866 |
+| RTX 2080Ti | shelfnet 1024x1024 (B=1) | inf     | 11.5464   |  86.6074  |   7.35396 |  135.981   |  6.37623 |   156.832 |
+| RTX 2080Ti | shelfnet 1024x1024 (B=1) | post    | 4.09058   |  244.464  |   3.91961 |  255.128   |  4.07343 |   245.493 |
+| RTX 2080Ti | shelfnet 1024x1024 (B=1) | tot     | 21.7556   |  45.9652  |   17.0882 |  58.5199   |  16.3366 |   61.2121 |
+| RTX 2080Ti | shelfnet 2048x2048 (B=4) | pre     | 25.435    |  39.3158  |   25.2953 |  39.5331   |  25.9303 |   38.565  | 
+| RTX 2080Ti | shelfnet 2048x2048 (B=4) | inf     | 36.5015   |  27.3961  |   17.0534 |  58.6395   |  15.6061 |   64.0773 |  
+| RTX 2080Ti | shelfnet 2048x2048 (B=4) | post    | 17.3917   |  57.4985  |   17.1649 |  58.2583   |  17.5539 |   56.9675 |  
+| RTX 2080Ti | shelfnet 2048x2048 (B=4) | tot     | 79.3283   |  12.6058  |   59.5136 |  16.8029   |  59.0903 |   16.9233 |  
+| AGX Xavier | shelfnet 1024x1024 (B=1) | pre     | 8.0174    |  124.729  |   7.5117  |  133.126   |  7.47333 |   133.809 |
+| AGX Xavier | shelfnet 1024x1024 (B=1) | inf     | 72.4173   |  13.8089  |   37.505  |  26.6631   |  31.3286 |   31.9197 |
+| AGX Xavier | shelfnet 1024x1024 (B=1) | post    | 8.89958   |  112.365  |   8.83576 |  113.176   |  9.42655 |   106.083 |
+| AGX Xavier | shelfnet 1024x1024 (B=1) | tot     | 89.3342   |  11.1939  |   53.8525 |  18.5692   |  48.2285 |   20.7346 |
+| AGX Xavier | shelfnet 2048x2048 (B=4) | pre     | 47.1454   |  21.211   |   21.6475 |  46.1947   |  21.4201 |   46.6851 | 
+| AGX Xavier | shelfnet 2048x2048 (B=4) | inf     | 266.537   |  3.75183  |   128.321 |  7.79293   |  107.621 |   9.29185 |  
+| AGX Xavier | shelfnet 2048x2048 (B=4) | post    | 44.0711   |  22.6906  |   40.1732 |  24.8922   |  39.873  |   25.0796 |  
+| AGX Xavier | shelfnet 2048x2048 (B=4) | tot     | 357.753   |  2.79522  |   190.142 |  5.25922   |  168.914 |   5.92016 |   -->
+

docs/demo.md

@@ -26,26 +26,26 @@ rm yolo4_fp32.rt        # be sure to delete(or move) old tensorRT files
 ```
 If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
 ```
-cmake .. -DDEBUG=True
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DDEBUG=True
 make
 ```
 
-Once you have successfully created your rt file, run the demo: 
+Once you have successfully created your rt file, run the demo:
 ```
-./demo <path-to-config>
+./ demo <path-to-config>
 ```
-In general the demo program takes 1 parameter, the ```<path-to-config>``` that is the path to che configuration file. The parameter is optional and its default value is ```"../demo/demoConfig.yaml"```. 
+In general the demo program takes 1 parameter, the ```<path-to-config>``` that is the path to che configuration file. The parameter is optional and its default value is ```"../demo/demoConfig.yaml"```.
 
 The config file is a yaml file with the following attributes:
-*  ```net``` is the rt file generated by a test
-*  ```input``` is the path to a video file or a camera input  (on Linux)
-*  ```win_input``` is the path to a video file or a camera input  (on Windows)
-*  ```ntype``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
-*  ```n_classes``` is the number of classes the network is trained on
-*  ```n_batch``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
-*  ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed.
-*  ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1)
-*  ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1)
+* ```net``` is the rt file generated by a test
+* ```input``` is the path to a video file or a camera input  (on Linux)
+* ```win_input``` is the path to a video file or a camera input  (on Windows)
+* ```ntype``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
+* ```n_classes``` is the number of classes the network is trained on
+* ```n_batch``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
+* ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed.
+* ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1)
+* ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1)
 
 N.B. By default it is used FP32 inference
 
@@ -58,10 +58,10 @@ N.B. By default it is used FP32 inference
 To run the demo with FP16 inference follow these steps (example with yolov3):
 ```
 export TKDNN_MODE=FP16  # set the half floating point optimization
-rm yolo3_fp16.rt        # be sure to delete(or move) old tensorRT files
-./test_yolo3            # run the yolo test (is slow)
-# set net: yolo3_fp16.rt in the config-file
-./demo 
+rm yolo4_fp16.rt        # be sure to delete(or move) old tensorRT files
+./test_yolo4            # run the yolo test (is slow)
+#set net: yolo4_fp16.rt in the config file
+./demo
 ```
 N.B. Using FP16 inference will lead to some errors in the results (first or second decimal). 
 
@@ -84,10 +84,10 @@ Then a complete example using yolo3 and COCO dataset would be:
 export TKDNN_MODE=INT8
 export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
 export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
-rm yolo3_int8.rt        # be sure to delete(or move) old tensorRT files
-./test_yolo3            # run the yolo test (is slow)
-# set net: yolo3_int8.rt in the config-file
-./demo 
+rm yolo4_int8.rt        # be sure to delete(or move) old tensorRT files
+./test_yolo4            # run the yolo test (is slow)
+#set net: yolo4_int8.rt in the config file
+./demo
 ```
 N.B. 
 

docs/exporting_weights.md

@@ -86,6 +86,18 @@ mkdir layer debug
 python export.py
 ```
 
+### 6)Export weights for monodepth2
+To get the weights needed to run Shelfnet tests use [this](https://github.com/perseusdg/monodepth2) fork of a Pytorch implementation of monodepth2 network. 
+
+```
+git clone https://github.com/perseusdg/monodepth2
+cd monodepth2
+mkdir models # Download the official weights and put depth.pth and encorder.pth inside this new folder
+conda env create --file monodepth.yaml
+conda activate monodepth2
+python exporter.py # you will find the weights inside the tkDNN_bin folder
+```
+
 ## Darknet Parser
 tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
 ```

docs/windows.md

@@ -7,17 +7,18 @@
  - [Run the demo on Windows](#run-the-demo-on-windows)
     - [FP16 inference windows](#fp16-inference-windows)
     - [INT8 inference windows](#int8-inference-windows)
+ - [Run tkDNN on WSL2 with cuda](#tkdnn-on-cuda-wsl)
  - [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows)
 
 ### Dependencies-Windows 
 This branch should work on every NVIDIA GPU supported in windows with the following dependencies:
 
-* WINDOWS 10 1803 or HIGHER 
-* CUDA 10.0 (Recommended CUDA 11.2 )
-* CUDNN 7.6 (Recommended CUDNN 8.1.1 )
-* TENSORRT 6.0.1 (Recommended TENSORRT 7.2.3.4 )
-* OPENCV 3.4 (Recommended OPENCV 4.2.0 )
-* MSVC 16.7 
+* WINDOWS 10 1803/WINDOWS 11 or HIGHER 
+* CUDA 11.2 
+* CUDNN 8.1.1
+* TENSORRT 7.2.3 
+* OPENCV 4.2  
+* MSVC 16.9+
 * YAML-CPP 
 * EIGEN3
 * 7ZIP (ADD TO PATH)
@@ -58,7 +59,7 @@ To run the object detection file create .rt file bu running:
 
 Once the rt file has been successfully create,run the demo using the following command:
 ```
-.\demo.exe yolo4tiny_fp32.rt ..\demo\yolo_test.mp4 y 
+.\demo.exe yolo4_fp32.rt ..\demo\yolo_test.mp4 y 80 ..\tests\darknet\cfg\yolo4.cfg ..\tests\darknet\names\cococ.names 
 ```
  For general info on more demo paramters,check Run the demo section on top 
  To run the test_all_tests.sh on windows,use git bash or msys2 
@@ -85,11 +86,17 @@ del /f  yolo4tiny_int8.rt        # be sure to delete(or move) old tensorRT files
 
 ```
 
+### Run tkDNN on WSL2 with cuda
+tkDNN works on wsl2 with cuda,although not all networks (centernet,mobilenet) work properly.
+If you encounter issues with running the network as a result of driver not found or cuda launch error,running the following command should solve the issue
+```cp /usr/lib/wsl/lib/lib* /usr/lib/x86_64-linux-gnu/ ```
+
+
+
 ### Known issues with tkDNN on Windows
 
-Mobilenet and Centernet demos work properly only when built with msvc 16.7 in Release Mode,when built in debug mode for the mentioned networks one might encounter opencv assert errors
+In theory all models (centernet,mobilenet,darknet,centertrack,cnet3d and shelfnet) should work on Windows.
 
-All Darknet models work properly with demo using MSVC version(16.7-16.9)
+On pascal cards(sm 6x) ,nvidia cuda wsl driver 510.06 don't work well with tkDNN both on windows and cuda wsl , Nvidia drivers >465+ and < 500 are completely supported .
 
-It is recommended to use Nvidia Driver(465+),Cuda unknown errors have been observed when using older drivers on pascal(SM 61) devices.
 

include/tkDNN/CenterTrack.h

@@ -13,6 +13,11 @@
 
 #include "TrackingNN.h"
 
+#ifdef _WIN32
+#define _USE_MATH_DEFINES
+#include <math.h>
+#endif 
+
 #include "kernelsThrust.h"
 
 

include/tkDNN/CenternetDetection3D.h

@@ -9,6 +9,11 @@
 #include <numeric>      // std::iota
 #include <algorithm>    // std::sort
 
+#ifdef _WIN32
+#define _USE_MATH_DEFINES
+#include <math.h>
+#endif 
+
 #include "DetectionNN3D.h"
 
 #include "kernelsThrust.h"

include/tkDNN/DarknetParser.h

@@ -47,5 +47,8 @@ namespace tk { namespace dnn {
                          std::vector<tk::dnn::Layer*> &netLayers, const std::vector<std::string>& names);
     std::vector<std::string> darknetReadNames(const std::string& names_file);
     tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file);
+    void loadYoloInfo(const std::string &cfg_file,int lineNo,std::vector<float> &mask,std::vector<float> &anchors,int &num,int &classes,float &nms_thresh,int &nms_kind,int &coords);
+    void loadYoloInitInfo(int &channels,int &width,int &height,const std::string &cfg_file);
+    std::vector<int> noYolosLine(const std::string &cfg_file);
 
 }}

include/tkDNN/DepthNN.h

@@ -0,0 +1,180 @@
+#ifndef DEPTHNN_H
+#define DEPTHNN_H
+
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>    
+#ifdef __linux__
+#include <unistd.h>
+#endif 
+
+#include <mutex>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "tkDNN/utils.h"
+#include "tkDNN/tkdnn.h"
+
+#include "NetworkViz.h"
+
+
+namespace tk { namespace dnn {
+
+class DepthNN {
+
+    public:
+        tk::dnn::NetworkRT *netRT = nullptr;
+        dnnType *input_h;
+        dnnType *input_d;
+        float* depth_h;
+
+        int output_w;
+        int output_h;
+
+        int nBatches = 1;
+
+        cv::Mat bgr[3];
+        cv::Mat imagePreproc;
+
+        std::vector<double> stats; /*keeps track of inference times (ms)*/
+        std::vector<std::vector<float>> depths;
+        std::vector<cv::Mat> depthMats;
+
+        DepthNN() {};
+        ~DepthNN(){};
+
+        /**
+         * Method used to initialize the class, allocate memory and compute 
+         * needed data.
+         * 
+         * @param tensor_path path to the rt file of the NN.
+         * @param n_batches maximum number of batches to use in inference
+         * @return true if everything is correct, false otherwise.
+         */
+        void init(const std::string& tensor_path, const int n_batches=1){
+            //create net
+            
+            std::cout<<(tensor_path).c_str()<<"\n";
+            nBatches = n_batches;
+            netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
+            
+            //allocate memory for NN input
+            checkCuda(cudaMallocHost(&input_h, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
+            checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
+
+            //allocate memory for NN output
+            depthMats.resize(nBatches);
+            depths.resize(nBatches);
+            for(int i=0; i< depths.size();++i)
+                depths[i].resize(netRT->buffersDIM[1].tot());
+
+            depth_h = (float *)malloc(netRT->buffersDIM[1].tot() * sizeof(float));
+
+            output_h = netRT->buffersDIM[1].h;
+            output_w = netRT->buffersDIM[1].w;
+            
+        }
+
+
+        /**
+         * This method preprocess the image, before feeding it to the NN.
+         *
+         * @param frame original frame to adapt for inference.
+         * @param bi batch index
+         */
+        void preprocess(cv::Mat &frame, const int bi=0) {
+            //resize image, remove mean, divide by std
+            cv::Mat frame_nomean;
+            resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+            frame.convertTo(frame_nomean, CV_32FC3);
+            frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 255.0, 0);
+
+            //copy image into tensor and copy it into GPU
+            cv::split(imagePreproc, bgr);
+            for (int i = 0; i < netRT->input_dim.c; i++){
+                int idx = i * imagePreproc.rows * imagePreproc.cols;
+                int ch = netRT->input_dim.c-1 -i;
+                memcpy((void *)&input_h[idx + netRT->input_dim.tot()*bi], (void *)bgr[ch].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType));
+            }
+            checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input_h + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
+        }
+
+        /**
+         * This method postprocess the output of the NN to obtain the correct 
+         * boundig boxes. 
+         * 
+         * @param bi batch index
+         * @param mAP set to true only if all the probabilities for a bounding 
+         *            box are needed, as in some cases for the mAP calculation
+         */
+        void postprocess(const int bi=0) {
+
+            dnnType *rt_out[1];
+            rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
+            checkCuda(cudaMemcpy(depth_h, rt_out[0], netRT->buffersDIM[1].tot()* sizeof(float), cudaMemcpyDeviceToHost));
+            memcpy(&depths[bi][0], &depth_h[0], netRT->buffersDIM[1].tot()* sizeof(float));
+
+            // cv::Mat d(netRT->buffersDIM[1].h, netRT->buffersDIM[1].w, CV_8UC1, depth_h);
+            // depthMats[bi] = d.clone();
+
+            cv::Mat depth_mat = vizData2Mat(rt_out[0], netRT->buffersDIM[1], netRT->buffersDIM[1].h, netRT->buffersDIM[1].w);
+            // cv::Mat depth_mat = vizData2Mat((dnnType *)netRT->buffersRT[0], netRT->buffersDIM[0], netRT->buffersDIM[0].h, netRT->buffersDIM[0].w);
+            depthMats[bi] = depth_mat.clone();
+
+        }
+        
+        /**
+         * This method performs the inference of the NN.
+         * 
+         * @param frames frames to build the embedding from.
+         * @param cur_batches number of batches to use in inference
+         */
+        void update(std::vector<cv::Mat>& frames, const int cur_batches=1){
+            if(cur_batches > nBatches)
+                FatalError("A batch size greater than nBatches cannot be used");
+
+            if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT feature extraction ", '=', 30); 
+            {
+                TKDNN_TSTART
+                for(int bi=0; bi<cur_batches;++bi){
+                    if(!frames[bi].data)
+                        FatalError("No image data feed to extract features");
+                    preprocess(frames[bi], bi);    
+                }
+                TKDNN_TSTOP
+            }
+
+            //do inference
+            tk::dnn::dataDim_t dim = netRT->input_dim;
+            dim.n = cur_batches;
+            {
+                if(TKDNN_VERBOSE) dim.print();
+                TKDNN_TSTART
+                netRT->infer(dim, input_d);
+                TKDNN_TSTOP
+                if(TKDNN_VERBOSE) dim.print();
+                stats.push_back(t_ns);
+            }
+
+            {
+                TKDNN_TSTART
+                for(int bi=0; bi<cur_batches;++bi)
+                    postprocess(bi);
+                TKDNN_TSTOP
+            }
+        }      
+
+        /**
+         * Method to draw the result.
+         * 
+         */
+        void draw() {        }
+
+};
+
+
+}}
+
+#endif /* DEPTHNN_H*/

include/tkDNN/Int8BatchStream.h

@@ -39,7 +39,7 @@ public:
 	float *getLabels() { return mLabels.data(); }
 	int getBatchesRead() const { return mBatchCount; }
 	int getBatchSize() const { return mBatchSize; }
-	nvinfer1::DimsNCHW getDims() const { return mDims; }
+	nvinfer1::Dims4 getDims() const { return mDims; }
 	float* getFileBatch() { return &mFileBatch[0]; }
 	float* getFileLabels() { return &mFileLabels[0]; }
 	void readInListFile(const std::string& dataFilePath, std::vector<std::string>& mListIn);
@@ -55,7 +55,7 @@ private:
 	int mFileBatchPos{ 0 };
 	int mImageSize{ 0 };
 
-	nvinfer1::DimsNCHW mDims;
+	nvinfer1::Dims4 mDims;
 	std::vector<float> mBatch;
 	std::vector<float> mLabels;
 	std::vector<float> mFileBatch;

include/tkDNN/Int8Calibrator.h

@@ -30,10 +30,10 @@ public:
 	Int8EntropyCalibrator(BatchStream& stream, int firstBatch, const std::string& calibTableFilePath, 
 							const std::string& inputBlobName, bool readCache = true);
 	virtual ~Int8EntropyCalibrator() { checkCuda(cudaFree(mDeviceInput)); }
-	int getBatchSize() const override { return mStream.getBatchSize(); }
-	bool getBatch(void* bindings[], const char* names[], int nbBindings) override;
-	const void* readCalibrationCache(size_t& length) override;
-	void writeCalibrationCache(const void* cache, size_t length) override;
+	int getBatchSize() const NOEXCEPT override { return mStream.getBatchSize(); }
+	bool getBatch(void* bindings[], const char* names[], int nbBindings) NOEXCEPT override;
+	const void* readCalibrationCache(size_t& length) NOEXCEPT override;
+	void writeCalibrationCache(const void* cache, size_t length) NOEXCEPT override;
 
 private:
 	BatchStream mStream;

include/tkDNN/Layer.h

@@ -31,7 +31,8 @@ enum layerType_t {
     LAYER_SHORTCUT,
     LAYER_UPSAMPLE,
     LAYER_REGION,
-    LAYER_YOLO
+    LAYER_YOLO,
+    LAYER_PADDING,
 };
 
 #define TKDNN_BN_MIN_EPSILON 1e-5
@@ -56,8 +57,8 @@ public:
 
     int id = 0;
     bool final;        //if the layer is the final one
-    uint n_params = 0;
-    uint feature_map_size = 0;
+    unsigned int n_params = 0;
+    unsigned int feature_map_size = 0;
     long unsigned MACC = 0;
 
 
@@ -87,6 +88,7 @@ public:
             case LAYER_UPSAMPLE:            return "Upsample";
             case LAYER_REGION:              return "Region";
             case LAYER_YOLO:                return "Yolo";
+            case LAYER_PADDING:             return "Padding";
             default:                        return "unknown";
         }
     }
@@ -423,6 +425,8 @@ public:
     virtual layerType_t getLayerType() { return LAYER_FLATTEN; };
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+    int c, h, w, rows, cols;
 };
 
 /**
@@ -436,6 +440,7 @@ public:
     virtual layerType_t getLayerType() { return LAYER_RESHAPE; };
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    int n,c,h,w;
 
 };
 
@@ -470,7 +475,6 @@ public:
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 
-protected:
     dnnType mul, add;
     dnnType *add_vector;
 };
@@ -497,6 +501,7 @@ public:
     int winH, winW;
     int strideH, strideW;
     int paddingH, paddingW;
+    int padding;
     bool size;
     tkdnnPoolingMode_t pool_mode;
 
@@ -516,9 +521,35 @@ protected:
     bool poolOn3d;
 };
 
+/**
+ * Padding Layers
+ * tkDNN supports reflection,constant and zero padding
+ */
+
+typedef enum {
+    PADDING_MODE_CONSTANT = 0,
+    PADDING_MODE_ZERO = 1,
+    PADDING_MODE_REFLECTION = 2
+} tkdnnPaddingMode_t;
+
+class Padding : public Layer {
+public:
+    Padding(Network *net,int32_t pad_h,int32_t pad_w,tkdnnPaddingMode_t padding_mode,float constant = 0.0);
+    virtual ~Padding();
+    virtual layerType_t getLayerType(){return LAYER_PADDING ;};
+    virtual dnnType* infer(dataDim_t& dim,dnnType* srcData);
+    int32_t paddingH,paddingW;
+    tkdnnPaddingMode_t padding_mode;
+    float constant;
+
+};
+
+
+
 /**
     Softmax layer
 */
+
 class Softmax : public Layer {
 
 public:
@@ -582,6 +613,8 @@ public:
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 
+    int c,h,w;
+
 public:
     Layer *backLayer;
     bool mul = false;
@@ -602,6 +635,7 @@ public:
 
     int stride;
     bool reverse;
+    int c,h,w;
 };
 
 struct box {
@@ -685,6 +719,7 @@ public:
     virtual layerType_t getLayerType() { return LAYER_REGION; };
 
     int classes, coords, num;
+    int c,h,w;
     
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 };

include/tkDNN/MobilenetDetection.h

@@ -65,7 +65,7 @@ public:
     MobilenetDetection() {};
     ~MobilenetDetection() {}; 
 
-    bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1, const float conf_thresh=0.3);
+    bool init(const std::string& tensor_path,const int n_classes, const int n_batches=1, const float conf_thresh=0.3);
     void preprocess(cv::Mat &frame, const int bi=0);
     void postprocess(const int bi=0,const bool mAP=false);
 };

include/tkDNN/NetworkRT.h

@@ -7,50 +7,29 @@
 #include "Layer.h"
 #include "NvInfer.h"
 #include <memory>
+#include <tkDNN/kernels.h>
+#include <pluginsRT/ActivationLeakyRT.h>
+#include <pluginsRT/ActivationLogisticRT.h>
+#include <pluginsRT/ActivationMishRT.h>
+#include <pluginsRT/ActivationReLUCeilingRT.h>
+#include <pluginsRT/DeformableConvRT.h>
+#include <pluginsRT/FlattenConcatRT.h>
+#include <pluginsRT/MaxPoolingFixedSizeRT.h>
+#include <pluginsRT/RegionRT.h>
+#include <pluginsRT/ReorgRT.h>
+#include <pluginsRT/ReshapeRT.h>
+#include <pluginsRT/ResizeLayerRT.h>
+#include <pluginsRT/RouteRT.h>
+#include <pluginsRT/ShortcutRT.h>
+#include <pluginsRT/UpsampleRT.h>
+#include <pluginsRT/YoloRT.h>
+#include <pluginsRT/ConstantPaddingRT.h>
+#include <pluginsRT/ReflectionPadding.h>
+
+
 
 namespace tk { namespace dnn {
 
-template<typename T> void writeBUF(char*& buffer, const T& val)
-{
-    *reinterpret_cast<T*>(buffer) = val;
-    buffer += sizeof(T);
-}
-
-template<typename T> T readBUF(const char*& buffer)
-{
-    T val = *reinterpret_cast<const T*>(buffer);
-    buffer += sizeof(T);
-    return val;
-}
-
-using namespace nvinfer1;
-#include "pluginsRT/ActivationLeakyRT.h"
-#include "pluginsRT/ActivationLogisticRT.h"
-#include "pluginsRT/ActivationReLUCeilingRT.h"
-#include "pluginsRT/ActivationMishRT.h"
-#include "pluginsRT/ReorgRT.h"
-#include "pluginsRT/RegionRT.h"
-#include "pluginsRT/RouteRT.h"
-#include "pluginsRT/ShortcutRT.h"
-#include "pluginsRT/YoloRT.h"
-#include "pluginsRT/UpsampleRT.h"
-#include "pluginsRT/ResizeLayerRT.h"
-#include "pluginsRT/DeformableConvRT.h"
-#include "pluginsRT/FlattenConcatRT.h"
-#include "pluginsRT/ReshapeRT.h"
-#include "pluginsRT/MaxPoolingFixedSizeRT.h"
-
-class PluginFactory : IPluginFactory
-{
-public:
-    YoloRT *yolos[16];
-    int n_yolos;
-
-	virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength);
-};
-
-
-
 class NetworkRT {
 
 public:
@@ -69,12 +48,12 @@ public:
     void* buffersRT[MAX_BUFFERS_RT];
     dataDim_t buffersDIM[MAX_BUFFERS_RT];
     int buf_input_idx, buf_output_idx;
-
+    bool builderActive = false;
     dataDim_t input_dim, output_dim;
     dnnType *output;
     cudaStream_t stream;
 
-    PluginFactory *pluginFactory;
+    std::vector<nvinfer1::YoloRT*> yolo_plugins; // yolo layers in network
 
     NetworkRT(Network *net, const char *name);
     virtual ~NetworkRT();
@@ -106,18 +85,26 @@ public:
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Region *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input,Padding *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor* input,MulAdd *l);
 
+#if NV_TENSORRT_MAJOR > 5 && NV_TENSORRT_MAJOR < 8
     bool serialize(const char *filename);
+#else
+    bool serialize(const char *filename,nvinfer1::IHostMemory *ptr);
+#endif
+
     bool deserialize(const char *filename);
+    void destroy();
 
 
 

include/tkDNN/NetworkViz.h

@@ -6,7 +6,7 @@
 namespace tk { namespace dnn {
 
 cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19);
-cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19);
+cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=0);
 cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000);
     
 }}

include/tkDNN/SegmentationNN.h

@@ -4,7 +4,9 @@
 #include <iostream>
 #include <signal.h>
 #include <stdlib.h>    
+#ifdef __linux__
 #include <unistd.h>
+#endif 
 #include <mutex>
 #include "utils.h"
 
@@ -181,6 +183,7 @@ class SegmentationNN {
 
             checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
             checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
+            return true;
 
             return true;
         }

include/tkDNN/Yolo3Detection.h

@@ -4,9 +4,9 @@
 #include "opencv2/opencv.hpp"
 
 #include "DetectionNN.h"
+#include "DarknetParser.h"
 
-namespace tk { namespace dnn { 
-
+namespace tk { namespace dnn {
 class Yolo3Detection : public DetectionNN
 {
 private:

include/tkDNN/demo_utils.h

@@ -9,12 +9,13 @@
 
 #ifdef __linux__
 #include <unistd.h>
+#endif
+
 #include <opencv2/core/core.hpp>
 #include <opencv2/highgui/highgui.hpp>
 #include <opencv2/imgproc/imgproc.hpp>
 
 #include <yaml-cpp/yaml.h>
-#endif
 
 
 void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat);

include/tkDNN/kernels.h

@@ -48,4 +48,11 @@ void dcnV2CudaForward(cublasStatus_t stat, cublasHandle_t handle,
                          const int dst_dim, cudaStream_t stream = cudaStream_t(0));
 
 void scalAdd(dnnType* dstData, int size, float alpha, float beta, int inc, cudaStream_t stream = cudaStream_t(0));
+
+void reflection_pad2d_out_forward(int32_t pad_h,int32_t pad_w,float *srcData,float *dstData,int32_t input_h,int32_t input_w,int32_t plane_dim,int32_t n_batch,cudaStream_t cudaStream = cudaStream_t(0));
+
+void constant_pad2d_forward(dnnType *srcData,dnnType *dstData,int32_t input_h,int32_t input_w,int32_t output_h,
+                            int32_t output_w,int32_t c,int32_t n,int32_t padT,int32_t padL,dnnType constant,cudaStream_t cudaStream = cudaStream_t(0));
+
+
 #endif //KERNELS_H

include/tkDNN/pluginsRT/ActivationLeakyRT.h

@@ -1,61 +1,88 @@
-#include<cassert>
+#include "NvInfer.h"
 #include "../kernels.h"
+#include <cassert>
+#include <vector>
 
-class ActivationLeakyRT : public IPlugin {
+namespace nvinfer1 {
+    class ActivationLeakyRT : public IPluginV2 {
 
-public:
-	ActivationLeakyRT(float s) {
-		slope = s;
-	}
+    public:
+        explicit ActivationLeakyRT(float s);
 
-	~ActivationLeakyRT(){
+        ActivationLeakyRT(const void *data, size_t length);
 
-	}
+        ~ActivationLeakyRT();
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        int getNbOutputs() const NOEXCEPT override;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return inputs[0];
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		size = 1;
-		for(int i=0; i<outputDims[0].nbDims; i++)
-			size *= outputDims[0].d[i];
-	}
+        void
+        configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
+                            PluginFormat format, int maxBatchSize) NOEXCEPT override;
 
-	int initialize() override {
+        int initialize() NOEXCEPT override;
 
-		return 0;
-	}
+        void terminate() NOEXCEPT override {}
 
-	virtual void terminate() override {
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, void const *const *inputs, void *const *outputs, void *workspace,
+                    cudaStream_t stream) NOEXCEPT override;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+        size_t getSerializationSize() const NOEXCEPT override;
 
-		activationLEAKYForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
-											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, slope, stream);
-		return 0;
-	}
+        void serialize(void *buffer) const NOEXCEPT override;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
+
+        const char *getPluginType() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        void destroy() NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        IPluginV2 *clone() const NOEXCEPT override;
+
+        int size;
+        float slope;
+
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class ActivationLeakyRTPluginCreator : public IPluginCreator {
+    public:
+        ActivationLeakyRTPluginCreator();
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
 
 
-	virtual size_t getSerializationSize() override {
-		return 1*sizeof(int) + 1*sizeof(float);
-	}
-
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, size);
-		assert(buf == a + getSerializationSize());
-	}
-
-	int size;
-	float slope;
-};
+    REGISTER_TENSORRT_PLUGIN(ActivationLeakyRTPluginCreator);
+};

include/tkDNN/pluginsRT/ActivationLogisticRT.h

@@ -1,60 +1,88 @@
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
 
-class ActivationLogisticRT : public IPlugin {
+namespace nvinfer1 {
 
-public:
-    ActivationLogisticRT() {
+    class ActivationLogisticRT : public IPluginV2 {
+
+    public:
+        ActivationLogisticRT() ;
+
+        ActivationLogisticRT(const void *data, size_t length) ;
+
+        ~ActivationLogisticRT() ;
+
+        int getNbOutputs() const NOEXCEPT override ;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
+
+        void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
+                            PluginFormat format, int maxBatchSize) NOEXCEPT override ;
+
+        int initialize() NOEXCEPT override ;
+
+        void terminate() NOEXCEPT override ;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                            cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
 
-    }
+        size_t getSerializationSize() const NOEXCEPT override ;
 
-    ~ActivationLogisticRT(){
+        void serialize(void *buffer) const NOEXCEPT override ;
 
-    }
+        const char *getPluginType() const NOEXCEPT override ;
 
-    int getNbOutputs() const override {
-        return 1;
-    }
+        const char *getPluginVersion() const NOEXCEPT override ;
 
-    Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-        return inputs[0];
-    }
+        void destroy() NOEXCEPT override ;
 
-    void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-        size = 1;
-        for(int i=0; i<outputDims[0].nbDims; i++)
-            size *= outputDims[0].d[i];
-    }
+        const char *getPluginNamespace() const NOEXCEPT override ;
 
-    int initialize() override {
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
 
-        return 0;
-    }
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
 
-    virtual void terminate() override {
-    }
+        IPluginV2 *clone() const NOEXCEPT override ;
 
-    virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-        return 0;
-    }
+        int size;
 
-    virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+    private:
+        std::string mPluginNamespace;
+    };
 
-        activationLOGISTICForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
-                                  reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
-        return 0;
-    }
+    class ActivationLogisticRTPluginCreator : public IPluginCreator {
+    public:
+        ActivationLogisticRTPluginCreator() ;
 
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
 
-    virtual size_t getSerializationSize() override {
-        return 1*sizeof(int);
-    }
+        IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
 
-    virtual void serialize(void* buffer) override {
-        char *buf = reinterpret_cast<char*>(buffer);
-        tk::dnn::writeBUF(buf, size);
-    }
+        const char *getPluginNamespace() const NOEXCEPT override ;
 
-    int size;
-};
+        IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ActivationLogisticRTPluginCreator);
+};

include/tkDNN/pluginsRT/ActivationMishRT.h

@@ -1,61 +1,82 @@
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
 
-class ActivationMishRT : public IPlugin {
+namespace nvinfer1 {
+    class ActivationMishRT : public IPluginV2 {
 
-public:
-	ActivationMishRT() {
+    public:
+        ActivationMishRT() ;
+
+        ~ActivationMishRT() ;
+
+        ActivationMishRT(const void *data, size_t length) ;
 
 
-	}
+        int getNbOutputs() const NOEXCEPT override ;
 
-	~ActivationMishRT(){
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
 
-	}
+        void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
+                            PluginFormat format, int maxBatchSize) NOEXCEPT override ;
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        int initialize() NOEXCEPT override ;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return inputs[0];
-	}
+        void terminate() NOEXCEPT override ;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		size = 1;
-		for(int i=0; i<outputDims[0].nbDims; i++)
-			size *= outputDims[0].d[i];
-	}
+         size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-	int initialize() override {
+        size_t getSerializationSize() const NOEXCEPT override ;
 
-		return 0;
-	}
+        void serialize(void *buffer) const NOEXCEPT override ;
 
-	virtual void terminate() override {
-	}
+        const char *getPluginType() const NOEXCEPT override ;
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
+        const char *getPluginVersion() const NOEXCEPT override ;
 
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+        void destroy() NOEXCEPT override { delete this; }
 
-		activationMishForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
-											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
-		return 0;
-	}
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
 
+        const char *getPluginNamespace() const NOEXCEPT override ;
 
-	virtual size_t getSerializationSize() override {
-		return 1*sizeof(int);
-	}
+        void setPluginNamespace(const char *plguinNamespace) NOEXCEPT override ;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, size);
-		assert(buf == a + getSerializationSize());
-	}
+        IPluginV2 *clone() const NOEXCEPT override ;
 
-	int size;
-};
+        int size;
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class ActivationMishRTPluginCreator : public IPluginCreator {
+    public:
+        ActivationMishRTPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ActivationMishRTPluginCreator);
+};

include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h

@@ -1,63 +1,81 @@
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
 
-class ActivationReLUCeiling : public IPlugin {
+namespace nvinfer1 {
+    class ActivationReLUCeiling : public IPluginV2 {
 
-public:
-	ActivationReLUCeiling(const float ceiling) {
-		this->ceiling = ceiling;
-	}
+    public:
+        explicit ActivationReLUCeiling(const float ceiling) ;
 
-	~ActivationReLUCeiling(){
+        ~ActivationReLUCeiling() ;
 
-	}
+        ActivationReLUCeiling(const void *data, size_t length) ;
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        int getNbOutputs() const NOEXCEPT override ;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return inputs[0];
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		size = 1;
-		for(int i=0; i<outputDims[0].nbDims; i++)
-			size *= outputDims[0].d[i];
-	}
+        void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,PluginFormat format, int maxBatchSize) NOEXCEPT override ;
 
-	int initialize() override {
+        int initialize() NOEXCEPT override ;
 
-		return 0;
-	}
+        void terminate() NOEXCEPT override ;
 
-	virtual void terminate() override {
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
+        size_t getSerializationSize() const NOEXCEPT override ;
 
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+        void serialize(void *buffer) const NOEXCEPT override ;
 
-		activationReLUCeilingForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
-											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, ceiling, stream);
-		return 0;
-	}
+        IPluginV2 *clone() const NOEXCEPT override ;
 
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
 
-	virtual size_t getSerializationSize() override {
-		return 1*sizeof(int) +  1*sizeof(float);
-	}
+        void destroy() NOEXCEPT override ;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, ceiling);
-		tk::dnn::writeBUF(buf, size);
-		assert(buf = a + getSerializationSize());
-		
-	}
+        const char *getPluginType() const NOEXCEPT override ;
 
-	int size;
-	float ceiling;
-};
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+        int size;
+        float ceiling;
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class ActivationReLUCeilingPluginCreator : public IPluginCreator {
+    public:
+        ActivationReLUCeilingPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    public:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ActivationReLUCeilingPluginCreator);
+};

include/tkDNN/pluginsRT/ConstantPaddingRT.h

@@ -0,0 +1,109 @@
+//
+// Created by perseusdg on 1/7/22.
+//
+
+#ifndef _CONSTANTPADDINGRT_PLUGIN_H
+#define _CONSTANTPADDINGRT_PLUGIN_H
+
+#include<cassert>
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
+#include <kernels.h>
+
+namespace nvinfer1{
+    class ConstantPaddingRT : public IPluginV2Ext {
+    public:
+        ConstantPaddingRT(int32_t padH,int32_t padW,int32_t n,int32_t c,int32_t i_h,int32_t i_w,int32_t o_h,int32_t o_w,float constant);
+
+        ConstantPaddingRT(const void *data,size_t length);
+
+        ~ConstantPaddingRT();
+
+        int getNbOutputs() const NOEXCEPT override;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
+
+        int initialize() NOEXCEPT override ;
+
+        void terminate() NOEXCEPT override ;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR <= 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override;
+
+        int32_t i_h,i_w,o_h,o_w,n,c,padH,padW;
+        float constant;
+    private:
+        std::string mPluginNamespace;
+
+    };
+
+    class ConstantPaddingRTPluginCreator : public IPluginCreator {
+    public:
+        ConstantPaddingRTPluginCreator();
+
+        void setPluginNamespace(const char* pluginNamespace) NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ConstantPaddingRTPluginCreator);
+};
+
+
+#endif //TKDNN_CONSTANTPADDINGRT_H

include/tkDNN/pluginsRT/DeformableConvRT.h

@@ -1,196 +1,137 @@
+#ifndef _DEFORMABLECONVRT_PLUGIN_H
+#define _DEFORMABLECONVRT_PLUGIN_H
+
+#include <NvInfer.h>
+#include <vector>
 #include<cassert>
 #include "../kernels.h"
+#include <tkdnn.h>
+
+namespace nvinfer1 {
+    class DeformableConvRT : public IPluginV2Ext {
 
 
-class DeformableConvRT : public IPlugin {
+    public:
+        DeformableConvRT(int chunk_dim, int kh, int kw, int sh, int sw, int ph, int pw,
+                         int deformableGroup, int i_n, int i_c, int i_h, int i_w,
+                         int o_n, int o_c, int o_h, int o_w,std::vector<dnnType> data_H,std::vector<dnnType> bias2_H,
+                         std::vector<dnnType> ones_d1_h,std::vector<dnnType> ones_d2_h,std::vector<dnnType> offsetH,std::vector<dnnType> maskH,int height_ones,
+                         int width_ones,int dim_ones);
+
+        ~DeformableConvRT();
+
+        DeformableConvRT(const void *data, size_t length) ;
+
+        int getNbOutputs() const NOEXCEPT override ;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
+
+        int initialize() NOEXCEPT override ;
+
+        void terminate() NOEXCEPT override ;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                            cudaStream_t stream) NOEXCEPT override;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
 
 
+        cublasStatus_t stat;
+        cublasHandle_t handle{nullptr};
+        int i_n, i_c, i_h, i_w;
+        int o_n, o_c, o_h, o_w;
+        int size;
+        int chunk_dim;
+        int kh, kw;
+        int sh, sw;
+        int ph, pw;
+        int deformableGroup;
+        int height_ones;
+        int width_ones;
+        int dim_ones;
 
-public:
-	DeformableConvRT(int chunk_dim, int kh, int kw, int sh, int sw, int ph, int pw, 
-					int deformableGroup, int i_n, int i_c, int i_h, int i_w, 
-					int o_n, int o_c, int o_h, int o_w, 
-					tk::dnn::DeformConv2d *deformable = nullptr) {
-		this->chunk_dim = chunk_dim;
-		this->kh = kh;
-		this->kw = kw;
-		this->sh = sh;
-		this->sw = sw;
-		this->ph = ph;
-		this->pw = pw;
-		this->deformableGroup = deformableGroup;
-		this->i_n = i_n;
-		this->i_c = i_c;
-		this->i_h = i_h;
-		this->i_w = i_w;
-		this->o_n = o_n;
-		this->o_c = o_c;
-		this->o_h = o_h;
-		this->o_w = o_w;
-		height_ones = (i_h + 2 * ph - (1 * (kh - 1) + 1)) / sh + 1;
-		width_ones = (i_w + 2 * pw - (1 * (kw - 1) + 1)) / sw + 1;
-		dim_ones = i_c * kh * kw * 1 * height_ones * width_ones;
-		
-		checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
-		checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
-		checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));
-		checkCuda( cudaMalloc(&offset, 2*chunk_dim*sizeof(dnnType)));
-		checkCuda( cudaMalloc(&mask, chunk_dim*sizeof(dnnType)));
-		checkCuda( cudaMalloc(&ones_d2, dim_ones*sizeof(dnnType)));
-		if(deformable != nullptr) {
-			this->defRT = deformable;
-			checkCuda( cudaMemcpy(data_d, deformable->data_d, sizeof(dnnType)*i_c * o_c * kh * kw * 1, cudaMemcpyDeviceToDevice) );
-			checkCuda( cudaMemcpy(bias2_d, deformable->bias2_d, sizeof(dnnType)*o_c, cudaMemcpyDeviceToDevice) );
-            checkCuda( cudaMemcpy(ones_d1, deformable->ones_d1, sizeof(dnnType)*height_ones*width_ones, cudaMemcpyDeviceToDevice) );
-            checkCuda( cudaMemcpy(offset, deformable->offset, sizeof(dnnType)*2*chunk_dim, cudaMemcpyDeviceToDevice) );
-            checkCuda( cudaMemcpy(mask, deformable->mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToDevice) );
-            checkCuda( cudaMemcpy(ones_d2, deformable->ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToDevice) );
-		}
-		stat = cublasCreate(&handle);
-		if (stat != CUBLAS_STATUS_SUCCESS)
-			FatalError("CUBLAS initialization failed\n");
-	}
-
-	~DeformableConvRT() {
-		checkCuda( cudaFree(data_d) );
-		checkCuda( cudaFree(bias2_d) );
-		checkCuda( cudaFree(ones_d1) );
-		checkCuda( cudaFree(offset) );
-		checkCuda( cudaFree(mask) );
-		checkCuda( cudaFree(ones_d2) );
-		cublasDestroy(handle);
-	}
-
-	int getNbOutputs() const override {
-		return 1;
-	}
-
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{defRT->output_dim.c, defRT->output_dim.h, defRT->output_dim.w};
-	}
-
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { }
-
-	int initialize() override {
-		return 0;
-	}
-
-	virtual void terminate() override { }
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *output_conv = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
-
-		// split conv2d outputs into offset to mask
-		for(int b=0; b<batchSize; b++) {
-			checkCuda(cudaMemcpy(offset, output_conv + b * 3 * chunk_dim, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); 
-			checkCuda(cudaMemcpy(mask, output_conv + b * 3 * chunk_dim + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); 
-			// kernel sigmoid
-			activationSIGMOIDForward(mask, mask, chunk_dim);
-			// deformable convolution
-			dcnV2CudaForward(stat, handle, 
-								srcData, data_d,
-								bias2_d, ones_d1,
-								offset, mask,
-								reinterpret_cast<dnnType*>(outputs[0]), ones_d2,
-								kh, kw,
-								sh, sw,
-								ph, pw,
-								1, 1,
-								deformableGroup, b,
-								i_n, i_c, i_h, i_w,
-								o_n, o_c, o_h, o_w,
-								chunk_dim);
-		}
-		return 0;
-	}
+        std::vector<dnnType> data_d_v;
+        std::vector<dnnType> bias2_d_v;
+        std::vector<dnnType> ones_d1_v;
+        std::vector<dnnType> offset_v;
+        std::vector<dnnType> mask_v;
+        std::vector<dnnType> ones_d2_v;
+        dnnType* data_d;
+        dnnType* bias2_d;
+        dnnType* ones_d1;
+        dnnType* offset;
+        dnnType* mask;
+        dnnType* ones_d2;
+        // dnnType *input_n;
+        // dnnType *offset_n;
+        // dnnType *mask_n;
+        // dnnType *output_n;
 
 
-	virtual size_t getSerializationSize() override {
-		return 16 * sizeof(int) + chunk_dim * 3 * sizeof(dnnType) + (i_c * o_c * kh * kw * 1 ) * sizeof(dnnType) +
-			   o_c * sizeof(dnnType) + height_ones * width_ones * sizeof(dnnType) + dim_ones * sizeof(dnnType);
-	}
+        tk::dnn::DeformConv2d *defRT;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, chunk_dim);
-		tk::dnn::writeBUF(buf, kh);
-		tk::dnn::writeBUF(buf, kw);
-		tk::dnn::writeBUF(buf, sh);
-		tk::dnn::writeBUF(buf, sw);
-		tk::dnn::writeBUF(buf, ph);
-		tk::dnn::writeBUF(buf, pw);
-		tk::dnn::writeBUF(buf, deformableGroup);
-		tk::dnn::writeBUF(buf, i_n);
-		tk::dnn::writeBUF(buf, i_c);
-		tk::dnn::writeBUF(buf, i_h);
-		tk::dnn::writeBUF(buf, i_w);
-		tk::dnn::writeBUF(buf, o_n);
-		tk::dnn::writeBUF(buf, o_c);
-		tk::dnn::writeBUF(buf, o_h);
-		tk::dnn::writeBUF(buf, o_w);
-		dnnType *aus = new dnnType[chunk_dim*2];
-		checkCuda( cudaMemcpy(aus, offset, sizeof(dnnType)*2*chunk_dim, cudaMemcpyDeviceToHost) );
-        for(int i=0; i<chunk_dim*2; i++)
-    		tk::dnn::writeBUF(buf, aus[i]);
-		free(aus);
-		aus = new dnnType[chunk_dim];
-		checkCuda( cudaMemcpy(aus, mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToHost) );
-        for(int i=0; i<chunk_dim; i++)
-    		tk::dnn::writeBUF(buf, aus[i]);
-		free(aus);
-		aus = new dnnType[(i_c * o_c * kh * kw * 1 )];
-		checkCuda( cudaMemcpy(aus, data_d, sizeof(dnnType)*(i_c * o_c * kh * kw * 1 ), cudaMemcpyDeviceToHost) );
-        for(int i=0; i<(i_c * o_c * kh * kw * 1 ); i++)
-    		tk::dnn::writeBUF(buf, aus[i]);
-		free(aus);
-		aus = new dnnType[o_c];
-		checkCuda( cudaMemcpy(aus, bias2_d, sizeof(dnnType)*o_c, cudaMemcpyDeviceToHost) );
-        for(int i=0; i < o_c; i++)
-    		tk::dnn::writeBUF(buf, aus[i]);
-		free(aus);
-		aus = new dnnType[height_ones * width_ones];
-		checkCuda( cudaMemcpy(aus, ones_d1, sizeof(dnnType)*height_ones * width_ones, cudaMemcpyDeviceToHost) );
-        for(int i=0; i<height_ones * width_ones; i++)
-    		tk::dnn::writeBUF(buf, aus[i]);
-		free(aus);
-		aus = new dnnType[dim_ones];
-		checkCuda( cudaMemcpy(aus, ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToHost) );
-        for(int i=0; i<dim_ones; i++)
-    		tk::dnn::writeBUF(buf, aus[i]);
-		free(aus);
-		assert(buf == a + getSerializationSize());
-	}
+    private:
+        std::string mPluginNamespace;
+    };
 
-	cublasStatus_t stat; 
-	cublasHandle_t handle; 
-	int i_n, i_c, i_h, i_w;
-	int o_n, o_c, o_h, o_w;
-	int size;
-	int chunk_dim;
-	int kh, kw;
-	int sh, sw;
-	int ph, pw;
-	int deformableGroup;
-	int height_ones;
-	int width_ones;
-	int dim_ones;
-		
-	dnnType *data_d;
-    dnnType *bias2_d;
-	dnnType *ones_d1;
-	dnnType * offset;
-	dnnType * mask;
-	dnnType *ones_d2;
-	// dnnType *input_n;
-	// dnnType *offset_n;
-	// dnnType *mask_n;
-	// dnnType *output_n;
-	
-	
-	tk::dnn::DeformConv2d *defRT;
+    class DeformableConvRTPluginCreator : public IPluginCreator {
+    public:
+        DeformableConvRTPluginCreator();
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+    REGISTER_TENSORRT_PLUGIN(DeformableConvRTPluginCreator);
 };
+#endif

include/tkDNN/pluginsRT/FlattenConcatRT.h

@@ -1,81 +1,100 @@
+#ifndef _FLATTENCONCATRT_PLUGIN_H
+#define _FLATTENCONCATRT_PLUGIN_H
+
 #include<cassert>
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
+namespace nvinfer1 {
+    class FlattenConcatRT : public IPluginV2Ext {
 
-class FlattenConcatRT : public IPlugin {
+    public:
+        FlattenConcatRT(int c,int h,int w,int rows,int cols) ;
 
-public:
-	FlattenConcatRT() {
-		stat = cublasCreate(&handle);
-		if (stat != CUBLAS_STATUS_SUCCESS) {
-			printf ("CUBLAS initialization failed\n");
-			return;
-  		}
-	}
+        FlattenConcatRT(const void *data, size_t length) ;
 
-	~FlattenConcatRT(){
+        ~FlattenConcatRT() ;
 
-	}
+        int getNbOutputs() const NOEXCEPT override ;
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{ inputs[0].d[0] * inputs[0].d[1] * inputs[0].d[2], 1, 1};
-	}
+        int initialize() NOEXCEPT override ;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		assert(nbOutputs == 1 && nbInputs ==1);
-		rows = inputDims[0].d[0];
-		cols = inputDims[0].d[1] * inputDims[0].d[2];
-		c = inputDims[0].d[0] * inputDims[0].d[1] * inputDims[0].d[2];
-		h = 1;
-		w = 1;
-	}
+        void terminate() NOEXCEPT override ;
 
-	int initialize() override {
-		return 0;
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
 
-	virtual void terminate() override {
-		checkERROR(cublasDestroy(handle));
-	}
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR <= 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
+        size_t getSerializationSize() const NOEXCEPT override ;
 
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*rows*cols*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+        void serialize(void *buffer) const NOEXCEPT override ;
 
-		checkERROR( cublasSetStream(handle, stream) );	
-		for(int i=0; i<batchSize; i++) {
-			float const alpha(1.0);
-			float const beta(0.0);
-			int offset = i*rows*cols;
-			checkERROR( cublasSgeam( handle, CUBLAS_OP_T, CUBLAS_OP_N, rows, cols, &alpha, srcData + offset, cols, &beta, srcData + offset, rows, dstData + offset, rows ));
-		}
-		return 0;
-	}
+        void destroy() NOEXCEPT override ;
 
+        const char *getPluginType() const NOEXCEPT override ;
 
-	virtual size_t getSerializationSize() override {
-		return 5*sizeof(int);
-	}
+        const char *getPluginVersion() const NOEXCEPT override;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a = buf;
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		tk::dnn::writeBUF(buf, rows);
-		tk::dnn::writeBUF(buf, cols);
-		assert(buf == a + getSerializationSize());
-	}
+        const char *getPluginNamespace() const NOEXCEPT override ;
 
-	int c, h, w;
-	int rows, cols;
-	cublasStatus_t stat; 
-	cublasHandle_t handle; 
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                            int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                            bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                            int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override;
+
+        int c, h, w;
+        int rows, cols;
+        cublasHandle_t handle{nullptr};
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class FlattenConcatRTPluginCreator : public IPluginCreator {
+    public:
+        FlattenConcatRTPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(FlattenConcatRTPluginCreator);
 };
+#endif

include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h

@@ -1,75 +1,105 @@
 #include<cassert>
 #include "../kernels.h"
-
-class MaxPoolFixedSizeRT : public IPlugin {
-
-public:
-	MaxPoolFixedSizeRT(int c, int h, int w, int n, int strideH, int strideW, int winSize, int padding) {
-		this->c = c;	
-		this->h = h;
-		this->w = w;
-		this->n = n;
-		this->stride_H = strideH;
-		this->stride_W = strideW;
-		this->winSize = winSize;
-		this->padding = padding;
-	}
-
-	~MaxPoolFixedSizeRT(){
-	}
-
-	int getNbOutputs() const override {
-		return 1;
-	}
-
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{this->c, this->h, this->w};
-	}
-
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {        
-	}
-
-	int initialize() override {
-		return 0;
-	}
-
-	virtual void terminate() override {
-	}
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-
-		//std::cout<<this->n<<"  "<<this->c<<"  "<<this->h<<"  "<<this->w<<"  "<<this->stride_H<<"  "<<this->stride_W<<"  "<<this->winSize<<"  "<<this->padding<<std::endl;
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-		MaxPoolingForward(srcData, dstData, batchSize, this->c, this->h, this->w, this->stride_H, this->stride_W, this->winSize, this->padding, stream);
-		return 0;
-	}
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
 
 
-	virtual size_t getSerializationSize() override {
-		return 8*sizeof(int);
-	}
+namespace nvinfer1 {
+    class MaxPoolFixedSizeRT : public IPluginV2Ext {
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    public:
+        MaxPoolFixedSizeRT(int c, int h, int w, int n, int strideH, int strideW, int winSize, int padding) ;
 
-		tk::dnn::writeBUF(buf, this->c);
-		tk::dnn::writeBUF(buf, this->h);
-		tk::dnn::writeBUF(buf, this->w);
-		tk::dnn::writeBUF(buf, this->n);
-		tk::dnn::writeBUF(buf, this->stride_H);
-		tk::dnn::writeBUF(buf, this->stride_W);
-		tk::dnn::writeBUF(buf, this->winSize);
-		tk::dnn::writeBUF(buf, this->padding);
-		assert(buf == a + getSerializationSize());
-	}
+        MaxPoolFixedSizeRT(const void *data, size_t length) ;
 
-	int n, c, h, w;
-	int stride_H, stride_W;
-	int winSize;
-	int padding;
+        ~MaxPoolFixedSizeRT() ;
+
+        int getNbOutputs() const NOEXCEPT override ;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
+
+        int initialize() NOEXCEPT override ;
+
+        void terminate() NOEXCEPT override ;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                            cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR <= 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+
+        int n, c, h, w;
+        int stride_H, stride_W;
+        int winSize;
+        int padding;
+
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class MaxPoolFixedSizeRTPluginCreator : public IPluginCreator {
+    public:
+        MaxPoolFixedSizeRTPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+
+    };
+
+    REGISTER_TENSORRT_PLUGIN(MaxPoolFixedSizeRTPluginCreator);
 };

include/tkDNN/pluginsRT/ReflectionPadding.h

@@ -0,0 +1,101 @@
+#ifndef _REFLECTIONPADDINGRT_PLUGIN_H
+#define _REFLECTIONPADDINGRT_PLUGIN_H
+
+#include<cassert>
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
+#include <kernels.h>
+
+namespace nvinfer1{
+    class ReflectionPaddingRT : public IPluginV2Ext {
+    public:
+        ReflectionPaddingRT(int32_t padH,int32_t padW,int32_t input_h,int32_t input_w,int32_t output_h,int32_t output_w,int32_t c,int32_t n);
+
+        ReflectionPaddingRT(const void *data,size_t length);
+
+        ~ReflectionPaddingRT();
+
+        int getNbOutputs() const NOEXCEPT override;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
+
+        int initialize() NOEXCEPT override ;
+
+        void terminate() NOEXCEPT override ;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR <= 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        bool supportsFormat (DataType type, PluginFormat format) const NOEXCEPT override;
+
+        int32_t padH,padW,input_h,input_w,output_h,output_w,n,c;
+    private:
+        std::string mPluginNamespace;
+
+    };
+
+    class ReflectionPaddingRTPluginCreator : public IPluginCreator {
+    public:
+        ReflectionPaddingRTPluginCreator();
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ReflectionPaddingRTPluginCreator);
+};
+#endif
+

include/tkDNN/pluginsRT/RegionRT.h

@@ -1,95 +1,110 @@
+#ifndef _REGIONRT_PLUGIN_H
+#define _REGIONRT_PLUGIN_H
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
 
-class RegionRT : public IPlugin {
+namespace nvinfer1 {
+    class RegionRT : public IPluginV2Ext {
 
-public:
-	RegionRT(int classes, int coords, int num) {
+    public:
+        RegionRT(int classes, int coords, int num,int c,int h,int w);
 
-		this->classes = classes;
-		this->coords = coords;
-		this->num = num;
-	}
+        ~RegionRT() ;
 
-	~RegionRT(){
+        RegionRT(const void *data, size_t length) ;
 
-	}
+        int getNbOutputs() const NOEXCEPT override ;
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return inputs[0];
-	}
-
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		c = inputDims[0].d[0];
-		h = inputDims[0].d[1];
-		w = inputDims[0].d[2];
-	}
-
-	int initialize() override {
-
-		return 0;
-	}
-
-	virtual void terminate() override {
-	}
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-
-		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
-
-		for (int b = 0; b < batchSize; ++b){
-			for(int n = 0; n < num; ++n){
-				int index = entry_index(b, n*w*h, 0);
-				activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
-				
-				index = entry_index(b, n*w*h, coords);
-				activationLOGISTICForward(srcData + index, dstData + index, w*h, stream);
-			}
-		}
-
-		//softmax start
-		int index = entry_index(0, 0, coords + 1);
-		softmaxForward(	srcData + index, classes, batchSize*num, 
-						(c*h*w)/num, 
-						w*h, 1, w*h, 1, dstData + index, stream);
-
-		return 0;
-	}
+        int initialize() NOEXCEPT override ;
 
 
-	virtual size_t getSerializationSize() override {
-		return 6*sizeof(int);
-	}
+        void terminate() NOEXCEPT override ;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, classes);
-		tk::dnn::writeBUF(buf, coords);
-		tk::dnn::writeBUF(buf, num);
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		assert(buf == a + getSerializationSize());
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
 
-	int c, h, w;
-    int classes, coords, num;
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                    cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-	int entry_index(int batch, int location, int entry) {
-		int n =   location / (w*h);
-		int loc = location % (w*h);
-		return batch*c*h*w + n*w*h*(coords+classes+1) + entry*w*h + loc;
-	}
 
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+        int c, h, w;
+        int classes, coords, num;
+
+        int entry_index(int batch, int location, int entry) {
+            int n = location / (w * h);
+            int loc = location % (w * h);
+            return batch * c * h * w + n * w * h * (coords + classes + 1) + entry * w * h + loc;
+        }
+
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class RegionRTPluginCreator : public IPluginCreator {
+    public:
+        RegionRTPluginCreator();
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(RegionRTPluginCreator);
 };
+
+#endif

include/tkDNN/pluginsRT/ReorgRT.h

@@ -1,64 +1,98 @@
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
 
-class ReorgRT : public IPlugin {
+namespace nvinfer1 {
+    class ReorgRT : public IPluginV2Ext {
 
-public:
-	ReorgRT(int stride) {
-		this->stride = stride;
-	}
+    public:
+        ReorgRT(int stride,int c,int h,int w);
 
-	~ReorgRT(){
+        ~ReorgRT();
 
-	}
+        ReorgRT(const void *data, size_t length);
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        int getNbOutputs() const NOEXCEPT override;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{inputs[0].d[0]*stride*stride, inputs[0].d[1]/stride, inputs[0].d[2]/stride};
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		c = inputDims[0].d[0];
-		h = inputDims[0].d[1];
-		w = inputDims[0].d[2];
-	}
+        int initialize() NOEXCEPT override;
 
-	int initialize() override {
+        void terminate() NOEXCEPT override;
 
-		return 0;
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
 
-	virtual void terminate() override {
-	}
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-
-		reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
-					  reinterpret_cast<dnnType*>(outputs[0]), 
-					  batchSize, c, h, w, stride, stream);
-		return 0;
-	}
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                    cudaStream_t stream) NOEXCEPT override;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
 
-	virtual size_t getSerializationSize() override {
-		return 4*sizeof(int);
-	}
+        size_t getSerializationSize() const NOEXCEPT override;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, stride);
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		assert(buf == a + getSerializationSize());
-	}
+        void serialize(void *buffer) const NOEXCEPT override;
 
-	int c, h, w, stride;
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
+
+        const char *getPluginType() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        void destroy() NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        IPluginV2Ext *clone() const NOEXCEPT override;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        int c, h, w, stride;
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class ReorgRTPluginCreator : public IPluginCreator {
+    public:
+        ReorgRTPluginCreator();
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
+
+        const char *getPluginName() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ReorgRTPluginCreator);
 };
+

include/tkDNN/pluginsRT/ReshapeRT.h

@@ -1,62 +1,101 @@
+#ifndef _RESHAPERT_PLUGIN_H
+#define _RESHAPERT_PLUGIN_H
+
 #include<cassert>
-
-class ReshapeRT : public IPlugin {
-
-public:
-	ReshapeRT(dataDim_t new_dim) {
-		n = new_dim.n;
-		c = new_dim.c;
-		h = new_dim.h;
-		w = new_dim.w;
-	}
-
-	~ReshapeRT(){
-
-	}
-
-	int getNbOutputs() const override {
-		return 1;
-	}
-
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{ c,h,w};
-	}
-
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-	}
-
-	int initialize() override {
-		return 0;
-	}
-
-	virtual void terminate() override {
-	}
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-
-		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
-		return 0;
-	}
+#include <NvInfer.h>
+#include <vector>
+#include <tkdnn.h>
 
 
-	virtual size_t getSerializationSize() override {
-		return 4*sizeof(int);
-	}
+namespace nvinfer1 {
+    class ReshapeRT : public IPluginV2Ext {
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a = buf;
-		tk::dnn::writeBUF(buf, n);
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		assert(buf == a + getSerializationSize());
-	}
+    public:
+        ReshapeRT(int n,int c,int h,int w) ;
 
-	int n, c, h, w;
+        ReshapeRT(const void *data, size_t length) ;
+
+        ~ReshapeRT() ;
+
+        int getNbOutputs() const NOEXCEPT override ;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
+
+
+        int initialize() NOEXCEPT override ;
+
+        void terminate() NOEXCEPT override ;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        int n, c, h, w;
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class ReshapeRTPluginCreator : public IPluginCreator {
+    public:
+        ReshapeRTPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ReshapeRTPluginCreator);
 };
+#endif

include/tkDNN/pluginsRT/ResizeLayerRT.h

@@ -1,68 +1,104 @@
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
+#include <utils.h>
 
-class ResizeLayerRT : public IPlugin {
+namespace nvinfer1 {
 
-public:
-	ResizeLayerRT(int c, int h, int w) {
-		o_c = c;
-		o_h = h;
-		o_w = w;	
-	}
+    class ResizeLayerRT : public IPluginV2Ext {
 
-	~ResizeLayerRT(){
-	}
+    public:
+        ResizeLayerRT(int oc, int oh, int ow,int ic,int ih,int iw) ;
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        ResizeLayerRT(const void *data, size_t length) ;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{o_c, o_h, o_w};
-	}
+        ~ResizeLayerRT() ;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		i_c = inputDims[0].d[0];
-		i_h = inputDims[0].d[1];
-		i_w = inputDims[0].d[2];        
-	}
+        int getNbOutputs() const NOEXCEPT override ;
 
-	int initialize() override {
-		return 0;
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
 
-	virtual void terminate() override {
-	}
+        int initialize() NOEXCEPT override ;
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
+        void terminate() NOEXCEPT override ;
 
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-    	// printf("%d %d %d %d %d %d\n", i_c, i_w, i_h, o_c, o_w, o_h);
-        resizeForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
-					  reinterpret_cast<dnnType*>(outputs[0]), 
-					  batchSize, i_c, i_h, i_w, o_c, o_h, o_w, stream);
-		return 0;
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                            cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR <= 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+        size_t getSerializationSize() const NOEXCEPT override ;
+
+        void serialize(void *buffer) const NOEXCEPT override ;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
+
+        const char *getPluginType() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        void destroy() NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+        int i_c, i_h, i_w, o_c, o_h, o_w;
+
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class ResizeLayerRTPluginCreator : public IPluginCreator {
+    public:
+        ResizeLayerRTPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
 
 
-	virtual size_t getSerializationSize() override {
-		return 6*sizeof(int);
-	}
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
 
-		tk::dnn::writeBUF(buf, o_c);
-		tk::dnn::writeBUF(buf, o_h);
-		tk::dnn::writeBUF(buf, o_w);
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
 
-		tk::dnn::writeBUF(buf, i_c);
-		tk::dnn::writeBUF(buf, i_h);
-		tk::dnn::writeBUF(buf, i_w);
-		assert(buf == a + getSerializationSize());
-	}
+    };
 
-	int i_c, i_h, i_w, o_c, o_h, o_w;
+    REGISTER_TENSORRT_PLUGIN(ResizeLayerRTPluginCreator);
 };
+

include/tkDNN/pluginsRT/RouteRT.h

@@ -1,96 +1,90 @@
 #include<cassert>
 #include "../kernels.h"
+#include <vector>
+#include <NvInfer.h>
 
-class RouteRT : public IPlugin {
+namespace nvinfer1 {
+    class RouteRT : public IPluginV2 {
 
-	/**
-		THIS IS NOT USED ANYMORE
-	*/
+        /**
+            THIS IS NOT USED ANYMORE
+        */
 
-public:
-	RouteRT(int groups, int group_id) {
-		this->groups = groups;
-		this->group_id = group_id;
-	}
+    public:
+        RouteRT(int groups, int group_id) ;
 
-	~RouteRT(){
+        ~RouteRT() ;
 
-	}
+        RouteRT(const void *data, size_t length) ;
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        int getNbOutputs() const NOEXCEPT override ;
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		int out_c = 0;
-		for(int i=0; i<nbInputDims; i++) out_c += inputs[i].d[0];
-		return DimsCHW{out_c/groups, inputs[0].d[1], inputs[0].d[2]};
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override ;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		in = nbInputs;
-		c = 0;
-		for(int i=0; i<nbInputs; i++) {
-			c_in[i] = inputDims[i].d[0]; 
-			c += inputDims[i].d[0];
-		}
-		h = inputDims[0].d[1];
-		w = inputDims[0].d[2];
-		c /= groups;
-	}
+        void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,PluginFormat format, int maxBatchSize) NOEXCEPT override ;
 
-	int initialize() override {
+        int initialize() NOEXCEPT override ;
 
-		return 0;
-	}
+        void terminate() NOEXCEPT override ;
 
-	virtual void terminate() override {
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override ;
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT override ;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-		
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+        size_t getSerializationSize() const NOEXCEPT override ;
 
-		for(int b=0; b<batchSize; b++) {
-			int offset = 0;
-			for(int i=0; i<in; i++) {
-				dnnType *input = (dnnType*)reinterpret_cast<const dnnType*>(inputs[i]);
-				int in_dim = c_in[i]*h*w;
-				int part_in_dim = in_dim / this->groups;
-				checkCuda( cudaMemcpyAsync(dstData + b*c*w*h + offset, input + b*c*w*h*groups + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) );
-				offset += part_in_dim;
-			}
-		}
+        void serialize(void *buffer) const NOEXCEPT override ;
 
-		return 0;
-	}
+        const char *getPluginType() const NOEXCEPT override ;
 
+        const char *getPluginVersion() const NOEXCEPT override ;
 
-	virtual size_t getSerializationSize() override {
-		return (6+MAX_INPUTS)*sizeof(int);
-	}
+        void destroy() NOEXCEPT override ;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, groups);
-		tk::dnn::writeBUF(buf, group_id);
-		tk::dnn::writeBUF(buf, in);
-		for(int i=0; i<MAX_INPUTS; i++)
-			tk::dnn::writeBUF(buf, c_in[i]);
+        const char *getPluginNamespace() const NOEXCEPT override ;
 
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		assert(buf == a + getSerializationSize());
-	}
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
 
-	static const int MAX_INPUTS = 4;
-	int in;
-	int c_in[MAX_INPUTS];
-	int c, h, w;
-	int groups, group_id;
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override ;
+
+        IPluginV2 *clone() const NOEXCEPT override ;
+
+        static const int MAX_INPUTS = 4;
+        int in;
+        int c_in[MAX_INPUTS];
+        int c, h, w;
+        int groups, group_id;
+    private:
+        std::string mPluginNamespace;
+    };
+
+    class RouteRTPluginCreator : public IPluginCreator {
+    public:
+        RouteRTPluginCreator() ;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override ;
+
+        const char *getPluginNamespace() const NOEXCEPT override ;
+
+        IPluginV2 *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override ;
+
+        IPluginV2 *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override ;
+
+        const char *getPluginName() const NOEXCEPT override ;
+
+        const char *getPluginVersion() const NOEXCEPT override ;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override ;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(RouteRTPluginCreator);
 };

include/tkDNN/pluginsRT/ShortcutRT.h

@@ -1,77 +1,109 @@
+#ifndef _SHORTCUTRT_PLUGIN_H
+#define _SHORTCUTRT_PLUGIN_H
+
 #include<cassert>
 #include "../kernels.h"
-
-class ShortcutRT : public IPlugin {
-
-public:
-	ShortcutRT(tk::dnn::dataDim_t bdim, bool mul) {
-		this->bc = bdim.c;
-		this->bh = bdim.h;
-		this->bw = bdim.w;
-		this->mul = mul;
-	}
-
-	~ShortcutRT(){
-
-	}
-
-	int getNbOutputs() const override {
-		return 1;
-	}
-
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW{inputs[0].d[0], inputs[0].d[1], inputs[0].d[2]};
-	}
-
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		c = inputDims[0].d[0];
-		h = inputDims[0].d[1];
-		w = inputDims[0].d[2];
-	}
-
-	int initialize() override {
-
-		return 0;
-	}
-
-	virtual void terminate() override {
-	}
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *srcDataBack = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-
-		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
-		shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream);
-
-		return 0;
-	}
+#include <NvInfer.h>
+#include <vector>
+#include <tkdnn.h>
 
 
-	virtual size_t getSerializationSize() override {
-		return 6*sizeof(int) + sizeof(bool);
-	}
+namespace nvinfer1 {
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, bc);
-		tk::dnn::writeBUF(buf, bh);
-		tk::dnn::writeBUF(buf, bw);
-		tk::dnn::writeBUF(buf, mul);
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		assert(buf == a + getSerializationSize());
-		
-	}
+    class ShortcutRT : public IPluginV2Ext {
+
+    public:
+        ShortcutRT(int bc,int bh,int bw,int c,int h,int w ,bool mul);
+
+        ~ShortcutRT();
+
+        ShortcutRT(const void *data, size_t length);
+
+        int getNbOutputs() const NOEXCEPT override;
+
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs,
+                              DataType const *inputTypes, DataType const *outputTypes, bool const *inputIsBroadcast,
+                              bool const *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch (int32_t outputIndex, bool const *inputIsBroadcasted, int32_t nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch (int32_t inputIndex) const NOEXCEPT override;
+
+        void attachToContext (cudnnContext *, cublasContext *, IGpuAllocator *) NOEXCEPT override;
+
+        void detachFromContext () NOEXCEPT override;
+
+        DataType getOutputDataType(int32_t index, nvinfer1::DataType const *inputTypes, int32_t nbInputs) const NOEXCEPT override;
+
+        int initialize() NOEXCEPT override;
+
+        void terminate() NOEXCEPT override;
+
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
+
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                    cudaStream_t stream) NOEXCEPT override;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
+
+
+        size_t getSerializationSize() const NOEXCEPT override;
+
+        void serialize(void *buffer) const NOEXCEPT override;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
+
+        const char *getPluginType() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        void destroy() NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        IPluginV2Ext *clone() const NOEXCEPT override;
+
+        int c, h, w;
+        int bc, bh, bw,bl;
+        bool mul;
+        tk::dnn::dataDim_t bDim;
+    private:
+        std::string mPluginNamespace;
+    };
+
+
+    class ShortcutRTPluginCreator : public IPluginCreator {
+    public:
+        ShortcutRTPluginCreator();
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
+
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
+
+        const char *getPluginName() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override;
+
+    public:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(ShortcutRTPluginCreator);
 
-	int c, h, w;
-	int bc, bh, bw;
-	bool mul;
 };
+
+#endif

include/tkDNN/pluginsRT/UpsampleRT.h

@@ -1,66 +1,103 @@
+#ifndef _UPSAMPLERT_PLUGIN_H
+#define _UPSAMPLERT_PLUGIN_H
+
 #include<cassert>
 #include "../kernels.h"
+#include <NvInfer.h>
+#include <vector>
 
-class UpsampleRT : public IPlugin {
+namespace nvinfer1 {
 
-public:
-	UpsampleRT(int stride) {
-		this->stride = stride;
-	}
+    class UpsampleRT : public IPluginV2Ext {
 
-	~UpsampleRT(){
+    public:
+        UpsampleRT(int stride,int c,int h,int w);
 
-	}
+        UpsampleRT(const void *data, size_t length);
 
-	int getNbOutputs() const override {
-		return 1;
-	}
+        ~UpsampleRT();
 
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return DimsCHW(inputs[0].d[0], inputs[0].d[1]*stride, inputs[0].d[2]*stride);
-	}
+        int getNbOutputs() const NOEXCEPT override;
 
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		c = inputDims[0].d[0];
-		h = inputDims[0].d[1];
-		w = inputDims[0].d[2];
-	}
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
 
-	int initialize() override {
+        int initialize() NOEXCEPT override;
 
-		return 0;
-	}
+        void terminate() NOEXCEPT override;
 
-	virtual void terminate() override {
-	}
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
 
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-	    
-		fill(dstData, batchSize*c*h*w*stride*stride, 0.0, stream);
-    	upsampleForward(srcData, dstData, batchSize, c, h, w, stride, 1, 1, stream);
-		return 0;
-	}
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                            cudaStream_t stream) NOEXCEPT override;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
 
-	virtual size_t getSerializationSize() override {
-		return 4*sizeof(int);
-	}
+        size_t getSerializationSize() const NOEXCEPT override;
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, stride);
-		tk::dnn::writeBUF(buf, c);
-		tk::dnn::writeBUF(buf, h);
-		tk::dnn::writeBUF(buf, w);
-		assert(buf == a + getSerializationSize());
-	}
+        void serialize(void *buffer) const NOEXCEPT override;
 
-	int c, h, w, stride;
-};
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
+
+        const char *getPluginType() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        void destroy() NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        IPluginV2Ext *clone() const NOEXCEPT override ;
+
+        bool isOutputBroadcastAcrossBatch (int32_t outputIndex, bool const *inputIsBroadcasted, int32_t nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch (int32_t inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs,
+                              DataType const *inputTypes, DataType const *outputTypes, bool const *inputIsBroadcast,
+                              bool const *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT override;
+
+        void attachToContext (cudnnContext *, cublasContext *, IGpuAllocator *) NOEXCEPT override;
+
+        void detachFromContext () NOEXCEPT override;
+
+        DataType getOutputDataType (int32_t index, nvinfer1::DataType const *inputTypes, int32_t nbInputs) const NOEXCEPT override;
+
+
+        int c, h, w, stride;
+    private:
+        std::string mPluginNamespace;
+    };
+
+        class UpsampleRTPluginCreator : public IPluginCreator {
+        public:
+            UpsampleRTPluginCreator();
+
+            void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+            const char *getPluginNamespace() const NOEXCEPT override;
+
+            IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
+
+            IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
+
+            const char *getPluginName() const NOEXCEPT override;
+
+            const char *getPluginVersion() const NOEXCEPT override;
+
+            const PluginFieldCollection *getFieldNames() NOEXCEPT override;
+
+        private:
+            static PluginFieldCollection mFC;
+            static std::vector<PluginField> mPluginAttributes;
+            std::string mPluginNamespace;
+        };
+
+        REGISTER_TENSORRT_PLUGIN(UpsampleRTPluginCreator);
+    };
+
+#endif

include/tkDNN/pluginsRT/YoloRT.h

@@ -1,143 +1,125 @@
+#ifndef _YOLORT_PLUGIN_H
+#define _YOLORT_PLUGIN_H
+
 #include<cassert>
+#include <vector>
 #include "../kernels.h"
+#include <NvInfer.h>
 
 #define YOLORT_CLASSNAME_W 256
 
-class YoloRT : public IPlugin {
+namespace nvinfer1 {
+    class YoloRT : public IPluginV2Ext {
+
+    public:
+        YoloRT(int classes, int num,int c,int h,int w, int n_masks = 3, float scale_xy = 1,
+               float nms_thresh = 0.45, int nms_kind = 0, int new_coords = 0);
+
+        YoloRT(const void *data, size_t length);
+
+        ~YoloRT();
 
 
+        int getNbOutputs() const NOEXCEPT override;
 
-public:
-	YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1, float nms_thresh=0.45, int nms_kind=0, int new_coords=0) {
+        Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT override;
 
-		this->classes = classes;
-		this->num = num;
-		this->n_masks = n_masks;
-		this->scaleXY = scale_xy;
-		this->nms_thresh = nms_thresh;
-		this->nms_kind = nms_kind;
-		this->new_coords = new_coords;
+        int initialize() NOEXCEPT override;
 
-        mask = new dnnType[n_masks];
-        bias = new dnnType[num*n_masks*2];
-        if(yolo != nullptr) {
-            memcpy(mask, yolo->mask_h, sizeof(dnnType)*n_masks);
-            memcpy(bias, yolo->bias_h, sizeof(dnnType)*num*n_masks*2);
-			classesNames = yolo->classesNames;
-        }
-	}
+        void terminate() NOEXCEPT override;
 
-	~YoloRT(){
-
-	}
-
-	int getNbOutputs() const override {
-		return 1;
-	}
-
-	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
-		return inputs[0];
-	}
-
-	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
-		c = inputDims[0].d[0];
-		h = inputDims[0].d[1];
-		w = inputDims[0].d[2];
-	}
-
-	int initialize() override {
-
-		return 0;
-	}
-
-	virtual void terminate() override {
-	}
-
-	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
-		return 0;
-	}
-
-	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
-
-		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
-		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
-
-		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+        size_t getWorkspaceSize(int maxBatchSize) const NOEXCEPT override;
 
 
-        for (int b = 0; b < batchSize; ++b){
-            for(int n = 0; n < n_masks; ++n){
-                int index = entry_index(b, n*w*h, 0);
-                if (new_coords == 1){
-                    if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
-                }
-                else{
-                    activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); //x,y
+#if NV_TENSORRT_MAJOR > 7
+        int enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                            cudaStream_t stream) NOEXCEPT override;
+#elif NV_TENSORRT_MAJOR == 7
+        int32_t enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) override;
+#endif
 
-                    if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
 
-                    index = entry_index(b, n*w*h, 4);
-                    activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream);
-                }
-            }
+        size_t getSerializationSize() const NOEXCEPT override;
+
+        bool supportsFormat(DataType type, PluginFormat format) const NOEXCEPT override;
+
+        void serialize(void *buffer) const NOEXCEPT override;
+
+        const char *getPluginType() const NOEXCEPT override;
+
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        void destroy() NOEXCEPT override;
+
+        const char *getPluginNamespace() const NOEXCEPT override;
+
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
+
+        IPluginV2Ext *clone() const NOEXCEPT override;
+
+        DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT override;
+
+        void attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT override;
+
+        bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT override;
+
+        bool canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT override;
+
+        void configurePlugin (Dims const *inputDims, int32_t nbInputs, Dims const *outputDims,
+                              int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes,
+                              bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat,
+                              int32_t maxBatchSize) NOEXCEPT override;
+
+        void detachFromContext() NOEXCEPT override;
+
+
+        int c, h, w;
+        int classes, num, n_masks;
+        float scaleXY;
+        float nms_thresh;
+        int nms_kind;
+        int new_coords;
+
+        std::vector<std::string> classesNames;
+        std::vector<dnnType> mask;
+        std::vector<dnnType> bias;
+
+
+        int entry_index(int batch, int location, int entry) {
+            int n = location / (w * h);
+            int loc = location % (w * h);
+            return batch * c * h * w + n * w * h * (4 + classes + 1) + entry * w * h + loc;
         }
 
-		//std::cout<<"YOLO END\n";
-		return 0;
-	}
+    private:
+        std::string mPluginNamespace;
 
+    };
 
-	virtual size_t getSerializationSize() override {
-		return 8*sizeof(int) + 2*sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
-	}
+    class YoloRTPluginCreator : public IPluginCreator {
+    public:
+        YoloRTPluginCreator();
 
-	virtual void serialize(void* buffer) override {
-		char *buf = reinterpret_cast<char*>(buffer),*a=buf;
-		tk::dnn::writeBUF(buf, classes); 	//std::cout << "Classes :" << classes << std::endl;
-		tk::dnn::writeBUF(buf, num); 		//std::cout << "Num : " << num << std::endl;
-		tk::dnn::writeBUF(buf, n_masks); 	//std::cout << "N_Masks" << n_masks << std::endl;
-		tk::dnn::writeBUF(buf, scaleXY); 	//std::cout << "ScaleXY :" << scaleXY << std::endl;
-		tk::dnn::writeBUF(buf, nms_thresh); //std::cout << "nms_thresh :" << nms_thresh << std::endl;
-		tk::dnn::writeBUF(buf, nms_kind); 	//std::cout << "nms_kind : " << nms_kind << std::endl;
-		tk::dnn::writeBUF(buf, new_coords); //std::cout << "new_coords : " << new_coords << std::endl;
-		tk::dnn::writeBUF(buf, c); 			//std::cout << "C : " << c << std::endl;
-		tk::dnn::writeBUF(buf, h); 			//std::cout << "H : " << h << std::endl;
-		tk::dnn::writeBUF(buf, w); 			//std::cout << "C : " << c << std::endl;
-		for (int i = 0; i < n_masks; i++)
-		{
-			tk::dnn::writeBUF(buf, mask[i]); //std::cout << "mask[i] : " << mask[i] << std::endl;
-		}
-		for (int i = 0; i < n_masks * 2 * num; i++)
-		{
-			tk::dnn::writeBUF(buf, bias[i]); //std::cout << "bias[i] : " << bias[i] << std::endl;
-		}
+        void setPluginNamespace(const char *pluginNamespace) NOEXCEPT override;
 
-		// save classes names
-		for(int i=0; i<classes; i++) {
-			char tmp[YOLORT_CLASSNAME_W];
-			strcpy(tmp, classesNames[i].c_str());
-			for(int j=0; j<YOLORT_CLASSNAME_W; j++) {
-				tk::dnn::writeBUF(buf, tmp[j]);
-			}
-		}
-		assert(buf == a + getSerializationSize());
-	}
+        const char *getPluginNamespace() const NOEXCEPT override;
 
-	int c, h, w;
-    int classes, num, n_masks;
-	float scaleXY;
-	float nms_thresh;
-	int nms_kind;
-	int new_coords;
-	std::vector<std::string> classesNames;
+        IPluginV2Ext *deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT override;
 
-    dnnType *mask;
-    dnnType *bias;
+        IPluginV2Ext *createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT override;
 
-	int entry_index(int batch, int location, int entry) {
-		int n =   location / (w*h);
-		int loc = location % (w*h);
-		return batch*c*h*w + n*w*h*(4+classes+1) + entry*w*h + loc;
-	}
+        const char *getPluginName() const NOEXCEPT override;
 
+        const char *getPluginVersion() const NOEXCEPT override;
+
+        const PluginFieldCollection *getFieldNames() NOEXCEPT override;
+
+    private:
+        static PluginFieldCollection mFC;
+        static std::vector<PluginField> mPluginAttributes;
+        std::string mPluginNamespace;
+    };
+
+    REGISTER_TENSORRT_PLUGIN(YoloRTPluginCreator);
 };
+#endif

include/tkDNN/tkdnn.h

@@ -5,4 +5,4 @@
 #include "Layer.h"
 #include "NetworkRT.h"
 
-#define TKDNN_VERSION 500
+#define TKDNN_VERSION 700

include/tkDNN/utils.h

@@ -6,16 +6,18 @@
 #include <fstream>
 #include <iomanip>
 #include <stdlib.h>
+#include <yaml-cpp/yaml.h>
+
 
 #include "cuda.h"
 #include "cuda_runtime_api.h"
 #include <cublas_v2.h>
 #include <cudnn.h>
+#include <NvInferVersion.h>
 
 
 #ifdef __linux__
 #include <unistd.h>
-
 #endif
 
 #include <ios>
@@ -23,8 +25,32 @@
 
 #include <yaml-cpp/yaml.h>
 
+
+
+#ifndef NOEXCEPT
+  #if NV_TENSORRT_MAJOR > 7
+  #define NOEXCEPT noexcept
+  #else
+  #define NOEXCEPT
+  #endif
+#endif
+
+
 #define dnnType float
 
+template<typename T> void writeBUF(char*& buffer, const T& val)
+{
+    *reinterpret_cast<T*>(buffer) = val;
+    buffer += sizeof(T);
+}
+
+template<typename T> T readBUF(const char*& buffer)
+{
+    T val = *reinterpret_cast<const T*>(buffer);
+    buffer += sizeof(T);
+    return val;
+}
+
 
 // Colored output
 #define COL_END "\033[0m"

scripts/install_OpenCV4.sh

@@ -27,17 +27,21 @@ sudo apt-get install -y build-essential \
     libgstreamer1.0-dev \
     libgstreamer-plugins-base1.0-dev \
     libdc1394-22-dev \
-    libavresample-dev
+    libavresample-dev \
+    libtbb-dev \
 
 git clone https://github.com/opencv/opencv.git
+cd opencv && git checkout 4.5.4 && cd ..
 git clone https://github.com/opencv/opencv_contrib.git
+cd opencv_contrib && git checkout 4.5.4 && cd ..
+
 
 python3 -m venv opencv4
 source opencv4/bin/activate
 pip install wheel
 pip install numpy
 
-cd opencv && mkdir build && cd build
+cd opencv &&  mkdir build && cd build
 
 cmake -D CMAKE_BUILD_TYPE=RELEASE \
     -D CMAKE_INSTALL_PREFIX=/usr/local \
@@ -56,6 +60,8 @@ cmake -D CMAKE_BUILD_TYPE=RELEASE \
     -D WITH_GSTREAMER=ON \
     -D WITH_GSTREAMER_0_10=OFF \
     -D WITH_TBB=ON \
+    -D WITH_OPENGL=ON \
+    -D WITH_VULKAN=ON \
     ../
 
 make -j4

src/Activation.cpp

@@ -56,6 +56,9 @@ dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
     else if(act_mode == ACTIVATION_LOGISTIC) {
         activationLOGISTICForward(srcData, dstData, dim.tot());
 
+    } else if(act_mode == ACTIVATION_ELU) {
+        activationELUForward(srcData, dstData, dim.tot());
+
     } else {
         dnnType alpha = dnnType(1);
         dnnType beta  = dnnType(0);

src/CenterTrack.cpp

@@ -17,7 +17,6 @@ bool CenterTrack::init(const std::string& tensor_path, const int n_classes, cons
     init_pre_inf();
     init_postprocessing();
     init_visualization(n_classes);
-
     return true;
 }
 
@@ -61,7 +60,6 @@ bool CenterTrack::init_preprocessing(){
     checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
     checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot()));
     checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) );
-
     return true;
 }
 
@@ -206,7 +204,6 @@ bool CenterTrack::init_postprocessing(){
     trRes.resize(nBatches);
     countTr.resize(nBatches, 0);
     trackId.resize(nBatches, 0);
-
     return true;
 }
 
@@ -316,7 +313,7 @@ void CenterTrack::preprocess(cv::Mat &frame, const int bi){
         }
         
         float c[] = {new_width / 2.0f, new_height /2.0f};
-        float s[] = {float(dim.w), float(dim.h)};
+        float s[] = {static_cast<float>(dim.w), static_cast<float>(dim.h)};
         // float s = new_width >= new_height ? new_width : new_height;
         // ----------- get_affine_transform
         // rot_rad = pi * 0 / 100 --> 0
@@ -421,9 +418,9 @@ cv::Mat CenterTrack::transform_preds_with_trans(float x1, float x2){
 }
 
 void CenterTrack::tracking(const int bi) {
-    float item_size[countDet];
-    int item_cl[countDet];
-    float dets[2*countDet];
+    std::vector<float> item_size(countDet);
+    std::vector<int> item_cl(countDet);
+    std::vector<float> dets(2*countDet);
     for(int i=0; i<countDet; i++){
         item_size[i] = (detRes[i].bb1.at<float>(0,0) - detRes[i].bb0.at<float>(0,0)) * 
                        (detRes[i].bb1.at<float>(0,1) - detRes[i].bb0.at<float>(0,1));
@@ -432,9 +429,9 @@ void CenterTrack::tracking(const int bi) {
         dets[i*2+1]  =  detRes[i].ct.at<float>(0,1);
     }
     
-    float track_size[countTr[bi]];
-    int track_cl[countTr[bi]];
-    float tracks[2*countTr[bi]];
+    std::vector<float> track_size(countTr[bi]);
+    std::vector<int> track_cl(countTr[bi]);
+    std::vector<float> tracks(2*countTr[bi]);
     for(int i=0; i<countTr[bi]; i++){
         track_size[i] = (trRes[bi][i].det_res.bb1.at<float>(0,0) - trRes[bi][i].det_res.bb0.at<float>(0,0)) * 
                         (trRes[bi][i].det_res.bb1.at<float>(0,1) - trRes[bi][i].det_res.bb0.at<float>(0,1));
@@ -442,7 +439,7 @@ void CenterTrack::tracking(const int bi) {
         tracks[i*2]   =  trRes[bi][i].det_res.ct.at<float>(0,0);
         tracks[i*2+1] =  trRes[bi][i].det_res.ct.at<float>(0,1);
     } 
-    float dist[countTr[bi]*countDet];
+    std::vector<float> dist(countTr[bi]*countDet);
     bool invalid;
     for(int i=0; i<countTr[bi]; i++){
         for(int j=0; j<countDet; j++){
@@ -454,7 +451,7 @@ void CenterTrack::tracking(const int bi) {
             dist[j*countTr[bi]+i] = dist[j*countTr[bi]+i] + invalid * (1 << 18);
         }
     }
-    int matched_indices[2*countTr[bi]];
+    std::vector<int> matched_indices(2*countTr[bi]);
     float min_tr;
     int min_idtr = -1;
     for(int i=0; i<countTr[bi]; i++) {
@@ -477,10 +474,10 @@ void CenterTrack::tracking(const int bi) {
         }
     }
     
-    bool unmatched_dets[countDet];
+    std::vector<bool> unmatched_dets(countDet);
     for(int i=0; i<countDet; i++)
         unmatched_dets[i] = false;
-    bool unmatched_tracks[countTr[bi]];
+    std::vector<bool> unmatched_tracks(countTr[bi]);
     for(int i=0; i<countTr[bi]; i++) 
         unmatched_tracks[i] = false;
     for(int i=0; i<countTr[bi]; i++) {

src/CenternetDetection.cpp

@@ -118,6 +118,7 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe
     
     dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
     dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
+    return true;
     
     return true;
 }
@@ -348,21 +349,21 @@ void CenternetDetection::postprocess(const int bi, const bool mAP){
         new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx0[i] +
                                 static_cast<float>(trans2.at<double>(0,1))*bby0[i] +
                                 static_cast<float>(trans2.at<double>(0,2))*1.0;
-        new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx0[i] +
+        new_pt1.at<float>(1,0)=static_cast<float>(trans2.at<double>(1,0))*bbx0[i] +
                                 static_cast<float>(trans2.at<double>(1,1))*bby0[i] +
                                 static_cast<float>(trans2.at<double>(1,2))*1.0;
 
         new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx1[i] +
                                 static_cast<float>(trans2.at<double>(0,1))*bby1[i] +
                                 static_cast<float>(trans2.at<double>(0,2))*1.0;
-        new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx1[i] +
+        new_pt2.at<float>(1,0)=static_cast<float>(trans2.at<double>(1,0))*bbx1[i] +
                                 static_cast<float>(trans2.at<double>(1,1))*bby1[i] +
                                 static_cast<float>(trans2.at<double>(1,2))*1.0;
 
         target_coords[i*4] = new_pt1.at<float>(0,0);
-        target_coords[i*4+1] = new_pt1.at<float>(0,1);
+        target_coords[i*4+1] = new_pt1.at<float>(1,0);
         target_coords[i*4+2] = new_pt2.at<float>(0,0);
-        target_coords[i*4+3] = new_pt2.at<float>(0,1);
+        target_coords[i*4+3] = new_pt2.at<float>(1,0);
     }
        
     detected.clear();

src/CenternetDetection3D.cpp

@@ -167,7 +167,6 @@ bool CenternetDetection3D::init(const std::string& tensor_path, const int n_clas
     faceId.push_back({2,3,7,6});
     faceId.push_back({3,0,4,7});
     // ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
-
     return true;
 }
 

src/DarknetParser.cpp

@@ -17,8 +17,8 @@ namespace tk { namespace dnn {
         if(sep == std::string::npos)
             return false;
         
-        name   = line.substr(0, sep);   
-        value  = line.substr(sep+1, line.size() - (sep+1));   
+        name   = line.substr(0, sep);
+        value  = line.substr(sep+1, line.size() - (sep+1));
         return true;
     }
 
@@ -32,6 +32,16 @@ namespace tk { namespace dnn {
         return values;
     }
 
+    std::vector<float> fromStringToFloatVec(const std::string& line, const char delimiter){
+        std::stringstream linestream(line);
+        std::string value;
+        std::vector<float> values;
+
+        while(getline(linestream,value,delimiter))
+            values.push_back(std::stof(value));
+        return values;
+    }
+
     bool darknetParseFields(const std::string& line, darknetFields_t& fields){
 
         std::string name,value;
@@ -268,7 +278,133 @@ namespace tk { namespace dnn {
         }
         return net;
     }
-        
-    
+    std::vector<int> noYolosLine(const std::string &cfg_file){
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+        std::string line;
+        std::vector<int> lineNo;
+        int count = 0;
+        while(std::getline(if_cfg,line)){
+            std::size_t found = line.find("#");
+            if ( found != std::string::npos ) {
+                line = line.substr(0, found);
+            }
+            // skip empty lines
+            if(line.empty())
+                continue;
+            if(line == "[yolo]"){
+                lineNo.push_back(count);
+
+
+            }
+            count++;
+        }
+        return lineNo;
+    }
+    void loadYoloInfo(const std::string &cfg_file,int lineNo,std::vector<float> &mask,std::vector<float> &anchors,int &num,int &classes,float &nms_thresh,int &nms_kind,int &coords){
+        std::vector<float> maskTemp,anchorsTemp;
+        int classesTemp,numTemp,nmsKindTemp;
+        int new_coordsTemp=0;
+        float nmsThreshTemp=0.45;
+
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+        std::string line;
+        int count = 0;
+        while(std::getline(if_cfg,line)){
+            std::string name,value;
+            std::size_t found = line.find("#");
+            if ( found != std::string::npos ) {
+                line = line.substr(0, found);
+            }
+            // skip empty lines
+            if(line.empty())
+                continue;
+            if(count > lineNo && count <=lineNo+30){
+                    divideNameAndValue(line,name,value);
+                    if(name == "mask "){
+                        maskTemp = fromStringToFloatVec(value,',');
+                    }
+                    if(name == "anchors "){
+                        anchorsTemp = fromStringToFloatVec(value,',');
+                    }
+                    if(name == "classes"){
+                        classesTemp = std::stoi(value);
+                    }
+                    if(name == "num"){
+                        numTemp = std::stoi(value);
+                    }
+                    if(name == "nms_kind"){
+                        if(value == "greedynms"){
+                            nmsKindTemp = 0;
+                        }else if(value == "diounms"){
+                            nmsKindTemp=1;
+                        }
+                        else{
+                            std::cout<<"NMS NOT SUPPORTED DEFAULTING TO GREEDYNMS"<<std::endl;
+                            nmsKindTemp=0;
+                        }
+                    }
+                    if(name == "new_coords"){
+                        new_coordsTemp = std::stoi(value);
+                    }
+                    if(name == "beta_nms"){
+                        nmsThreshTemp = std::stof(value);
+                    }
+            }
+            count++;
+        }
+        mask = maskTemp;
+        anchors = anchorsTemp;
+        num = numTemp;
+        nms_kind = nmsKindTemp;
+        nms_thresh = nmsThreshTemp;
+        coords = new_coordsTemp;
+        classes = classesTemp;
+
+    }
+    void loadYoloInitInfo(int &channels,int &width,int &height,const std::string &cfg_file){
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+        std::string line;
+        int count = 0;
+
+        while(std::getline(if_cfg,line)){
+            if(count == 7){
+                std::string name,value;
+                divideNameAndValue(line,name,value);
+                if(name == "width"){
+                    width = std::stoi(value);
+                }
+            }
+
+            if(count == 8){
+                std::string name,value;
+                divideNameAndValue(line,name,value);
+                if(name == "height"){
+                    height = std::stoi(value);
+                }
+            }
+
+            if(count == 9){
+                std::string name,value;
+                divideNameAndValue(line,name,value);
+                if(name == "channels"){
+                    channels = std::stoi(value);
+                    break;
+                }
+                else{
+                    std::cerr<<"EXITING PROGRAM DUE TO INSUFFICENT DATA FROM CFG"<<std::endl;
+                    break;
+                }
+            }
+
+            count++;
+        }
+
+    }
 
 }}

src/Flatten.cpp

@@ -15,6 +15,11 @@ Flatten::Flatten(Network *net) : Layer(net) {
     output_dim.w = 1;
     output_dim.l = 1;
 
+    this->h = 1;
+    this->w = 1;
+    this->rows = input_dim.c;
+    this->cols = input_dim.h * input_dim.w;
+    this->c = input_dim.w * input_dim.h * input_dim.c;
 }
 
 Flatten::~Flatten() {

src/Int8BatchStream.cpp

@@ -8,14 +8,14 @@
 BatchStream::BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches, const std::string& fileimglist, const std::string& filelabellist) {
     mBatchSize = batchSize;
     mMaxBatches = maxBatches;
-    mDims = nvinfer1::DimsNCHW{ dim.n, dim.c, dim.h, dim.w };
+    mDims = nvinfer1::Dims4{ dim.n, dim.c, dim.h, dim.w };
     mHeight = dim.h;
     mWidth = dim.w;
-    mImageSize = mDims.c()*mDims.h()*mDims.w();
+    mImageSize = mDims.d[1]*mDims.d[2]*mDims.d[3];
     mBatch.resize(mBatchSize*mImageSize, 0);
     mLabels.resize(mBatchSize, 0);
-    mFileBatch.resize(mDims.n()*mImageSize, 0);
-    mFileLabels.resize(mDims.n(), 0);
+    mFileBatch.resize(mDims.d[0]*mImageSize, 0);
+    mFileLabels.resize(mDims.d[0], 0);
     mFileImgList = fileimglist;
     readInListFile(fileimglist, mListImg);
     mFileLabelList = filelabellist;
@@ -27,7 +27,7 @@ BatchStream::BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches,
 void BatchStream::reset(int firstBatch) {
     mBatchCount = 0;
     mFileCount = 0;
-    mFileBatchPos = mDims.n();
+    mFileBatchPos = mDims.d[0];
     skip(firstBatch);
 }
 
@@ -37,11 +37,11 @@ bool BatchStream::next() {
         return false;
 
     for (int csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize) {
-        assert(mFileBatchPos > 0 && mFileBatchPos <= mDims.n());
-        if (mFileBatchPos == mDims.n() && !update())
+        assert(mFileBatchPos > 0 && mFileBatchPos <= mDims.d[0]);
+        if (mFileBatchPos == mDims.d[0] && !update())
             return false;
 
-        csize = std::min(mBatchSize - batchPos, mDims.n() - mFileBatchPos);
+        csize = std::min(mBatchSize - batchPos, mDims.d[0] - mFileBatchPos);
         std::copy_n(getFileBatch() + mFileBatchPos * mImageSize, csize * mImageSize, getBatch() + batchPos * mImageSize);
         std::copy_n(getFileLabels() + mFileBatchPos, csize, getLabels() + batchPos);
     }
@@ -50,8 +50,8 @@ bool BatchStream::next() {
 }
 
 void BatchStream::skip(int skipCount) {
-    if (mBatchSize >= mDims.n() && mBatchSize%mDims.n() == 0 && mFileBatchPos == mDims.n()) {
-        mFileCount += skipCount * mBatchSize / mDims.n();
+    if (mBatchSize >= mDims.d[0] && mBatchSize%mDims.d[0] == 0 && mFileBatchPos == mDims.d[0]) {
+        mFileCount += skipCount * mBatchSize / mDims.d[0];
         return;
     }
 

src/Int8Calibrator.cpp

@@ -8,13 +8,13 @@ Int8EntropyCalibrator::Int8EntropyCalibrator(BatchStream& stream, int firstBatch
     mCalibTableFilePath(calibTableFilePath),
     mInputBlobName(inputBlobName.c_str()),
     mReadCache(readCache) {
-    nvinfer1::DimsNCHW dims = mStream.getDims();
-    mInputCount = mStream.getBatchSize() * dims.c() * dims.h() * dims.w();
+    nvinfer1::Dims4 dims = mStream.getDims();
+    mInputCount = mStream.getBatchSize() + dims.d[1]*dims.d[2]*dims.d[3];
     checkCuda(cudaMalloc(&mDeviceInput, mInputCount * sizeof(float)));
     mStream.reset(firstBatch);
 }
 
-bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int nbBindings) {
+bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int nbBindings) NOEXCEPT {
     if (!mStream.next())
         return false;
 
@@ -24,7 +24,7 @@ bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int
     return true;
 }
 
-const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) {
+const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) NOEXCEPT {
     mCalibrationCache.clear();
     assert(!mCalibTableFilePath.empty());
     std::ifstream input(mCalibTableFilePath, std::ios::binary);
@@ -38,7 +38,7 @@ const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) {
     return length ? &mCalibrationCache[0] : nullptr;
 }
 
-void Int8EntropyCalibrator::writeCalibrationCache(const void* cache, size_t length) {
+void Int8EntropyCalibrator::writeCalibrationCache(const void* cache, size_t length) NOEXCEPT {
     assert(!mCalibTableFilePath.empty());
     std::ofstream output(mCalibTableFilePath, std::ios::binary);
     output.write(reinterpret_cast<const char*>(cache), length);

src/NetworkViz.cpp

@@ -383,7 +383,7 @@ cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) {
     default:
         // expand your range to 0..255. Similar to histEq();
         map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min); 
-        applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET);
+        applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_PARULA);
     }
     return falseColorsMap;
 }

src/Padding.cpp

@@ -0,0 +1,45 @@
+//
+// Created by perseusdg on 03/01/22.
+//
+
+#include <iostream>
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk{ namespace dnn {
+    Padding::Padding(Network *net, int32_t pad_h, int32_t pad_w, tkdnnPaddingMode_t padding_mode,float constant) : Layer(net) {
+        this->paddingH = pad_h;
+        this->paddingW = pad_w;
+        this->padding_mode = padding_mode;
+        output_dim.c = input_dim.c;
+        output_dim.n = input_dim.n;
+        output_dim.h = input_dim.h + 2 * (this->paddingH);
+        output_dim.w = input_dim.w + 2 * (this->paddingW);
+        if(padding_mode == tkdnnPaddingMode_t::PADDING_MODE_CONSTANT){
+            this->constant = constant;
+        }else{
+            this->constant = 0;
+        }
+        checkCuda(cudaMalloc(&dstData,output_dim.tot()*sizeof(dnnType)));
+    }
+
+    Padding::~Padding() {
+        checkCuda(cudaFree(dstData));
+    }
+    dnnType* Padding::infer(dataDim_t &dim, float *srcData) {
+        fill(dstData,output_dim.tot(),0.0);
+        if(padding_mode == tkdnnPaddingMode_t::PADDING_MODE_REFLECTION)
+        {
+            reflection_pad2d_out_forward(paddingH, paddingW, srcData, dstData, input_dim.h, input_dim.w, input_dim.c,
+                                         input_dim.n);
+        }
+        else if(padding_mode == tkdnnPaddingMode_t::PADDING_MODE_CONSTANT){
+            constant_pad2d_forward(srcData,dstData,input_dim.h,input_dim.w,output_dim.h,output_dim.w,input_dim.c,
+                                   input_dim.n,paddingH,paddingW,constant);
+        }
+
+        dim = output_dim;
+        return dstData;
+    }
+
+}}

src/Pooling.cpp

@@ -17,6 +17,7 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
     this->pool_mode = pool_mode;
     this->paddingH = paddingH;
     this->paddingW = paddingW;
+    this->padding = winH -1;
 
     checkCUDNN( cudnnCreatePoolingDescriptor(&poolingDesc) );
 

src/Region.cpp

@@ -16,7 +16,6 @@ Region::Region(Network *net, int classes, int coords, int num) :
     this->classes = classes;
     this->coords = coords;
     this->num = num;
-
     // same
     output_dim.n = input_dim.n;
     output_dim.c = input_dim.c;

src/Reshape.cpp

@@ -8,7 +8,10 @@ namespace tk { namespace dnn {
 Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) {
 
     checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
-
+    this->n = new_dim.n;
+    this->c = new_dim.c;
+    this->h = new_dim.h;
+    this->w = new_dim.w;
     output_dim.n = new_dim.n;
     output_dim.c = new_dim.c;
     output_dim.h = new_dim.h;

src/Shortcut.cpp

@@ -9,6 +9,9 @@ Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) {
 
     this->backLayer = backLayer;
     this->mul = mul;
+    this->c = input_dim.c;
+    this->h = input_dim.h;
+    this->w = input_dim.w;
     checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 
     if( ( backLayer->output_dim.c != input_dim.c && mul ) ||

src/Upsample.cpp

@@ -14,6 +14,9 @@ Upsample::Upsample(Network *net, int stride) : Layer(net) {
     output_dim.h = input_dim.h*stride;
     output_dim.w = input_dim.w*stride;
     output_dim.l = input_dim.l;
+    this->c = input_dim.c;
+    this->h = input_dim.h;
+    this->w = input_dim.w;
     
     checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 }

src/Yolo.cpp

@@ -133,7 +133,7 @@ void correct_yolo_boxes(Yolo::detection *dets, int n, int w, int h, int netw, in
     }
 }
 
-int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int new_coords) {
+int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int newCoords) {
 
     if(predictions == nullptr)
         predictions = new dnnType[output_dim.tot()];
@@ -157,7 +157,7 @@ int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int net
             if(objectness <= thresh) continue;
             int box_index  = entry_index(0, n*lw*lh + i, 0, classes, input_dim, output_dim);
             
-            dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh, new_coords);
+            dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh, newCoords);
             dets[count].objectness = objectness;
             dets[count].classes = classes;
             for(j = 0; j < classes; ++j){

src/Yolo3Detection.cpp

@@ -3,23 +3,23 @@
 
 namespace tk { namespace dnn {
 
-bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
+    bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
 
     //convert network to tensorRT
     std::cout<<(tensor_path).c_str()<<"\n";
-    netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
+    netRT = new tk::dnn::NetworkRT(nullptr, (tensor_path).c_str() );
 
     nBatches = n_batches;
     confThreshold = conf_thresh;
     tk::dnn::dataDim_t idim = netRT->input_dim;    
     idim.n = nBatches;
 
-    if(netRT->pluginFactory->n_yolos < 2 ) {
+    if(netRT->yolo_plugins.size() < 2 ) {
         FatalError("this is not yolo3");
     }
 
-    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
-        YoloRT *yRT = netRT->pluginFactory->yolos[i];
+    for(int i=0; i<netRT->yolo_plugins.size(); i++) {
+        nvinfer1::YoloRT *yRT = netRT->yolo_plugins[i];
         classes = yRT->classes;
         num = yRT->num;
         nMasks = yRT->n_masks;
@@ -28,8 +28,8 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
         yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias
         yolo[i]->mask_h = new dnnType[nMasks];
         yolo[i]->bias_h = new dnnType[num*nMasks*2];
-        memcpy(yolo[i]->mask_h, yRT->mask, sizeof(dnnType)*nMasks);
-        memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*nMasks*2);
+        memcpy(yolo[i]->mask_h, yRT->mask.data(), sizeof(dnnType)*nMasks);
+        memcpy(yolo[i]->bias_h, yRT->bias.data(), sizeof(dnnType)*num*nMasks*2);
         yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w);
         yolo[i]->classesNames = yRT->classesNames;
         yolo[i]->nms_thresh = yRT->nms_thresh;
@@ -93,10 +93,15 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
 
 void Yolo3Detection::postprocess(const int bi, const bool mAP){
 
+
+
     //get yolo outputs
+    if(netRT->yolo_plugins.size() < 2){
+        FatalError("YOLOS WRONG!!");
+    }
     std::vector<float *> rt_out;
     //dnnType *rt_out[netRT->pluginFactory->n_yolos];
-    for(int i=0; i<netRT->pluginFactory->n_yolos; i++)
+    for(int i=0; i<netRT->yolo_plugins.size(); i++)
         rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi);
 
     float x_ratio =  float(originalSize[bi].width) / float(netRT->input_dim.w);
@@ -104,7 +109,7 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
 
     // compute dets
     nDets = 0;
-    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
+    for(int i=0; i<netRT->yolo_plugins.size(); i++) {
         yolo[i]->dstData = rt_out[i];
         yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords);
     }

src/kernels/padding.cu

@@ -0,0 +1,110 @@
+#include "kernels.h"
+#include <thrust/pair.h>
+#include <stdio.h>
+
+/*
+ * Reflection padding is from https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/native/cuda/ReflectionPad.cu
+ */
+__device__
+inline thrust::pair<int32_t,int32_t> get_index_mapping2d(
+        int32_t input_dim_x,int32_t input_dim_y,int32_t output_dim_x,
+        int32_t output_dim_y,int32_t pad_l,int32_t pad_t,int32_t output_xy,
+        int32_t y_shift,int32_t z_shift,int32_t n_plane){
+    auto input_offset = ((blockIdx.y + y_shift) + (blockIdx.z + z_shift)*n_plane)*input_dim_x*input_dim_y;
+    auto output_offset = ((blockIdx.y + y_shift) + (blockIdx.z + z_shift)*n_plane)*output_dim_x*output_dim_y;
+    auto output_x = output_xy % output_dim_x;
+    auto output_y = output_xy/output_dim_x;
+
+    auto i_start_x = ::max(int32_t(0),-pad_l);
+    auto i_start_y = ::max(int32_t(0),-pad_t);
+    auto o_start_x = ::max(int32_t(0),pad_l);
+    auto o_start_y = ::max(int32_t(0),pad_t);
+
+    auto input_x = ::abs(output_x - pad_l) - ::abs(output_x - (input_dim_x + pad_l -1)) -output_x + 2*pad_l + input_dim_x -1 -o_start_x + i_start_x;
+    auto input_y = ::abs(output_y - pad_t) - ::abs(output_y - (input_dim_y + pad_t -1)) -output_y + 2*pad_t + input_dim_y -1 -o_start_y + i_start_y;
+
+    return thrust::make_pair<int32_t,int32_t>(input_offset + input_y*input_dim_x + input_x,output_offset + output_y*output_dim_x+output_x);
+}
+
+__global__
+void reflection_pad2d_out_kernel(
+        float* input,float* output,int32_t input_dim_x,
+        int32_t input_dim_y,int32_t pad_t,int32_t pad_b,int32_t pad_l,
+        int32_t pad_r,int32_t y_shift,int32_t z_shift,int32_t n_plane){
+    auto output_xy = threadIdx.x + blockIdx.x * blockDim.x;
+    auto output_dim_x = input_dim_x + pad_l + pad_r;
+    auto output_dim_y = input_dim_y + pad_t + pad_b;
+
+    if(output_xy < output_dim_x*output_dim_y){
+        auto index_pair = get_index_mapping2d(input_dim_x,input_dim_y,output_dim_x,output_dim_y,pad_l,pad_t,output_xy,y_shift,z_shift,n_plane);
+        output[index_pair.second] = input[index_pair.first];
+    }
+}
+
+int32_t ceilDiv(int32_t a,int32_t b){
+    return (a+b-1)/b;
+}
+
+
+void reflection_pad2d_out_forward(int32_t pad_h,int32_t pad_w,float *srcData,float *dstData,int32_t input_h,int32_t input_w,int32_t plane_dim,int32_t n_batch,cudaStream_t cudaStream){
+    int32_t pad_l = pad_w;
+    int32_t pad_r = pad_w;
+    int32_t pad_t = pad_h;
+    int32_t pad_b = pad_w;
+    int32_t output_h = input_h + pad_t + pad_b;
+    int32_t output_w = input_w + pad_l + pad_r;
+    int32_t size_y = plane_dim;
+    int32_t size_z = n_batch;
+    int32_t output_plane_size = output_h*output_w;
+    dim3 block_size(output_plane_size>256 ?256:output_plane_size);
+    for(int32_t block_y=0;block_y<size_y;block_y += 65535){
+        int32_t block_y_size = std::min(size_y - block_y,static_cast<int32_t>(65535));
+        for(int32_t block_z=0;block_z<size_z;block_z += 65535){
+            int32_t block_z_size = std::min(size_z -block_z,static_cast<int32_t>(65535));
+
+            dim3 grid_size(ceilDiv(output_plane_size,static_cast<int32_t>(256)),block_y_size,block_z_size);
+            reflection_pad2d_out_kernel<<<grid_size,block_size,0,cudaStream>>>(srcData,dstData,input_w,input_h,pad_t,pad_b,pad_l,pad_r,block_y,block_z,plane_dim);
+        }
+    }
+
+}
+
+/*
+ * constant padding is inspired from https://github.com/apache/incubator-mxnet/blob/master/src/operator/pad.cu
+ */
+
+__global__
+void constant_pad2d_kernel(dnnType *srcData,dnnType *dstData,const int32_t padT,const int32_t padL,float constant,int32_t n,int32_t c,int32_t i_h,int32_t i_w,int32_t o_h,int32_t o_w){
+    int outputPointId = threadIdx.x + blockIdx.x * blockDim.x;
+    if(outputPointId >= o_h*o_w){
+        return ;
+    }
+
+    int Ny = i_h;
+    int Nx = i_w;
+
+    int plane =   blockIdx.y;
+    int batch = blockIdx.z;
+    int outputPointX = outputPointId % o_w;
+    int outputPointY = outputPointId / o_w;
+    int checkT       = max(0, outputPointY - padT + 1);
+    int checkB       = max(0, padT + Ny - outputPointY);
+    int checkL       = max(0, outputPointX - padL + 1);
+    int checkR       = max(0, padL + Nx - outputPointX);
+    int inputPointX  = min(max(outputPointX - padL, 0), Nx - 1);
+    int inputPointY  = min(max(outputPointY - padT, 0), Ny - 1);
+    int need_pad     = !(checkT * checkB * checkL * checkR);
+    float value_to_copy = srcData[batch*c*i_h*i_w + plane*i_h*i_w + inputPointY*i_w + inputPointX];
+    dstData[batch*c*o_w*o_h + plane*o_h*o_w + outputPointY*o_w + outputPointX] = value_to_copy * (!need_pad) + need_pad*constant;
+
+}
+
+void constant_pad2d_forward(dnnType *srcData,dnnType *dstData,int32_t input_h,int32_t input_w,int32_t output_h,
+                            int32_t output_w,int32_t c,int32_t n,int32_t padT,int32_t padL,dnnType constant,cudaStream_t cudaStream){
+    int32_t output_plane_size = output_h*output_w;
+    dim3 block_size(output_plane_size>256 ?256:output_plane_size);
+    dim3 grid_size(ceilDiv(output_plane_size,static_cast<int32_t>(256)),c,n);
+    constant_pad2d_kernel<<<grid_size,block_size,0,cudaStream>>>(srcData,dstData,padT,padL,constant,n,c,input_h,input_w,output_h,output_w);
+
+}
+

src/pluginsRT/ActivationLeakyRT.cpp

@@ -0,0 +1,149 @@
+#include <tkDNN/pluginsRT/ActivationLeakyRT.h>
+using namespace nvinfer1;
+
+
+std::vector<PluginField> ActivationLeakyRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ActivationLeakyRTPluginCreator::mFC{};
+
+ActivationLeakyRT::ActivationLeakyRT(float s) {
+    slope = s;
+}
+
+ActivationLeakyRT::ActivationLeakyRT(const void *data, size_t length) {
+    std::cout << "DESERIALIZE LEAKYRT" << std::endl;
+    const char *buf = reinterpret_cast<const char *>(data), *bufCheck = buf;
+    slope = readBUF<float>(buf);
+    size = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+ActivationLeakyRT::~ActivationLeakyRT() {}
+
+int ActivationLeakyRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ActivationLeakyRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return inputs[0];
+}
+
+void ActivationLeakyRT::configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs,
+                                            DataType type, PluginFormat format, int maxBatchSize) NOEXCEPT {
+    assert(type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+    size = 1;
+    for (int i = 0; i < outputDims[0].nbDims; i++)
+        size *= outputDims[0].d[i];
+}
+int ActivationLeakyRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+size_t ActivationLeakyRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int ActivationLeakyRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                               cudaStream_t stream) NOEXCEPT {
+    activationLEAKYForward(
+            (dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+            reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, slope,
+            stream);
+    return 0;
+
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t ActivationLeakyRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                   cudaStream_t stream) {
+    activationLEAKYForward(
+            (dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+            reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, slope,
+            stream);
+    return 0;
+}
+#endif
+
+
+size_t ActivationLeakyRT::getSerializationSize() const NOEXCEPT {
+    return 1 * sizeof(int) + 1 * sizeof(float);
+}
+
+void ActivationLeakyRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char *>(buffer), *a = buf;
+    writeBUF(buf, size);
+    assert(buf == a + getSerializationSize());
+}
+
+bool ActivationLeakyRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *ActivationLeakyRT::getPluginType() const NOEXCEPT {
+    return "ActivationLeakyRT_tkDNN";
+}
+
+const char *ActivationLeakyRT::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+void ActivationLeakyRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *ActivationLeakyRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+
+}
+
+void ActivationLeakyRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2* ActivationLeakyRT::clone() const NOEXCEPT {
+    auto *p = new ActivationLeakyRT(slope);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+ActivationLeakyRTPluginCreator::ActivationLeakyRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ActivationLeakyRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2* ActivationLeakyRTPluginCreator::deserializePlugin(const char *name, const void *serialData,size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ActivationLeakyRT(serialData, serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char* ActivationLeakyRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2* ActivationLeakyRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 1);
+    assert(fields[0].type == PluginFieldType::kFLOAT32);
+    float slope = *(static_cast<const float *>(fields[0].data));
+    auto *pluginObj = new ActivationLeakyRT(slope);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char* ActivationLeakyRTPluginCreator::getPluginName() const NOEXCEPT {
+    return "ActivationLeakyRT_tkDNN";
+}
+
+const char* ActivationLeakyRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+const PluginFieldCollection* ActivationLeakyRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+

src/pluginsRT/ActivationLogisticRT.cpp

@@ -0,0 +1,134 @@
+#include <tkDNN/pluginsRT/ActivationLogisticRT.h>
+using namespace nvinfer1;
+std::vector<PluginField> ActivationLogisticRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ActivationLogisticRTPluginCreator::mFC{};
+
+ActivationLogisticRT::ActivationLogisticRT() {}
+
+ActivationLogisticRT::ActivationLogisticRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char *>(data), *bufCheck = buf;
+    size = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+ActivationLogisticRT::~ActivationLogisticRT() {}
+
+int ActivationLogisticRT::getNbOutputs() const NOEXCEPT {
+return 1;
+}
+
+Dims ActivationLogisticRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return inputs[0];
+}
+
+void ActivationLogisticRT::configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims,
+                                               int nbOutputs, DataType type, PluginFormat format,
+                                               int maxBatchSize) NOEXCEPT {
+    size = 1;
+    for (int i = 0; i < outputDims[0].nbDims; i++)
+        size *= outputDims[0].d[i];
+}
+
+int ActivationLogisticRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ActivationLogisticRT::terminate() NOEXCEPT {}
+
+size_t ActivationLogisticRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+#if NV_TENSORRT_MAJOR > 7
+int ActivationLogisticRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                                  cudaStream_t stream) NOEXCEPT {
+    activationLOGISTICForward((dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+                              reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t ActivationLogisticRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                      cudaStream_t stream) {
+    activationLOGISTICForward((dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+                              reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, stream);
+    return 0;
+}
+#endif
+
+size_t ActivationLogisticRT::getSerializationSize() const NOEXCEPT {
+    return 1 * sizeof(int);
+}
+
+void ActivationLogisticRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char *>(buffer);
+    writeBUF(buf, size);
+}
+
+const char* ActivationLogisticRT::getPluginType() const NOEXCEPT {
+    return "ActivationLogisticRT_tkDNN";
+}
+
+const char* ActivationLogisticRT::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+void ActivationLogisticRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char* ActivationLogisticRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ActivationLogisticRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+bool ActivationLogisticRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return true;
+    //todo assert
+}
+
+IPluginV2* ActivationLogisticRT::clone() const NOEXCEPT {
+    auto *p = new ActivationLogisticRT();
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+ActivationLogisticRTPluginCreator::ActivationLogisticRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ActivationLogisticRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2* ActivationLogisticRTPluginCreator::deserializePlugin(const char *name, const void *serialData,
+                                                                size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ActivationLogisticRT(serialData, serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char* ActivationLogisticRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2* ActivationLogisticRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    auto *pluginObj = new ActivationLogisticRT();
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char* ActivationLogisticRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+const PluginFieldCollection* ActivationLogisticRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+const char *ActivationLogisticRTPluginCreator::getPluginName() const NOEXCEPT  {
+    return "ActivationLogisticRT_tkDNN";
+}

src/pluginsRT/ActivationMishRT.cpp

@@ -0,0 +1,133 @@
+//
+// Created by perseusdg on 9/4/21.
+//
+#include <tkDNN/pluginsRT/ActivationMishRT.h>
+using namespace  nvinfer1;
+std::vector<PluginField> ActivationMishRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ActivationMishRTPluginCreator::mFC{};
+
+ActivationMishRT::ActivationMishRT() {
+
+}
+
+ActivationMishRT::~ActivationMishRT() {
+
+}
+
+ActivationMishRT::ActivationMishRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char *>(data), *bufCheck = buf;
+    size = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+int ActivationMishRT::getNbOutputs() const NOEXCEPT  { return 1; }
+
+Dims ActivationMishRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT  { return inputs[0]; }
+
+void ActivationMishRT::configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
+                         PluginFormat format, int maxBatchSize) NOEXCEPT {
+assert(format == PluginFormat::kLINEAR);
+size = 1;
+for (int i = 0; i < outputDims[0].nbDims; i++)
+size *= outputDims[0].d[i];
+}
+
+int ActivationMishRT::initialize() NOEXCEPT  { return 0; }
+
+void ActivationMishRT::terminate() NOEXCEPT {}
+
+size_t ActivationMishRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT  { return 0; }
+
+#if NV_TENSORRT_MAJOR > 7
+int ActivationMishRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+            cudaStream_t stream) NOEXCEPT {
+    activationMishForward((dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+    reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t ActivationMishRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                  cudaStream_t stream) {
+    activationMishForward((dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+                          reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, stream);
+    return 0;
+}
+#endif
+
+size_t ActivationMishRT::getSerializationSize() const NOEXCEPT  {
+    return 1 * sizeof(int);
+}
+
+void ActivationMishRT::serialize(void *buffer) const NOEXCEPT  {
+    char *buf = reinterpret_cast<char *>(buffer), *a = buf;
+    writeBUF(buf, size);
+    assert(buf == a + getSerializationSize());
+}
+
+const char* ActivationMishRT::getPluginType() const NOEXCEPT  {
+    return "ActivationMishRT_tkDNN";
+}
+
+const char *ActivationMishRT::getPluginVersion() const NOEXCEPT  {
+    return "1";
+}
+
+bool ActivationMishRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+        return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *ActivationMishRT::getPluginNamespace() const NOEXCEPT  {
+    return mPluginNamespace.c_str();
+}
+
+void ActivationMishRT::setPluginNamespace(const char *plguinNamespace) NOEXCEPT  {
+    mPluginNamespace = plguinNamespace;
+}
+
+IPluginV2 *ActivationMishRT::clone() const NOEXCEPT {
+    auto *p = new ActivationMishRT();
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+
+
+ActivationMishRTPluginCreator::ActivationMishRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ActivationMishRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ActivationMishRTPluginCreator::getPluginNamespace() const NOEXCEPT  {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2 *ActivationMishRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT  {
+    auto *pluginObj = new ActivationMishRT(serialData, serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2 *ActivationMishRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT  {
+    const PluginField *fields = fc->fields;
+    auto *pluginObj = new ActivationMishRT();
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ActivationMishRTPluginCreator::getPluginName() const NOEXCEPT  {
+    return "ActivationMishRT_tkDNN";
+}
+
+const char *ActivationMishRTPluginCreator::getPluginVersion() const NOEXCEPT{
+    return "1";
+}
+
+const PluginFieldCollection *ActivationMishRTPluginCreator::getFieldNames() NOEXCEPT  {
+    return &mFC;
+}
+

src/pluginsRT/ActivationReLUCeilingRT.cpp

@@ -0,0 +1,138 @@
+#include <tkDNN/pluginsRT/ActivationReLUCeilingRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> ActivationReLUCeilingPluginCreator::mPluginAttributes;
+PluginFieldCollection ActivationReLUCeilingPluginCreator::mFC{};
+
+ActivationReLUCeiling::ActivationReLUCeiling(const float ceiling) {
+    this->ceiling = ceiling;
+}
+
+ActivationReLUCeiling::~ActivationReLUCeiling() {
+
+}
+
+ActivationReLUCeiling::ActivationReLUCeiling(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char *>(data), *bufCheck = buf;
+    ceiling = readBUF<float>(buf);
+    size = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+int ActivationReLUCeiling::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ActivationReLUCeiling::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT { return inputs[0]; }
+
+void ActivationReLUCeiling::configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs,DataType type, PluginFormat format, int maxBatchSize) NOEXCEPT {
+    assert(type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+    size = 1;
+    for (int i = 0; i < outputDims[0].nbDims; i++)
+        size *= outputDims[0].d[i];
+}
+
+int ActivationReLUCeiling::initialize() NOEXCEPT  { return 0; }
+
+void ActivationReLUCeiling::terminate() NOEXCEPT {}
+
+size_t ActivationReLUCeiling::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int ActivationReLUCeiling::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT  {
+    activationReLUCeilingForward((dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+    reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, ceiling, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t ActivationReLUCeiling::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                       cudaStream_t stream) {
+    activationReLUCeilingForward((dnnType *) reinterpret_cast<const dnnType *>(inputs[0]),
+                                 reinterpret_cast<dnnType *>(outputs[0]), batchSize * size, ceiling, stream);
+    return 0;
+}
+#endif
+
+size_t ActivationReLUCeiling::getSerializationSize() const NOEXCEPT {
+    return 1 * sizeof(int) + 1 * sizeof(float);
+}
+
+void ActivationReLUCeiling::serialize(void *buffer) const NOEXCEPT  {
+    char *buf = reinterpret_cast<char *>(buffer), *a = buf;
+    writeBUF(buf, ceiling);
+    writeBUF(buf, size);
+    assert(buf = a + getSerializationSize());
+}
+
+IPluginV2 *ActivationReLUCeiling::clone() const NOEXCEPT {
+    auto *p = new ActivationReLUCeiling(ceiling);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+bool ActivationReLUCeiling::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT  {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+void ActivationReLUCeiling::destroy() NOEXCEPT { delete this; }
+
+const char *ActivationReLUCeiling::getPluginType() const NOEXCEPT {
+    return "ActivationReLUCeilingRT_tkDNN";
+}
+
+const char *ActivationReLUCeiling::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+const char *ActivationReLUCeiling::getPluginNamespace() const NOEXCEPT  {
+    return mPluginNamespace.c_str();
+}
+
+void ActivationReLUCeiling::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+ActivationReLUCeilingPluginCreator::ActivationReLUCeilingPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ActivationReLUCeilingPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT  {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ActivationReLUCeilingPluginCreator::getPluginNamespace() const NOEXCEPT  {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2 *ActivationReLUCeilingPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ActivationReLUCeiling(serialData, serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2 *ActivationReLUCeilingPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT  {
+    const PluginField *fields = fc->fields;
+    float ceiling = *(static_cast<const float *>(fields[0].data));
+    auto *pluginObj = new ActivationReLUCeiling(ceiling);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ActivationReLUCeilingPluginCreator::getPluginName() const NOEXCEPT  {
+    return "ActivationReLUCeilingRT_tkDNN";
+}
+
+const char *ActivationReLUCeilingPluginCreator::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+const PluginFieldCollection *ActivationReLUCeilingPluginCreator::getFieldNames() NOEXCEPT  {
+    return &mFC;
+}
+
+
+

src/pluginsRT/ConstantPaddingRT.cpp

@@ -0,0 +1,204 @@
+#include <tkDNN/pluginsRT/ConstantPaddingRT.h>
+
+using namespace nvinfer1;
+
+std::vector<PluginField> ConstantPaddingRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ConstantPaddingRTPluginCreator::mFC{};
+
+static const char* CONSTANTPADDINGRT_PLUGIN_VERSION{"1"};
+static const char* CONSTANTPADDINGRT_PLUGIN_NAME{"ConstantPaddingRT_tkDNN"};
+
+ConstantPaddingRT::ConstantPaddingRT(int32_t padH, int32_t padW, int32_t n, int32_t c, int32_t i_h, int32_t i_w,
+                                     int32_t o_h, int32_t o_w, float constant) {
+    this->padH = padH;
+    this->padW = padW;
+    this->n = n;
+    this->c = c;
+    this->i_h = i_h;
+    this->i_w = i_w;
+    this->o_h = o_h;
+    this->o_w = o_w;
+    this->constant = constant;
+
+}
+
+ConstantPaddingRT::ConstantPaddingRT(const void *data, size_t length) {
+    const char* buf = reinterpret_cast<const char*>(data),*bufcheck=buf;
+    padH = readBUF<int32_t>(buf);
+    padW = readBUF<int32_t>(buf);
+    i_h = readBUF<int32_t>(buf);
+    i_w = readBUF<int32_t>(buf);
+    o_h = readBUF<int32_t>(buf);
+    o_w = readBUF<int32_t>(buf);
+    n = readBUF<int32_t>(buf);
+    c = readBUF<int32_t>(buf);
+    constant = readBUF<float>(buf);
+    assert(buf = bufcheck + length);
+}
+
+ConstantPaddingRT::~ConstantPaddingRT() {}
+
+int ConstantPaddingRT::getNbOutputs() const NOEXCEPT{
+    return 1;
+}
+
+Dims ConstantPaddingRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{c,o_h,o_w};
+}
+
+int ConstantPaddingRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ConstantPaddingRT::terminate() NOEXCEPT {
+
+}
+
+size_t ConstantPaddingRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int ConstantPaddingRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) NOEXCEPT {
+     dnnType* srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+     dnnType* dstData = reinterpret_cast<dnnType*>(outputs[0]);
+     constant_pad2d_forward(srcData,dstData,i_h,i_w,o_h,o_w,c,n,padH,padW,constant,stream);
+     return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t ConstantPaddingRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                   cudaStream_t stream) {
+    dnnType* srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType* dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    constant_pad2d_forward(srcData,dstData,i_h,i_w,o_h,o_w,c,n,padH,padW,constant,stream);
+    return 0;
+}
+#endif
+
+size_t ConstantPaddingRT::getSerializationSize() const NOEXCEPT {
+    return (8*sizeof(int32_t) + 1*sizeof(float));
+}
+
+void ConstantPaddingRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf,padH);
+    writeBUF(buf,padW);
+    writeBUF(buf,i_h);
+    writeBUF(buf,i_w);
+    writeBUF(buf,o_h);
+    writeBUF(buf,o_w);
+    writeBUF(buf,n);
+    writeBUF(buf,c);
+    writeBUF(buf,constant);
+}
+
+void ConstantPaddingRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char* ConstantPaddingRT::getPluginType() const NOEXCEPT {
+    return CONSTANTPADDINGRT_PLUGIN_NAME;
+}
+
+const char* ConstantPaddingRT::getPluginVersion() const NOEXCEPT {
+    return CONSTANTPADDINGRT_PLUGIN_VERSION;
+}
+
+const char* ConstantPaddingRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ConstantPaddingRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *ConstantPaddingRT::clone() const NOEXCEPT {
+    auto *p = new ConstantPaddingRT(padH,padW,n,c,i_h,i_w,o_h,o_w,constant);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType ConstantPaddingRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes,
+                                              int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void ConstantPaddingRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                                        IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+bool ConstantPaddingRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted,
+                                                     int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool ConstantPaddingRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void ConstantPaddingRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims,
+                                        int32_t nbOutputs, const DataType *inputTypes, const DataType *outputTypes,
+                                        const bool *inputIsBroadcast, const bool *outputIsBroadcast,
+                                        PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void ConstantPaddingRT::detachFromContext() NOEXCEPT {
+
+}
+
+bool ConstantPaddingRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+
+
+ConstantPaddingRTPluginCreator::ConstantPaddingRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ConstantPaddingRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ConstantPaddingRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *ConstantPaddingRTPluginCreator::deserializePlugin(const char *name, const void *serialData,
+                                                                size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ConstantPaddingRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *ConstantPaddingRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    int padH = *(static_cast<const int32_t*>(fields[0].data));
+    int padW = *(static_cast<const int32_t*>(fields[1].data));
+    int inputH = *(static_cast<const int32_t*>(fields[2].data));
+    int inputW = *(static_cast<const int32_t*>(fields[3].data));
+    int outputH = *(static_cast<const int32_t*>(fields[4].data));
+    int outputW = *(static_cast<const int32_t*>(fields[5].data));
+    int n = *(static_cast<const int32_t*>(fields[6].data));
+    int c = *(static_cast<const int32_t*>(fields[7].data));
+    float constant = *(static_cast<const float*>(fields[8].data));
+    auto *pluginObj = new ConstantPaddingRT(padH,padW,n,c,inputH,inputW,outputH,outputW,constant);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ConstantPaddingRTPluginCreator::getPluginName() const NOEXCEPT {
+    return CONSTANTPADDINGRT_PLUGIN_NAME;
+}
+
+const char *ConstantPaddingRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return CONSTANTPADDINGRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *ConstantPaddingRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}

src/pluginsRT/DeformableConvRT.cpp

@@ -0,0 +1,361 @@
+#include <tkDNN/pluginsRT/DeformableConvRT.h>
+
+#include <utility>
+using namespace nvinfer1;
+using namespace tk::dnn;
+
+std::vector<PluginField> DeformableConvRTPluginCreator::mPluginAttributes;
+PluginFieldCollection DeformableConvRTPluginCreator::mFC{};
+
+static const char* DEFORMABLECONVRT_PLUGIN_VERSION{"1"};
+static const char* DEFORMABLECONVRT_PLUGIN_NAME{"DeformableConvRT_tkDNN"};
+
+
+DeformableConvRT::DeformableConvRT(int chunk_dim, int kh, int kw, int sh, int sw, int ph, int pw, int deformableGroup,
+                                   int i_n, int i_c, int i_h, int i_w, int o_n, int o_c, int o_h, int o_w,std::vector<dnnType> data_H,std::vector<dnnType> bias2_H,
+                                   std::vector<dnnType> ones_d1_h,std::vector<dnnType> ones_d2_h,std::vector<dnnType> offsetH,std::vector<dnnType> maskH,int height_ones,int width_ones,int dim_ones) {
+    this->chunk_dim = chunk_dim;
+    this->kh = kh;
+    this->kw = kw;
+    this->sh = sh;
+    this->sw = sw;
+    this->ph = ph;
+    this->pw = pw;
+    this->deformableGroup = deformableGroup;
+    this->i_n = i_n;
+    this->i_c = i_c;
+    this->i_h = i_h;
+    this->i_w = i_w;
+    this->o_n = o_n;
+    this->o_c = o_c;
+    this->o_h = o_h;
+    this->o_w = o_w;
+    this->mask_v = std::move(maskH);
+    this->offset_v = std::move(offsetH);
+    this->ones_d2_v = std::move(ones_d2_h);
+    this->ones_d1_v = std::move(ones_d1_h);
+    this->data_d_v = std::move(data_H);
+    this->bias2_d_v = std::move(bias2_H);
+    this->height_ones = height_ones;
+    this->width_ones = width_ones;
+    this->dim_ones = dim_ones;
+
+    checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
+    checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
+    checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));
+    checkCuda( cudaMalloc(&offset, 2*chunk_dim*sizeof(dnnType)));
+    checkCuda( cudaMalloc(&mask, chunk_dim*sizeof(dnnType)));
+    checkCuda( cudaMalloc(&ones_d2, dim_ones*sizeof(dnnType)));
+    if(!data_d_v.empty() && !bias2_d_v.empty() && !ones_d1_v.empty() && !ones_d2_v.empty() && !mask_v.empty() && !offset_v.empty()) {
+        checkCuda(cudaMemcpy(data_d, data_d_v.data(), sizeof(dnnType) * data_d_v.size(), cudaMemcpyHostToDevice));
+        checkCuda(cudaMemcpy(bias2_d, bias2_d_v.data(), sizeof(dnnType) * bias2_d_v.size(), cudaMemcpyHostToDevice));
+        checkCuda(cudaMemcpy(ones_d1, ones_d1_v.data(), sizeof(dnnType) * ones_d1_v.size(), cudaMemcpyHostToDevice));
+        checkCuda(cudaMemcpy(offset, offset_v.data(), sizeof(dnnType) * offset_v.size(), cudaMemcpyHostToDevice));
+        checkCuda(cudaMemcpy(mask, mask_v.data(), sizeof(dnnType) * mask_v.size(), cudaMemcpyHostToDevice));
+        checkCuda(cudaMemcpy(ones_d2, ones_d2_v.data(), sizeof(dnnType) * ones_d2_v.size(), cudaMemcpyHostToDevice));
+    }
+
+
+}
+
+DeformableConvRT::~DeformableConvRT() {
+    checkCuda( cudaFree(data_d) );
+    checkCuda( cudaFree(bias2_d) );
+    checkCuda( cudaFree(ones_d1) );
+    checkCuda( cudaFree(offset) );
+    checkCuda( cudaFree(mask) );
+    checkCuda( cudaFree(ones_d2) );
+}
+
+DeformableConvRT::DeformableConvRT(const void *data, size_t length) {
+    const char* buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    chunk_dim = readBUF<int>(buf);
+    kh = readBUF<int>(buf);
+    kw = readBUF<int>(buf);
+    sh = readBUF<int>(buf);
+    sw = readBUF<int>(buf);
+    ph = readBUF<int>(buf);
+    pw = readBUF<int>(buf);
+    deformableGroup = readBUF<int>(buf);
+    i_n = readBUF<int>(buf);
+    i_c = readBUF<int>(buf);
+    i_h = readBUF<int>(buf);
+    i_w = readBUF<int>(buf);
+    o_n = readBUF<int>(buf);
+    o_c = readBUF<int>(buf);
+    o_h = readBUF<int>(buf);
+    o_w = readBUF<int>(buf);
+    height_ones = readBUF<int>(buf);
+    width_ones = readBUF<int>(buf);
+    dim_ones = readBUF<int>(buf);
+    offset_v.resize(chunk_dim*2);
+    for(int i=0;i<chunk_dim*2;i++)
+        offset_v[i] = readBUF<dnnType>(buf);
+    mask_v.resize(chunk_dim);
+    for(int i=0;i<chunk_dim;i++)
+        mask_v[i] = readBUF<dnnType>(buf);
+    data_d_v.resize(i_c*o_c*kh*kw*1);
+    for(int i=0;i<(i_c*o_c*kh*kw*1);i++)
+        data_d_v[i] = readBUF<dnnType>(buf);
+    bias2_d_v.resize(o_c);
+    for(int i=0; i < o_c; i++)
+        bias2_d_v[i] = readBUF<dnnType>(buf);
+    ones_d1_v.resize(height_ones*width_ones);
+    for(int i=0; i<height_ones * width_ones; i++)
+        ones_d1_v[i] = readBUF<dnnType>(buf);
+    ones_d2_v.resize(dim_ones);
+    for(int i=0; i<dim_ones; i++)
+        ones_d2_v[i] = readBUF<dnnType>(buf);
+    assert(buf == bufCheck + length);
+
+}
+
+int DeformableConvRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims DeformableConvRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{o_c, o_h, o_w};
+}
+
+int DeformableConvRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void DeformableConvRT::terminate() NOEXCEPT {}
+
+size_t DeformableConvRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {return 0;}
+
+#if NV_TENSORRT_MAJOR > 7
+int DeformableConvRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                              cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *output_conv = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
+
+    // split conv2d outputs into offset to mask
+    for(int b=0; b<batchSize; b++) {
+        checkCuda(cudaMemcpy(offset, output_conv + b * 3 * chunk_dim, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+        checkCuda(cudaMemcpy(mask, output_conv + b * 3 * chunk_dim + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+        // kernel sigmoid
+        activationSIGMOIDForward(mask, mask, chunk_dim);
+        // deformable convolution
+        dcnV2CudaForward(stat, handle,
+                         srcData, data_d,
+                         bias2_d, ones_d1,
+                         offset, mask,
+                         reinterpret_cast<dnnType*>(outputs[0]), ones_d2,
+                         kh, kw,
+                         sh, sw,
+                         ph, pw,
+                         1, 1,
+                         deformableGroup, b,
+                         i_n, i_c, i_h, i_w,
+                         o_n, o_c, o_h, o_w,
+                         chunk_dim);
+    }
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t DeformableConvRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                  cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *output_conv = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
+
+    // split conv2d outputs into offset to mask
+    for(int b=0; b<batchSize; b++) {
+        checkCuda(cudaMemcpy(offset, output_conv + b * 3 * chunk_dim, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+        checkCuda(cudaMemcpy(mask, output_conv + b * 3 * chunk_dim + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+        // kernel sigmoid
+        activationSIGMOIDForward(mask, mask, chunk_dim);
+        // deformable convolution
+        dcnV2CudaForward(stat, handle,
+                         srcData, data_d,
+                         bias2_d, ones_d1,
+                         offset, mask,
+                         reinterpret_cast<dnnType*>(outputs[0]), ones_d2,
+                         kh, kw,
+                         sh, sw,
+                         ph, pw,
+                         1, 1,
+                         deformableGroup, b,
+                         i_n, i_c, i_h, i_w,
+                         o_n, o_c, o_h, o_w,
+                         chunk_dim);
+    }
+    return 0;
+}
+#endif
+
+size_t DeformableConvRT::getSerializationSize() const NOEXCEPT {
+    return 19 * sizeof(int) + chunk_dim * 3 * sizeof(dnnType) + (i_c * o_c * kh * kw * 1 ) * sizeof(dnnType) +
+           o_c * sizeof(dnnType) + height_ones * width_ones * sizeof(dnnType) + dim_ones * sizeof(dnnType);
+}
+
+void DeformableConvRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, chunk_dim);
+    writeBUF(buf, kh);
+    writeBUF(buf, kw);
+    writeBUF(buf, sh);
+    writeBUF(buf, sw);
+    writeBUF(buf, ph);
+    writeBUF(buf, pw);
+    writeBUF(buf, deformableGroup);
+    writeBUF(buf, i_n);
+    writeBUF(buf, i_c);
+    writeBUF(buf, i_h);
+    writeBUF(buf, i_w);
+    writeBUF(buf, o_n);
+    writeBUF(buf, o_c);
+    writeBUF(buf, o_h);
+    writeBUF(buf, o_w);
+    writeBUF(buf,height_ones);
+    writeBUF(buf,width_ones);
+    writeBUF(buf,dim_ones);
+    for(int i=0; i<offset_v.size(); i++)
+        writeBUF(buf, offset_v[i]);
+    for(int i=0; i<mask_v.size(); i++)
+        writeBUF(buf, mask_v[i]);
+    for(int i=0; i<data_d_v.size(); i++)
+        writeBUF(buf, data_d_v[i]);
+    for(int i=0; i < bias2_d_v.size(); i++)
+        writeBUF(buf, bias2_d_v[i]);
+    for(int i=0; i<ones_d1_v.size(); i++)
+        writeBUF(buf, ones_d1_v[i]);
+    for(int i=0; i<ones_d2_v.size(); i++)
+        writeBUF(buf, ones_d2_v[i]);
+
+    assert(buf == a + getSerializationSize());
+}
+
+void DeformableConvRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+
+const char *DeformableConvRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void DeformableConvRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *DeformableConvRT::getPluginType() const NOEXCEPT {
+    return DEFORMABLECONVRT_PLUGIN_NAME;
+}
+
+const char *DeformableConvRT::getPluginVersion() const NOEXCEPT {
+    return DEFORMABLECONVRT_PLUGIN_VERSION;
+}
+
+IPluginV2Ext *DeformableConvRT::clone() const NOEXCEPT {
+    auto *p = new DeformableConvRT(chunk_dim,kh,kw,sh,sw,ph,pw,deformableGroup,i_n,i_c,i_h,i_w,o_n,o_c,o_h,o_w,data_d_v,bias2_d_v,ones_d1_v,ones_d2_v,offset_v,mask_v,height_ones,width_ones,dim_ones);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType
+DeformableConvRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void DeformableConvRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                                       IGpuAllocator *gpuAllocator) NOEXCEPT {
+    handle = cublasContext;
+
+}
+
+bool DeformableConvRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted,
+                                                    int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool DeformableConvRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void DeformableConvRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                  const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                                  const bool *outputIsBroadcast, PluginFormat floatFormat,
+                                  int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void DeformableConvRT::detachFromContext() NOEXCEPT {
+
+}
+
+bool DeformableConvRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+        return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+DeformableConvRTPluginCreator::DeformableConvRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void DeformableConvRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *DeformableConvRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *DeformableConvRTPluginCreator::deserializePlugin(const char *name, const void *serialData,
+                                                            size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new DeformableConvRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *DeformableConvRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    int chunk_dim = *(static_cast<const int *>(fields[0].data));
+    int kh = *(static_cast<const int *>(fields[1].data));
+    int kw = *(static_cast<const int *>(fields[2].data));
+    int sh = *(static_cast<const int *>(fields[3].data));
+    int sw = *(static_cast<const int *>(fields[4].data));
+    int ph = *(static_cast<const int *>(fields[5].data));
+    int pw = *(static_cast<const int *>(fields[6].data));
+    int deformableGroup = *(static_cast<const int *>(fields[7].data));
+    int i_n = *(static_cast<const int *>(fields[8].data));
+    int i_c = *(static_cast<const int *>(fields[9].data));
+    int i_h = *(static_cast<const int *>(fields[10].data));
+    int i_w = *(static_cast<const int *>(fields[11].data));
+    int o_n = *(static_cast<const int *>(fields[12].data));
+    int o_c = *(static_cast<const int *>(fields[13].data));
+    int o_h = *(static_cast<const int *>(fields[14].data));
+    int o_w = *(static_cast<const int *>(fields[15].data));
+    std::vector<dnnType> mask_v(static_cast<const dnnType*>(fields[16].data),static_cast<const dnnType*>(fields[16].data)+fields[16].length);
+    std::vector<dnnType> offset_v(static_cast<const dnnType*>(fields[17].data),static_cast<const dnnType*>(fields[17].data)+fields[17].length);
+    std::vector<dnnType> ones_d2_v(static_cast<const dnnType*>(fields[18].data),static_cast<const dnnType*>(fields[18].data)+fields[18].length);
+    std::vector<dnnType> ones_d1_v(static_cast<const dnnType*>(fields[19].data),static_cast<const dnnType*>(fields[19].data)+fields[19].length);
+    std::vector<dnnType> data_d_v(static_cast<const dnnType*>(fields[20].data),static_cast<const dnnType*>(fields[20].data)+fields[20].length);
+    std::vector<dnnType> bias2_d_v(static_cast<const dnnType*>(fields[21].data),static_cast<const dnnType*>(fields[21].data)+fields[21].length);
+    int height_ones = *(static_cast<const int *>(fields[22].data));
+    int width_ones = *(static_cast<const int *>(fields[23].data));
+    int dim_ones = *(static_cast<const int *>(fields[24].data));
+    auto *pluginObj = new DeformableConvRT(chunk_dim,kh,kw,sh,sw,ph,pw,deformableGroup,i_n,i_c,i_h,i_w,o_n,o_c,o_h,o_w,data_d_v,bias2_d_v,ones_d1_v,ones_d2_v,offset_v,mask_v,height_ones,width_ones,dim_ones);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *DeformableConvRTPluginCreator::getPluginName() const NOEXCEPT {
+    return DEFORMABLECONVRT_PLUGIN_NAME;
+}
+
+const char *DeformableConvRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return DEFORMABLECONVRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *DeformableConvRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+

src/pluginsRT/FlattenConcatRT.cpp

@@ -0,0 +1,215 @@
+#include <tkDNN/pluginsRT/FlattenConcatRT.h>
+using namespace  nvinfer1;
+
+std::vector<PluginField> FlattenConcatRTPluginCreator::mPluginAttributes;
+PluginFieldCollection FlattenConcatRTPluginCreator::mFC{};
+
+static const char* FLATTENCONCATRT_PLUGIN_VERSION{"1"};
+static const char* FLATTENCONCATRT_PLUGIN_NAME{"FlattenConcatRT_tkDNN"};
+
+FlattenConcatRT::FlattenConcatRT(int c, int h, int w, int rows, int cols) {
+    this->c = c;
+    this->h = h;
+    this->w = w;
+    this->rows = rows;
+    this->cols = cols;
+}
+
+FlattenConcatRT::FlattenConcatRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char *>(data),*bufCheck=buf;
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    rows = readBUF<int>(buf);
+    cols = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+FlattenConcatRT::~FlattenConcatRT() {}
+
+int FlattenConcatRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims FlattenConcatRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{ inputs[0].d[0] * inputs[0].d[1] * inputs[0].d[2], 1, 1};
+}
+
+int FlattenConcatRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void FlattenConcatRT::terminate() NOEXCEPT {
+
+}
+
+size_t FlattenConcatRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int FlattenConcatRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                             cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*rows*cols*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+    checkERROR( cublasSetStream(handle, stream) );
+    for(int i=0; i<batchSize; i++) {
+        float const alpha(1.0);
+        float const beta(0.0);
+        int offset = i*rows*cols;
+        checkERROR( cublasSgeam( handle, CUBLAS_OP_T, CUBLAS_OP_N, rows, cols, &alpha, srcData + offset, cols, &beta, srcData + offset, rows, dstData + offset, rows ));
+    }
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t FlattenConcatRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                 cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*rows*cols*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+    checkERROR( cublasSetStream(handle, stream) );
+    for(int i=0; i<batchSize; i++) {
+        float const alpha(1.0);
+        float const beta(0.0);
+        int offset = i*rows*cols;
+        checkERROR( cublasSgeam( handle, CUBLAS_OP_T, CUBLAS_OP_N, rows, cols, &alpha, srcData + offset, cols, &beta, srcData + offset, rows, dstData + offset, rows ));
+    }
+    return 0;
+}
+#endif
+
+
+size_t FlattenConcatRT::getSerializationSize() const NOEXCEPT {
+    return 5*sizeof(int);
+}
+
+void FlattenConcatRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a = buf;
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    writeBUF(buf, rows);
+    writeBUF(buf, cols);
+    assert(buf == a + getSerializationSize());
+}
+
+void FlattenConcatRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+
+
+const char *FlattenConcatRT::getPluginType() const NOEXCEPT {
+    return FLATTENCONCATRT_PLUGIN_NAME;
+}
+
+const char *FlattenConcatRT::getPluginVersion() const NOEXCEPT {
+    return FLATTENCONCATRT_PLUGIN_VERSION;
+}
+
+const char *FlattenConcatRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void FlattenConcatRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *FlattenConcatRT::clone() const NOEXCEPT {
+    auto* p = new FlattenConcatRT(c, h, w, rows, cols);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType FlattenConcatRT::getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const NOEXCEPT
+{
+    return DataType::kFLOAT;
+}
+
+
+void FlattenConcatRT::attachToContext(cudnnContext* cudnnContext, cublasContext* cublasContext, IGpuAllocator* gpuAllocator) NOEXCEPT
+{
+    handle = cublasContext;
+}
+
+bool FlattenConcatRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const NOEXCEPT
+{
+    return false;
+}
+
+bool FlattenConcatRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT
+{
+    return false;
+}
+
+void FlattenConcatRT::detachFromContext() NOEXCEPT
+{
+}
+
+void
+FlattenConcatRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                 const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                                 const bool *outputIsBroadcast, PluginFormat floatFormat,
+                                 int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+bool FlattenConcatRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+FlattenConcatRTPluginCreator::FlattenConcatRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void FlattenConcatRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *FlattenConcatRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *FlattenConcatRTPluginCreator::deserializePlugin(const char *name, const void *serialData,
+                                                           size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new FlattenConcatRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *FlattenConcatRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField* fields = fc->fields;
+    int c = *(static_cast<const int*>(fields[0].data));
+    int h = *(static_cast<const int*>(fields[1].data));
+    int w = *(static_cast<const int*>(fields[2].data));
+    int rows = *(static_cast<const int*>(fields[3].data));
+    int cols = *(static_cast<const int*>(fields[4].data));
+    auto* pluginObj = new FlattenConcatRT(c, h, w, rows, cols);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *FlattenConcatRTPluginCreator::getPluginName() const NOEXCEPT {
+    return FLATTENCONCATRT_PLUGIN_NAME;
+}
+
+const char *FlattenConcatRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return FLATTENCONCATRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *FlattenConcatRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+

src/pluginsRT/MaxPoolingSizeRT.cpp

@@ -0,0 +1,212 @@
+#include <tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> MaxPoolFixedSizeRTPluginCreator::mPluginAttributes;
+PluginFieldCollection MaxPoolFixedSizeRTPluginCreator::mFC{};
+
+static const char* MAXPOOLFIXEDSIZERT_PLUGIN_VERSION{"1"};
+static const char* MAXPOOLFIXEDSIZERT_PLUGIN_NAME{"MaxPoolingFixedSizeRT_tkDNN"};
+
+MaxPoolFixedSizeRT::MaxPoolFixedSizeRT(int c, int h, int w, int n, int strideH, int strideW, int winSize, int padding){
+    this->c = c;
+    this->h = h;
+    this->w = w;
+    this->n = n;
+    this->stride_H = strideH;
+    this->stride_W = strideW;
+    this->winSize = winSize;
+    this->padding = padding;
+}
+
+MaxPoolFixedSizeRT::MaxPoolFixedSizeRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    n = readBUF<int>(buf);
+    stride_H = readBUF<int>(buf);
+    stride_W = readBUF<int>(buf);
+    winSize  = readBUF<int>(buf);
+    padding = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+MaxPoolFixedSizeRT::~MaxPoolFixedSizeRT() {
+
+}
+
+int MaxPoolFixedSizeRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims MaxPoolFixedSizeRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{this->c, this->h, this->w};
+}
+
+int MaxPoolFixedSizeRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void MaxPoolFixedSizeRT::terminate() NOEXCEPT {
+
+}
+
+size_t MaxPoolFixedSizeRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int MaxPoolFixedSizeRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                                cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    MaxPoolingForward(srcData, dstData, batchSize, this->c, this->h, this->w, this->stride_H, this->stride_W, this->winSize, this->padding, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t MaxPoolFixedSizeRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                                    cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    MaxPoolingForward(srcData, dstData, batchSize, this->c, this->h, this->w, this->stride_H, this->stride_W, this->winSize, this->padding, stream);
+    return 0;
+}
+#endif
+
+
+size_t MaxPoolFixedSizeRT::getSerializationSize() const NOEXCEPT {
+    return 8*sizeof(int);
+}
+
+void MaxPoolFixedSizeRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, this->c);
+    writeBUF(buf, this->h);
+    writeBUF(buf, this->w);
+    writeBUF(buf, this->n);
+    writeBUF(buf, this->stride_H);
+    writeBUF(buf, this->stride_W);
+    writeBUF(buf, this->winSize);
+    writeBUF(buf, this->padding);
+    assert(buf == a + getSerializationSize());
+}
+
+void MaxPoolFixedSizeRT::destroy() NOEXCEPT {
+delete this;
+}
+
+bool MaxPoolFixedSizeRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *MaxPoolFixedSizeRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void MaxPoolFixedSizeRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *MaxPoolFixedSizeRT::getPluginType() const NOEXCEPT {
+    return MAXPOOLFIXEDSIZERT_PLUGIN_NAME;
+}
+
+const char *MaxPoolFixedSizeRT::getPluginVersion() const NOEXCEPT {
+    return MAXPOOLFIXEDSIZERT_PLUGIN_VERSION;
+}
+
+IPluginV2Ext *MaxPoolFixedSizeRT::clone() const NOEXCEPT {
+    auto *p = new MaxPoolFixedSizeRT(c,h,w,n,stride_H,stride_W,winSize,padding);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType
+MaxPoolFixedSizeRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void MaxPoolFixedSizeRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                                         IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+bool MaxPoolFixedSizeRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted,
+                                                      int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool MaxPoolFixedSizeRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void
+MaxPoolFixedSizeRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                    const DataType *inputTypes, const DataType *outputTypes,
+                                    const bool *inputIsBroadcast, const bool *outputIsBroadcast,
+                                    PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void MaxPoolFixedSizeRT::detachFromContext() NOEXCEPT {
+    IPluginV2Ext::detachFromContext();
+}
+
+MaxPoolFixedSizeRTPluginCreator::MaxPoolFixedSizeRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void MaxPoolFixedSizeRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *MaxPoolFixedSizeRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *MaxPoolFixedSizeRTPluginCreator::deserializePlugin(const char *name, const void *serialData,size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new MaxPoolFixedSizeRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *MaxPoolFixedSizeRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    int c = *(static_cast<const int *>(fields[0].data));
+    int h = *(static_cast<const int *>(fields[1].data));
+    int w = *(static_cast<const int *>(fields[2].data));
+    int n = *(static_cast<const int *>(fields[3].data));
+    int stride_H = *(static_cast<const int *>(fields[4].data));
+    int stride_W = *(static_cast<const int *>(fields[5].data));
+    int winSize = *(static_cast<const int *>(fields[6].data));
+    int padding = *(static_cast<const int *>(fields[7].data));
+    auto *pluginObj = new MaxPoolFixedSizeRT(c,h,w,n,stride_H,stride_W,winSize,padding);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *MaxPoolFixedSizeRTPluginCreator::getPluginName() const NOEXCEPT {
+    return MAXPOOLFIXEDSIZERT_PLUGIN_NAME;
+}
+
+const char *MaxPoolFixedSizeRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return MAXPOOLFIXEDSIZERT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *MaxPoolFixedSizeRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/ReflectionPadding.cpp

@@ -0,0 +1,202 @@
+#include <tkDNN/pluginsRT/ReflectionPadding.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> ReflectionPaddingRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ReflectionPaddingRTPluginCreator::mFC{};
+
+static const char* REFLECTIONPADDINGRT_PLUGIN_VERSION{"1"};
+static const char* REFLECTIONPADDINGRT_PLUGIN_NAME{"ReflectionPaddingRT_tkDNN"};
+
+ReflectionPaddingRT::ReflectionPaddingRT(int32_t padH, int32_t padW, int32_t input_h, int32_t input_w, int32_t output_h,
+                                         int32_t output_w, int32_t c, int32_t n) {
+    this->padH = padH;
+    this->padW = padW;
+    this->input_h  = input_h;
+    this->input_w = input_w;
+    this->output_h = output_h;
+    this->output_w = output_w;
+    this->n = n;
+    this->c = c;
+}
+
+ReflectionPaddingRT::ReflectionPaddingRT(const void *data, size_t length) {
+    const char* buf = reinterpret_cast<const char*>(data),*bufcheck=buf;
+    padH = readBUF<int32_t>(buf);
+    padW = readBUF<int32_t>(buf);
+    input_h = readBUF<int32_t>(buf);
+    input_w = readBUF<int32_t>(buf);
+    output_h = readBUF<int32_t>(buf);
+    output_w = readBUF<int32_t>(buf);
+    n = readBUF<int32_t>(buf);
+    c = readBUF<int32_t>(buf);
+    assert(buf = bufcheck + length);
+}
+
+ReflectionPaddingRT::~ReflectionPaddingRT() {}
+
+int ReflectionPaddingRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ReflectionPaddingRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{c,output_h,output_w};
+}
+
+int ReflectionPaddingRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ReflectionPaddingRT::terminate() NOEXCEPT {
+
+}
+
+size_t ReflectionPaddingRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int ReflectionPaddingRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                                 cudaStream_t stream) NOEXCEPT {
+    dnnType* srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType* dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    reflection_pad2d_out_forward(padH,padW,srcData,dstData,input_h,input_w,c,n,stream);
+    return 0;
+}
+
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t ReflectionPaddingRT::enqueue (int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream){
+    dnnType* srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType* dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    reflection_pad2d_out_forward(padH,padW,srcData,dstData,input_h,input_w,c,n,stream);
+    return 0;
+}
+#endif
+
+
+size_t ReflectionPaddingRT::getSerializationSize() const NOEXCEPT {
+    return 8*sizeof(int32_t);
+}
+
+void ReflectionPaddingRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf,padH);
+    writeBUF(buf,padW);
+    writeBUF(buf,input_h);
+    writeBUF(buf,input_w);
+    writeBUF(buf,output_h);
+    writeBUF(buf,output_w);
+    writeBUF(buf,n);
+    writeBUF(buf,c);
+}
+
+void ReflectionPaddingRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *ReflectionPaddingRT::getPluginType() const NOEXCEPT {
+    return REFLECTIONPADDINGRT_PLUGIN_NAME;
+}
+
+const char *ReflectionPaddingRT::getPluginVersion() const NOEXCEPT {
+    return REFLECTIONPADDINGRT_PLUGIN_VERSION;
+}
+
+const char *ReflectionPaddingRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ReflectionPaddingRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *ReflectionPaddingRT::clone() const NOEXCEPT {
+    auto *p = new ReflectionPaddingRT(padH,padW,input_h,input_w,output_h,output_w,c,n);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType
+ReflectionPaddingRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void ReflectionPaddingRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                                          IGpuAllocator *gpuAllocator) NOEXCEPT {
+}
+
+bool ReflectionPaddingRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted,
+                                                       int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool ReflectionPaddingRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void
+ReflectionPaddingRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                     const DataType *inputTypes, const DataType *outputTypes,
+                                     const bool *inputIsBroadcast, const bool *outputIsBroadcast,
+                                     PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void ReflectionPaddingRT::detachFromContext() NOEXCEPT {
+
+}
+
+bool ReflectionPaddingRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+
+ReflectionPaddingRTPluginCreator::ReflectionPaddingRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ReflectionPaddingRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ReflectionPaddingRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *ReflectionPaddingRTPluginCreator::deserializePlugin(const char *name, const void *serialData,
+                                                                  size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ReflectionPaddingRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *
+ReflectionPaddingRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    int padH = *(static_cast<const int32_t*>(fields[0].data));
+    int padW = *(static_cast<const int32_t*>(fields[1].data));
+    int inputH = *(static_cast<const int32_t*>(fields[2].data));
+    int inputW = *(static_cast<const int32_t*>(fields[3].data));
+    int outputH = *(static_cast<const int32_t*>(fields[4].data));
+    int outputW = *(static_cast<const int32_t*>(fields[5].data));
+    int n = *(static_cast<const int32_t*>(fields[6].data));
+    int c = *(static_cast<const int32_t*>(fields[7].data));
+    auto *pluginObj = new ReflectionPaddingRT(padH,padW,inputH,inputW,outputH,outputW,c,n);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ReflectionPaddingRTPluginCreator::getPluginName() const NOEXCEPT {
+    return REFLECTIONPADDINGRT_PLUGIN_NAME;
+}
+
+const char *ReflectionPaddingRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return REFLECTIONPADDINGRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *ReflectionPaddingRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+

src/pluginsRT/RegionRT.cpp

@@ -0,0 +1,229 @@
+#include <tkDNN/pluginsRT/RegionRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> RegionRTPluginCreator::mPluginAttributes;
+PluginFieldCollection RegionRTPluginCreator::mFC{};
+
+static const char* REGIONRT_PLUGIN_VERSION{"1"};
+static const char* REGIONRT_PLUGIN_NAME{"RegionRT_tkDNN"};
+
+RegionRT::RegionRT(int classes, int coords, int num,int c,int h,int w) {
+    this->classes = classes;
+    this->coords = coords;
+    this->num = num;
+    this->c = c;
+    this->h = h;
+    this->w = w;
+}
+
+RegionRT::~RegionRT() {}
+
+RegionRT::RegionRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char*>(data),*bufCheck=buf;
+    classes = readBUF<int>(buf);
+    coords = readBUF<int>(buf);
+    num = readBUF<int>(buf);
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    assert(buf == bufCheck+length);
+}
+
+int RegionRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims RegionRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return inputs[0];
+}
+
+
+int RegionRT::initialize() NOEXCEPT {return 0;}
+
+void RegionRT::terminate() NOEXCEPT {}
+
+size_t RegionRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT { return 0; }
+
+#if NV_TENSORRT_MAJOR > 7
+int RegionRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                      cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+    for (int b = 0; b < batchSize; ++b){
+        for(int n = 0; n < num; ++n){
+            int index = entry_index(b, n*w*h, 0);
+            activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
+
+            index = entry_index(b, n*w*h, coords);
+            activationLOGISTICForward(srcData + index, dstData + index, w*h, stream);
+        }
+    }
+
+    //softmax start
+    int index = entry_index(0, 0, coords + 1);
+    softmaxForward(	srcData + index, classes, batchSize*num,
+                       (c*h*w)/num,
+                       w*h, 1, w*h, 1, dstData + index, stream);
+
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t RegionRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+    for (int b = 0; b < batchSize; ++b){
+        for(int n = 0; n < num; ++n){
+            int index = entry_index(b, n*w*h, 0);
+            activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
+
+            index = entry_index(b, n*w*h, coords);
+            activationLOGISTICForward(srcData + index, dstData + index, w*h, stream);
+        }
+    }
+
+    //softmax start
+    int index = entry_index(0, 0, coords + 1);
+    softmaxForward(	srcData + index, classes, batchSize*num,
+                       (c*h*w)/num,
+                       w*h, 1, w*h, 1, dstData + index, stream);
+
+    return 0;
+}
+#endif
+
+size_t RegionRT::getSerializationSize() const NOEXCEPT {
+    return 6*sizeof(int);
+}
+
+void RegionRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, classes);
+    writeBUF(buf, coords);
+    writeBUF(buf, num);
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    assert(buf == a + getSerializationSize());
+}
+
+const char *RegionRT::getPluginType() const NOEXCEPT {
+    return REGIONRT_PLUGIN_NAME;
+}
+
+const char *RegionRT::getPluginVersion() const NOEXCEPT {
+    return REGIONRT_PLUGIN_VERSION;
+}
+
+void RegionRT::destroy() NOEXCEPT { delete this; }
+
+const char *RegionRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void RegionRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+bool RegionRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+IPluginV2Ext *RegionRT::clone() const NOEXCEPT {
+    auto *p = new RegionRT(classes,coords,num,c,h,w);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType RegionRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void RegionRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                               IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+bool RegionRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool RegionRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void RegionRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                               const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                               const bool *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void RegionRT::detachFromContext() NOEXCEPT {
+
+}
+
+
+RegionRTPluginCreator::RegionRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void RegionRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *RegionRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *RegionRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new RegionRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *RegionRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 6);
+    for(int i=0;i<6;i++){
+        assert(fields[i].type == PluginFieldType::kINT32);
+    }
+    int classes = *(static_cast<const int*>(fields[0].data));
+    int coords = *(static_cast<const int*>(fields[1].data));
+    int num = *(static_cast<const int*>(fields[2].data));
+    int c = *(static_cast<const int*>(fields[3].data));
+    int h = *(static_cast<const int*>(fields[4].data));
+    int w = *(static_cast<const int*>(fields[5].data));
+    auto *pluginObj = new RegionRT(classes,coords,num,c,h,w);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *RegionRTPluginCreator::getPluginName() const NOEXCEPT {
+    return REGIONRT_PLUGIN_NAME;
+}
+
+const char *RegionRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return REGIONRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *RegionRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/ReorgRT.cpp

@@ -0,0 +1,194 @@
+#include <tkDNN/pluginsRT/ReorgRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> ReorgRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ReorgRTPluginCreator::mFC{};
+
+static const char* REORGRT_PLUGIN_VERSION{"1"};
+static const char* REORGRT_PLUGIN_NAME{"ReorgRT_tkDNN"};
+
+ReorgRT::ReorgRT(int stride,int c,int h,int w) {
+    this->stride = stride;
+    this->c = c;
+    this->h = h;
+    this->w = w;
+}
+
+ReorgRT::~ReorgRT() {}
+
+ReorgRT::ReorgRT(const void *data, size_t length) {
+    const char* buf  = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    stride = readBUF<int>(buf);
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+int ReorgRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ReorgRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{inputs[0].d[0]*stride*stride, inputs[0].d[1]/stride, inputs[0].d[2]/stride};
+}
+
+
+int ReorgRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ReorgRT::terminate() NOEXCEPT {}
+
+size_t ReorgRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int ReorgRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,cudaStream_t stream) NOEXCEPT {
+    reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
+                 reinterpret_cast<dnnType*>(outputs[0]),
+                 batchSize, c, h, w, stride, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t ReorgRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) {
+    reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
+                 reinterpret_cast<dnnType*>(outputs[0]),
+                 batchSize, c, h, w, stride, stream);
+    return 0;
+}
+#endif
+
+
+size_t ReorgRT::getSerializationSize() const NOEXCEPT {
+    return 4*sizeof(int);
+}
+
+void ReorgRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, stride);
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    assert(buf == a + getSerializationSize());
+}
+
+bool ReorgRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *ReorgRT::getPluginType() const NOEXCEPT {
+    return REORGRT_PLUGIN_NAME;
+}
+
+const char *ReorgRT::getPluginVersion() const NOEXCEPT {
+    return REORGRT_PLUGIN_VERSION;
+}
+
+void ReorgRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *ReorgRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ReorgRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *ReorgRT::clone() const NOEXCEPT {
+    auto *p = new ReorgRT(stride,c,h,w);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType ReorgRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void ReorgRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                              IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+bool ReorgRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool ReorgRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void ReorgRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                              const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                              const bool *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void ReorgRT::detachFromContext() NOEXCEPT {
+
+}
+
+ReorgRTPluginCreator::ReorgRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ReorgRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ReorgRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *ReorgRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ReorgRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *ReorgRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 4);
+    for(int i=0;i<4;i++){
+        assert(fields[1].type == PluginFieldType::kINT32);
+    }
+    int stride = *(static_cast<const int *>(fields[0].data));
+    int c = *(static_cast<const int *>(fields[1].data));
+    int h = *(static_cast<const int *>(fields[2].data));
+    int w = *(static_cast<const int *>(fields[3].data));
+
+    auto *pluginObj = new ReorgRT(stride,c,h,w);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ReorgRTPluginCreator::getPluginName() const NOEXCEPT {
+    return REORGRT_PLUGIN_NAME;
+}
+
+const char *ReorgRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return REORGRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *ReorgRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/ReshapeRT.cpp

@@ -0,0 +1,197 @@
+#include <tkDNN/pluginsRT/ReshapeRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> ReshapeRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ReshapeRTPluginCreator::mFC{};
+
+static const char* RESHAPERT_PLUGIN_VERSION{"1"};
+static const char* RESHAPERT_PLUGIN_NAME{"ReshapeRT_tkDNN"};
+
+ReshapeRT::ReshapeRT(int n,int c,int h,int w) {
+    this->n = n;
+    this->c = c;
+    this->h = h;
+    this->w = w;
+}
+
+ReshapeRT::ReshapeRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    n = readBUF<int>(buf);
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+ReshapeRT::~ReshapeRT() {}
+
+int ReshapeRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ReshapeRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{ c,h,w} ;
+}
+
+int ReshapeRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ReshapeRT::terminate() NOEXCEPT {}
+
+size_t ReshapeRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int ReshapeRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                       cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t ReshapeRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    std::cout << "C : " << c << "H : " << h << "w :" << w << std::endl;
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    return 0;
+}
+#endif
+
+
+size_t ReshapeRT::getSerializationSize() const NOEXCEPT {
+    return 4*sizeof(int);
+}
+
+void ReshapeRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a = buf;
+    writeBUF(buf, n);
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    assert(buf == a + getSerializationSize());
+}
+
+bool ReshapeRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *ReshapeRT::getPluginType() const NOEXCEPT {
+    return RESHAPERT_PLUGIN_NAME;
+}
+
+const char *ReshapeRT::getPluginVersion() const NOEXCEPT {
+    return RESHAPERT_PLUGIN_VERSION;
+}
+
+void ReshapeRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *ReshapeRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ReshapeRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *ReshapeRT::clone() const NOEXCEPT {
+    auto *p = new ReshapeRT(n,c,h,w);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType ReshapeRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void ReshapeRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                                IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+bool
+ReshapeRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool ReshapeRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void ReshapeRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                                const bool *outputIsBroadcast, PluginFormat floatFormat,
+                                int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void ReshapeRT::detachFromContext() NOEXCEPT {
+
+}
+
+ReshapeRTPluginCreator::ReshapeRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ReshapeRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ReshapeRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *ReshapeRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ReshapeRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *ReshapeRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 4);
+    for(int i=0;i<4;i++){
+        assert(fields[1].type == PluginFieldType::kINT32);
+    }
+    int n = *(static_cast<const int *>(fields[0].data));
+    int c = *(static_cast<const int *>(fields[1].data));
+    int h = *(static_cast<const int *>(fields[2].data));
+    int w = *(static_cast<const int *>(fields[3].data));
+
+    auto *pluginObj = new ReshapeRT(n,c,h,w);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ReshapeRTPluginCreator::getPluginName() const NOEXCEPT {
+    return RESHAPERT_PLUGIN_NAME;
+}
+
+const char *ReshapeRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return RESHAPERT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *ReshapeRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/ResizeLayerRT.cpp

@@ -0,0 +1,198 @@
+#include <tkDNN/pluginsRT/ResizeLayerRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> ResizeLayerRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ResizeLayerRTPluginCreator::mFC{};
+
+
+ResizeLayerRT::ResizeLayerRT(int oc, int oh, int ow,int ic,int ih,int iw) {
+   this->o_c = oc;
+   this->o_h = oh;
+   this->o_w = ow;
+   this->i_c = ic;
+   this->i_h = ih;
+   this->i_w = iw;
+}
+
+ResizeLayerRT::ResizeLayerRT(const void *data, size_t length) {
+    const char *buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    o_c = readBUF<int>(buf);
+    o_h = readBUF<int>(buf);
+    o_w = readBUF<int>(buf);
+    i_c = readBUF<int>(buf);
+    i_h = readBUF<int>(buf);
+    i_w = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+ResizeLayerRT::~ResizeLayerRT() {}
+
+int ResizeLayerRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ResizeLayerRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{o_c, o_h, o_w};
+}
+
+int ResizeLayerRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ResizeLayerRT::terminate() NOEXCEPT {}
+
+size_t ResizeLayerRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT { return 0; }
+
+#if NV_TENSORRT_MAJOR > 7
+int ResizeLayerRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                           cudaStream_t stream) NOEXCEPT {
+    resizeForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
+                  reinterpret_cast<dnnType*>(outputs[0]),
+                  batchSize, i_c, i_h, i_w, o_c, o_h, o_w, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t ResizeLayerRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                               cudaStream_t stream) {
+    resizeForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
+                  reinterpret_cast<dnnType*>(outputs[0]),
+                  batchSize, i_c, i_h, i_w, o_c, o_h, o_w, stream);
+    return 0;
+}
+#endif
+
+size_t ResizeLayerRT::getSerializationSize() const NOEXCEPT {
+    return 6*sizeof(int);
+}
+
+void ResizeLayerRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, o_c);
+    writeBUF(buf, o_h);
+    writeBUF(buf, o_w);
+    writeBUF(buf, i_c);
+    writeBUF(buf, i_h);
+    writeBUF(buf, i_w);
+    assert(buf == a + getSerializationSize());
+}
+
+bool ResizeLayerRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *ResizeLayerRT::getPluginType() const NOEXCEPT {
+    return "ResizeLayerRT_tkDNN";
+}
+
+const char *ResizeLayerRT::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+void ResizeLayerRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *ResizeLayerRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ResizeLayerRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *ResizeLayerRT::clone() const NOEXCEPT {
+    auto *p = new ResizeLayerRT(o_c,o_h,o_w,i_c,i_h,i_w);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType
+ResizeLayerRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void ResizeLayerRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                                    IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+bool ResizeLayerRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted,
+                                                 int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool ResizeLayerRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void ResizeLayerRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                    const DataType *inputTypes, const DataType *outputTypes,
+                                    const bool *inputIsBroadcast, const bool *outputIsBroadcast,
+                                    PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void ResizeLayerRT::detachFromContext() NOEXCEPT {
+
+}
+
+ResizeLayerRTPluginCreator::ResizeLayerRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ResizeLayerRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ResizeLayerRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *ResizeLayerRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ResizeLayerRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *ResizeLayerRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 6);
+    for(int i=0;i<6;i++){
+        assert(fields[i].type == PluginFieldType::kINT32);
+    }
+    int oc = *(static_cast<const int *>(fields[0].data));
+    int oh = *(static_cast<const int *>(fields[1].data));
+    int ow = *(static_cast<const int *>(fields[2].data));
+    int ic = *(static_cast<const int *>(fields[3].data));
+    int ih = *(static_cast<const int *>(fields[4].data));
+    int iw = *(static_cast<const int *>(fields[5].data));
+    auto *pluginObj = new ResizeLayerRT(oc,oh,ow,ic,ih,iw);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ResizeLayerRTPluginCreator::getPluginName() const NOEXCEPT {
+    return "ResizeLayerRT_tkDNN";
+}
+
+const char *ResizeLayerRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+const PluginFieldCollection *ResizeLayerRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/RouteRT.cpp

@@ -0,0 +1,197 @@
+#include <tkDNN/pluginsRT/RouteRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> RouteRTPluginCreator::mPluginAttributes;
+PluginFieldCollection RouteRTPluginCreator::mFC{};
+
+
+RouteRT::RouteRT(int groups, int group_id) {
+    this->groups = groups;
+    this->group_id = group_id;
+}
+
+RouteRT::~RouteRT() {}
+
+RouteRT::RouteRT(const void *data, size_t length) {
+    const char* buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    groups = readBUF<int>(buf);
+    group_id = readBUF<int>(buf);
+    in = readBUF<int>(buf);
+    for(int i=0;i <MAX_INPUTS;i++){
+        c_in[i] = readBUF<int>(buf);
+    }
+    c= readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+int RouteRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims RouteRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    int out_c = 0;
+    for(int i=0; i<nbInputDims; i++) out_c += inputs[i].d[0];
+    return Dims3{out_c/groups, inputs[0].d[1], inputs[0].d[2]};
+}
+
+void
+RouteRT::configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type,
+                             PluginFormat format, int maxBatchSize) NOEXCEPT {
+    in = nbInputs;
+    c = 0;
+    for(int i=0; i<nbInputs; i++) {
+        c_in[i] = inputDims[i].d[0];
+        c += inputDims[i].d[0];
+    }
+    h = inputDims[0].d[1];
+    w = inputDims[0].d[2];
+    c /= groups;
+}
+
+int RouteRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void RouteRT::terminate() NOEXCEPT {}
+
+size_t RouteRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int RouteRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                     cudaStream_t stream) NOEXCEPT {
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    for(int b=0; b<batchSize; b++) {
+        int offset = 0;
+        for(int i=0; i<in; i++) {
+            dnnType *input = (dnnType*)reinterpret_cast<const dnnType*>(inputs[i]);
+            int in_dim = c_in[i]*h*w;
+            int part_in_dim = in_dim / this->groups;
+            checkCuda( cudaMemcpyAsync(dstData + b*c*w*h + offset, input + b*c*w*h*groups + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) );
+            offset += part_in_dim;
+        }
+    }
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t RouteRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) {
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+    for(int b=0; b<batchSize; b++) {
+        int offset = 0;
+        for(int i=0; i<in; i++) {
+            dnnType *input = (dnnType*)reinterpret_cast<const dnnType*>(inputs[i]);
+            int in_dim = c_in[i]*h*w;
+            int part_in_dim = in_dim / this->groups;
+            checkCuda( cudaMemcpyAsync(dstData + b*c*w*h + offset, input + b*c*w*h*groups + this->group_id*part_in_dim, part_in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) );
+            offset += part_in_dim;
+        }
+    }
+    return 0;
+}
+#endif
+
+size_t RouteRT::getSerializationSize() const NOEXCEPT {
+    return (6+MAX_INPUTS)*sizeof(int);
+}
+
+void RouteRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, groups);
+    writeBUF(buf, group_id);
+    writeBUF(buf, in);
+    for(int i=0; i<MAX_INPUTS; i++)
+        writeBUF(buf, c_in[i]);
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    assert(buf == a + getSerializationSize());
+}
+
+const char *RouteRT::getPluginType() const NOEXCEPT {
+    return "RouteRT_tkDNN";
+}
+
+const char *RouteRT::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+void RouteRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *RouteRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void RouteRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+bool RouteRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+IPluginV2 *RouteRT::clone() const NOEXCEPT {
+    auto *p = new RouteRT(groups,group_id);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+RouteRTPluginCreator::RouteRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void RouteRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *RouteRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2 *RouteRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new RouteRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2 *RouteRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 2);
+    assert(fields[0].type == PluginFieldType::kINT32);
+    assert(fields[1].type == PluginFieldType::kINT32);
+    int groups = *(static_cast<const int *>(fields[0].data));
+    int group_id = *(static_cast<const int *>(fields[1].data));
+    RouteRT *pluginObj = new RouteRT(groups,group_id);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *RouteRTPluginCreator::getPluginName() const NOEXCEPT {
+    return "RouteRT_tkDNN";
+}
+
+const char *RouteRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return "1";
+}
+
+const PluginFieldCollection *RouteRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/ShortcutRT.cpp

@@ -0,0 +1,211 @@
+#include <tkDNN/pluginsRT/ShortcutRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> ShortcutRTPluginCreator::mPluginAttributes;
+PluginFieldCollection ShortcutRTPluginCreator::mFC{};
+
+static const char* SHORTCUTRT_PLUGIN_VERSION{"1"};
+static const char* SHORTCUTRT_PLUGIN_NAME{"ShortcutRT_tkDNN"};
+
+ShortcutRT::ShortcutRT(int bc,int bh,int bw,int c,int h,int w,bool mul) {
+    this->bc = bc;
+    this->bh = bh;
+    this->bw = bw;
+    this->mul = mul;
+    this->c = c;
+    this->h = h;
+    this->w = w;
+}
+
+ShortcutRT::~ShortcutRT() {}
+
+ShortcutRT::ShortcutRT(const void *data, size_t length) {
+    const char* buf =reinterpret_cast<const char*>(data),*bufCheck = buf;
+    bc = readBUF<int>(buf);
+    bh = readBUF<int>(buf);
+    bw = readBUF<int>(buf);
+    mul = readBUF<bool>(buf);
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+int ShortcutRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims ShortcutRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3{inputs[0].d[0], inputs[0].d[1], inputs[0].d[2]};
+}
+
+int ShortcutRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void ShortcutRT::terminate() NOEXCEPT {}
+
+size_t ShortcutRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT { return 0; }
+
+#if NV_TENSORRT_MAJOR > 7
+int ShortcutRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                        cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *srcDataBack = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream);
+
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t ShortcutRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                            cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *srcDataBack = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+    checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream);
+
+    return 0;
+}
+#endif
+
+
+size_t ShortcutRT::getSerializationSize() const NOEXCEPT {
+    return 6*sizeof(int) + sizeof(bool);
+}
+
+void ShortcutRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, bc);
+    writeBUF(buf, bh);
+    writeBUF(buf, bw);
+    writeBUF(buf, mul);
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    assert(buf == a + getSerializationSize());
+}
+
+bool ShortcutRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *ShortcutRT::getPluginType() const NOEXCEPT {
+    return SHORTCUTRT_PLUGIN_NAME;
+}
+
+const char *ShortcutRT::getPluginVersion() const NOEXCEPT {
+    return SHORTCUTRT_PLUGIN_VERSION;
+}
+
+void ShortcutRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *ShortcutRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void ShortcutRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *ShortcutRT::clone() const NOEXCEPT {
+    auto *p = new ShortcutRT(bc,bh,bw,c,h,w,mul);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+void ShortcutRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                 const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                                 const bool *outputIsBroadcast, PluginFormat floatFormat,
+                                 int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+bool ShortcutRT::isOutputBroadcastAcrossBatch(int32_t outputIndex, const bool *inputIsBroadcasted,
+                                              int32_t nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool ShortcutRT::canBroadcastInputAcrossBatch(int32_t inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void ShortcutRT::attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *) NOEXCEPT {
+
+}
+
+void ShortcutRT::detachFromContext() NOEXCEPT {
+
+}
+
+DataType ShortcutRT::getOutputDataType(int32_t index, const nvinfer1::DataType *inputTypes, int32_t nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+
+ShortcutRTPluginCreator::ShortcutRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void ShortcutRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *ShortcutRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *ShortcutRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new ShortcutRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *ShortcutRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    assert(fc->nbFields == 7);
+    int bc = *(static_cast<const int *>(fields[0].data));
+    int bh = *(static_cast<const int *>(fields[1].data));
+    int bw = *(static_cast<const int *>(fields[2].data));
+    bool mul = *(static_cast<const bool *>(fields[3].data));
+    int c = *(static_cast<const int *>(fields[4].data));
+    int h = *(static_cast<const int *>(fields[5].data));
+    int w = *(static_cast<const int *>(fields[6].data));
+    auto *pluginObj = new ShortcutRT(bc,bh,bw,c,h,w,mul);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *ShortcutRTPluginCreator::getPluginName() const NOEXCEPT {
+    return SHORTCUTRT_PLUGIN_NAME;
+}
+
+const char *ShortcutRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return SHORTCUTRT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *ShortcutRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/UpsampleRT.cpp

@@ -0,0 +1,197 @@
+#include <tkDNN/pluginsRT/UpsampleRT.h>
+using namespace nvinfer1;
+
+std::vector<PluginField> UpsampleRTPluginCreator::mPluginAttributes;
+PluginFieldCollection UpsampleRTPluginCreator::mFC{};
+
+static const char* UPSAMPLERT_PLUGIN_VERSION{"1"};
+static const char* UPSAMPLERT_PLUGIN_NAME{"UpSample_tkDNN"};
+
+UpsampleRT::UpsampleRT(int stride,int c,int h,int w) {
+    this->stride = stride;
+    this->h = h;
+    this->c = c;
+    this->w = w;
+}
+
+UpsampleRT::UpsampleRT(const void *data, size_t length) {
+    const char* buf = reinterpret_cast<const char*>(data),*bufCheck=buf;
+    stride = readBUF<int>(buf);
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+    assert(buf == bufCheck + length);
+}
+
+UpsampleRT::~UpsampleRT() {}
+
+int UpsampleRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims UpsampleRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return Dims3(inputs[0].d[0], inputs[0].d[1]*stride, inputs[0].d[2]*stride);
+}
+
+
+
+int UpsampleRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void UpsampleRT::terminate() NOEXCEPT {}
+
+size_t UpsampleRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int UpsampleRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                        cudaStream_t stream) NOEXCEPT {
+    auto *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    auto *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+    fill(dstData, batchSize*c*h*w*stride*stride, 0.0, stream);
+    upsampleForward(srcData, dstData, batchSize, c, h, w, stride, 1, 1, stream);
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR <= 7
+int32_t UpsampleRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace,
+                            cudaStream_t stream) {
+    dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+    fill(dstData, batchSize*c*h*w*stride*stride, 0.0, stream);
+    upsampleForward(srcData, dstData, batchSize, c, h, w, stride, 1, 1, stream);
+    return 0;
+}
+#endif
+
+size_t UpsampleRT::getSerializationSize() const NOEXCEPT {
+    return 4*sizeof(int);
+}
+
+void UpsampleRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char*>(buffer),*a=buf;
+    writeBUF(buf, stride);
+    writeBUF(buf, c);
+    writeBUF(buf, h);
+    writeBUF(buf, w);
+    assert(buf == a + getSerializationSize());
+}
+
+bool UpsampleRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+const char *UpsampleRT::getPluginType() const NOEXCEPT {
+    return UPSAMPLERT_PLUGIN_NAME;
+}
+
+const char *UpsampleRT::getPluginVersion() const NOEXCEPT {
+    return UPSAMPLERT_PLUGIN_VERSION;
+}
+
+void UpsampleRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *UpsampleRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void UpsampleRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *UpsampleRT::clone() const NOEXCEPT {
+    auto *p = new UpsampleRT(stride,c,h,w);
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+bool UpsampleRT::isOutputBroadcastAcrossBatch(int32_t outputIndex, const bool *inputIsBroadcasted,
+                                              int32_t nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool UpsampleRT::canBroadcastInputAcrossBatch(int32_t inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void UpsampleRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                                 const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                                 const bool *outputIsBroadcast, PluginFormat floatFormat,
+                                 int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+void UpsampleRT::attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *) NOEXCEPT {
+}
+
+void UpsampleRT::detachFromContext() NOEXCEPT {
+
+}
+
+DataType UpsampleRT::getOutputDataType(int32_t index, const nvinfer1::DataType *inputTypes, int32_t nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+UpsampleRTPluginCreator::UpsampleRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void UpsampleRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *UpsampleRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *UpsampleRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new UpsampleRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *UpsampleRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    int stride = *(static_cast<const int *>(fields[0].data));
+    int c = *(static_cast<const int*>(fields[1].data));
+    int h = *(static_cast<const int*>(fields[2].data));
+    int w = *(static_cast<const int*>(fields[3].data));
+    auto *pluginObj = new UpsampleRT(stride,c,h,w);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+const char *UpsampleRTPluginCreator::getPluginName() const NOEXCEPT {
+    return UPSAMPLERT_PLUGIN_NAME;
+}
+
+const char *UpsampleRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return UPSAMPLERT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *UpsampleRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/pluginsRT/YoloRT.cpp

@@ -0,0 +1,318 @@
+#include <tkDNN/pluginsRT/YoloRT.h>
+
+#include <utility>
+#include <mutex>
+using namespace nvinfer1;
+
+// used to retrive Yolo plugin during network deserialization
+std::mutex gYoloPlugins_mutex;
+std::vector<YoloRT*> gYoloPlugins;
+
+std::vector<PluginField> YoloRTPluginCreator::mPluginAttributes;
+PluginFieldCollection YoloRTPluginCreator::mFC{};
+
+static const char* YOLORT_PLUGIN_VERSION{"1"};
+static const char* YOLORT_PLUGIN_NAME{"YoloRT_tkDNN"};
+
+YoloRT::YoloRT(int classes, int num, int c,int h,int w,int n_masks, float scale_xy,
+               float nms_thresh, int nms_kind,
+               int new_coords) {
+    this->c = c;
+    this->h = h;
+    this->w = w;
+    this->classes = classes;
+    this->num = num;
+    this->n_masks = n_masks;
+    this->scaleXY = scale_xy;
+    this->nms_thresh = nms_thresh;
+    this->nms_kind = nms_kind;
+    this->new_coords = new_coords;
+
+    bias.clear();
+    mask.clear();
+    classesNames.clear();
+}
+
+YoloRT::YoloRT(const void *data, size_t length) {
+    const char* buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
+    classes = readBUF<int>(buf);
+    num = readBUF<int>(buf);
+    n_masks = readBUF<int>(buf);
+    scaleXY = readBUF<float>(buf);
+    nms_thresh = readBUF<float>(buf);
+    nms_kind = readBUF<int>(buf);
+    new_coords = readBUF<int>(buf);
+    c = readBUF<int>(buf);
+    h = readBUF<int>(buf);
+    w = readBUF<int>(buf);
+
+    mask.resize(n_masks);
+    for(int i=0; i<n_masks; i++)
+        mask[i] = readBUF<dnnType>(buf);
+    bias.resize(n_masks*2*num);
+    for(int i=0; i<n_masks*2*num; i++)
+        bias[i] = readBUF<dnnType>(buf);
+
+    // save classes names
+    classesNames.resize(classes);
+    for(int i=0; i<classes; i++) {
+        char tmp[YOLORT_CLASSNAME_W];
+        for(int j=0; j<YOLORT_CLASSNAME_W; j++)
+            tmp[j] = readBUF<char>(buf);
+        classesNames[i] = std::string(tmp);
+    }
+    assert(buf == bufCheck + length);
+    gYoloPlugins.push_back(this);
+}
+
+YoloRT::~YoloRT() {}
+
+int YoloRT::getNbOutputs() const NOEXCEPT {
+    return 1;
+}
+
+Dims YoloRT::getOutputDimensions(int index, const Dims *inputs, int nbInputDims) NOEXCEPT {
+    return inputs[0];
+}
+
+
+
+int YoloRT::initialize() NOEXCEPT {
+    return 0;
+}
+
+void YoloRT::terminate() NOEXCEPT {}
+
+size_t YoloRT::getWorkspaceSize(int maxBatchSize) const NOEXCEPT {
+    return 0;
+}
+
+#if NV_TENSORRT_MAJOR > 7
+int YoloRT::enqueue(int batchSize, const void *const *inputs, void *const *outputs, void *workspace,
+                    cudaStream_t stream) NOEXCEPT {
+    dnnType *srcData = (dnnType *) reinterpret_cast<const dnnType *>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType *>(outputs[0]);
+
+    checkCuda(cudaMemcpyAsync(dstData, srcData, batchSize * c * h * w * sizeof(dnnType), cudaMemcpyDeviceToDevice,
+                              stream));
+
+
+    for (int b = 0; b < batchSize; ++b) {
+        for (int n = 0; n < n_masks; ++n) {
+            int index = entry_index(b, n * w * h, 0);
+            if (new_coords == 1) {
+                if (this->scaleXY != 1)
+                    scalAdd(dstData + index, 2 * w * h, this->scaleXY, -0.5 * (this->scaleXY - 1), 1);
+            } else {
+                activationLOGISTICForward(srcData + index, dstData + index, 2 * w * h, stream); //x,y
+
+                if (this->scaleXY != 1)
+                    scalAdd(dstData + index, 2 * w * h, this->scaleXY, -0.5 * (this->scaleXY - 1), 1);
+
+                index = entry_index(b, n * w * h, 4);
+                activationLOGISTICForward(srcData + index, dstData + index, (1 + classes) * w * h, stream);
+            }
+        }
+    }
+
+    //std::cout<<"YOLO END\n";
+    return 0;
+}
+#elif NV_TENSORRT_MAJOR == 7
+int32_t YoloRT::enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream) {
+    dnnType *srcData = (dnnType *) reinterpret_cast<const dnnType *>(inputs[0]);
+    dnnType *dstData = reinterpret_cast<dnnType *>(outputs[0]);
+
+    checkCuda(cudaMemcpyAsync(dstData, srcData, batchSize * c * h * w * sizeof(dnnType), cudaMemcpyDeviceToDevice,
+                              stream));
+
+
+    for (int b = 0; b < batchSize; ++b) {
+        for (int n = 0; n < n_masks; ++n) {
+            int index = entry_index(b, n * w * h, 0);
+            if (new_coords == 1) {
+                if (this->scaleXY != 1)
+                    scalAdd(dstData + index, 2 * w * h, this->scaleXY, -0.5 * (this->scaleXY - 1), 1);
+            } else {
+                activationLOGISTICForward(srcData + index, dstData + index, 2 * w * h, stream); //x,y
+
+                if (this->scaleXY != 1)
+                    scalAdd(dstData + index, 2 * w * h, this->scaleXY, -0.5 * (this->scaleXY - 1), 1);
+
+                index = entry_index(b, n * w * h, 4);
+                activationLOGISTICForward(srcData + index, dstData + index, (1 + classes) * w * h, stream);
+            }
+        }
+    }
+
+    //std::cout<<"YOLO END\n";
+    return 0;
+}
+#endif
+
+
+size_t YoloRT::getSerializationSize() const NOEXCEPT {
+    return 8 * sizeof(int) + 2 * sizeof(float) + n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
+}
+
+bool YoloRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
+    return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR);
+}
+
+void YoloRT::serialize(void *buffer) const NOEXCEPT {
+    char *buf = reinterpret_cast<char *>(buffer), *a = buf;
+    writeBUF(buf, classes);    //std::cout << "Classes :" << classes << std::endl;
+    writeBUF(buf, num);        //std::cout << "Num : " << num << std::endl;
+    writeBUF(buf, n_masks);    //std::cout << "N_Masks" << n_masks << std::endl;
+    writeBUF(buf, scaleXY);    //std::cout << "ScaleXY :" << scaleXY << std::endl;
+    writeBUF(buf, nms_thresh); //std::cout << "nms_thresh :" << nms_thresh << std::endl;
+    writeBUF(buf, nms_kind);    //std::cout << "nms_kind : " << nms_kind << std::endl;
+    writeBUF(buf, new_coords); //std::cout << "new_coords : " << new_coords << std::endl;
+    writeBUF(buf, c);            //std::cout << "C : " << c << std::endl;
+    writeBUF(buf, h);            //std::cout << "H : " << h << std::endl;
+    writeBUF(buf, w);            //std::cout << "C : " << c << std::endl;
+    for (int i = 0; i < n_masks; i++)
+        writeBUF(buf, mask[i]); //std::cout << "mask[i] : " << mask[i] << std::endl;
+    for (int i = 0; i < n_masks * 2 * num; i++)
+        writeBUF(buf, bias[i]); //std::cout << "bias[i] : " << bias[i] << std::endl;
+
+    // save classes names
+    for(int i=0; i<classes; i++) {
+        char tmp[YOLORT_CLASSNAME_W];
+        strcpy(tmp, classesNames[i].c_str());
+        for(int j=0; j<YOLORT_CLASSNAME_W; j++) {
+            writeBUF(buf, tmp[j]);
+        }
+    }
+
+    assert(buf == a + getSerializationSize());
+}
+
+const char *YoloRT::getPluginType() const NOEXCEPT {
+    return YOLORT_PLUGIN_NAME;
+}
+
+const char *YoloRT::getPluginVersion() const NOEXCEPT {
+    return YOLORT_PLUGIN_VERSION;
+}
+
+void YoloRT::destroy() NOEXCEPT {
+    delete this;
+}
+
+const char *YoloRT::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+void YoloRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+IPluginV2Ext *YoloRT::clone() const NOEXCEPT {
+    auto *p = new YoloRT(classes, num,c,h,w,n_masks, scaleXY, nms_thresh, nms_kind, new_coords);
+    p->mask = mask;
+    p->bias = bias;
+    p->classesNames = classesNames;
+    p->setPluginNamespace(mPluginNamespace.c_str());
+    return p;
+}
+
+DataType YoloRT::getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const NOEXCEPT {
+    return DataType::kFLOAT;
+}
+
+void YoloRT::attachToContext(cudnnContext *cudnnContext, cublasContext *cublasContext,
+                             IGpuAllocator *gpuAllocator) NOEXCEPT {
+
+}
+
+void YoloRT::configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs,
+                             const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast,
+                             const bool *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) NOEXCEPT {
+
+}
+
+bool YoloRT::isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, int nbInputs) const NOEXCEPT {
+    return false;
+}
+
+bool YoloRT::canBroadcastInputAcrossBatch(int inputIndex) const NOEXCEPT {
+    return false;
+}
+
+void YoloRT::detachFromContext() NOEXCEPT {
+
+}
+
+YoloRTPluginCreator::YoloRTPluginCreator() {
+    mPluginAttributes.clear();
+    mFC.nbFields = mPluginAttributes.size();
+    mFC.fields = mPluginAttributes.data();
+}
+
+void YoloRTPluginCreator::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
+    mPluginNamespace = pluginNamespace;
+}
+
+const char *YoloRTPluginCreator::getPluginNamespace() const NOEXCEPT {
+    return mPluginNamespace.c_str();
+}
+
+IPluginV2Ext *YoloRTPluginCreator::deserializePlugin(const char *name, const void *serialData, size_t serialLength) NOEXCEPT {
+    auto *pluginObj = new YoloRT(serialData,serialLength);
+    pluginObj->setPluginNamespace(mPluginNamespace.c_str());
+    return pluginObj;
+}
+
+IPluginV2Ext *YoloRTPluginCreator::createPlugin(const char *name, const PluginFieldCollection *fc) NOEXCEPT {
+    const PluginField *fields = fc->fields;
+    int classes = *(static_cast<const int *>(fields[0].data));
+    int num = *(static_cast<const int *>(fields[1].data));
+    int c = *(static_cast<const int *>(fields[2].data));
+    int h = *(static_cast<const int *>(fields[3].data));
+    int w = *(static_cast<const int *>(fields[4].data));
+    int n_masks = *(static_cast<const int *>(fields[5].data));
+    dnnType scaleXY = *(static_cast<const float*>(fields[6].data));
+    dnnType nmsThresh = *(static_cast<const float*>(fields[7].data));
+    int nms_kind = *(static_cast<const int*>(fields[8].data));
+    int new_coords = *(static_cast<const int*>(fields[9].data));
+    auto *pluginObj = new YoloRT(classes,num,c,h,w,n_masks,scaleXY,nmsThresh,nms_kind,new_coords);
+
+    // fill additional data
+    pluginObj->mask.resize(fields[10].length*sizeof(float));
+    memcpy(pluginObj->mask.data(), fields[10].data, fields[10].length*sizeof(float));
+    pluginObj->bias.resize(fields[11].length*sizeof(float));
+    memcpy(pluginObj->bias.data(), fields[11].data, fields[11].length*sizeof(float));
+    pluginObj->classesNames.resize(classes);
+    for(int i=0; i<classes; i++) {
+        pluginObj->classesNames[i].resize(fields[12+i].length);
+        memcpy(&pluginObj->classesNames[i][0], fields[12+i].data, fields[12+i].length*sizeof(char));
+    }
+    return pluginObj;
+}
+
+const char *YoloRTPluginCreator::getPluginName() const NOEXCEPT {
+    return YOLORT_PLUGIN_NAME;
+}
+
+const char *YoloRTPluginCreator::getPluginVersion() const NOEXCEPT {
+    return YOLORT_PLUGIN_VERSION;
+}
+
+const PluginFieldCollection *YoloRTPluginCreator::getFieldNames() NOEXCEPT {
+    return &mFC;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/utils.cpp

@@ -23,7 +23,7 @@ bool fileExist(const char *fname) {
 void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url){
     if(!fileExist(input_bin.c_str())){
         std::string mkdir_cmd = "mkdir " + test_folder; 
-        std::string wget_cmd = "curl " + weights_url + " --output " + test_folder + "/weights.zip";
+        std::string wget_cmd = "curl -tlsv1 -C - " + weights_url + " --output " + test_folder + "/weights.zip --user user:pass -O --retry 999 --retry-max-time 0";
 #ifdef __linux__
         std::string unzip_cmd = "unzip " + test_folder + "/weights.zip -d" + test_folder;
         std::string rm_cmd = "rm " + test_folder + "/weights.zip";

tests/centernet/dla34_cnet/dla34_cnet.cpp

@@ -540,5 +540,6 @@ int main()
         std::cout<<"CUDNN vs TRT    "; 
         ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
     }
+    netRT.destroy();
     return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
 }

tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp

@@ -558,5 +558,6 @@ int main()
         std::cout<<"CUDNN vs TRT    "; 
         ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
     }
+    netRT.destroy();
     return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
 }

tests/centernet/resnet101_cnet/resnet101_cnet.cpp

@@ -421,5 +421,6 @@ int main()
         std::cout<<"CUDNN vs TRT    "; 
         ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
     }
+    netRT.destroy();
     return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
 }

tests/darknet/csresnext50-panet-spp.cpp

@@ -29,6 +29,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/csresnext50-panet-spp_berkeley.cpp

@@ -29,6 +29,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/yolo2_voc.cpp

@@ -27,6 +27,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/yolo2tiny.cpp

@@ -28,6 +28,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/yolo3.cpp

@@ -29,6 +29,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);    
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/yolo3_512.cpp

@@ -41,6 +41,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/yolo3_berkeley.cpp

@@ -29,6 +29,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }

tests/darknet/yolo3_coco4.cpp

@@ -29,6 +29,7 @@ int main() {
     int ret = testInference(input_bins, output_bins, net, netRT);
     net->releaseLayers();
     delete net;
+    netRT->destroy();
     delete netRT;
     return ret;
 }