update tensorrt8 branch

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.15)
 project (tkDNN)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 if(UNIX)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations -Wno-unused-variable ")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations -Wno-unused-variable")
 endif()
 if(WIN32)
 set(CMAKE_CXX_STANDARD 11)
@@ -31,7 +31,7 @@ endif()
 find_package(CUDA 9.0 REQUIRED)
 SET(CUDA_SEPARABLE_COMPILATION ON)
 #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'")
-set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
+set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32 -G -g)
 
 
 find_package(CUDNN REQUIRED)
@@ -41,8 +41,7 @@ include_directories(${CUDNN_INCLUDE_DIR})
 # compile
 file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu" )
 cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
-cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
+
-target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES})
 
 
 #-------------------------------------------------------------------------------
@@ -65,12 +64,12 @@ find_package(yaml-cpp REQUIRED)
 # Build Libraries
 #-------------------------------------------------------------------------------
 file(GLOB tkdnn_SRC "src/*.cpp")
-set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp)
+set(tkdnn_LIBS ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES})
-add_library(tkDNN SHARED ${tkdnn_SRC})
+cuda_add_library(tkDNN SHARED ${tkdnn_SRC} ${tkdnn_CUSRC})
-target_link_libraries(tkDNN ${tkdnn_LIBS})
+target_link_libraries(tkDNN ${tkdnn_LIBS} ${CUDA_CUBLAS_LIBRAY})
 
 #static 
 #add_library(tkDNN_static STATIC ${tkdnn_SRC})
@@ -169,7 +168,7 @@ target_link_libraries(seg_demo tkDNN)
 #endif()
 message("install dir:" ${CMAKE_INSTALL_PREFIX})
 install(DIRECTORY include/ DESTINATION include/)
-install(TARGETS tkDNN kernels DESTINATION lib)
+install(TARGETS tkDNN  DESTINATION lib)
 install(TARGETS test_simple test_mnist test_mnistRT test_rtinference demo map_demo DESTINATION bin)
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory
         DESTINATION "share/tkDNN/cmake/" # target directory

include/tkDNN/NetworkRT.h

@@ -119,6 +119,7 @@ public:
 
     bool serialize(const char *filename);
     bool deserialize(const char *filename);
+    void destroy();
 
 
 

include/tkDNN/pluginsRT/ActivationLeakyRT.h

@@ -8,6 +8,7 @@ public:
 
     ActivationLeakyRT(const void *data, size_t length)
     {
+        std::cout<<"DESERIALIZE LEAKYRT"<<std::endl;
         const char* buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
         slope = readBUF<float>(buf);
         size = readBUF<int>(buf);
@@ -139,8 +140,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ActivationLogisticRT.h

@@ -139,8 +139,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ActivationMishRT.h

@@ -9,6 +9,7 @@ public:
     ~ActivationMishRT() {}
 
     ActivationMishRT(const void *data, size_t length) {
+        std::cout<<"DESERIALIZE MISH"<<std::endl;
         const char *buf = reinterpret_cast<const char *>(data), *bufCheck = buf;
         size = readBUF<int>(buf);
         assert(buf == bufCheck + length);
@@ -126,8 +127,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h

@@ -14,6 +14,7 @@ public:
     }
 
     ActivationReLUCeiling(const void *data, size_t length) {
+        std::cout<<"RELU CEILING DESERIALIZE"<<std::endl;
         const char *buf = reinterpret_cast<const char *>(data), *bufCheck = buf;
         ceiling = readBUF<float>(buf);
         size = readBUF<int>(buf);
@@ -140,9 +141,9 @@ public:
         return &mFC;
     }
 
-private:
+public:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/DeformableConvRT.h

@@ -352,10 +352,9 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 
-REGISTER_TENSORRT_PLUGIN(DeformableConvRTPluginCreator);
 

include/tkDNN/pluginsRT/FlattenConcatRT.h

@@ -154,8 +154,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h

@@ -173,8 +173,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 
 };

include/tkDNN/pluginsRT/RegionRT.h

@@ -179,8 +179,8 @@ public:
         return &mFC;
     }
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ReorgRT.h

@@ -134,8 +134,8 @@ public:
         return &mFC;
     }
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ReshapeRT.h

@@ -140,8 +140,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ResizeLayerRT.h

@@ -155,8 +155,8 @@ public:
 
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 
 };

include/tkDNN/pluginsRT/RouteRT.h

@@ -173,8 +173,8 @@ public:
         return &mFC;
     }
  private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/ShortcutRT.h

@@ -157,9 +157,9 @@ public:
     const PluginFieldCollection *getFieldNames() NOEXCEPT override{
         return &mFC;
     }
-private:
+public:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

include/tkDNN/pluginsRT/UpsampleRT.h

@@ -73,7 +73,8 @@ public:
 	}
 
 	const char *getPluginVersion() const NOEXCEPT override{
-	    return "UpsampleRT_tkDNN";
+	    static const char* UPSAMPLE_RT_PLUGIN = "UpsampleRT_TRT";
+	    return UPSAMPLE_RT_PLUGIN;
 	}
 
 	void destroy() NOEXCEPT override{delete this;}
@@ -128,7 +129,8 @@ public:
     }
 
     const char *getPluginName() const NOEXCEPT override{
-        return "UpsampleRT_tkDNN";
+        static const char* UPSAMPLE_RT_PLUGIN = "UpsampleRT_TRT";
+        return UPSAMPLE_RT_PLUGIN;
     }
 
     const char *getPluginVersion() const NOEXCEPT override{
@@ -139,9 +141,10 @@ public:
         return &mFC;
     }
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 
 REGISTER_TENSORRT_PLUGIN(UpsampleRTPluginCreator);
+

include/tkDNN/pluginsRT/YoloRT.h

@@ -1,4 +1,5 @@
 #include<cassert>
+#include <vector>
 #include "../kernels.h"
 #define YOLORT_CLASSNAME_W 256
 
@@ -27,11 +28,12 @@ public:
     }
 
     YoloRT(const void *data,size_t length){
+        std::vector<float> maskTemp,biasTemp;
+        std::cout<<"LENGTH : "<<length<<std::endl;
         const char* buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
         classes = readBUF<int>(buf);
         num = readBUF<int>(buf);
         n_masks = readBUF<int>(buf);
-        std::cout<<n_masks<<std::endl;
         scaleXY = readBUF<float>(buf);
         nms_thresh = readBUF<float>(buf);
         nms_kind = readBUF<int>(buf);
@@ -39,10 +41,16 @@ public:
         c = readBUF<int>(buf);
         h = readBUF<int>(buf);
         w = readBUF<int>(buf);
-        for(int i=0;i<n_masks;i++)
+        for(int i=0;i<n_masks;i++){
-            mask[i] = readBUF<dnnType>(buf);
+            maskTemp.push_back(readBUF<dnnType>(buf));
-        for(int i=0;i<n_masks*2*num;i++)
+            std::cout<<maskTemp[i]<<std::endl;
-            bias[i] = readBUF<dnnType>(buf);
+        }
+        for(int i=0;i<n_masks*2*num;i++){
+            biasTemp.push_back(readBUF<dnnType>(buf));
+            std::cout<<biasTemp[i]<<std::endl;
+        }
+        mask = maskTemp.data();
+        bias = biasTemp.data();
         classesNames.resize(classes);
         for(int i=0;i<classes;i++){
             char tmp[YOLORT_CLASSNAME_W];
@@ -133,6 +141,7 @@ public:
         char *buf = reinterpret_cast<char *>(buffer), *a = buf;
         tk::dnn::writeBUF(buf, classes);    //std::cout << "Classes :" << classes << std::endl;
         tk::dnn::writeBUF(buf, num);        //std::cout << "Num : " << num << std::endl;
+        std::cout<<num<<std::endl;
         tk::dnn::writeBUF(buf, n_masks);    //std::cout << "N_Masks" << n_masks << std::endl;
         tk::dnn::writeBUF(buf, scaleXY);    //std::cout << "ScaleXY :" << scaleXY << std::endl;
         tk::dnn::writeBUF(buf, nms_thresh); //std::cout << "nms_thresh :" << nms_thresh << std::endl;
@@ -265,8 +274,8 @@ public:
     }
 
 private:
-    static PluginFieldCollection mFC;
+    PluginFieldCollection mFC;
-    static std::vector<PluginField> mPluginAttributes;
+    std::vector<PluginField> mPluginAttributes;
     std::string mPluginNamespace;
 };
 

src/NetworkRT.cpp

@@ -8,45 +8,13 @@
 
 #include "utils.h"
 #include "NvInfer.h"
+
 #include "NetworkRT.h"
 #include "Int8Calibrator.h"
 
 
 using namespace nvinfer1;
 
-PluginFieldCollection tk::dnn::ActivationLeakyRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ActivationReLUCeilingPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ActivationMishRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ActivationLogisticRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::DeformableConvRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::RegionRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ReorgRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::UpsampleRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ShortcutRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ReshapeRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::MaxPoolFixedSizeRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::ResizeLayerRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::YoloRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::RouteRTPluginCreator::mFC{};
-PluginFieldCollection tk::dnn::FlattenConcatRTPluginCreator::mFC{};
-
-
-std::vector<PluginField> tk::dnn::ActivationLeakyRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ActivationReLUCeilingPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ActivationMishRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ActivationLogisticRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::DeformableConvRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::RegionRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ReorgRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::UpsampleRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ShortcutRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ReshapeRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::MaxPoolFixedSizeRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::ResizeLayerRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::YoloRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::RouteRTPluginCreator::mPluginAttributes;
-std::vector<PluginField> tk::dnn::FlattenConcatRTPluginCreator::mPluginAttributes;
-
 // Logger for info/warning/errors
 class Logger : public ILogger {
     void log(Severity severity, const char* msg) NOEXCEPT override {
@@ -590,7 +558,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
 ILayer* NetworkRT::convert_layer(ITensor *input, Upsample *l) {
     //std::cout<<"convert Upsample\n";
 
-    //std::cout<<"New plugin UPSAMPLE\n";
+    std::cout<<"New plugin UPSAMPLE\n";
     IPluginV2 *plugin = new UpsampleRT(l->stride);
     IPluginV2Layer *lRT = networkRT->addPluginV2(&input, 1, *plugin);
     checkNULL(lRT);
@@ -681,11 +649,21 @@ bool NetworkRT::deserialize(const char *filename) {
 
     runtimeRT = createInferRuntime(loggerRT);
     engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size);
+    std::cout<<size<<std::endl;
     //if (gieModelStream) delete [] gieModelStream;
 
     return true;
 }
 
+void NetworkRT::destroy() {
+    contextRT->destroy();
+    configRT->destroy();
+    engineRT->destroy();
+    builderRT->destroy();
+}
+
+
+
 
 /*
 IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, size_t serialLength) {
@@ -735,7 +713,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->w = readBUF<int>(buf);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     if(name.find("Reorg") == 0) {
         int strideTemp = readBUF<int>(buf);
@@ -745,7 +723,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->w = readBUF<int>(buf);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     if(name.find("Shortcut") == 0) {
         tk::dnn::dataDim_t bdim;
@@ -760,7 +738,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->w = readBUF<int>(buf);
         return r;
         assert(buf == bufCheck + serialLength);
-    } 
+    }
 
     if(name.find("Pooling") == 0) {
         int cTemp = readBUF<int>(buf);
@@ -788,10 +766,10 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->i_w = readBUF<int>(buf);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     if(name.find("Flatten") == 0) {
-        FlattenConcatRT *r = new FlattenConcatRT(); 
+        FlattenConcatRT *r = new FlattenConcatRT();
         r->c = readBUF<int>(buf);
         r->h = readBUF<int>(buf);
         r->w = readBUF<int>(buf);
@@ -799,7 +777,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->cols = readBUF<int>(buf);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     if(name.find("Reshape") == 0) {
 
@@ -808,11 +786,11 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         new_dim.c = readBUF<int>(buf);
         new_dim.h = readBUF<int>(buf);
         new_dim.w = readBUF<int>(buf);
-        ReshapeRT *r = new ReshapeRT(new_dim); 
+        ReshapeRT *r = new ReshapeRT(new_dim);
         assert(buf == bufCheck + serialLength);
-        
+
         return r;
-    } 
+    }
 
     if(name.find("Yolo") == 0) {
 
@@ -824,7 +802,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         int nms_kind_temp = readBUF<int>(buf);
         int new_coords_temp = readBUF<int>(buf);
 
-       YoloRT *r = new YoloRT(classes_temp,num_temp,nullptr,n_masks_temp,scale_xy_temp,nms_thresh_temp,nms_kind_temp,new_coords_temp);  
+       YoloRT *r = new YoloRT(classes_temp,num_temp,nullptr,n_masks_temp,scale_xy_temp,nms_thresh_temp,nms_kind_temp,new_coords_temp);
 
 
 
@@ -848,7 +826,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
 
         yolos[n_yolos++] = r;
         return r;
-    } 
+    }
     if(name.find("Upsample") == 0) {
         int strideTemp = readBUF<int>(buf);
         UpsampleRT* r = new UpsampleRT(strideTemp);
@@ -857,7 +835,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->w = readBUF<int>(buf);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     if(name.find("Route") == 0) {
         int groupsTemp = readBUF<int>(buf);
@@ -871,7 +849,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         r->w = readBUF<int>(buf);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     if(name.find("Deformable") == 0) {
         int chuck_dimTemp = readBUF<int>(buf);
@@ -924,7 +902,7 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         free(aus);
         assert(buf == bufCheck + serialLength);
         return r;
-    } 
+    }
 
     FatalError("Cant deserialize Plugin");
     return NULL;

src/Yolo3Detection.cpp

@@ -7,7 +7,7 @@ namespace tk { namespace dnn {
 
     //convert network to tensorRT
     std::cout<<(tensor_path).c_str()<<"\n";
-    netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
+    netRT = new tk::dnn::NetworkRT(nullptr, (tensor_path).c_str() );
 
     nBatches = n_batches;
     confThreshold = conf_thresh;

tests/darknet/yolo4tiny.cpp

@@ -26,8 +26,15 @@ int main() {
     tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
     
     int ret = testInference(input_bins, output_bins, net, netRT);
+    std::cout<<ret<<std::endl;
+    std::cout<<"Releasing Layers"<<std::endl;
     net->releaseLayers();
+    std::cout<<"DELETING NET"<<std::endl;
     delete net;
+    std::cout<<"DELETING NETRT"<<std::endl;
+    netRT->destroy();
     delete netRT;
+
+    std::cout<<"RETRUNING RET"<<std::endl;
     return ret;
 }