bug fixes for dla networks and ported optimization from different pull request

CMakeLists.txt

@@ -6,36 +6,9 @@ set(CMAKE_CXX_STANDARD 14)
 option(ENABLE_OPENCV_CUDA_CONTRIB "Enable OpenCV CUDA Contrib" OFF )
 
 find_package(CUDA 9.0 REQUIRED)
-if (CUDA_FOUND)
-    set(OUTPUTFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cuda_script) # No suffix required
-    execute_process(COMMAND "rm ${OUTPUTFILE}")
-    set(CUDAFILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/getCudaArch.cu)
-    execute_process(COMMAND ${CUDA_NVCC_EXECUTABLE} -lcuda ${CUDAFILE} -o ${OUTPUTFILE})
-    execute_process(COMMAND ${OUTPUTFILE}
-            RESULT_VARIABLE CUDA_RETURN_CODE
-            OUTPUT_VARIABLE ARCH)
-
-    if(${CUDA_RETURN_CODE} EQUAL 0)
-        set(CUDA_SUCCESS "TRUE")
-    else()
-        set(CUDA_SUCCESS "FALSE")
-    endif()
-
-    if (${CUDA_SUCCESS})
-        message(STATUS "CUDA Architecture: ${ARCH}")
-        message(STATUS "CUDA Version: ${CUDA_VERSION_STRING}")
-        message(STATUS "CUDA Path: ${CUDA_TOOLKIT_ROOT_DIR}")
-        message(STATUS "CUDA Libararies: ${CUDA_LIBRARIES}")
-        message(STATUS "CUDA Performance Primitives: ${CUDA_npp_LIBRARY}")
-        set(CUDA_NVCC_FLAGS "${ARCH}")
-    else()
-        message(WARNING ${ARCH})
-    endif()
-endif()
 
 SET(CUDA_SEPARABLE_COMPILATION ON)
 
-
 if(UNIX)
     if(CMAKE_BUILD_TYPE MATCHES Release)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -fPIC -Wno-deprecated-declarations -Wno-unused-variable -O3")

include/tkDNN/NetworkRT.h

@@ -48,7 +48,7 @@ public:
     void* buffersRT[MAX_BUFFERS_RT];
     dataDim_t buffersDIM[MAX_BUFFERS_RT];
     int buf_input_idx, buf_output_idx;
-
+    bool builderActive = false;
     dataDim_t input_dim, output_dim;
     dnnType *output;
     cudaStream_t stream;
@@ -84,14 +84,14 @@ public:
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Region *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
-    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
+    nvinfer1::IPluginV2Layer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l);
 
     bool serialize(const char *filename);

include/tkDNN/pluginsRT/YoloRT.h

@@ -13,8 +13,7 @@ namespace nvinfer1 {
     class YoloRT : public IPluginV2Ext {
 
     public:
-        YoloRT(int classes, int num,int c,int h,int w,std::vector<std::string> classNames,
+        YoloRT(int classes, int num,int c,int h,int w, int n_masks = 3, float scale_xy = 1,
-               std::vector<float> masks_v,std::vector<float> bias_v, int n_masks = 3, float scale_xy = 1,
                float nms_thresh = 0.45, int nms_kind = 0, int new_coords = 0);
 
         YoloRT(const void *data, size_t length);
@@ -84,8 +83,6 @@ namespace nvinfer1 {
         int NUM = 0;
         std::vector<std::string> classesNames;
 
-        std::vector<dnnType> mask;
-        std::vector<dnnType> bias;
 
         int entry_index(int batch, int location, int entry) {
             int n = location / (w * h);

src/NetworkRT.cpp

@@ -148,8 +148,10 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
         // we don't need the network any more
         //networkRT->destroy();
         std::cout<<"serialize net\n";
+        builderActive = true;
         serialize(name);
     } else {
+        builderActive = false;
         deserialize(name);
     }
 
@@ -386,6 +388,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) {
         auto *lRT = networkRT->addPluginV2(&input, 1, *plugin);
         checkNULL(lRT);
         return lRT;
+
     }
     else
     {
@@ -433,14 +436,15 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
         return lRT;
     } 
     else if(l->act_mode == ACTIVATION_MISH) {
-        IPluginV2 *plugin = new ActivationMishRT();
+        IActivationLayer *lRT1 = networkRT->addActivation(*input, ActivationType::kSOFTPLUS);
-        IPluginV2Layer *lRT = networkRT->addPluginV2(&input, 1, *plugin);
+        lRT1->setAlpha(1);
-        checkNULL(lRT);
+        lRT1->setBeta(1);
-        return lRT;
+        IActivationLayer *lRT2 = networkRT->addActivation(*lRT1->getOutput(0), ActivationType::kTANH);
+        IElementWiseLayer *lRT3 = networkRT->addElementWise(*input, *lRT2->getOutput(0), ElementWiseOperation::kPROD);
+        return lRT3;
     }
     else if(l->act_mode == ACTIVATION_LOGISTIC) {
-        IPluginV2 *plugin = new ActivationLogisticRT();
+        IActivationLayer *lRT = networkRT->addActivation(*input,ActivationType::kSIGMOID);
-        IPluginV2Layer *lRT = networkRT->addPluginV2(&input, 1, *plugin);
         checkNULL(lRT);
         return lRT;
     }
@@ -484,7 +488,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Route *l) {
     return lRT;
 }
 
-ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) {
+IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Flatten *l) {
     auto creator = getPluginRegistry()->getPluginCreator("FlattenConcatRT_tkDNN","1");
     std::vector<PluginField> mPluginAttributes;
     PluginFieldCollection mFC{};
@@ -495,14 +499,13 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) {
     mPluginAttributes.emplace_back(PluginField("cols",&l->cols,PluginFieldType::kINT32,1));
     mFC.nbFields = mPluginAttributes.size();
     mFC.fields = mPluginAttributes.data();
-
     auto *plugin = creator->createPlugin(l->getLayerName().c_str(),&mFC);
     auto *lRT = networkRT->addPluginV2(&input, 1, *plugin);
     checkNULL(lRT);
     return lRT;
 }
 
-ILayer* NetworkRT::convert_layer(ITensor *input, Reshape *l) {
+IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Reshape *l) {
     // std::cout<<"convert Reshape\n";
     auto creator = getPluginRegistry()->getPluginCreator("ReshapeRT_tkDNN","1");
     std::vector<PluginField> mPluginAttributes;
@@ -530,7 +533,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Resize *l) {
     return lRT;
 }
 
-ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
+IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
     //std::cout<<"convert Reorg\n";
 
     //std::cout<<"New plugin REORG\n";
@@ -549,7 +552,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
     return lRT;
 }
 
-ILayer* NetworkRT::convert_layer(ITensor *input, Region *l) {
+IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Region *l) {
     //std::cout<<"convert Region\n";
 
     //std::cout<<"New plugin REGION\n";
@@ -608,10 +611,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
     }
 }
 
-ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
+IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
 
-    std::vector<dnnType> mask_h(l->mask_h,l->mask_h+sizeof(dnnType)*l->n_masks);
-    std::vector<dnnType> bias_h(l->bias_h,l->bias_h+sizeof(dnnType)*2*l->n_masks*l->num);
     auto creator = getPluginRegistry()->getPluginCreator("YoloRT_tkDNN","1");
     std::vector<PluginField> mPluginAttributes;
     PluginFieldCollection mFC{};
@@ -620,9 +621,6 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
     mPluginAttributes.emplace_back(PluginField("c",&l->input_dim.c,PluginFieldType::kINT32,1));
     mPluginAttributes.emplace_back(PluginField("h",&l->input_dim.h,PluginFieldType::kINT32,1));
     mPluginAttributes.emplace_back(PluginField("w",&l->input_dim.w,PluginFieldType::kINT32,1));
-    mPluginAttributes.emplace_back(PluginField("classNames",&l->classesNames[0],PluginFieldType::kUNKNOWN,l->classesNames.size()));
-    mPluginAttributes.emplace_back(PluginField("mask_v",&mask_h[0],PluginFieldType::kFLOAT32,mask_h.size()));
-    mPluginAttributes.emplace_back(PluginField("bias_v",&bias_h[0],PluginFieldType::kFLOAT32,bias_h.size()));
     mPluginAttributes.emplace_back(PluginField("n_masks",&l->n_masks,PluginFieldType::kINT32,1));
     mPluginAttributes.emplace_back(PluginField("scale_xy",&l->scaleXY,PluginFieldType::kFLOAT32,1));
     mPluginAttributes.emplace_back(PluginField("nms_thresh",&l->nms_thresh,PluginFieldType::kFLOAT32,1));
@@ -636,9 +634,8 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
     return lRT;
 }
 
-ILayer* NetworkRT::convert_layer(ITensor *input, Upsample *l) {
+IPluginV2Layer* NetworkRT::convert_layer(ITensor *input, Upsample *l) {
     //std::cout<<"convert Upsample\n";
-
     auto creator = getPluginRegistry()->getPluginCreator("UpSample_tkDNN","1");
     std::vector<PluginField> mPluginAttributes;
     PluginFieldCollection mFC{};
@@ -789,8 +786,10 @@ bool NetworkRT::deserialize(const char *filename) {
 
 void NetworkRT::destroy() {
     contextRT->destroy();
+    if(builderActive) {
         engineRT->destroy();
         builderRT->destroy();
     }
+}
 
 }}

src/pluginsRT/YoloRT.cpp

@@ -9,8 +9,7 @@ PluginFieldCollection YoloRTPluginCreator::mFC{};
 static const char* YOLORT_PLUGIN_VERSION{"1"};
 static const char* YOLORT_PLUGIN_NAME{"YoloRT_tkDNN"};
 
-YoloRT::YoloRT(int classes, int num, int c,int h,int w,std::vector<std::string> classNames,
+YoloRT::YoloRT(int classes, int num, int c,int h,int w,int n_masks, float scale_xy,
-               std::vector<float> masks_v,std::vector<float> bias_v,int n_masks, float scale_xy,
                float nms_thresh, int nms_kind,
                int new_coords) {
     this->c = c;
@@ -23,14 +22,9 @@ YoloRT::YoloRT(int classes, int num, int c,int h,int w,std::vector<std::string>
     this->nms_thresh = nms_thresh;
     this->nms_kind = nms_kind;
     this->new_coords = new_coords;
-    this->classesNames = std::move(classNames);
-    this->mask = std::move(masks_v);
-    this->bias = std::move(bias_v);
-
 }
 
 YoloRT::YoloRT(const void *data, size_t length) {
-    std::vector<float> maskTemp,biasTemp;
     const char* buf = reinterpret_cast<const char*>(data),*bufCheck = buf;
     classes = readBUF<int>(buf);
     num = readBUF<int>(buf);
@@ -42,21 +36,6 @@ YoloRT::YoloRT(const void *data, size_t length) {
     c = readBUF<int>(buf);
     h = readBUF<int>(buf);
     w = readBUF<int>(buf);
-    mask.resize(n_masks);
-    for(int i=0;i<n_masks;i++){
-        mask[i] = readBUF<dnnType>(buf);
-    }
-    bias.resize(n_masks*2*num);
-    for(int i=0;i<n_masks*2*num;i++){
-        bias[i] = readBUF<dnnType>(buf);
-    }
-    classesNames.resize(classes);
-    for(int i=0;i<classes;i++){
-        char tmp[YOLORT_CLASSNAME_W];
-        for(int j=0;j<YOLORT_CLASSNAME_W;j++)
-            tmp[j] = readBUF<char>(buf);
-        classesNames[1] = std::string(tmp);
-    }
     assert(buf == bufCheck + length);
 }
 
@@ -147,8 +126,7 @@ int32_t YoloRT::enqueue(int32_t batchSize, const void *const *inputs, void **out
 
 
 size_t YoloRT::getSerializationSize() const NOEXCEPT {
-    return 8 * sizeof(int) + 2 * sizeof(float) + n_masks * sizeof(dnnType) + num * n_masks * 2 * sizeof(dnnType) +
+    return 8 * sizeof(int) + 2 * sizeof(float) ;
-           YOLORT_CLASSNAME_W * classes * sizeof(char);
 }
 
 bool YoloRT::supportsFormat(DataType type, PluginFormat format) const NOEXCEPT {
@@ -167,21 +145,7 @@ void YoloRT::serialize(void *buffer) const NOEXCEPT {
     writeBUF(buf, c);            //std::cout << "C : " << c << std::endl;
     writeBUF(buf, h);            //std::cout << "H : " << h << std::endl;
     writeBUF(buf, w);            //std::cout << "C : " << c << std::endl;
-    for (int i = 0; i < n_masks; i++) {
-        writeBUF(buf, mask[i]); //std::cout << "mask[i] : " << mask[i] << std::endl;
-    }
-    for (int i = 0; i < n_masks * 2 * num; i++) {
-        writeBUF(buf, bias[i]); //std::cout << "bias[i] : " << bias[i] << std::endl;
-    }
 
-    // save classes names
-    for (int i = 0; i < classes; i++) {
-        char tmp[YOLORT_CLASSNAME_W];
-        strcpy(tmp, classesNames[i].c_str());
-        for (int j = 0; j < YOLORT_CLASSNAME_W; j++) {
-            writeBUF(buf, tmp[j]);
-        }
-    }
     assert(buf == a + getSerializationSize());
 }
 
@@ -206,7 +170,7 @@ void YoloRT::setPluginNamespace(const char *pluginNamespace) NOEXCEPT {
 }
 
 IPluginV2Ext *YoloRT::clone() const NOEXCEPT {
-    auto *p = new YoloRT(classes, num,c,h,w,classesNames,mask,bias, n_masks, scaleXY, nms_thresh, nms_kind, new_coords);
+    auto *p = new YoloRT(classes, num,c,h,w,n_masks, scaleXY, nms_thresh, nms_kind, new_coords);
     p->setPluginNamespace(mPluginNamespace.c_str());
     return p;
 }
@@ -265,15 +229,12 @@ IPluginV2Ext *YoloRTPluginCreator::createPlugin(const char *name, const PluginFi
     int c = *(static_cast<const int *>(fields[2].data));
     int h = *(static_cast<const int *>(fields[3].data));
     int w = *(static_cast<const int *>(fields[4].data));
-    std::vector<std::string> classNames(static_cast<const std::string *>(fields[5].data),static_cast<const std::string *>(fields[5].data) + fields[5].length);
+    int n_masks = *(static_cast<const int *>(fields[5].data));
-    std::vector<dnnType> mask_v(static_cast<const dnnType*>(fields[6].data),static_cast<const dnnType*>(fields[6].data) + fields[6].length);
+    dnnType scaleXY = *(static_cast<const float*>(fields[6].data));
-    std::vector<dnnType> bias_v(static_cast<const dnnType*>(fields[7].data),static_cast<const dnnType*>(fields[7].data) + fields[7].length);
+    dnnType nmsThresh = *(static_cast<const float*>(fields[7].data));
-    int n_masks = *(static_cast<const int *>(fields[8].data));
+    int nms_kind = *(static_cast<const int*>(fields[8].data));
-    dnnType scaleXY = *(static_cast<const float*>(fields[9].data));
+    int new_coords = *(static_cast<const int*>(fields[9].data));
-    dnnType nmsThresh = *(static_cast<const float*>(fields[10].data));
+    auto *pluginObj = new YoloRT(classes,num,c,h,w,n_masks,scaleXY,nmsThresh,nms_kind,new_coords);
-    int nms_kind = *(static_cast<const int*>(fields[11].data));
-    int new_coords = *(static_cast<const int*>(fields[12].data));
-    auto *pluginObj = new YoloRT(classes,num,c,h,w,classNames,mask_v,bias_v,n_masks,scaleXY,nmsThresh,nms_kind,new_coords);
     return pluginObj;
 }
 

tests/centernet/dla34_cnet/dla34_cnet.cpp

@@ -492,7 +492,7 @@ int main()
 //    }
 
     //convert network to tensorRT
-    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(&net, net.getNetworkRTName("dla34_cnet"));
+    tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet"));
 
     tk::dnn::dataDim_t dim1 = dim; //input dim
     printCenteredTitle(" CUDNN inference ", '=', 30);
@@ -509,7 +509,7 @@ int main()
     {
         dim2.print();
         TKDNN_TSTART
-        netRT->infer(dim2, data);
+        netRT.infer(dim2, data);
         TKDNN_TSTOP
         dim2.print();
     }
@@ -528,7 +528,7 @@ int main()
 
         dnnType *cudnn_out, *rt_out;
         cudnn_out = outs[i]->dstData;
-        rt_out = (dnnType *)netRT->buffersRT[i+out_count];
+        rt_out = (dnnType *)netRT.buffersRT[i+out_count];
         // there is the maxpool. It isn't an output but it is necessary for the process section
         if(i==0)
             out_count ++;
@@ -540,6 +540,6 @@ int main()
         std::cout<<"CUDNN vs TRT    "; 
         ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
     }
-    netRT->destroy();
+    netRT.destroy();
     return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
 }

tests/shelfnet/shelfnet_berkeley.cpp

@@ -290,6 +290,6 @@ int main()
 
     cv::Mat viz = vizLayer2Mat(&net, net.num_layers-1);
     cv::imwrite("test.png", viz);
-    netRT.destroy();
+
     return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
 }