Mnist works at the moment with trt8,others like yolo4tiny and mobilenet generate the engine files but crash after throwing nvifer1::CudaRuntimeError and when demo is being run ,it doesnt deserialize properly and crashes

CMakeLists.txt

@@ -33,12 +33,13 @@ SET(CUDA_SEPARABLE_COMPILATION ON)
 #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'")
 set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
 
+
 find_package(CUDNN REQUIRED)
 include_directories(${CUDNN_INCLUDE_DIR})
 
 
 # compile
-file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu")
+file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu" )
 cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
 cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
 target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES})

demo/demo/demo.cpp

@@ -23,6 +23,18 @@ int main(int argc, char *argv[]) {
 
 
     std::string net = "yolo4tiny_fp32.rt";
+    #ifdef __linux__
+        std::string cfgPath = "../tests/darknet/cfg/yolo4tiny.cfg";
+    #elif _WIN32
+         std::string cfgPath = "..\\tests\\darknet\\cfg\\yolo4tiny.cfg";
+    #endif
+
+    #ifdef __linux__
+             std::string namePath = "../tests/darknet/names/coco.names";
+    #elif _WIN32
+            std::string namePath = "..\\tests\\darknet\\names\\coco.names";
+    #endif
+
     if(argc > 1)
         net = argv[1]; 
     #ifdef __linux__ 
@@ -31,23 +43,28 @@ int main(int argc, char *argv[]) {
         std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
     #endif
 
+
     if(argc > 2)
-        input = argv[2]; 
+        cfgPath = argv[2];
-    char ntype = 'y';
     if(argc > 3)
-        ntype = argv[3][0]; 
+        namePath = argv[3];
-    int n_classes = 80;
     if(argc > 4)
-        n_classes = atoi(argv[4]); 
+        input = argv[4];
-    int n_batch = 1;
+    char ntype = 'y';
     if(argc > 5)
-        n_batch = atoi(argv[5]); 
+        ntype = argv[5][0];
-    bool show = true;
+    int n_classes = 80;
     if(argc > 6)
-        show = atoi(argv[6]); 
+        n_classes = atoi(argv[6]);
-    float conf_thresh=0.3;
+    int n_batch = 1;
     if(argc > 7)
-        conf_thresh = atof(argv[7]);     
+        n_batch = atoi(argv[7]);
+    bool show = true;
+    if(argc > 8)
+        show = atoi(argv[8]);
+    float conf_thresh=0.3;
+    if(argc > 9)
+        conf_thresh = atof(argv[9]);
 
     if(n_batch < 1 || n_batch > 64)
         FatalError("Batch dim not supported");
@@ -56,8 +73,8 @@ int main(int argc, char *argv[]) {
         SAVE_RESULT = true;
 
     tk::dnn::Yolo3Detection yolo;
-    tk::dnn::CenternetDetection cnet;
+    //tk::dnn::CenternetDetection cnet;
-    tk::dnn::MobilenetDetection mbnet;  
+    //tk::dnn::MobilenetDetection mbnet;
 
     tk::dnn::DetectionNN *detNN;  
 
@@ -67,17 +84,17 @@ int main(int argc, char *argv[]) {
             detNN = &yolo;
             break;
         case 'c':
-            detNN = &cnet;
+            //detNN = &cnet;
             break;
         case 'm':
-            detNN = &mbnet;
+            //detNN = &mbnet;
             n_classes++;
             break;
         default:
         FatalError("Network type not allowed (3rd parameter)\n");
     }
 
-    detNN->init(net, n_classes, n_batch, conf_thresh);
+    detNN->init(net,cfgPath,namePath,n_classes,n_batch,conf_thresh);
 
     gRun = true;
 

include/tkDNN/DarknetParser.h

@@ -47,5 +47,8 @@ namespace tk { namespace dnn {
                          std::vector<tk::dnn::Layer*> &netLayers, const std::vector<std::string>& names);
     std::vector<std::string> darknetReadNames(const std::string& names_file);
     tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file);
+    void loadYoloInfo(const std::string &cfg_file,int lineNo,std::vector<float> &mask,std::vector<float> &anchors,int &num,int &classes,float &nms_thresh,int &nms_kind,int &coords);
+    void loadYoloInitInfo(int &channels,int &width,int &height,const std::string &cfg_file);
+    std::vector<int> noYolosLine(const std::string &cfg_file);
 
 }}

include/tkDNN/DetectionNN.h

@@ -87,7 +87,7 @@ class DetectionNN {
          * @param n_batches maximum number of batches to use in inference
          * @return true if everything is correct, false otherwise.
          */
-        virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0;
+        virtual bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0;
         
         /**
          * This method performs the whole detection of the NN.

include/tkDNN/NetworkRT.h

@@ -7,6 +7,7 @@
 #include "Layer.h"
 #include "NvInfer.h"
 #include <memory>
+#include <tkDNN/kernels.h>
 
 namespace tk { namespace dnn {
 
@@ -52,7 +53,6 @@ public:
 
 
 
-
 class NetworkRT {
 
 public:

include/tkDNN/Yolo3Detection.h

@@ -4,9 +4,9 @@
 #include "opencv2/opencv.hpp"
 
 #include "DetectionNN.h"
+#include "DarknetParser.h"
 
-namespace tk { namespace dnn { 
+namespace tk { namespace dnn {
-
 class Yolo3Detection : public DetectionNN
 {
 private:
@@ -19,12 +19,13 @@ private:
     tk::dnn::Yolo* getYoloLayer(int n=0);
 
     cv::Mat bgr_h;
+    std::vector<int> noYolos;
     
 public:
     Yolo3Detection() {};
     ~Yolo3Detection() {}; 
 
-    bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3);
+    bool init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path,const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3);
     void preprocess(cv::Mat &frame, const int bi=0);
     void postprocess(const int bi=0,const bool mAP=false);
 };

include/tkDNN/pluginsRT/ReorgRT.h

@@ -41,7 +41,7 @@ public:
 
 	virtual int enqueue(int batchSize, const void*const * inputs, void* const* outputs, void* workspace, cudaStream_t stream) NOEXCEPT override {
 
-		reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+		reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
 					  reinterpret_cast<dnnType*>(outputs[0]), 
 					  batchSize, c, h, w, stride, stream);
 		return 0;

include/tkDNN/pluginsRT/YoloRT.h

@@ -31,6 +31,7 @@ public:
         classes = readBUF<int>(buf);
         num = readBUF<int>(buf);
         n_masks = readBUF<int>(buf);
+        std::cout<<n_masks<<std::endl;
         scaleXY = readBUF<float>(buf);
         nms_thresh = readBUF<float>(buf);
         nms_kind = readBUF<int>(buf);

src/DarknetParser.cpp

@@ -32,6 +32,16 @@ namespace tk { namespace dnn {
         return values;
     }
 
+    std::vector<float> fromStringToFloatVec(const std::string& line, const char delimiter){
+        std::stringstream linestream(line);
+        std::string value;
+        std::vector<float> values;
+
+        while(getline(linestream,value,delimiter))
+            values.push_back(std::stof(value));
+        return values;
+    }
+
     bool darknetParseFields(const std::string& line, darknetFields_t& fields){
 
         std::string name,value;
@@ -268,7 +278,134 @@ namespace tk { namespace dnn {
         }
         return net;
     }
-        
+    std::vector<int> noYolosLine(const std::string &cfg_file){
-    
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+        std::string line;
+        std::vector<int> lineNo;
+        int count = 0;
+        while(std::getline(if_cfg,line)){
+            std::size_t found = line.find("#");
+            if ( found != std::string::npos ) {
+                line = line.substr(0, found);
+            }
+            // skip empty lines
+            if(line.empty())
+                continue;
+            if(line == "[yolo]"){
+                lineNo.push_back(count);
+
+
+            }
+            count++;
+        }
+        return lineNo;
+    }
+    void loadYoloInfo(const std::string &cfg_file,int lineNo,std::vector<float> &mask,std::vector<float> &anchors,int &num,int &classes,float &nms_thresh,int &nms_kind,int &coords){
+        std::vector<float> maskTemp,anchorsTemp;
+        int classesTemp,numTemp,nmsKindTemp;
+        int new_coordsTemp=0;
+        float nmsThreshTemp=0.45;
+
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+        std::string line;
+        int count = 0;
+        while(std::getline(if_cfg,line)){
+            std::string name,value;
+            std::size_t found = line.find("#");
+            if ( found != std::string::npos ) {
+                line = line.substr(0, found);
+            }
+            // skip empty lines
+            if(line.empty())
+                continue;
+            if(count > lineNo && count <=20){
+                    divideNameAndValue(line,name,value);
+                    if(name == "mask "){
+                        maskTemp = fromStringToFloatVec(value,',');
+                    }
+                    if(name == "anchors "){
+                        anchorsTemp = fromStringToFloatVec(value,',');
+                    }
+                    if(name == "classes"){
+                        classesTemp = std::stoi(value);
+                    }
+                    if(name == "num"){
+                        numTemp = std::stoi(value);
+                    }
+                    if(name == "nms_kind"){
+                        if(value == "greedynms"){
+                            nmsKindTemp = 0;
+                        }else if(value == "diounms"){
+                            nmsKindTemp=1;
+                        }
+                        else{
+                            std::cout<<"NMS NOT SUPPORTED DEFAULTING TO GREEDYNMS"<<std::endl;
+                            nmsKindTemp=0;
+                        }
+                    }
+                    if(name == "new_coords"){
+                        new_coordsTemp = std::stoi(value);
+                    }
+                    if(name == "beta_nms"){
+                        nmsThreshTemp = std::stof(value);
+                    }
+
+                count++;
+            }
+        }
+        mask = maskTemp;
+        anchors = anchorsTemp;
+        num = numTemp;
+        nms_kind = nmsKindTemp;
+        nms_thresh = nmsThreshTemp;
+        coords = new_coordsTemp;
+        classes = classesTemp;
+
+    }
+    void loadYoloInitInfo(int &channels,int &width,int &height,const std::string &cfg_file){
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+        std::string line;
+        int count = 0;
+
+        while(std::getline(if_cfg,line)){
+            if(count == 7){
+                std::string name,value;
+                divideNameAndValue(line,name,value);
+                if(name == "width"){
+                    width = std::stoi(value);
+                }
+            }
+
+            if(count == 8){
+                std::string name,value;
+                divideNameAndValue(line,name,value);
+                if(name == "height"){
+                    height = std::stoi(value);
+                }
+            }
+
+            if(count == 9){
+                std::string name,value;
+                divideNameAndValue(line,name,value);
+                if(name == "channels"){
+                    channels = std::stoi(value);
+                    break;
+                }
+                else{
+                    std::cerr<<"EXITING PROGRAM DUE TO INSUFFICENT DATA FROM CFG"<<std::endl;
+                    break;
+                }
+            }
+
+            count++;
+        }
+
+    }
 
 }}

src/NetworkRT.cpp

@@ -11,8 +11,42 @@
 #include "NetworkRT.h"
 #include "Int8Calibrator.h"
 
+
 using namespace nvinfer1;
 
+PluginFieldCollection tk::dnn::ActivationLeakyRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ActivationReLUCeilingPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ActivationMishRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ActivationLogisticRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::DeformableConvRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::RegionRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ReorgRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::UpsampleRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ShortcutRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ReshapeRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::MaxPoolFixedSizeRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::ResizeLayerRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::YoloRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::RouteRTPluginCreator::mFC{};
+PluginFieldCollection tk::dnn::FlattenConcatRTPluginCreator::mFC{};
+
+
+std::vector<PluginField> tk::dnn::ActivationLeakyRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ActivationReLUCeilingPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ActivationMishRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ActivationLogisticRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::DeformableConvRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::RegionRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ReorgRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::UpsampleRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ShortcutRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ReshapeRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::MaxPoolFixedSizeRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::ResizeLayerRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::YoloRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::RouteRTPluginCreator::mPluginAttributes;
+std::vector<PluginField> tk::dnn::FlattenConcatRTPluginCreator::mPluginAttributes;
+
 // Logger for info/warning/errors
 class Logger : public ILogger {
     void log(Severity severity, const char* msg) NOEXCEPT override {

src/Yolo3Detection.cpp

@@ -3,7 +3,7 @@
 
 namespace tk { namespace dnn {
 
-bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
+    bool Yolo3Detection::init(const std::string& tensor_path,const std::string& cfg_path,const std::string& name_path,const int n_classes, const int n_batches, const float conf_thresh) {
 
     //convert network to tensorRT
     std::cout<<(tensor_path).c_str()<<"\n";
@@ -14,28 +14,42 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
     tk::dnn::dataDim_t idim = netRT->input_dim;    
     idim.n = nBatches;
 
+    std::vector<int> yolosLine = noYolosLine(cfg_path);
+    noYolos = yolosLine;
+    int channels,height,width;
+    loadYoloInitInfo(channels,width,height,cfg_path);
 
-    if(netRT->pluginFactory->n_yolos < 2 ) {
+
+
+    if(yolosLine.size() < 2 ) {
         FatalError("this is not yolo3");
     }
 
-    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
-        YoloRT *yRT = netRT->pluginFactory->yolos[i];
-        classes = yRT->classes;
-        num = yRT->num;
-        nMasks = yRT->n_masks;
 
+    for(int i=0; i<noYolos.size(); i++) {
+        std::vector<float> maskTemp,anchorsTemp;
+        std::vector<std::string> classNamesTemp;
+        int classes,nms_kind,coords,numTemp;
+        float nmsthresh;
+        loadYoloInfo(cfg_path,yolosLine[i],maskTemp,anchorsTemp,numTemp,classes,nmsthresh,nms_kind,coords);
+        classNamesTemp = darknetReadNames(name_path);
+        num = numTemp/maskTemp.size();
+        nMasks = maskTemp.size();
+        dnnType* maskTempF;
+        dnnType* biasTempF;
+        maskTempF = maskTemp.data();
+        biasTempF = anchorsTemp.data();
         // make a yolo layer to interpret predictions
         yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias
         yolo[i]->mask_h = new dnnType[nMasks];
         yolo[i]->bias_h = new dnnType[num*nMasks*2];
-        memcpy(yolo[i]->mask_h, yRT->mask, sizeof(dnnType)*nMasks);
+        memcpy(yolo[i]->mask_h, maskTempF, sizeof(dnnType)*nMasks);
-        memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*nMasks*2);
+        memcpy(yolo[i]->bias_h, biasTempF, sizeof(dnnType)*num*nMasks*2);
-        yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w);
+        yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, channels, height, width);
-        yolo[i]->classesNames = yRT->classesNames;
+        yolo[i]->classesNames = classNamesTemp;
-        yolo[i]->nms_thresh = yRT->nms_thresh;
+        yolo[i]->nms_thresh = nmsthresh;
-        yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) yRT->nms_kind;
+        yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) nms_kind;
-        yolo[i]->new_coords = yRT->new_coords;
+        yolo[i]->new_coords = coords;
     }
 
     dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
@@ -94,10 +108,15 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
 
 void Yolo3Detection::postprocess(const int bi, const bool mAP){
 
+
+
     //get yolo outputs
+    if(noYolos.size() < 2){
+        FatalError("YOLOS WRONG!!");
+    }
     std::vector<float *> rt_out;
     //dnnType *rt_out[netRT->pluginFactory->n_yolos];
-    for(int i=0; i<netRT->pluginFactory->n_yolos; i++)
+    for(int i=0; i<noYolos.size(); i++)
         rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi);
 
     float x_ratio =  float(originalSize[bi].width) / float(netRT->input_dim.w);
@@ -105,7 +124,7 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
 
     // compute dets
     nDets = 0;
-    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
+    for(int i=0; i<noYolos.size(); i++) {
         yolo[i]->dstData = rt_out[i];
         yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords);
     }

tests/mnist/test_mnistRT.cpp

@@ -128,7 +128,7 @@ int main() {
 	builder->setMaxBatchSize(1);
 	config->setMaxWorkspaceSize(1 << 20);
 
-	auto engine = builder->buildCudaEngine(*network);
+	auto engine = builder->buildEngineWithConfig(*network,*config);
 	// we don't need the network any more
 	network->destroy();