add DLA, plugin for shortcut and leaky. new verison 0.4

Signed-off-by: xavier <micaelaverucchi@gmail.com>

.gitignore

@@ -2,4 +2,10 @@
 build/
 .vscode/
 *.bin
-*.pyc
+*.pyc
+*.prototxt
+*.caffemodel
+*.h5
+*.tar.gz
+*.weights
+.idea/

CMakeLists.txt

@@ -1,20 +1,126 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.5)
 
 project (tkDNN)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN)
 
-find_package(CUDA QUIET REQUIRED)
+# project specific flags
+if(DEBUG)
+    add_definitions(-DDEBUG)
+endif()
 
-cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS})
-cuda_add_library(kernels SHARED src/kernels/activation_elu.cu)
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS})
+#-------------------------------------------------------------------------------
-add_library(tkDNN SHARED src/Layer.cpp src/LayerWgs.cpp
+# CUDA
-    src/Dense.cpp src/Activation.cpp src/Conv2d.cpp src/Flatten.cpp src/MulAdd.cpp src/Pooling.cpp src/Softmax.cpp
+#-------------------------------------------------------------------------------
-    src/Network.cpp src/utils.cpp)
+find_package(CUDA 9.0 REQUIRED)
-target_link_libraries(tkDNN kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} -lcudnn)
+SET(CUDA_SEPARABLE_COMPILATION ON)
+#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'")
 
-add_executable(test_simple tests/test/test.cpp)
+find_package(CUDNN REQUIRED)
+
+# compile
+file(GLOB tkdnn_CUSRC "src/kernels/*.cu")
+cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
+cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
+
+
+#-------------------------------------------------------------------------------
+# External Libraries
+#-------------------------------------------------------------------------------
+find_package(OpenCV REQUIRED)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV")
+
+
+#-------------------------------------------------------------------------------
+# Build Libraries
+#-------------------------------------------------------------------------------
+file(GLOB tkdnn_SRC "src/*.cpp")
+set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS})
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES})
+add_library(tkDNN SHARED ${tkdnn_SRC})
+target_link_libraries(tkDNN ${tkdnn_LIBS})
+
+#static 
+#add_library(tkDNN_static STATIC ${tkdnn_SRC})
+#target_link_libraries(tkDNN_static ${tkdnn_LIBS})
+
+add_executable(test_simple tests/simple/test_simple.cpp)
 target_link_libraries(test_simple tkDNN)
 
-add_executable(test_mnist tests/mnist/test.cpp)
+add_executable(test_mnist tests/mnist/test_mnist.cpp)
 target_link_libraries(test_mnist tkDNN)
+
+add_executable(test_mnistRT tests/mnist/test_mnistRT.cpp)
+target_link_libraries(test_mnistRT tkDNN)
+
+## YOLO NETS
+add_executable(test_yolo tests/yolo/yolo.cpp)
+target_link_libraries(test_yolo tkDNN)
+
+add_executable(test_yolo_voc tests/yolo_voc/yolo_voc.cpp)
+target_link_libraries(test_yolo_voc tkDNN)
+
+add_executable(test_yolo_tiny tests/yolo_tiny/yolo_tiny.cpp)
+target_link_libraries(test_yolo_tiny tkDNN)
+
+add_executable(test_yolo_relu tests/yolo_relu/yolo_relu.cpp)
+target_link_libraries(test_yolo_relu tkDNN)
+
+
+add_executable(test_yolo_224 tests/yolo_224/yolo_224.cpp)
+target_link_libraries(test_yolo_224 tkDNN)
+
+add_executable(test_yolo_berkeley tests/yolo_berkeley/yolo_berkeley.cpp)
+target_link_libraries(test_yolo_berkeley tkDNN)
+
+add_executable(test_yolo3_coco4 tests/yolo3_coco4/yolo3_coco4.cpp)
+target_link_libraries(test_yolo3_coco4 tkDNN)
+
+add_executable(test_yolo3_berkeley tests/yolo3_berkeley/yolo3_berkeley.cpp)
+target_link_libraries(test_yolo3_berkeley tkDNN)
+
+add_executable(test_yolo3_flir tests/yolo3_flir/yolo3_flir.cpp)
+target_link_libraries(test_yolo3_flir tkDNN)
+################################################################################
+
+
+add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
+target_link_libraries(test_rtinference tkDNN)
+
+add_executable(yolo3_demo demo/demo/demo.cpp)
+target_link_libraries(yolo3_demo tkDNN)
+
+
+#-------------------------------------------------------------------------------
+# Install
+#-------------------------------------------------------------------------------
+#if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+#    set (CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" 
+#         CACHE PATH "default install path" FORCE)
+#endif()
+message("install dir:" ${CMAKE_INSTALL_PREFIX})
+install(DIRECTORY include/ DESTINATION include/)
+install(TARGETS tkDNN kernels DESTINATION lib)
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory
+        DESTINATION "share/tkDNN/cmake/" # target directory
+)
+
+
+#-------------------------------------------------------------------------------
+# Prepare for test
+#-------------------------------------------------------------------------------
+set(TEST_DATA true CACHE BOOL "If true download deps")
+if( ${TEST_DATA} )
+	message("Launching pre-build dependency installer script...")
+
+    execute_process (COMMAND bash -c "bash build_models.sh download"
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+
+	set(TEST_DATA false CACHE BOOL "If true download deps" FORCE)
+	message("Finished dowloading test weights")
+endif()
+

README.md

@@ -1,15 +1,12 @@
 # tkDNN
-tkDNN is a Deep Neural Network library built with cuDNN primitives specifically thought to work on NVIDIA TK1 board.<br>
+tkDNN is a Deep Neural Network library built with cuDNN primitives specifically thought to work on NVIDIA TK1(and all successive) board.<br>
 The main scope is to do high performance inference on already trained models.
-Currently supports the following layers:
 
-* Dense, fully interconnected
+this branch actually work on every NVIDIA GPU that support the dependencies:
-* Activation (RELU, ELU, SIGMOID, TANH)
+* CUDA 10.0
-* Convolutional 2D
+* CUDNN 7.603
-* Convolutional 3D
+* TENSORRT 6.01
-* Max and Average Pooling
+* OPENCV 4.1
-* Flatten 
-* Data preprocessing
 
 ## Workflow
 The recommended workflow follow these step:
@@ -24,62 +21,31 @@ Build with cmake
 mkdir build
 cd build
 cmake ..
+# use -DTEST_DATA=False to skip dataset download
 make
 ```
+during the cmake configuration it will be dowloaded the weights needed for running
+the tests
 
 ## Test
-There is a ready to use example on *test* directory, to try it you must generate the weights with Keras
+Assumiung you have correctly builded the library these are the test ready to exec:
-```
+* test_simple: a simple convolutional and dense network (CUDNN only)
-cd tests
+* test_mnist: the famous mnist netwok (CUDNN and TENSORRT)
-python test_model.py
+* test_mnistRT: the mnist network hardcoded in using tensorRT apis (TENSORRT only)
-```
+* test_yolo: YOLO detection network (CUDNN and TENSORRT)
-And then execute the inference on build directory
+* test_yolo_tiny: smaller version of YOLO (CUDNN and TENSRRT)
-```
+* test_yolo3_berkeley: our yolo3 version trained with BDD100K dateset 
-cd build
-./tkDNNtest 
-```
-this should output the same prediction as Keras.
 
-## Simple example
+## yolo3 berkeley demo detection
-Here is a example of the entire workflow on a simple model.
+For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process:
-Using the following Keras model save it to a file
-```python
-model = Sequential()
-model.add(Reshape((20, 1), input_shape=(20)))
-model.add(Dense(256))
-model.compile()
-
-# save model
-model.save("path/to/model.h5")
 ```
-
+export TKDNN_MODE=FP16   # set the half floating point optimization
-After the model is created the weights can be exported for tkDNN inference
+rm yolo3_berkeley.rt		 # be sure to delete(or move) old tensorRT files
+./test_yolo3_berkeley              # run the yolo test (is slow)
+# with f16 inference the result will be a bit incorrect
 ```
-python weights_exporter model.h5 dense --output=weights/path
+this will genereate a yolo3_berkeley.rt file that can be used for live detection:
+```
+./yolo3_demo                               # launch detection on a demo video
+./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0
 ```
-the exporter take as arguments, in order:
-* input model
-* layer type ["dense", "conv2d", conv3d"]
-* { layer type ["dense", "conv2d", conv3d"] for each layer to export }
-* optional argument --output define path where export weights   
-
-Then we can create a c++ program to do inference on tk1
-```c++
-#include<tkdnn.h>   //library include
-
-//Network object 
-tkDNN::Network net;
-//input dimension
-tkDNN::dataDim_t dim(1, 20, 1, 1, 1);
-//Dense layer
-tkDNN::Dense d0(&net, dim, 256, "weights/path", "bias/path");
-
-//here load the input data to CUDA
-//value_type is an alias of "float"
-value_type *data_d = [...] 
-
-//do inference
-value_type *output_d = d0.infer(dim, data_d);
-//dim will be updated with the output dimension
-``` 
-The result is finally stored on output_d in device memory.

cmake/FindCUDNN.cmake

@@ -0,0 +1,33 @@
+# Find the header files
+
+find_path(CUDNN_INCLUDE_DIR
+        ${CMAKE_SYSROOT}/usr/local/include
+        ${CMAKE_SYSROOT}/usr/include
+        /usr/local/nvidia/tensorrt/include/
+        NO_DEFAULT_PATH
+)
+
+set(OLD_ROOT ${CMAKE_FIND_ROOT_PATH})
+list(APPEND CMAKE_FIND_ROOT_PATH /)
+list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.7)
+list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.5)
+find_library(CUDNN_LIB
+    NAMES cudnn
+    PATHS
+    /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib
+    /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/
+    NO_DEFAULT_PATH
+)
+find_library(CUDNN_NVLIB
+    NAMES "nvinfer"
+    PATHS
+    /usr/local/driveworks/targets/${CMAKE_SYSTEM_PROCESSOR}-Linux/lib
+    /usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/
+    NO_DEFAULT_PATH
+)
+set(CMAKE_FIND_ROOT_PATH ${OLD_ROOT})
+
+set(CUDNN_LIBRARIES ${CUDNN_LIB} ${CUDNN_NVLIB})
+message("-- Found CUDNN: "  ${CUDNN_LIB})
+message("-- Found NVINFER: "  ${CUDNN_NVLIB})
+set(CUDNN_FOUND true)

cmake/tkDNNConfig.cmake

@@ -0,0 +1,24 @@
+message("-- Found tkDNN")
+set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
+set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} --std=c++11 -fPIC")
+
+find_package(CUDA REQUIRED)
+find_package(OpenCV REQUIRED)
+find_package(CUDNN REQUIRED)
+
+set(tkDNN_INCLUDE_DIRS 
+	${CUDA_INCLUDE_DIRS} 
+	${OPENCV_INCLUDE_DIRS} 
+    ${CUDNN_INCLUDE_DIRS}
+)
+
+set(tkDNN_LIBRARIES 
+    tkDNN 
+    kernels 
+    ${CUDA_LIBRARIES} 
+    ${CUDA_CUBLAS_LIBRARIES}
+	${CUDNN_LIBRARIES}
+	${OpenCV_LIBS}
+)
+
+set(tkDNN_FOUND true)

demo/demo/demo.cpp

@@ -0,0 +1,109 @@
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>     /* srand, rand */
+#include <unistd.h>
+#include <mutex>
+#include "utils.h"
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "Yolo3Detection.h"
+
+bool 			gRun;
+bool SAVE_RESULT = false;
+
+void sig_handler(int signo) {
+    std::cout<<"request gateway stop\n";
+    gRun = false;
+}
+
+int main(int argc, char *argv[]) {
+
+    std::cout<<"detection\n";
+    signal(SIGINT, sig_handler);
+
+
+    char *net = "yolo3_berkeley.rt";
+    if(argc > 1)
+        net = argv[1]; 
+    char *input = "../demo/yolo_test.mp4";
+    if(argc > 2)
+        input = argv[2]; 
+
+    tk::dnn::Yolo3Detection yolo;
+    yolo.init(net);
+
+    gRun = true;
+
+    cv::VideoCapture cap(input);
+    if(!cap.isOpened())
+        gRun = false; 
+    else
+        std::cout<<"camera started\n";
+
+
+    cv::VideoWriter resultVideo;
+    if(SAVE_RESULT) {
+        int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
+        int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
+        resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
+    }
+
+    cv::Mat frame;
+    cv::Mat dnn_input;
+    cv::namedWindow("detection", cv::WINDOW_NORMAL);
+    
+    while(gRun) {
+        cap >> frame; 
+        if(!frame.data) {
+            break;
+        }  
+ 
+        // this will be resized to the net format
+        dnn_input = frame.clone();
+        // TODO: async infer
+        yolo.update(dnn_input);
+
+        // draw dets
+        for(int i=0; i<yolo.detected.size(); i++) {
+            tk::dnn::box b = yolo.detected[i];
+            int x0   				= b.x;
+            int x1   				= b.x + b.w;
+            int y0   				= b.y;
+            int y1   				= b.y + b.h;
+            std::string det_class 	= yolo.getYoloLayer()->classesNames[b.cl];
+            float prob 				= b.prob;
+
+            std::cout<<det_class<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
+			// draw rectangle
+            cv::rectangle(frame, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[b.cl], 2); 
+
+	        // draw label
+            int baseline = 0;
+            float fontScale = 0.5;
+            int thickness = 2;
+            cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
+            cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);                      
+            cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
+        }
+    
+        cv::imshow("detection", frame);
+        cv::waitKey(1);
+        if(SAVE_RESULT)
+            resultVideo << frame;
+    }
+
+    std::cout<<"detection end\n";   
+
+
+    std::cout<<COL_GREENB<<"\n\nTime stats:\n";
+    std::cout<<"Min: "<<*std::min_element(yolo.stats.begin(), yolo.stats.end())<<" ms\n";    
+    std::cout<<"Max: "<<*std::max_element(yolo.stats.begin(), yolo.stats.end())<<" ms\n";    
+    double mean = 0; for(int i=0; i<yolo.stats.size(); i++) mean += yolo.stats[i]; mean /= yolo.stats.size();
+    std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;    
+    return 0;
+}
+

include/Layer.h

@@ -1,226 +0,0 @@
-#ifndef LAYER_H
-#define LAYER_H
-
-#include<iostream>
-#include "utils.h"
-#include "Network.h"
-
-namespace tkDNN {
-
-/**
-    Data rapresentation beetween layers
-    n = batch size
-    c = channels
-    h = heigth (lines)
-    w = width  (rows)
-    l = lenght (3rd dimension)
-*/
-struct dataDim_t {
-
-    int n, c, h, w, l;
-
-    dataDim_t() : n(1), c(1), h(1), w(1), l(1) {};
-
-    dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) :
-        n(_n), c(_c), h(_h), w(_w), l(_l) {};
-
-    void print() {
-        std::cout<<"Data dim: "<<n<<" "<<c<<" "<<h<<" "<<w<<" "<<l<<"\n";
-    }
-
-    int tot() {
-        return n*c*h*w*l;
-    }
-};
-
-
-/** 
-    Simple layer Father class
-*/
-class Layer {
-
-public:
-    Layer(Network *net, dataDim_t input_dim);
-    virtual ~Layer();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData) {
-        std::cout<<"No infer action for this layer\n";
-        return NULL;
-    }
-
-    dataDim_t input_dim, output_dim;
-
-protected:
-    Network *net;
-    cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc;
-};
-
-
-/**
-    Father class of all layer that need to load trained weights
-*/
-class LayerWgs : public Layer {
-
-public:
-    LayerWgs(Network *net, dataDim_t input_dim, 
-             int inputs, int outputs, int kh, int kw, int kt, 
-             const char* fname_weights, const char* fname_bias); 
-    virtual ~LayerWgs();
-
-protected:
-    int inputs, outputs;
-    std::string weights_path, bias_path;
-
-    value_type *data_h, *data_d;
-    value_type *bias_h, *bias_d;
-};
-
-
-/**
-    Dense (full interconnection) layer
-*/
-class Dense : public LayerWgs {
-
-public:
-    Dense(Network *net, dataDim_t in_dim, int out_ch, 
-          const char* fname_weights, const char* fname_bias); 
-    virtual ~Dense();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-    value_type *dstData;  //where results will be putted
-};
-
-
-/**
-    Activation layer (it doesnt need weigths)
-*/
-class Activation : public Layer {
-
-public:
-    Activation(Network *net, dataDim_t input_dim, cudnnActivationMode_t act_mode); 
-    virtual ~Activation();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-    cudnnActivationMode_t act_mode;
-    cudnnActivationDescriptor_t activDesc;
-    value_type *dstData;  //where results will be putted
-};
-
-
-/**
-    Convolutional 2D layer
-*/
-class Conv2d : public LayerWgs {
-
-public:
-    Conv2d(Network *net, dataDim_t in_dim, int out_ch,
-            int kernelH, int kernelW, int strideH, int strideW,
-            const char* fname_weights, const char* fname_bias); 
-    virtual ~Conv2d();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-    value_type *dstData;  //where results will be putted
-    int kernelH, kernelW, strideH, strideW;
-
-    cudnnFilterDescriptor_t filterDesc;
-    cudnnConvolutionDescriptor_t convDesc;
-    cudnnConvolutionFwdAlgo_t algo;
-    cudnnTensorDescriptor_t biasTensorDesc;
-
-    void*  workSpace;
-    size_t ws_sizeInBytes;
-};
-
-
-/**
-    Flatten layer
-    is actually a matrix transposition
-*/
-class Flatten : public Layer {
-
-public:
-    Flatten(Network *net, dataDim_t input_dim); 
-    virtual ~Flatten();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-    value_type *dstData;  //where results will be putted
-};
-
-
-/**
-    MulAdd layer
-    apply a multiplication and then an addition for each data
-*/
-class MulAdd : public Layer {
-
-public:
-    MulAdd(Network *net, dataDim_t input_dim, value_type mul, value_type add); 
-    virtual ~MulAdd();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-    value_type mul, add;
-    value_type *dstData, *add_vector;  //where results will be putted
-};
-
-
-
-/**
-    Avaible pooling functions (padding on tkDNN is not supported)
-*/
-typedef enum {
-    POOLING_MAX     = 0,
-    POOLING_AVERAGE = 1,                  // count for average includes padded values
-    POOLING_AVERAGE_EXCLUDE_PADDING = 2   // count for average does not include padded values
-} tkdnnPoolingMode_t;
-
-/**
-    Pooling layer
-    currenty supported only 2d pooing (also on 3d input)
-*/
-class Pooling : public Layer {
-
-public:
-    Pooling(Network *net, dataDim_t input_dim, int winH, int winW, 
-            int strideH, int strideW, tkdnnPoolingMode_t pool_mode); 
-    virtual ~Pooling();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-
-    cudnnPoolingDescriptor_t poolingDesc;
-
-    int winH, winW;
-    int strideH, strideW;
-    tkdnnPoolingMode_t pool_mode;
-    value_type *dstData, *tmpInputData, *tmpOutputData;  //where results will be putted
-    bool poolOn3d;
-};
-
-/**
-    Softmax layer
-*/
-class Softmax : public Layer {
-
-public:
-    Softmax(Network *net, dataDim_t input_dim); 
-    virtual ~Softmax();
-
-    virtual value_type* infer(dataDim_t &dim, value_type* srcData);
-
-protected:
-    value_type *dstData;  //where results will be putted
-};
-
-}
-#endif //LAYER_H

include/Network.h

@@ -1,36 +0,0 @@
-#ifndef NETWORK_H
-#define NETWORK_H
-
-#include "utils.h"
-
-namespace tkDNN {
-
-struct dataDim_t;
-class Layer;
-const int MAX_LAYERS = 256;
-
-class Network {
-
-public:
-    Network();
-    virtual ~Network();
-
-    /**
-        Do inferece for every added layer
-    */
-    value_type* infer(dataDim_t &dim, value_type* data);
-
-    bool addLayer(Layer *l);
-
-    cudnnDataType_t dataType;
-    cudnnTensorFormat_t tensorFormat;
-    cudnnHandle_t cudnnHandle;
-    cublasHandle_t cublasHandle;
-
-private:
-    Layer* layers[MAX_LAYERS];  //contains layers of the net
-    int num_layers; //current number of layers
-};
-
-}
-#endif //NETWORK_H

include/kernels.h

@@ -1,3 +0,0 @@
-#include "utils.h"
-
-void activationELUForward(value_type* srcData, value_type* dstData, int size);

include/tkDNN/Layer.h

@@ -0,0 +1,423 @@
+#ifndef LAYER_H
+#define LAYER_H
+
+#include<iostream>
+#include<vector>
+#include "utils.h"
+#include "Network.h"
+
+namespace tk { namespace dnn {
+
+enum layerType_t {
+    LAYER_DENSE,
+    LAYER_CONV2D,
+    LAYER_ACTIVATION,
+    LAYER_FLATTEN,
+    LAYER_MULADD,
+    LAYER_POOLING,
+    LAYER_SOFTMAX,
+    LAYER_ROUTE,
+    LAYER_REORG,
+    LAYER_SHORTCUT,
+    LAYER_UPSAMPLE,
+    LAYER_REGION,
+    LAYER_YOLO
+};
+
+#define TKDNN_BN_MIN_EPSILON 1e-5
+
+/** 
+    Simple layer Father class
+*/
+class Layer {
+
+public:
+    Layer(Network *net);
+    virtual ~Layer();
+    virtual layerType_t getLayerType() = 0;
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) {
+        std::cout<<"No infer action for this layer\n";
+        return NULL;
+    }
+
+    dataDim_t input_dim, output_dim;
+    dnnType *dstData;  //where results will be putted
+
+    std::string getLayerName() {
+        layerType_t type = getLayerType();
+        switch(type) {
+            case LAYER_DENSE:       return "Dense";
+            case LAYER_CONV2D:      return "Conv2d";
+            case LAYER_ACTIVATION:  return "Activation";
+            case LAYER_FLATTEN:     return "Flatten";
+            case LAYER_MULADD:      return "MulAdd";
+            case LAYER_POOLING:     return "Pooling";
+            case LAYER_SOFTMAX:     return "Softmax";
+            case LAYER_ROUTE:       return "Route";            
+            case LAYER_REORG:       return "Reorg";
+            case LAYER_SHORTCUT:    return "Shortcut";
+            case LAYER_UPSAMPLE:    return "Upsample";
+            case LAYER_REGION:      return "Region";
+            case LAYER_YOLO:        return "Yolo";
+            default:                return "unknown";
+        }
+    }
+
+protected:
+    Network *net;
+    cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc;
+
+};
+
+
+/**
+    Father class of all layer that need to load trained weights
+*/
+class LayerWgs : public Layer {
+
+public:
+    LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt, 
+             std::string fname_weights, bool batchnorm = false); 
+    virtual ~LayerWgs();
+
+    int inputs, outputs;
+    std::string weights_path;
+
+    dnnType *data_h, *data_d;
+    dnnType *bias_h, *bias_d;
+
+    //batchnorm
+    bool batchnorm;
+    dnnType *power_h;
+    dnnType *scales_h,   *scales_d;
+    dnnType *mean_h,     *mean_d;
+    dnnType *variance_h, *variance_d;
+
+    //fp16
+    __half *data16_h, *bias16_h;
+    __half *data16_d, *bias16_d;
+
+    __half *power16_h,    *power16_d;
+    __half *scales16_h,   *scales16_d;
+    __half *mean16_h,     *mean16_d;
+    __half *variance16_h, *variance16_d;
+};
+
+
+/**
+    Dense (full interconnection) layer
+*/
+class Dense : public LayerWgs {
+
+public:
+    Dense(Network *net, int out_ch, std::string fname_weights); 
+    virtual ~Dense();
+    virtual layerType_t getLayerType() { return LAYER_DENSE; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+};
+
+
+/**
+    Avaible activation functions
+*/
+typedef enum {
+    ACTIVATION_ELU     = 100,
+    ACTIVATION_LEAKY   = 101
+} tkdnnActivationMode_t;
+
+/**
+    Activation layer (it doesnt need weigths)
+*/
+class Activation : public Layer {
+
+public:
+    int act_mode;
+
+    Activation(Network *net, int act_mode); 
+    virtual ~Activation();
+    virtual layerType_t getLayerType() { return LAYER_ACTIVATION; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+protected:
+    cudnnActivationDescriptor_t activDesc;
+};
+
+
+/**
+    Convolutional 2D layer
+*/
+class Conv2d : public LayerWgs {
+
+public:
+    Conv2d( Network *net, int out_ch, int kernelH, int kernelW, 
+                int strideH, int strideW, int paddingH, int paddingW,
+                std::string fname_weights, bool batchnorm = false); 
+    virtual ~Conv2d();
+    virtual layerType_t getLayerType() { return LAYER_CONV2D; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+    int kernelH, kernelW, strideH, strideW, paddingH, paddingW;
+
+protected:
+    cudnnFilterDescriptor_t filterDesc;
+    cudnnConvolutionDescriptor_t convDesc;
+    cudnnConvolutionFwdAlgo_t algo;
+    cudnnTensorDescriptor_t biasTensorDesc;
+
+    void*  workSpace;
+    size_t ws_sizeInBytes;
+};
+
+
+/**
+    Flatten layer
+    is actually a matrix transposition
+*/
+class Flatten : public Layer {
+
+public:
+    Flatten(Network *net); 
+    virtual ~Flatten();
+    virtual layerType_t getLayerType() { return LAYER_FLATTEN; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+};
+
+
+/**
+    MulAdd layer
+    apply a multiplication and then an addition for each data
+*/
+class MulAdd : public Layer {
+
+public:
+    MulAdd(Network *net, dnnType mul, dnnType add); 
+    virtual ~MulAdd();
+    virtual layerType_t getLayerType() { return LAYER_MULADD; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+protected:
+    dnnType mul, add;
+    dnnType *add_vector;
+};
+
+
+
+/**
+    Avaible pooling functions (padding on tkDNN is not supported)
+*/
+typedef enum {
+    POOLING_MAX     = 0,
+    POOLING_AVERAGE = 1,                  // count for average includes padded values
+    POOLING_AVERAGE_EXCLUDE_PADDING = 2   // count for average does not include padded values
+} tkdnnPoolingMode_t;
+
+/**
+    Pooling layer
+    currenty supported only 2d pooing (also on 3d input)
+*/
+class Pooling : public Layer {
+
+public:
+    int winH, winW;
+    int strideH, strideW;
+    int paddingH, paddingW;
+
+    Pooling(Network *net, int winH, int winW, 
+            int strideH, int strideW, tkdnnPoolingMode_t pool_mode); 
+    virtual ~Pooling();
+    virtual layerType_t getLayerType() { return LAYER_POOLING; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+protected:
+
+    cudnnPoolingDescriptor_t poolingDesc;
+    tkdnnPoolingMode_t pool_mode;
+    dnnType *tmpInputData, *tmpOutputData;
+    bool poolOn3d;
+};
+
+/**
+    Softmax layer
+*/
+class Softmax : public Layer {
+
+public:
+    Softmax(Network *net); 
+    virtual ~Softmax();
+    virtual layerType_t getLayerType() { return LAYER_SOFTMAX; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+};
+
+/**
+    Route layer
+    Merge a list of layers
+*/
+class Route : public Layer {
+
+public:
+    Route(Network *net, Layer **layers, int layers_n); 
+    virtual ~Route();
+    virtual layerType_t getLayerType() { return LAYER_ROUTE; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+public:
+    Layer **layers;  //ids of layers to be merged
+    int layers_n; //number of layers
+};
+
+
+/**
+    Reorg layer
+    Mantain same dimension but change C*H*W distribution
+*/
+class Reorg : public Layer {
+
+public:
+    Reorg(Network *net, int stride);
+    virtual ~Reorg();
+    virtual layerType_t getLayerType() { return LAYER_REORG; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+    int stride;
+};
+
+/**
+    Shortcut layer
+    sum with stride another layer
+*/
+class Shortcut : public Layer {
+
+public:
+    Shortcut(Network *net, Layer *backLayer); 
+    virtual ~Shortcut();
+    virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+public:
+    Layer *backLayer;
+};
+
+/**
+    Upsample layer
+    Mantain same dimension but change C*H*W distribution
+*/
+class Upsample : public Layer {
+
+public:
+    Upsample(Network *net, int stride);
+    virtual ~Upsample();
+    virtual layerType_t getLayerType() { return LAYER_UPSAMPLE; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+    int stride;
+    bool reverse;
+};
+
+struct box {
+    int cl;
+    float x, y, w, h;
+    float prob;
+};
+struct sortable_bbox {
+    int index;
+    int cl;
+    float **probs;
+};
+
+/**
+    Yolo3 layer
+*/
+class Yolo : public Layer {
+
+public:
+    struct box {
+        float x, y, w, h;
+    };
+
+    struct detection{
+        Yolo::box bbox;
+        int classes;
+        float *prob;
+        float *mask;
+        float objectness;
+        int sort_class;
+    };
+
+    Yolo(Network *net, int classes, int num, std::string fname_weights);
+    virtual ~Yolo();
+    virtual layerType_t getLayerType() { return LAYER_YOLO; };
+
+    int classes, num;
+    dnnType *mask_h, *mask_d; //anchors
+    dnnType *bias_h, *bias_d; //anchors
+    std::vector<std::string> classesNames;
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh);
+
+    dnnType *predictions;
+
+    static const int MAX_DETECTIONS = 256;
+    static Yolo::detection *allocateDetections(int nboxes, int classes);
+    static void             mergeDetections(Yolo::detection *dets, int ndets, int classes);
+};
+
+/**
+    Region layer
+*/
+class Region : public Layer {
+
+public:
+    Region(Network *net, int classes, int coords, int num);
+    virtual ~Region();
+    virtual layerType_t getLayerType() { return LAYER_REGION; };
+
+    int classes, coords, num;
+    
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+};
+
+class RegionInterpret {
+
+public:
+    RegionInterpret(dataDim_t input_dim, dataDim_t output_dim, 
+                    int classes, int coords, int num, float thresh, std::string fname_weights);
+    ~RegionInterpret();
+
+    dataDim_t input_dim, output_dim;
+    dnnType *bias_h, *bias_d; //anchors
+    int classes, coords, num;
+    float thresh;
+
+    
+    box *boxes;
+    float **probs;
+    sortable_bbox *s;
+    box res_boxes[256];
+    int res_boxes_n;
+
+    box get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride);
+    void get_region_boxes(  float *input, int w, int h, int netw, int neth, float thresh, 
+                            float **probs, box *boxes, int only_objectness, 
+                            int *map, float tree_thresh, int relative); 
+    void correct_region_boxes(box *boxes, int n, int w, int h, int netw, int neth, int relative);
+    void interpretData(dnnType *data_h, int imageW = 0, int imageH = 0);
+    void showImageResult(dnnType *input_h);
+
+    static float box_iou(box a, box b);
+};
+
+}}
+#endif //LAYER_H

include/tkDNN/Network.h

@@ -0,0 +1,66 @@
+#ifndef NETWORK_H
+#define NETWORK_H
+
+#include "utils.h"
+
+namespace tk { namespace dnn {
+
+/**
+    Data rapresentation beetween layers
+    n = batch size
+    c = channels
+    h = heigth (lines)
+    w = width  (rows)
+    l = lenght (3rd dimension)
+*/
+struct dataDim_t {
+
+    int n, c, h, w, l;
+
+    dataDim_t() : n(1), c(1), h(1), w(1), l(1) {};
+
+    dataDim_t(int _n, int _c, int _h, int _w, int _l = 1) :
+        n(_n), c(_c), h(_h), w(_w), l(_l) {};
+
+    void print() {
+        std::cout<<"Data dim: "<<n<<" "<<c<<" "<<h<<" "<<w<<" "<<l<<"\n";
+    }
+
+    int tot() {
+        return n*c*h*w*l;
+    }
+};
+
+class Layer;
+const int MAX_LAYERS = 256;
+
+class Network {
+
+public:
+    Network(dataDim_t input_dim);
+    virtual ~Network();
+
+    /**
+        Do inferece for every added layer
+    */
+    dnnType* infer(dataDim_t &dim, dnnType* data);
+
+    bool addLayer(Layer *l);
+    void print();
+
+    cudnnDataType_t dataType;
+    cudnnTensorFormat_t tensorFormat;
+    cudnnHandle_t cudnnHandle;
+    cublasHandle_t cublasHandle;
+
+    Layer* layers[MAX_LAYERS];  //contains layers of the net
+    int num_layers; //current number of layers
+
+    dataDim_t input_dim;
+    dataDim_t getOutputDim();
+
+    bool fp16, dla;
+};
+
+}}
+#endif //NETWORK_H

include/tkDNN/NetworkRT.h

@@ -0,0 +1,94 @@
+#ifndef NETWORKRT_H
+#define NETWORKRT_H
+
+#include <string.h> // memcpy
+#include "utils.h"
+#include "Network.h"
+#include "Layer.h"
+#include "NvInfer.h"
+
+namespace tk { namespace dnn {
+
+template<typename T> void writeBUF(char*& buffer, const T& val)
+{
+    *reinterpret_cast<T*>(buffer) = val;
+    buffer += sizeof(T);
+}
+
+template<typename T> T readBUF(const char*& buffer)
+{
+    T val = *reinterpret_cast<const T*>(buffer);
+    buffer += sizeof(T);
+    return val;
+}
+
+using namespace nvinfer1;
+#include "pluginsRT/ActivationLeakyRT.h"
+#include "pluginsRT/ReorgRT.h"
+#include "pluginsRT/RegionRT.h"
+//#include "pluginsRT/RouteRT.h"
+#include "pluginsRT/ShortcutRT.h"
+#include "pluginsRT/YoloRT.h"
+#include "pluginsRT/UpsampleRT.h"
+//#include "pluginsRT/Int8Calibrator.h"
+
+class PluginFactory : IPluginFactory
+{
+public:
+    YoloRT *yolos[16];
+    int n_yolos;
+
+	virtual IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength);
+};
+
+
+
+class NetworkRT {
+
+public:
+    nvinfer1::DataType dtRT;
+    nvinfer1::IBuilder *builderRT;
+    nvinfer1::IRuntime *runtimeRT;
+    nvinfer1::INetworkDefinition *networkRT; 
+    
+    nvinfer1::ICudaEngine *engineRT;
+    nvinfer1::IExecutionContext *contextRT;
+
+    const static int MAX_BUFFERS_RT = 10;
+    void* buffersRT[MAX_BUFFERS_RT];
+    int buf_input_idx, buf_output_idx;
+
+    dataDim_t input_dim, output_dim;
+    dnnType *output;
+    cudaStream_t stream;
+
+    PluginFactory *pluginFactory;
+
+    NetworkRT(Network *net, const char *name);
+    virtual ~NetworkRT();
+
+    /**
+        Do inferece
+    */
+    dnnType* infer(dataDim_t &dim, dnnType* data);
+    void enqueue();    
+
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Activation *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Dense *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
+
+    bool serialize(const char *filename);
+    bool deserialize(const char *filename);
+};
+
+}}
+#endif //NETWORKRT_H

include/tkDNN/Yolo3Detection.h

@@ -0,0 +1,67 @@
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>     /* srand, rand */
+#include <unistd.h>
+#include <mutex>
+#include "utils.h"
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "tkdnn.h"
+
+namespace tk { namespace dnn {
+
+/**
+ * 
+ * @author Francesco Gatti
+ */
+class Yolo3Detection {
+
+    private:
+        tk::dnn::NetworkRT *netRT = nullptr;
+        tk::dnn::Yolo* yolo[3];
+        dnnType *input, *input_d;
+
+        int ndets = 0;
+        tk::dnn::Yolo::detection *dets = nullptr;
+
+        cv::Mat imageF;
+        cv::Mat bgr[3]; 
+
+    public:
+        int classes = 0;
+        int num = 0;
+        float thresh = 0.3;
+        cv::Scalar colors[256];
+
+        // this is filled with results
+        std::vector<tk::dnn::box> detected;
+
+        // keep track of inference times (ms)
+        std::vector<double> stats;
+
+        Yolo3Detection() {}
+
+        virtual ~Yolo3Detection() {}
+
+        /**
+         * Method used for inizialize the class
+         * 
+         * @return Success of the initialization
+         */
+        bool init(std::string tensor_path);
+
+        void update(cv::Mat &frame);
+
+        tk::dnn::Yolo* getYoloLayer(int n=0) {
+            if(n<3)
+                return yolo[n];
+            else 
+                return nullptr;
+        }
+
+};
+
+}}

include/tkDNN/kernels.h

@@ -0,0 +1,27 @@
+#ifndef KERNELS_H
+#define KERNELS_H
+
+#include "utils.h"
+
+void activationELUForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationLOGISTICForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream = cudaStream_t(0));
+
+void fill(dnnType* data, int size, dnnType val, cudaStream_t stream = cudaStream_t(0));
+
+void reorgForward(  dnnType* srcData, dnnType* dstData, 
+                    int n, int c, int h, int w, int stride, cudaStream_t stream = cudaStream_t(0));
+void softmaxForward(float *input, int n, int batch, int batch_offset, 
+                    int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream = cudaStream_t(0));
+
+
+void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1,
+                                                         int n2, int c2, int h2, int w2, int s2, 
+                     cudaStream_t stream = cudaStream_t(0));
+
+void upsampleForward(dnnType* srcData, dnnType* dstData, 
+                     int n, int c, int h, int w, int s, int forward, float scale,                                   
+                     cudaStream_t stream = cudaStream_t(0));
+
+void float2half(float* srcData, __half* dstData, int size, const cudaStream_t stream = cudaStream_t(0));
+#endif //KERNELS_H

include/tkDNN/models/Yolo3.h

@@ -0,0 +1,289 @@
+int preYoloFilters = (classes+5)*3; 
+
+std::string input_bin = bin_path + "/layers/input.bin";
+std::vector<std::string> output_bins = {
+    bin_path + "/debug/layer82_out.bin",
+    bin_path + "/debug/layer94_out.bin",
+    bin_path + "/debug/layer106_out.bin"
+};
+std::string c0_bin    = bin_path + "/layers/c0.bin";
+std::string c1_bin    = bin_path + "/layers/c1.bin";
+std::string c2_bin    = bin_path + "/layers/c2.bin";
+std::string c3_bin    = bin_path + "/layers/c3.bin";
+std::string c5_bin    = bin_path + "/layers/c5.bin";
+std::string c6_bin    = bin_path + "/layers/c6.bin";
+std::string c7_bin    = bin_path + "/layers/c7.bin";
+std::string c9_bin    = bin_path + "/layers/c9.bin";
+std::string c10_bin   = bin_path + "/layers/c10.bin";
+std::string c12_bin   = bin_path + "/layers/c12.bin";
+std::string c13_bin   = bin_path + "/layers/c13.bin";
+std::string c14_bin   = bin_path + "/layers/c14.bin";
+std::string c16_bin   = bin_path + "/layers/c16.bin";
+std::string c17_bin   = bin_path + "/layers/c17.bin";
+std::string c19_bin   = bin_path + "/layers/c19.bin";
+std::string c20_bin   = bin_path + "/layers/c20.bin";
+std::string c22_bin   = bin_path + "/layers/c22.bin";
+std::string c23_bin   = bin_path + "/layers/c23.bin";
+std::string c25_bin   = bin_path + "/layers/c25.bin";
+std::string c26_bin   = bin_path + "/layers/c26.bin";
+std::string c28_bin   = bin_path + "/layers/c28.bin";
+std::string c29_bin   = bin_path + "/layers/c29.bin";
+std::string c31_bin   = bin_path + "/layers/c31.bin";
+std::string c32_bin   = bin_path + "/layers/c32.bin";
+std::string c34_bin   = bin_path + "/layers/c34.bin";
+std::string c35_bin   = bin_path + "/layers/c35.bin";
+std::string c37_bin   = bin_path + "/layers/c37.bin";
+std::string c38_bin   = bin_path + "/layers/c38.bin";
+std::string c39_bin   = bin_path + "/layers/c39.bin";
+std::string c41_bin   = bin_path + "/layers/c41.bin";
+std::string c42_bin   = bin_path + "/layers/c42.bin";
+std::string c44_bin   = bin_path + "/layers/c44.bin";
+std::string c45_bin   = bin_path + "/layers/c45.bin";
+std::string c47_bin   = bin_path + "/layers/c47.bin";
+std::string c48_bin   = bin_path + "/layers/c48.bin";
+std::string c50_bin   = bin_path + "/layers/c50.bin";
+std::string c51_bin   = bin_path + "/layers/c51.bin";
+std::string c53_bin   = bin_path + "/layers/c53.bin";
+std::string c54_bin   = bin_path + "/layers/c54.bin";
+std::string c56_bin   = bin_path + "/layers/c56.bin";
+std::string c57_bin   = bin_path + "/layers/c57.bin";
+std::string c59_bin   = bin_path + "/layers/c59.bin";
+std::string c60_bin   = bin_path + "/layers/c60.bin";
+std::string c62_bin   = bin_path + "/layers/c62.bin";
+std::string c63_bin   = bin_path + "/layers/c63.bin";
+std::string c64_bin   = bin_path + "/layers/c64.bin";
+std::string c66_bin   = bin_path + "/layers/c66.bin";
+std::string c67_bin   = bin_path + "/layers/c67.bin";
+std::string c69_bin   = bin_path + "/layers/c69.bin";
+std::string c70_bin   = bin_path + "/layers/c70.bin";
+std::string c72_bin   = bin_path + "/layers/c72.bin";
+std::string c73_bin   = bin_path + "/layers/c73.bin";
+std::string c75_bin   = bin_path + "/layers/c75.bin";
+std::string c76_bin   = bin_path + "/layers/c76.bin";
+std::string c77_bin   = bin_path + "/layers/c77.bin";
+std::string c78_bin   = bin_path + "/layers/c78.bin";
+std::string c79_bin   = bin_path + "/layers/c79.bin";
+std::string c80_bin   = bin_path + "/layers/c80.bin";
+std::string c81_bin   = bin_path + "/layers/c81.bin";
+std::string g82_bin   = bin_path + "/layers/g82.bin";
+std::string c84_bin   = bin_path + "/layers/c84.bin";
+std::string c87_bin   = bin_path + "/layers/c87.bin";
+std::string c88_bin   = bin_path + "/layers/c88.bin";
+std::string c89_bin   = bin_path + "/layers/c89.bin";
+std::string c90_bin   = bin_path + "/layers/c90.bin";
+std::string c91_bin   = bin_path + "/layers/c91.bin";
+std::string c92_bin   = bin_path + "/layers/c92.bin";
+std::string c93_bin   = bin_path + "/layers/c93.bin";
+std::string g94_bin   = bin_path + "/layers/g94.bin";
+std::string c96_bin   = bin_path + "/layers/c96.bin";
+std::string c99_bin   = bin_path + "/layers/c99.bin";
+std::string c100_bin  = bin_path + "/layers/c100.bin";
+std::string c101_bin  = bin_path + "/layers/c101.bin";
+std::string c102_bin  = bin_path + "/layers/c102.bin";
+std::string c103_bin  = bin_path + "/layers/c103.bin";
+std::string c104_bin  = bin_path + "/layers/c104.bin";
+std::string c105_bin  = bin_path + "/layers/c105.bin";
+std::string g106_bin  = bin_path + "/layers/g106.bin";
+
+tk::dnn::Conv2d     c0   (&net,  32, 3, 3, 1, 1, 1, 1,  c0_bin, true);
+tk::dnn::Activation a0   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c1   (&net,  64, 3, 3, 2, 2, 1, 1,  c1_bin, true);
+tk::dnn::Activation a1   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c2   (&net,  32, 1, 1, 1, 1, 0, 0,  c2_bin, true);
+tk::dnn::Activation a2   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c3   (&net,  64, 3, 3, 1, 1, 1, 1,  c3_bin, true);
+tk::dnn::Activation a3   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s4   (&net, &a1);
+tk::dnn::Conv2d     c5   (&net, 128, 3, 3, 2, 2, 1, 1,  c5_bin, true);
+tk::dnn::Activation a5   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c6   (&net,  64, 1, 1, 1, 1, 0, 0,  c6_bin, true);
+tk::dnn::Activation a6   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c7   (&net, 128, 3, 3, 1, 1, 1, 1,  c7_bin, true);
+tk::dnn::Activation a7   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s8   (&net, &a5);
+tk::dnn::Conv2d     c9   (&net,  64, 1, 1, 1, 1, 0, 0,  c9_bin, true);
+tk::dnn::Activation a9   (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c10  (&net, 128, 3, 3, 1, 1, 1, 1, c10_bin, true);
+tk::dnn::Activation a10  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s11  (&net, &s8);
+
+tk::dnn::Conv2d     c12  (&net, 256, 3, 3, 2, 2, 1, 1, c12_bin, true);
+tk::dnn::Activation a12  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c13  (&net, 128, 1, 1, 1, 1, 0, 0, c13_bin, true);
+tk::dnn::Activation a13  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c14  (&net, 256, 3, 3, 1, 1, 1, 1, c14_bin, true);
+tk::dnn::Activation a14  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s15  (&net, &a12);
+
+tk::dnn::Conv2d     c16  (&net, 128, 1, 1, 1, 1, 0, 0, c16_bin, true);
+tk::dnn::Activation a16  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c17  (&net, 256, 3, 3, 1, 1, 1, 1, c17_bin, true);
+tk::dnn::Activation a17  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s18  (&net, &s15);
+tk::dnn::Conv2d     c19  (&net, 128, 1, 1, 1, 1, 0, 0, c19_bin, true);
+tk::dnn::Activation a19  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c20  (&net, 256, 3, 3, 1, 1, 1, 1, c20_bin, true);
+tk::dnn::Activation a20  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s21  (&net, &s18);
+tk::dnn::Conv2d     c22  (&net, 128, 1, 1, 1, 1, 0, 0, c22_bin, true);
+tk::dnn::Activation a22  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c23  (&net, 256, 3, 3, 1, 1, 1, 1, c23_bin, true);
+tk::dnn::Activation a23  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s24  (&net, &s21);
+tk::dnn::Conv2d     c25  (&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true);
+tk::dnn::Activation a25  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c26  (&net, 256, 3, 3, 1, 1, 1, 1, c26_bin, true);
+tk::dnn::Activation a26  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s27  (&net, &s24);
+tk::dnn::Conv2d     c28  (&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true);
+tk::dnn::Activation a28  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c29  (&net, 256, 3, 3, 1, 1, 1, 1, c29_bin, true);
+tk::dnn::Activation a29  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s30  (&net, &s27);
+tk::dnn::Conv2d     c31  (&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true);
+tk::dnn::Activation a31  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c32  (&net, 256, 3, 3, 1, 1, 1, 1, c32_bin, true);
+tk::dnn::Activation a32  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s33  (&net, &s30);
+tk::dnn::Conv2d     c34  (&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true);
+tk::dnn::Activation a34  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c35  (&net, 256, 3, 3, 1, 1, 1, 1, c35_bin, true);
+tk::dnn::Activation a35  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s36  (&net, &s33);
+
+tk::dnn::Conv2d     c37  (&net, 512, 3, 3, 2, 2, 1, 1, c37_bin, true);
+tk::dnn::Activation a37  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c38  (&net, 256, 1, 1, 1, 1, 0, 0, c38_bin, true);
+tk::dnn::Activation a38  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c39  (&net, 512, 3, 3, 1, 1, 1, 1, c39_bin, true);
+tk::dnn::Activation a39  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s40  (&net, &a37);
+
+tk::dnn::Conv2d     c41  (&net, 256, 1, 1, 1, 1, 0, 0, c41_bin, true);
+tk::dnn::Activation a41  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c42  (&net, 512, 3, 3, 1, 1, 1, 1, c42_bin, true);
+tk::dnn::Activation a42  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s43  (&net, &s40);
+tk::dnn::Conv2d     c44  (&net, 256, 1, 1, 1, 1, 0, 0, c44_bin, true);
+tk::dnn::Activation a44  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c45  (&net, 512, 3, 3, 1, 1, 1, 1, c45_bin, true);
+tk::dnn::Activation a45  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s46  (&net, &s43);
+tk::dnn::Conv2d     c47  (&net, 256, 1, 1, 1, 1, 0, 0, c47_bin, true);
+tk::dnn::Activation a47  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c48  (&net, 512, 3, 3, 1, 1, 1, 1, c48_bin, true);
+tk::dnn::Activation a48  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s49  (&net, &s46);
+tk::dnn::Conv2d     c50  (&net, 256, 1, 1, 1, 1, 0, 0, c50_bin, true);
+tk::dnn::Activation a50  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c51  (&net, 512, 3, 3, 1, 1, 1, 1, c51_bin, true);
+tk::dnn::Activation a51  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s52  (&net, &s49);
+tk::dnn::Conv2d     c53  (&net, 256, 1, 1, 1, 1, 0, 0, c53_bin, true);
+tk::dnn::Activation a53  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c54  (&net, 512, 3, 3, 1, 1, 1, 1, c54_bin, true);
+tk::dnn::Activation a54  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s55  (&net, &s52);
+tk::dnn::Conv2d     c56  (&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true);
+tk::dnn::Activation a56  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c57  (&net, 512, 3, 3, 1, 1, 1, 1, c57_bin, true);
+tk::dnn::Activation a57  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s58  (&net, &s55);
+tk::dnn::Conv2d     c59  (&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true);
+tk::dnn::Activation a59  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c60  (&net, 512, 3, 3, 1, 1, 1, 1, c60_bin, true);
+tk::dnn::Activation a60  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s61  (&net, &s58);
+
+tk::dnn::Conv2d     c62  (&net,1024, 3, 3, 2, 2, 1, 1, c62_bin, true);
+tk::dnn::Activation a62  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c63  (&net, 512, 1, 1, 1, 1, 0, 0, c63_bin, true);
+tk::dnn::Activation a63  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c64  (&net,1024, 3, 3, 1, 1, 1, 1, c64_bin, true);
+tk::dnn::Activation a64  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s65  (&net, &a62);
+
+tk::dnn::Conv2d     c66  (&net, 512, 1, 1, 1, 1, 0, 0, c66_bin, true);
+tk::dnn::Activation a66  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c67  (&net,1024, 3, 3, 1, 1, 1, 1, c67_bin, true);
+tk::dnn::Activation a67  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s68  (&net, &s65);
+
+tk::dnn::Conv2d     c69  (&net, 512, 1, 1, 1, 1, 0, 0, c69_bin, true);
+tk::dnn::Activation a69  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c70  (&net,1024, 3, 3, 1, 1, 1, 1, c70_bin, true);
+tk::dnn::Activation a70  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s71  (&net, &s68);
+
+tk::dnn::Conv2d     c72  (&net, 512, 1, 1, 1, 1, 0, 0, c72_bin, true);
+tk::dnn::Activation a72  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c73  (&net,1024, 3, 3, 1, 1, 1, 1, c73_bin, true);
+tk::dnn::Activation a73  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Shortcut   s74  (&net, &s71);
+
+tk::dnn::Conv2d     c75  (&net, 512, 1, 1, 1, 1, 0, 0, c75_bin, true);
+tk::dnn::Activation a75  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c76  (&net,1024, 3, 3, 1, 1, 1, 1, c76_bin, true);
+tk::dnn::Activation a76  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c77  (&net, 512, 1, 1, 1, 1, 0, 0, c77_bin, true);
+tk::dnn::Activation a77  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c78  (&net,1024, 3, 3, 1, 1, 1, 1, c78_bin, true);
+tk::dnn::Activation a78  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c79  (&net, 512, 1, 1, 1, 1, 0, 0, c79_bin, true);
+tk::dnn::Activation a79  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c80  (&net,1024, 3, 3, 1, 1, 1, 1, c80_bin, true);
+tk::dnn::Activation a80  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c81  (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c81_bin, false);
+tk::dnn::Yolo     yolo0  (&net, classes, 3, g82_bin);
+
+tk::dnn::Layer *m83_layers[1] = { &a79 };
+tk::dnn::Route      m83  (&net, m83_layers, 1);
+tk::dnn::Conv2d     c84  (&net, 256, 1, 1, 1, 1, 0, 0, c84_bin, true);
+tk::dnn::Activation a84  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Upsample   u85  (&net, 2);
+
+tk::dnn::Layer *m86_layers[2] = { &u85, &s61 };
+tk::dnn::Route      m86  (&net, m86_layers, 2);
+tk::dnn::Conv2d     c87  (&net, 256, 1, 1, 1, 1, 0, 0, c87_bin, true);
+tk::dnn::Activation a87  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c88  (&net, 512, 3, 3, 1, 1, 1, 1, c88_bin, true);
+tk::dnn::Activation a88  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c89  (&net, 256, 1, 1, 1, 1, 0, 0, c89_bin, true);
+tk::dnn::Activation a89  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c90  (&net, 512, 3, 3, 1, 1, 1, 1, c90_bin, true);
+tk::dnn::Activation a90  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c91  (&net, 256, 1, 1, 1, 1, 0, 0, c91_bin, true);
+tk::dnn::Activation a91  (&net, tk::dnn::ACTIVATION_LEAKY);
+
+tk::dnn::Conv2d     c92  (&net, 512, 3, 3, 1, 1, 1, 1, c92_bin, true);
+tk::dnn::Activation a92  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c93  (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c93_bin, false);
+tk::dnn::Yolo     yolo1  (&net, classes, 3, g94_bin);
+
+tk::dnn::Layer *m95_layers[1] = { &a91 };
+tk::dnn::Route      m95  (&net, m95_layers, 1);
+tk::dnn::Conv2d     c96  (&net, 128, 1, 1, 1, 1, 0, 0, c96_bin, true);
+tk::dnn::Activation a96  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Upsample   u97  (&net, 2);
+
+tk::dnn::Layer *m98_layers[2] = { &u97, &s36 };
+tk::dnn::Route      m98  (&net, m98_layers, 2);
+tk::dnn::Conv2d     c99  (&net, 128, 1, 1, 1, 1, 0, 0, c99_bin, true);
+tk::dnn::Activation a99  (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c100 (&net, 256, 3, 3, 1, 1, 1, 1, c100_bin, true);
+tk::dnn::Activation a100 (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c101 (&net, 128, 1, 1, 1, 1, 0, 0, c101_bin, true);
+tk::dnn::Activation a101 (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c102 (&net, 256, 3, 3, 1, 1, 1, 1, c102_bin, true);
+tk::dnn::Activation a102 (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c103 (&net, 128, 1, 1, 1, 1, 0, 0, c103_bin, true);
+tk::dnn::Activation a103 (&net, tk::dnn::ACTIVATION_LEAKY);
+
+tk::dnn::Conv2d     c104 (&net, 256, 3, 3, 1, 1, 1, 1, c104_bin, true);
+tk::dnn::Activation a104 (&net, tk::dnn::ACTIVATION_LEAKY);
+tk::dnn::Conv2d     c105 (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c105_bin, false);
+tk::dnn::Yolo      yolo2 (&net, classes, 3, g106_bin);
+
+yolo[0] = &yolo0;
+yolo[1] = &yolo1;
+yolo[2] = &yolo2;

include/tkDNN/pluginsRT/ActivationLeakyRT.h

@@ -0,0 +1,60 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ActivationLeakyRT : public IPlugin {
+
+public:
+	ActivationLeakyRT() {
+
+
+	}
+
+	~ActivationLeakyRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return inputs[0];
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		size = 1;
+		for(int i=0; i<outputDims[0].nbDims; i++)
+			size *= outputDims[0].d[i];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		activationLEAKYForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+											reinterpret_cast<dnnType*>(outputs[0]), size, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 1*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, size);
+	}
+
+	int size;
+};

include/tkDNN/pluginsRT/Int8Calibrator.h

@@ -0,0 +1,168 @@
+#include <vector>
+#include <assert.h>
+#include <algorithm>
+#include <iterator>
+
+#include "NvInfer.h"
+
+class BatchStream
+{
+public:
+	BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches)
+	{
+        mBatchSize = batchSize;
+        mMaxBatches = maxBatches;
+		mDims = nvinfer1::DimsNCHW{ dim.n, dim.c, dim.h, dim.w };
+		mImageSize = mDims.c()*mDims.h()*mDims.w();
+		mBatch.resize(mBatchSize*mImageSize, 0);
+		mLabels.resize(mBatchSize, 0);
+		mFileBatch.resize(mDims.n()*mImageSize, 0);
+		mFileLabels.resize(mDims.n(), 0);
+		reset(0);
+	}
+
+	void reset(int firstBatch)
+	{
+		mBatchCount = 0;
+		mFileCount = 0;
+		mFileBatchPos = mDims.n();
+		skip(firstBatch);
+	}
+
+	bool next()
+	{
+        std::cout<<"Next batch: "<<mBatchCount<<" of "<<mMaxBatches<<"\n";
+		if (mBatchCount == mMaxBatches)
+			return false;
+
+		for (int csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize)
+		{
+			assert(mFileBatchPos > 0 && mFileBatchPos <= mDims.n());
+			if (mFileBatchPos == mDims.n() && !update())
+				return false;
+
+			// copy the smaller of: elements left to fulfill the request, or elements left in the file buffer.
+			csize = std::min(mBatchSize - batchPos, mDims.n() - mFileBatchPos);
+			std::copy_n(getFileBatch() + mFileBatchPos * mImageSize, csize * mImageSize, getBatch() + batchPos * mImageSize);
+			std::copy_n(getFileLabels() + mFileBatchPos, csize, getLabels() + batchPos);
+		}
+		mBatchCount++;
+		return true;
+	}
+
+	void skip(int skipCount)
+	{
+		if (mBatchSize >= mDims.n() && mBatchSize%mDims.n() == 0 && mFileBatchPos == mDims.n())
+		{
+			mFileCount += skipCount * mBatchSize / mDims.n();
+            std::cout<<mFileCount<<"\n";
+			return;
+		}
+
+		int x = mBatchCount;
+		for (int i = 0; i < skipCount; i++)
+			next();
+		mBatchCount = x;
+	}
+
+	float *getBatch() { return &mBatch[0]; }
+	float *getLabels() { return &mLabels[0]; }
+	int getBatchesRead() const { return mBatchCount; }
+	int getBatchSize() const { return mBatchSize; }
+	nvinfer1::DimsNCHW getDims() const { return mDims; }
+private:
+	float* getFileBatch() { return &mFileBatch[0]; }
+	float* getFileLabels() { return &mFileLabels[0]; }
+
+	bool update()
+	{
+		std::string inputFileName = std::string("calibBatches/batch") + std::to_string(mFileCount++);
+		FILE * file = fopen(inputFileName.c_str(), "rb");
+		if (!file) {
+            FatalError("cant open batch calib file: " + inputFileName);
+        	return false;
+        }
+
+		size_t readInputCount = fread(getFileBatch(), sizeof(float), mDims.n()*mImageSize, file);
+		size_t readLabelCount = fread(getFileLabels(), sizeof(float), mDims.n(), file);;
+		assert(readInputCount == size_t(mDims.n()*mImageSize) && readLabelCount == size_t(mDims.n()));
+
+		fclose(file);
+		mFileBatchPos = 0;
+		return true;
+	}
+
+	int mBatchSize{ 0 };
+	int mMaxBatches{ 0 };
+	int mBatchCount{ 0 };
+
+	int mFileCount{ 0 }, mFileBatchPos{ 0 };
+	int mImageSize{ 0 };
+
+	nvinfer1::DimsNCHW mDims;
+	std::vector<float> mBatch;
+	std::vector<float> mLabels;
+	std::vector<float> mFileBatch;
+	std::vector<float> mFileLabels;
+};
+
+
+
+class Int8EntropyCalibrator : public IInt8EntropyCalibrator
+{
+public:
+	Int8EntropyCalibrator(BatchStream& stream, int firstBatch, bool readCache = true)
+		: mStream(stream), mReadCache(readCache)
+	{
+		DimsNCHW dims = mStream.getDims();
+		mInputCount = mStream.getBatchSize() * dims.c() * dims.h() * dims.w();
+		checkCuda(cudaMalloc(&mDeviceInput, mInputCount * sizeof(float)));
+		mStream.reset(firstBatch);
+	}
+
+	virtual ~Int8EntropyCalibrator()
+	{
+		checkCuda(cudaFree(mDeviceInput));
+	}
+
+	int getBatchSize() const override { return mStream.getBatchSize(); }
+
+	bool getBatch(void* bindings[], const char* names[], int nbBindings) override
+	{
+        std::cout<<"CALIB request batch\n";
+		if (!mStream.next())
+			return false;
+
+		checkCuda(cudaMemcpy(mDeviceInput, mStream.getBatch(), mInputCount * sizeof(float), cudaMemcpyHostToDevice));
+		bindings[0] = mDeviceInput;
+		return true;
+	}
+
+	const void* readCalibrationCache(size_t& length) override
+	{
+		mCalibrationCache.clear();
+		std::ifstream input("table.calib", std::ios::binary);
+		input >> std::noskipws;
+		
+        FatalError("rewrite different");
+        //if (mReadCache && input.good())
+		//	std::copy(std::istream_iterator<char>(input), std::istream_iterator<char>(), std::back_inserter(mCalibrationCache));
+
+		length = mCalibrationCache.size();
+		return length ? &mCalibrationCache[0] : nullptr;
+	}
+
+	void writeCalibrationCache(const void* cache, size_t length) override
+	{
+		std::ofstream output("table.calib", std::ios::binary);
+		output.write(reinterpret_cast<const char*>(cache), length);
+	}
+
+private:
+	BatchStream mStream;
+	bool mReadCache{ true };
+
+	size_t mInputCount;
+	void* mDeviceInput{ nullptr };
+	std::vector<char> mCalibrationCache;
+};

include/tkDNN/pluginsRT/RegionRT.h

@@ -0,0 +1,94 @@
+#include<cassert>
+#include "../kernels.h"
+
+class RegionRT : public IPlugin {
+
+public:
+	RegionRT(int classes, int coords, int num) {
+
+		this->classes = classes;
+		this->coords = coords;
+		this->num = num;
+	}
+
+	~RegionRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return inputs[0];
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		c = inputDims[0].d[0];
+		h = inputDims[0].d[1];
+		w = inputDims[0].d[2];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+		for (int b = 0; b < batchSize; ++b){
+			for(int n = 0; n < num; ++n){
+				int index = entry_index(b, n*w*h, 0, batchSize);
+				activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
+				
+				index = entry_index(b, n*w*h, coords, batchSize);
+				activationLOGISTICForward(srcData + index, dstData + index, w*h, stream);
+			}
+		}
+
+		//softmax start
+		int index = entry_index(0, 0, coords + 1, batchSize);
+		softmaxForward(	srcData + index, classes, batchSize*num, 
+						(batchSize*c*h*w)/num, 
+						w*h, 1, w*h, 1, dstData + index, stream);
+
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 6*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, classes);
+		tk::dnn::writeBUF(buf, coords);
+		tk::dnn::writeBUF(buf, num);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+	}
+
+	int c, h, w;
+    int classes, coords, num;
+
+	int entry_index(int batch, int location, int entry, int batchSize) {
+		int n =   location / (w*h);
+		int loc = location % (w*h);
+		return batch*c*h*w*batchSize + n*w*h*(coords+classes+1) + entry*w*h + loc;
+	}
+
+};

include/tkDNN/pluginsRT/ReorgRT.h

@@ -0,0 +1,63 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ReorgRT : public IPlugin {
+
+public:
+	ReorgRT(int stride) {
+		this->stride = stride;
+	}
+
+	~ReorgRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{inputs[0].d[0]*stride*stride, inputs[0].d[1]/stride, inputs[0].d[2]/stride};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		c = inputDims[0].d[0];
+		h = inputDims[0].d[1];
+		w = inputDims[0].d[2];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		reorgForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+					  reinterpret_cast<dnnType*>(outputs[0]), 
+					  batchSize, c, h, w, stride, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 4*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, stride);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+	}
+
+	int c, h, w, stride;
+};

include/tkDNN/pluginsRT/RouteRT.h

@@ -0,0 +1,82 @@
+#include<cassert>
+#include "../kernels.h"
+
+class RouteRT : public IPlugin {
+
+public:
+	RouteRT() {
+	}
+
+	~RouteRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		int out_c = 0;
+		for(int i=0; i<nbInputDims; i++) out_c += inputs[i].d[0];
+		return DimsCHW{out_c, inputs[0].d[1], inputs[0].d[2]};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		in = nbInputs;
+		c = 0;
+		for(int i=0; i<nbInputs; i++) {
+			c_in[i] = inputDims[i].d[0]; 
+			c += inputDims[i].d[0];
+		}
+		h = inputDims[0].d[1];
+		w = inputDims[0].d[2];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+		int offset = 0;
+		for(int i=0; i<in; i++) {
+			dnnType *input = (dnnType*)reinterpret_cast<const dnnType*>(inputs[i]);
+			int in_dim = c_in[i]*h*w;
+			checkCuda( cudaMemcpyAsync(dstData + offset, input, in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream) );
+			offset += in_dim;
+		}
+
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return (4+MAX_INPUTS)*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, in);
+		for(int i=0; i<MAX_INPUTS; i++)
+			tk::dnn::writeBUF(buf, c_in[i]);
+
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+	}
+
+	static const int MAX_INPUTS = 4;
+	int in;
+	int c_in[MAX_INPUTS];
+	int c, h, w;
+};

include/tkDNN/pluginsRT/ShortcutRT.h

@@ -0,0 +1,65 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ShortcutRT : public IPlugin {
+
+public:
+	ShortcutRT() {
+	}
+
+	~ShortcutRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{inputs[0].d[0], inputs[0].d[1], inputs[0].d[2]};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		c = inputDims[0].d[0];
+		h = inputDims[0].d[1];
+		w = inputDims[0].d[2];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *srcDataBack = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+		shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, c, h, w, 1, stream);
+
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 3*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+	}
+
+	int c, h, w;
+};

include/tkDNN/pluginsRT/UpsampleRT.h

@@ -0,0 +1,65 @@
+#include<cassert>
+#include "../kernels.h"
+
+class UpsampleRT : public IPlugin {
+
+public:
+	UpsampleRT(int stride) {
+		this->stride = stride;
+	}
+
+	~UpsampleRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW(inputs[0].d[0], inputs[0].d[1]*stride, inputs[0].d[2]*stride);
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		c = inputDims[0].d[0];
+		h = inputDims[0].d[1];
+		w = inputDims[0].d[2];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+	    
+		fill(dstData, batchSize*c*h*w*stride*stride, 0.0, stream);
+    	upsampleForward(srcData, dstData, batchSize, c, h, w, stride, 1, 1, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 4*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, stride);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+	}
+
+	int c, h, w, stride;
+};

include/tkDNN/pluginsRT/YoloRT.h

@@ -0,0 +1,116 @@
+#include<cassert>
+#include "../kernels.h"
+
+#define YOLORT_CLASSNAME_W 256
+
+class YoloRT : public IPlugin {
+
+
+
+public:
+	YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr) {
+
+		this->classes = classes;
+		this->num = num;
+
+        mask = new dnnType[num];
+        bias = new dnnType[num*3*2];
+        if(yolo != nullptr) {
+            memcpy(mask, yolo->mask_h, sizeof(dnnType)*num);
+            memcpy(bias, yolo->bias_h, sizeof(dnnType)*num*3*2);
+			classesNames = yolo->classesNames;
+        }
+	}
+
+	~YoloRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return inputs[0];
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		c = inputDims[0].d[0];
+		h = inputDims[0].d[1];
+		w = inputDims[0].d[2];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+		for (int b = 0; b < batchSize; ++b){
+			for(int n = 0; n < num; ++n){
+				int index = entry_index(b, n*w*h, 0, batchSize);
+				activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
+				
+				index = entry_index(b, n*w*h, 4, batchSize);
+				activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream);
+			}
+		}
+
+		//std::cout<<"YOLO END\n";
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 5*sizeof(int) + num*sizeof(dnnType) + num*3*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, classes);
+		tk::dnn::writeBUF(buf, num);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+        for(int i=0; i<num; i++)
+    		tk::dnn::writeBUF(buf, mask[i]);
+        for(int i=0; i<3*2*num; i++)
+    		tk::dnn::writeBUF(buf, bias[i]);
+
+		// save classes names
+		for(int i=0; i<classes; i++) {
+			char tmp[YOLORT_CLASSNAME_W];
+			strcpy(tmp, classesNames[i].c_str());
+			for(int j=0; j<YOLORT_CLASSNAME_W; j++) {
+				tk::dnn::writeBUF(buf, tmp[j]);
+			}
+		}
+	}
+
+	int c, h, w;
+    int classes, num;
+	std::vector<std::string> classesNames;
+
+    dnnType *mask;
+    dnnType *bias;
+
+	int entry_index(int batch, int location, int entry, int batchSize) {
+		int n =   location / (w*h);
+		int loc = location % (w*h);
+		return batch*c*h*w*batchSize + n*w*h*(4+classes+1) + entry*w*h + loc;
+	}
+
+};

include/tkDNN/tkdnn.h

@@ -3,14 +3,6 @@
 */
 #include "Network.h"
 #include "Layer.h"
+#include "NetworkRT.h"
 
-namespace tkDNN {
+#define TKDNN_VERSION 400
-
-    /**
-        Return the tkDNN version
-    */
-    int getVersion() {
-
-        return 100;
-    }
-}

include/tkDNN/utils.h

@@ -12,17 +12,35 @@
 #include <cublas_v2.h>
 #include <cudnn.h>
 
-#define value_type float
+#define dnnType float
 
+// Colored output
+#define COL_END "\033[0m"
 
+#define COL_RED "\033[31m"
+#define COL_GREEN "\033[32m"
+#define COL_ORANGE "\033[33m"
+#define COL_BLUE "\033[34m"
+#define COL_PURPLE "\033[35m"
+#define COL_CYAN "\033[36m"
+
+#define COL_REDB "\033[1;31m"
+#define COL_GREENB "\033[1;32m"
+#define COL_ORANGEB "\033[1;33m"
+#define COL_BLUEB "\033[1;34m"
+#define COL_PURPLEB "\033[1;35m"
+#define COL_CYANB "\033[1;36m"
+
+// Simple Timer 
 #define TIMER_START timespec start, end;                               \
                     clock_gettime(CLOCK_MONOTONIC, &start);            
 
-#define TIMER_STOP  clock_gettime(CLOCK_MONOTONIC, &end);              \
+#define TIMER_STOP_C(col)  clock_gettime(CLOCK_MONOTONIC, &end);       \
     double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 +       \
                   (double)(end.tv_nsec - start.tv_nsec))/1.0e6;        \
-    std::cout<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"; 
+    std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END; 
 
+#define TIMER_STOP TIMER_STOP_C(COL_CYANB)
 
 /********************************************************
  * Prints the error message, and exits
@@ -62,12 +80,23 @@
     }                                                                  \
 }
 
-void readBinaryFile(const char* fname, int size, value_type** data_h, value_type** data_d);
+#define checkNULL(ptr) {                                               \
-void printDeviceVector(int size, value_type* vec_d);
+    std::stringstream _error;                                          \
-void resize(int size, value_type **data);
+    if (ptr == nullptr) {                                              \
+      _error << "Null pointer";                                        \
+      FatalError(_error.str());                                        \
+    }                                                                  \
+}
 
-void matrixTranspose(cublasHandle_t handle, value_type* srcData, value_type* dstData, int rows, int cols);
+void printCenteredTitle(const char *title, char fill, int dim);
+bool fileExist(const char *fname);
+void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
+int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true);
+void printDeviceVector(int size, dnnType* vec_d, bool device = true);
+void resize(int size, dnnType **data);
 
-void matrixMulAdd(  cublasHandle_t handle, value_type* srcData, value_type* dstData, 
+void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols);
-                    value_type* add_vector, int dim, value_type mul);
+
-#endif //UTILS_H
+void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData, 
+                    dnnType* add_vector, int dim, dnnType mul);
+#endif //UTILS_H

src/Activation.cpp

@@ -3,55 +3,64 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Activation::Activation(Network *net, dataDim_t input_dim, cudnnActivationMode_t act_mode) : 
+Activation::Activation(Network *net, int act_mode) : 
-    Layer(net, input_dim) {
+    Layer(net) {
 
     this->act_mode = act_mode;
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 
-    checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
+    if(int(act_mode) < 100) {
-                                        net->tensorFormat,
+
-                                        net->dataType,
+        checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
-                                        input_dim.n*input_dim.l, 
-                                        input_dim.c,
-                                        input_dim.h, input_dim.w) );
-    checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
                                             net->tensorFormat,
                                             net->dataType,
                                             input_dim.n*input_dim.l, 
                                             input_dim.c,
                                             input_dim.h, input_dim.w) );
+        checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
+                                                net->tensorFormat,
+                                                net->dataType,
+                                                input_dim.n*input_dim.l, 
+                                                input_dim.c,
+                                                input_dim.h, input_dim.w) );
 
 
-    checkCUDNN( cudnnCreateActivationDescriptor(&activDesc) );
+        checkCUDNN( cudnnCreateActivationDescriptor(&activDesc) );
-    checkCUDNN( cudnnSetActivationDescriptor(activDesc,
+        checkCUDNN( cudnnSetActivationDescriptor(activDesc,
-                                             act_mode,
+                                                (cudnnActivationMode_t) act_mode,
-                                             CUDNN_PROPAGATE_NAN,
+                                                CUDNN_PROPAGATE_NAN,
-                                             0.0) );
+                                                0.0) );
+    }
 }
 
 Activation::~Activation() {
 
     checkCuda( cudaFree(dstData) );
 
-    checkCUDNN( cudnnDestroyActivationDescriptor(activDesc) );
+    if(int(act_mode) < 100)
+        checkCUDNN( cudnnDestroyActivationDescriptor(activDesc) );
 }
 
-value_type* Activation::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
 
-    value_type alpha = value_type(1);
+    if(act_mode == ACTIVATION_LEAKY) {
-    value_type beta  = value_type(0);
+        activationLEAKYForward(srcData, dstData, dim.tot());
-    checkCUDNN( cudnnActivationForward(net->cudnnHandle,
+
-                                        activDesc,
+    } else {
-                                        &alpha,
+        dnnType alpha = dnnType(1);
-                                        srcTensorDesc,
+        dnnType beta  = dnnType(0);
-                                        srcData,
+        checkCUDNN( cudnnActivationForward(net->cudnnHandle,
-                                        &beta,
+                                            activDesc,
-                                        dstTensorDesc,
+                                            &alpha,
-                                        dstData) );    
+                                            srcTensorDesc,
+                                            srcData,
+                                            &beta,
+                                            dstTensorDesc,
+                                            dstData) );   
+    } 
     return dstData;
 }
 
-}
+}}

src/Conv2d.cpp

@@ -2,19 +2,21 @@
 
 #include "Layer.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Conv2d::Conv2d( Network *net, dataDim_t in_dim, int out_ch, 
+Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, 
-                int kernelH, int kernelW, int strideH, int strideW,
+                int strideH, int strideW, int paddingH, int paddingW,
-                const char* fname_weights, const char* fname_bias) : 
+                std::string fname_weights, bool batchnorm) : 
     
-    LayerWgs(net, in_dim, in_dim.c, out_ch, kernelH, kernelW, 1, 
+    LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, 
-             fname_weights, fname_bias) {
+             fname_weights, batchnorm) {
 
     this->kernelH = kernelH;
     this->kernelW = kernelW;
     this->strideH = strideH;
     this->strideW = strideW;
+    this->paddingH = paddingH;
+    this->paddingW = paddingW;
 
     checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) );
     checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) );
@@ -33,10 +35,10 @@ Conv2d::Conv2d( Network *net, dataDim_t in_dim, int out_ch,
                 kernelH, kernelW) );
 
     checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc,
-                0,0, // padding
+                paddingH, paddingW, // padding
                 strideH, strideW, // stride
                 1,1, // upscale
-                CUDNN_CROSS_CORRELATION) );
+                CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) );
 
     // find dimension of convolution output
     checkCUDNN( cudnnGetConvolution2dForwardOutputDim(
@@ -74,7 +76,7 @@ Conv2d::Conv2d( Network *net, dataDim_t in_dim, int out_ch,
     output_dim.l = 1;
 
     //allocate data for infer result
-    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 }
 
 Conv2d::~Conv2d() {
@@ -89,28 +91,38 @@ Conv2d::~Conv2d() {
     checkCuda( cudaFree(dstData) );
 }
 
-value_type* Conv2d::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* Conv2d::infer(dataDim_t &dim, dnnType* srcData) {
 
 
     // convolution
-    value_type alpha = value_type(1);
+    dnnType alpha = dnnType(1);
-    value_type beta  = value_type(0);
+    dnnType beta  = dnnType(0);
     checkCUDNN( cudnnConvolutionForward(net->cudnnHandle,
                 &alpha, srcTensorDesc, srcData, filterDesc,
                 data_d, convDesc, algo, workSpace, ws_sizeInBytes,
                 &beta, dstTensorDesc, dstData) );
 
-    // bias
+    if(!batchnorm) {
-    alpha = value_type(1);
+        // bias
-    beta  = value_type(1);
+        alpha = dnnType(1);
-    checkCUDNN( cudnnAddTensor(net->cudnnHandle,
+        beta  = dnnType(1);
-                &alpha, biasTensorDesc, bias_d,
+        checkCUDNN( cudnnAddTensor(net->cudnnHandle,
-                &beta, dstTensorDesc, dstData) );
+                    &alpha, biasTensorDesc, bias_d,
-
+                    &beta, dstTensorDesc, dstData) );
+    } else {
+        float one = 1;
+        float zero = 0;
+        cudnnBatchNormalizationForwardInference(net->cudnnHandle,
+            CUDNN_BATCHNORM_SPATIAL, &one, &zero, 
+            dstTensorDesc, dstData, dstTensorDesc, 
+            dstData, biasTensorDesc, //same tensor descriptor as bias 
+            scales_d, bias_d, mean_d, variance_d, 
+            TKDNN_BN_MIN_EPSILON);
+    }
     //update data dimensions    
     dim = output_dim;
 
     return dstData;
 }
 
-}
+}}

src/Dense.cpp

@@ -2,11 +2,10 @@
 
 #include "Layer.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Dense::Dense(Network *net, dataDim_t in_dim, 
+Dense::Dense(Network *net, int out_ch, std::string fname_weights) : 
-    int out_ch, const char* fname_weights, const char* fname_bias) : 
+    LayerWgs(net, net->getOutputDim().tot(), out_ch, 1, 1, 1, fname_weights) {
-    LayerWgs(net, in_dim, in_dim.tot(), out_ch, 1, 1, 1, fname_weights, fname_bias) {
 
     output_dim.n = 1;
     output_dim.c = out_ch;
@@ -15,7 +14,7 @@ Dense::Dense(Network *net, dataDim_t in_dim,
     output_dim.l = 1;
 
     //allocate data for infer result
-    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 }
 
 Dense::~Dense() {
@@ -23,7 +22,7 @@ Dense::~Dense() {
     checkCuda( cudaFree(dstData) );
 }
 
-value_type* Dense::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
 
     if (dim.n != 1)
         FatalError("Not Implemented"); 
@@ -34,9 +33,9 @@ value_type* Dense::infer(dataDim_t &dim, value_type* srcData) {
     if (dim_x != input_dim.tot())
         FatalError("Input mismatch");
 
-    value_type alpha = value_type(1), beta = value_type(1);
+    dnnType alpha = dnnType(1), beta = dnnType(1);
     // place bias into dstData
-    checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(value_type), cudaMemcpyDeviceToDevice) );
+    checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
     
     //do matrix moltiplication
     checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
@@ -56,4 +55,4 @@ value_type* Dense::infer(dataDim_t &dim, value_type* srcData) {
     return dstData;
 }
 
-}
+}}

src/Flatten.cpp

@@ -3,12 +3,11 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Flatten::Flatten(Network *net, dataDim_t input_dim) : 
+Flatten::Flatten(Network *net) : Layer(net) {
-    Layer(net, input_dim) {
 
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 
     output_dim.n = 1;
     output_dim.c = input_dim.tot();
@@ -23,7 +22,7 @@ Flatten::~Flatten() {
     checkCuda( cudaFree(dstData) );
 }
 
-value_type* Flatten::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* Flatten::infer(dataDim_t &dim, dnnType* srcData) {
 
     //transpose per channel
     matrixTranspose(net->cublasHandle, srcData, dstData, dim.c, dim.h*dim.w*dim.l);
@@ -34,4 +33,4 @@ value_type* Flatten::infer(dataDim_t &dim, value_type* srcData) {
     return dstData;
 }
 
-}
+}}

src/Layer.cpp

@@ -2,19 +2,22 @@
 
 #include "Layer.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Layer::Layer(Network *net, dataDim_t in_dim) {
+Layer::Layer(Network *net) {
 
     this->net = net;
-    this->input_dim = in_dim;
-    this->output_dim = in_dim;
-    
-    checkCUDNN( cudnnCreateTensorDescriptor(&srcTensorDesc) );
-    checkCUDNN( cudnnCreateTensorDescriptor(&dstTensorDesc) );
 
-    if(!net->addLayer(this))
+    if(net != nullptr) {
-        FatalError("Net reached max number of layers");    
+        this->input_dim = net->getOutputDim();
+        this->output_dim = input_dim;
+        
+        checkCUDNN( cudnnCreateTensorDescriptor(&srcTensorDesc) );
+        checkCUDNN( cudnnCreateTensorDescriptor(&dstTensorDesc) );
+
+        if(!net->addLayer(this))
+            FatalError("Net reached max number of layers");    
+    }
 }
 
 Layer::~Layer() {
@@ -23,4 +26,4 @@ Layer::~Layer() {
     checkCUDNN( cudnnDestroyTensorDescriptor(dstTensorDesc) );
 }
 
-}
+}}

src/LayerWgs.cpp

@@ -1,21 +1,100 @@
 #include <iostream>
+#include <string.h>
 
 #include "Layer.h"
+#include "kernels.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-LayerWgs::LayerWgs(Network *net, dataDim_t in_dim, 
+LayerWgs::LayerWgs(Network *net, int inputs, int outputs, 
-                int inputs, int outputs, int kh, int kw, int kl, 
+                   int kh, int kw, int kl, 
-                const char* fname_weights, const char* fname_bias) : Layer(net, in_dim) {
+                   std::string fname_weights, bool batchnorm) : Layer(net) {
 
     this->inputs  = inputs;
     this->outputs = outputs;    
     this->weights_path  = std::string(fname_weights);
-    this->bias_path     = std::string(fname_bias);
+   
-
     std::cout<<"Reading weights: I="<<inputs<<" O="<<outputs<<" KERNEL="<<kh<<"x"<<kw<<"x"<<kl<<"\n";
-    readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d);
+    int seek = 0;
-    readBinaryFile(bias_path.c_str(), outputs, &bias_h, &bias_d);
+    readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
+    seek += inputs*outputs*kh*kw*kl;
+    readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
+    
+    this->batchnorm = batchnorm;
+    if(batchnorm) {
+        seek += outputs;
+        readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek);
+        seek += outputs;
+        readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek);
+        seek += outputs;
+        readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek);
+
+        float eps = TKDNN_BN_MIN_EPSILON;
+
+        power_h = new dnnType[outputs];
+        for(int i=0; i<outputs; i++) power_h[i] = 1.0f;
+
+        for(int i=0; i<outputs; i++)
+            mean_h[i] = mean_h[i] / -sqrt(eps + variance_h[i]); 
+
+        for(int i=0; i<outputs; i++)
+            variance_h[i] = 1.0f / sqrt(eps + variance_h[i]);
+    }
+
+
+    if(!net->fp16)
+        return;
+
+    //convert to fp16
+    int w_size = inputs*outputs*kh*kw*kl;
+    data16_h = new __half[w_size];
+    cudaMalloc(&data16_d, w_size*sizeof(__half));
+    float2half(data_d, data16_d, w_size);
+    cudaMemcpy(data16_h, data16_d, w_size*sizeof(__half), cudaMemcpyDeviceToHost);
+
+    int b_size = outputs;
+    bias16_h = new __half[b_size];
+    cudaMalloc(&bias16_d, w_size*sizeof(__half));
+    float2half(bias_d, bias16_d, b_size);
+    cudaMemcpy(bias16_h, bias16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+
+    if(batchnorm) {
+
+        power16_h    = new __half[b_size];  
+        mean16_h     = new __half[b_size];
+        variance16_h = new __half[b_size];
+        scales16_h   = new __half[b_size];
+
+        cudaMalloc(&power16_d, b_size*sizeof(__half));
+        cudaMalloc(&mean16_d, b_size*sizeof(__half));
+        cudaMalloc(&variance16_d, b_size*sizeof(__half));
+        cudaMalloc(&scales16_d, b_size*sizeof(__half));
+
+        //temporary buffers
+        float *tmp_d;
+        cudaMalloc(&tmp_d, b_size*sizeof(float));
+
+        //init power array of ones
+        cudaMemcpy(tmp_d, power_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
+        float2half(tmp_d, power16_d, b_size);
+        cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+
+        //mean array
+
+        cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
+        float2half(tmp_d, mean16_d, b_size);
+        cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+
+        //convert variance
+ 
+        cudaMemcpy(tmp_d, variance_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
+        float2half(tmp_d, variance16_d, b_size);
+        cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+
+        //conver scales
+        float2half(scales_d, scales16_d, b_size);
+        cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+    }
 }
 
 LayerWgs::~LayerWgs() {
@@ -24,6 +103,15 @@ LayerWgs::~LayerWgs() {
     delete [] bias_h;
     checkCuda( cudaFree(data_d) );
     checkCuda( cudaFree(bias_d) );
+
+    if(batchnorm) {
+        delete [] scales_h;
+        delete [] mean_h;
+        delete [] variance_h;
+        checkCuda( cudaFree(scales_d) );
+        checkCuda( cudaFree(mean_d) );
+        checkCuda( cudaFree(variance_d) );
+    }
 }
 
-}
+}}

src/MulAdd.cpp

@@ -3,10 +3,9 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-MulAdd::MulAdd(Network *net, dataDim_t input_dim, value_type mul, value_type add) : 
+MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) {
-    Layer(net, input_dim) {
 
     this->mul = mul;
     this->add = add;
@@ -14,16 +13,16 @@ MulAdd::MulAdd(Network *net, dataDim_t input_dim, value_type mul, value_type add
     int size = input_dim.tot();
 
     // create a vector with all value setted to add 
-    value_type *add_vector_h = new value_type[size];
+    dnnType *add_vector_h = new dnnType[size];
     for(int i=0; i<size; i++)
         add_vector_h[i] = add;
 
-    checkCuda( cudaMalloc(&add_vector, size*sizeof(value_type)));
+    checkCuda( cudaMalloc(&add_vector, size*sizeof(dnnType)));
-    checkCuda( cudaMemcpy(add_vector, add_vector_h, size*sizeof(value_type), cudaMemcpyHostToDevice));
+    checkCuda( cudaMemcpy(add_vector, add_vector_h, size*sizeof(dnnType), cudaMemcpyHostToDevice));
     delete [] add_vector_h;
 
 
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 }
 
 MulAdd::~MulAdd() {
@@ -32,7 +31,7 @@ MulAdd::~MulAdd() {
     checkCuda( cudaFree(dstData) );
 }
 
-value_type* MulAdd::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* MulAdd::infer(dataDim_t &dim, dnnType* srcData) {
 
     matrixMulAdd(net->cublasHandle, srcData, dstData, add_vector, input_dim.tot(), mul);
     
@@ -42,4 +41,4 @@ value_type* MulAdd::infer(dataDim_t &dim, value_type* srcData) {
     return dstData;
 }
 
-}
+}}

src/Network.cpp

@@ -1,24 +1,44 @@
 #include <iostream>
+#include <string.h>
 
 #include "tkdnn.h"
 #include "Network.h"
 #include "Layer.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Network::Network() {
+Network::Network(dataDim_t input_dim) {
+    this->input_dim = input_dim;
 
-    float tk_ver = float(tkDNN::getVersion())/1000;
+    float tk_ver = float(TKDNN_VERSION)/1000;
     float cu_ver = float(cudnnGetVersion())/1000;
 
-    std::cout<<"New NETWORK (tkDNN v"<<tk_ver<<", CUDNN v"<<cu_ver<<")\n";
+    std::cout<<"New NETWORK (tkDNN v"<<tk_ver
+             <<", CUDNN v"<<cu_ver<<")\n";
     dataType = CUDNN_DATA_FLOAT;
     tensorFormat = CUDNN_TENSOR_NCHW;
+    num_layers = 0;
+
+    fp16 = false;
+    dla = false;
+    if(const char* env_p = std::getenv("TKDNN_MODE")) {
+        if(strcmp(env_p, "FP16") == 0)
+            fp16 = true;
+	else if(strcmp(env_p, "DLA") == 0) {
+    	    dla = true;
+	    fp16 = true;	
+	}
+    }
+   
+    if(fp16)
+        std::cout<<COL_REDB<<"!! FP16 INERENCE ENABLED !!"<<COL_END<<"\n";
+    if(dla)
+        std::cout<<COL_GREENB<<"!! DLA INERENCE ENABLED !!"<<COL_END<<"\n";
+
 
     checkCUDNN( cudnnCreate(&cudnnHandle) );
     checkERROR( cublasCreate(&cublasHandle) );
 
-    num_layers = 0;
 }
 
 Network::~Network() {
@@ -27,12 +47,13 @@ Network::~Network() {
     checkERROR( cublasDestroy(cublasHandle) );
 }
 
-value_type* Network::infer(dataDim_t &dim, value_type* data) {
+dnnType* Network::infer(dataDim_t &dim, dnnType* data) {
 
     //do infer for every layer
-    for(int i=0; i<num_layers; i++)
+    for(int i=0; i<num_layers; i++) {
         data = layers[i]->infer(dim, data);
-
+    }
+    checkCuda(cudaDeviceSynchronize());
     return data;
 }
 
@@ -44,4 +65,47 @@ bool Network::addLayer(Layer *l) {
     return true;
 }
 
-}
+dataDim_t Network::getOutputDim() {
+
+        if(num_layers == 0)
+            return input_dim;
+        else
+            return layers[num_layers-1]->output_dim;
+}
+
+void Network::print() {
+
+    printCenteredTitle(" NETWORK MODEL ", '=', 60);
+    std::cout.width(3); std::cout<<std::left<<"N.";
+    std::cout<<" ";
+    std::cout.width(17); std::cout<<std::left<<"Layer type";
+    std::cout.width(22); std::cout<<std::left<<"input (H*W,CH)";
+    std::cout.width(16); std::cout<<std::left<<"output (H*W,CH)";
+    std::cout<<"\n";
+
+    for(int i=0; i<num_layers; i++) {
+        dataDim_t in = layers[i]->input_dim;
+        dataDim_t out = layers[i]->output_dim;
+
+        std::cout.width(3); std::cout<<std::right<<i;
+        std::cout<<" ";
+        std::cout.width(16); std::cout<<std::left<<layers[i]->getLayerName();
+        std::cout.width(4);  std::cout<<std::right<<in.h;
+        std::cout<<" x ";
+        std::cout.width(4);  std::cout<<std::right<<in.w;
+        std::cout<<", ";
+        std::cout.width(4);  std::cout<<std::right<<in.c;
+        std::cout<<"  -> ";
+        std::cout.width(4);  std::cout<<std::right<<out.h;
+        std::cout<<" x ";
+        std::cout.width(4);  std::cout<<std::right<<out.w;
+        std::cout<<", ";
+        std::cout.width(4);  std::cout<<std::right<<out.c;
+        std::cout<<"\n";
+    }
+    printCenteredTitle("", '=', 60);
+    std::cout<<"\n";
+}
+
+
+}}

src/NetworkRT.cpp

@@ -0,0 +1,514 @@
+#include <iostream>
+#include <map>
+#include <errno.h>
+#include <string.h> // memcpy
+#include <stdlib.h>
+
+#include "kernels.h"
+
+#include "utils.h"
+#include "NvInfer.h"
+#include "NetworkRT.h"
+
+using namespace nvinfer1;
+
+// Logger for info/warning/errors
+class Logger : public ILogger {
+    void log(Severity severity, const char* msg) override {
+#ifdef DEBUG
+        std::cout <<"TENSORRT LOG: "<< msg << std::endl;
+#endif
+    }
+} loggerRT;
+
+namespace tk { namespace dnn {
+
+std::map<Layer*, nvinfer1::ITensor*>tensors; 
+
+NetworkRT::NetworkRT(Network *net, const char *name) {
+
+    float rt_ver = float(NV_TENSORRT_MAJOR) + 
+                   float(NV_TENSORRT_MINOR)/10 + 
+                   float(NV_TENSORRT_PATCH)/100;
+    std::cout<<"New NetworkRT (TensorRT v"<<rt_ver<<")\n";
+  
+    builderRT = createInferBuilder(loggerRT);
+    std::cout<<"Float16 support: "<<builderRT->platformHasFastFp16()<<"\n";
+    std::cout<<"Int8 support: "<<builderRT->platformHasFastInt8()<<"\n";
+    std::cout<<"DLAs: "<<builderRT->getNbDLACores()<<"\n";
+    networkRT = builderRT->createNetwork();
+
+    if(!fileExist(name)) {
+
+        //input and dataType
+        dataDim_t dim = net->layers[0]->input_dim;
+        dtRT = DataType::kFLOAT;
+
+        builderRT->setMaxBatchSize(1);
+        builderRT->setMaxWorkspaceSize(1 << 30);
+
+        if(net->fp16 && builderRT->platformHasFastFp16()) {
+            dtRT = DataType::kHALF;
+            builderRT->setHalf2Mode(true);
+        }
+        if(net->dla && builderRT->getNbDLACores() > 0) {
+            dtRT = DataType::kHALF;
+            builderRT->setFp16Mode(true);
+            builderRT->allowGPUFallback(true);
+            builderRT->setDefaultDeviceType(DeviceType::kDLA);
+            builderRT->setDLACore(0);
+        }
+       
+        //add input layer
+        ITensor *input = networkRT->addInput("data", DataType::kFLOAT, 
+                        DimsCHW{ dim.c, dim.h, dim.w});
+        checkNULL(input);
+
+        //add other layers
+        for(int i=0; i<net->num_layers; i++) {
+            Layer *l = net->layers[i];
+            ILayer *Ilay = convert_layer(input, l);
+            Ilay->setName( (l->getLayerName() + std::to_string(i)).c_str() );
+            
+            input = Ilay->getOutput(0);
+            input->setName( (l->getLayerName() + std::to_string(i) + "_out").c_str() );
+            
+            if(l->getLayerType() == LAYER_YOLO)
+                networkRT->markOutput(*input);
+            tensors[l] = input;
+        }
+        if(input == NULL)
+            FatalError("conversion failed");
+
+        //build tensorRT
+        input->setName("out");
+        networkRT->markOutput(*input);
+
+        std::cout<<"Building tensorRT cuda engine...\n";
+        engineRT = builderRT->buildCudaEngine(*networkRT);
+        // we don't need the network any more
+        //networkRT->destroy();
+        serialize(name);
+    } else {
+        deserialize(name);
+    }
+
+    std::cout<<"create execution context\n";
+	contextRT = engineRT->createExecutionContext();
+
+	// input and output buffer pointers that we pass to the engine - the engine requires exactly IEngine::getNbBindings(),
+	std::cout<<"Input/outputs numbers: "<<engineRT->getNbBindings()<<"\n";
+    if(engineRT->getNbBindings() > MAX_BUFFERS_RT)
+        FatalError("over RT buffer array size");
+
+	// In order to bind the buffers, we need to know the names of the input and output tensors.
+	// note that indices are guaranteed to be less than IEngine::getNbBindings()
+	buf_input_idx = engineRT->getBindingIndex("data"); 
+    buf_output_idx = engineRT->getBindingIndex("out");
+    std::cout<<"input idex = "<<buf_input_idx<<" -> output index = "<<buf_output_idx<<"\n";
+
+
+    Dims iDim = engineRT->getBindingDimensions(buf_input_idx);
+    input_dim.n = 1;
+    input_dim.c = iDim.d[0];
+    input_dim.h = iDim.d[1];
+    input_dim.w = iDim.d[2];
+    input_dim.print();
+
+    Dims oDim = engineRT->getBindingDimensions(buf_output_idx);
+    output_dim.n = 1;
+    output_dim.c = oDim.d[0];
+    output_dim.h = oDim.d[1];
+    output_dim.w = oDim.d[2];
+    output_dim.print();
+	
+    // create GPU buffers and a stream
+    for(int i=0; i<engineRT->getNbBindings(); i++) {
+        Dims dim = engineRT->getBindingDimensions(i);
+        checkCuda(cudaMalloc(&buffersRT[i], dim.d[0]*dim.d[1]*dim.d[2]*sizeof(dnnType)));
+    }
+    checkCuda(cudaMalloc(&output, output_dim.tot()*sizeof(dnnType)));
+	checkCuda(cudaStreamCreate(&stream));
+}
+
+NetworkRT::~NetworkRT() {
+
+}
+
+dnnType* NetworkRT::infer(dataDim_t &dim, dnnType* data) {
+
+    checkCuda(cudaMemcpyAsync(buffersRT[buf_input_idx], data, input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    contextRT->enqueue(1, buffersRT, stream, nullptr);
+    checkCuda(cudaMemcpyAsync(output, buffersRT[buf_output_idx], output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    cudaStreamSynchronize(stream);
+
+    dim = output_dim;
+
+    return output;
+}
+
+void NetworkRT::enqueue() {
+    contextRT->enqueue(1, buffersRT, stream, nullptr);
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
+
+    layerType_t type = l->getLayerType();
+
+    if(type == LAYER_DENSE)
+        return convert_layer(input, (Dense*) l);
+    if(type == LAYER_CONV2D)
+        return convert_layer(input, (Conv2d*) l);
+    if(type == LAYER_POOLING)
+        return convert_layer(input, (Pooling*) l);
+    if(type == LAYER_ACTIVATION)
+        return convert_layer(input, (Activation*) l);
+    if(type == LAYER_SOFTMAX)
+        return convert_layer(input, (Softmax*) l);
+    if(type == LAYER_ROUTE)
+        return convert_layer(input, (Route*) l);
+    if(type == LAYER_REORG)
+        return convert_layer(input, (Reorg*) l);
+    if(type == LAYER_REGION)
+        return convert_layer(input, (Region*) l);
+    if(type == LAYER_SHORTCUT)
+        return convert_layer(input, (Shortcut*) l);
+    if(type == LAYER_YOLO)
+        return convert_layer(input, (Yolo*) l);
+    if(type == LAYER_UPSAMPLE)
+        return convert_layer(input, (Upsample*) l);
+
+    FatalError("Layer not implemented in tensorRT");
+    return NULL;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Dense *l) {
+    //std::cout<<"convert Dense\n";
+    void *data_b, *bias_b;
+    if(dtRT == DataType::kHALF) {
+        data_b     = l->data16_h;    
+        bias_b     = l->bias16_h;
+    } else {
+        data_b     = l->data_h;    
+        bias_b     = l->bias_h;
+    }
+
+    Weights w { dtRT, data_b, l->inputs*l->outputs};
+    Weights b = { dtRT, bias_b, l->outputs};
+    IFullyConnectedLayer *lRT = networkRT->addFullyConnected(*input, l->outputs, w, b);
+
+    checkNULL(lRT);
+    return lRT;
+}
+
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
+    //std::cout<<"convert conv2D\n";
+
+    void *data_b, *bias_b, *power_b, *mean_b, *variance_b, *scales_b;
+    if(dtRT == DataType::kHALF) {
+        data_b     = l->data16_h;    
+        bias_b     = l->bias16_h;
+        power_b    = l->power16_h;
+        mean_b     = l->mean16_h;
+        variance_b = l->variance16_h;
+        scales_b   = l->scales16_h;
+    } else {
+        data_b     = l->data_h;    
+        bias_b     = l->bias_h;
+        power_b    = l->power_h;
+        mean_b     = l->mean_h;
+        variance_b = l->variance_h;
+        scales_b   = l->scales_h;
+    }
+
+
+    Weights w { dtRT, data_b, l->inputs*l->outputs*l->kernelH*l->kernelW};
+    Weights b;
+    if(!l->batchnorm)
+        b = { dtRT, bias_b, l->outputs};
+    else
+        b = { dtRT, nullptr, 0}; //on batchnorm bias are added later
+
+    // Add a convolution layer with 20 outputs and a 5x5 filter.
+    IConvolutionLayer *lRT = networkRT->addConvolution(*input, 
+               l->outputs, DimsHW{l->kernelH, l->kernelW}, w, b);
+    checkNULL(lRT);
+
+    lRT->setStride(DimsHW{l->strideH, l->strideW});
+    lRT->setPadding(DimsHW{l->paddingH, l->paddingW});
+
+    if(l->batchnorm) {
+        Weights power{dtRT, power_b, l->outputs};
+        Weights shift{dtRT, mean_b, l->outputs};
+        Weights scale{dtRT, variance_b, l->outputs};
+        IScaleLayer *lRT2 = networkRT->addScale(*lRT->getOutput(0), ScaleMode::kCHANNEL, 
+                    shift, scale, power);
+        checkNULL(lRT2);
+
+        Weights shift2{dtRT, bias_b, l->outputs};
+        Weights scale2{dtRT, scales_b, l->outputs};
+        IScaleLayer *lRT3 = networkRT->addScale(*lRT2->getOutput(0), ScaleMode::kCHANNEL, 
+                    shift2, scale2, power);
+        checkNULL(lRT3);
+
+        return lRT3;
+    }
+
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) {
+    //std::cout<<"convert Pooling\n";
+
+    IPoolingLayer *lRT = networkRT->addPooling(*input, 
+        PoolingType::kMAX, DimsHW{l->winH, l->winW});
+    checkNULL(lRT);
+    lRT->setStride(DimsHW{l->strideH, l->strideW});
+
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
+    //std::cout<<"convert Activation\n";
+
+    if(l->act_mode == ACTIVATION_LEAKY) {
+        //std::cout<<"New plugin LEAKY\n";
+        
+        /*
+        // plugin version
+        IPlugin *plugin = new ActivationLeakyRT();
+        IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+        checkNULL(lRT);
+        return lRT;
+        */
+
+        IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU);
+        lRT->setAlpha(0.1);
+        checkNULL(lRT);
+        return lRT;
+
+    } else if(l->act_mode == CUDNN_ACTIVATION_RELU) {
+        IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kRELU);
+        checkNULL(lRT);
+        return lRT;
+    
+    } else {
+        FatalError("this Activation mode is not yet implemented");
+        return NULL;
+    }
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Softmax *l) {
+    //std::cout<<"convert softmax\n";
+
+    ISoftMaxLayer *lRT = networkRT->addSoftMax(*input);
+    checkNULL(lRT);
+
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Route *l) {
+    //std::cout<<"convert route\n";
+
+    ITensor **tens = new ITensor*[l->layers_n];
+    for(int i=0; i<l->layers_n; i++) {
+        tens[i] = tensors[l->layers[i]];
+    }
+    IConcatenationLayer *lRT = networkRT->addConcatenation(tens, l->layers_n);
+    //IPlugin *plugin = new RouteRT();
+    //IPluginLayer *lRT = networkRT->addPlugin(tens, l->layers_n, *plugin);
+    checkNULL(lRT);
+
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
+    //std::cout<<"convert Reorg\n";
+
+    //std::cout<<"New plugin REORG\n";
+    IPlugin *plugin = new ReorgRT(l->stride);
+    IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+    checkNULL(lRT);
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Region *l) {
+    //std::cout<<"convert Region\n";
+
+    //std::cout<<"New plugin REGION\n";
+    IPlugin *plugin = new RegionRT(l->classes, l->coords, l->num);
+    IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+    checkNULL(lRT);
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
+    //std::cout<<"convert Shortcut\n";
+
+    //std::cout<<"New plugin Shortcut\n";
+    
+    ITensor *back_tens = tensors[l->backLayer];
+    /*
+    // plugin version
+    IPlugin *plugin = new ShortcutRT();
+    ITensor **inputs = new ITensor*[2];
+    inputs[0] = input;
+    inputs[1] = back_tens; 
+    IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin);
+    checkNULL(lRT);
+    */
+
+    IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM);
+    checkNULL(lRT);
+    
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
+    //std::cout<<"convert Yolo\n";
+
+    //std::cout<<"New plugin YOLO\n";
+    IPlugin *plugin = new YoloRT(l->classes, l->num, l);
+    IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+    checkNULL(lRT);
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Upsample *l) {
+    //std::cout<<"convert Upsample\n";
+
+    //std::cout<<"New plugin UPSAMPLE\n";
+    IPlugin *plugin = new UpsampleRT(l->stride);
+    IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+    checkNULL(lRT);
+    return lRT;
+}
+
+bool NetworkRT::serialize(const char *filename) {
+
+    std::ofstream p(filename);
+    if (!p) {
+        FatalError("could not open plan output file");
+        return false;
+    }
+
+    IHostMemory *ptr = engineRT->serialize();
+    if(ptr == nullptr)
+        FatalError("Cant serialize network");
+
+    p.write(reinterpret_cast<const char*>(ptr->data()), ptr->size());
+    ptr->destroy();
+    return true;
+}
+
+bool NetworkRT::deserialize(const char *filename) {
+
+    char *gieModelStream{nullptr};
+    size_t size{0};
+    std::ifstream file(filename, std::ios::binary);
+    if (file.good()) {
+        file.seekg(0, file.end);
+        size = file.tellg();
+        file.seekg(0, file.beg);
+        gieModelStream = new char[size];
+        file.read(gieModelStream, size);
+        file.close();
+    }
+
+    pluginFactory = new PluginFactory();
+    runtimeRT = createInferRuntime(loggerRT);
+    engineRT = runtimeRT->deserializeCudaEngine(gieModelStream, size, (IPluginFactory *) pluginFactory);
+    //if (gieModelStream) delete [] gieModelStream;
+
+    return true;
+}
+
+
+
+IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, size_t serialLength) {
+    const char * buf = reinterpret_cast<const char*>(serialData);
+    
+    std::string name(layerName);
+
+    if(name.find("Activation") == 0) {
+        ActivationLeakyRT *a = new ActivationLeakyRT();
+        a->size = readBUF<int>(buf);
+        return a;
+    }
+
+    if(name.find("Region") == 0) {
+        RegionRT *r = new RegionRT(readBUF<int>(buf),    //classes
+                                    readBUF<int>(buf),    //coords
+                                    readBUF<int>(buf));   //num
+
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        return r;
+    } 
+
+    if(name.find("Reorg") == 0) {
+        ReorgRT *r = new ReorgRT(readBUF<int>(buf)); //stride
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        return r;
+    } 
+
+    if(name.find("Shortcut") == 0) {
+        ShortcutRT *r = new ShortcutRT();
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        return r;
+    } 
+
+    if(name.find("Yolo") == 0) {
+        YoloRT *r = new YoloRT(readBUF<int>(buf),    //classes
+                                readBUF<int>(buf));   //num
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        for(int i=0; i<r->num; i++)
+            r->mask[i] = readBUF<dnnType>(buf);
+        for(int i=0; i<3*2*r->num; i++)
+            r->bias[i] = readBUF<dnnType>(buf);
+
+		// save classes names
+        r->classesNames.resize(r->classes);
+		for(int i=0; i<r->classes; i++) {
+            char tmp[YOLORT_CLASSNAME_W];
+			for(int j=0; j<YOLORT_CLASSNAME_W; j++)
+				tmp[j] = readBUF<char>(buf);
+            r->classesNames[i] = std::string(tmp);
+		}
+
+        yolos[n_yolos++] = r;
+        return r;
+    } 
+
+    if(name.find("Upsample") == 0) {
+        UpsampleRT *r = new UpsampleRT(readBUF<int>(buf)); //stride
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        return r;
+    } 
+/*
+    if(name.find("Route") == 0) {
+        RouteRT *r = new RouteRT();
+        r->in = readBUF<int>(buf);
+        for(int i=0; i<RouteRT::MAX_INPUTS; i++)
+            r->c_in[i] = readBUF<int>(buf);
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        return r;
+    } 
+*/
+    FatalError("Cant deserialize Plugin");
+    return NULL;
+}
+
+}}

src/Pooling.cpp

@@ -3,22 +3,20 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Pooling::Pooling(   Network *net, dataDim_t input_dim, 
+Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
-                    int winH, int winW, int strideH, int strideW, tkdnnPoolingMode_t pool_mode) : 
+                  tkdnnPoolingMode_t pool_mode) : 
-    Layer(net, input_dim) {
+    Layer(net) {
-
-
-    if(winH != strideH || winW != strideW)
-        FatalError("stride pooling not yet implemented");
 
     this->winH = winH;
     this->winW = winW;
     this->strideH = strideH;
     this->strideW = strideW;
     this->pool_mode = pool_mode;
-    
+    this->paddingH = 0;
+    this->paddingW = 0;
+
     checkCUDNN( cudnnCreatePoolingDescriptor(&poolingDesc) );
 
     int n = input_dim.n;
@@ -46,27 +44,28 @@ Pooling::Pooling(   Network *net, dataDim_t input_dim,
                 net->tensorFormat, net->dataType, n, c, h, w) );
 
     //get out dim
-    h = h / winH; w = w / winW;
+    checkCUDNN( cudnnGetPooling2dForwardOutputDim(poolingDesc, srcTensorDesc, &n, &c, &h, &w)); 
-    
+    //h = (h + winH*this->paddingH)/strideH;
+    //w = (w + winW*this->paddingW)/strideW;
+
     checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
                 net->tensorFormat, net->dataType, n, c, h, w) );
        
-
     output_dim.n = n;
     output_dim.c = c;
     output_dim.h = h;
     output_dim.w = w;
     output_dim.l = l;
 
-    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 
     //pool on 3d data need transposition at the enter and on the exit
     //allocate for initial and final transposition
     if(poolOn3d) {
         output_dim.n = 1;
 
-        checkCuda( cudaMalloc(&tmpInputData, input_dim.tot()*sizeof(value_type)) );
+        checkCuda( cudaMalloc(&tmpInputData, input_dim.tot()*sizeof(dnnType)) );
-        checkCuda( cudaMalloc(&tmpOutputData, output_dim.tot()*sizeof(value_type)) );
+        checkCuda( cudaMalloc(&tmpOutputData, output_dim.tot()*sizeof(dnnType)) );
     }
 
 }
@@ -82,10 +81,10 @@ Pooling::~Pooling() {
     checkCuda( cudaFree(dstData) );
 }
 
-value_type* Pooling::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* Pooling::infer(dataDim_t &dim, dnnType* srcData) {
 
-    value_type *poolSrc = srcData;
+    dnnType *poolSrc = srcData;
-    value_type *poolDst = dstData;
+    dnnType *poolDst = dstData;
 
     if(poolOn3d) {
         matrixTranspose(net->cublasHandle, srcData, tmpInputData, dim.h*dim.w*dim.c, dim.l);
@@ -93,8 +92,8 @@ value_type* Pooling::infer(dataDim_t &dim, value_type* srcData) {
         poolDst = tmpOutputData;
     }
 
-    value_type alpha = value_type(1);
+    dnnType alpha = dnnType(1);
-    value_type beta = value_type(0);
+    dnnType beta = dnnType(0);
     checkCUDNN( cudnnPoolingForward(net->cudnnHandle, poolingDesc,
                 &alpha, srcTensorDesc, poolSrc,
                 &beta, dstTensorDesc, poolDst) );
@@ -108,4 +107,4 @@ value_type* Pooling::infer(dataDim_t &dim, value_type* srcData) {
     return dstData;
 }
 
-}
+}}

src/Region.cpp

@@ -0,0 +1,342 @@
+#include <iostream>
+
+#ifdef OPENCV
+    #include <opencv2/core/core.hpp>
+    #include <opencv2/highgui/highgui.hpp>
+    #include <opencv2/imgproc/imgproc.hpp>
+#endif
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Region::Region(Network *net, int classes, int coords, int num) : 
+    Layer(net) {
+
+    this->classes = classes;
+    this->coords = coords;
+    this->num = num;
+
+    // same
+    output_dim.n = input_dim.n;
+    output_dim.c = input_dim.c;
+    output_dim.h = input_dim.h;
+    output_dim.w = input_dim.w;
+    output_dim.l = input_dim.l;
+
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+}
+
+Region::~Region() {
+    checkCuda( cudaFree(dstData) );
+}
+
+int entry_index(int batch, int location, int entry, 
+            int coords, int classes, dataDim_t &input_dim, dataDim_t &output_dim) {
+    int n =   location / (input_dim.w*input_dim.h);
+    int loc = location % (input_dim.w*input_dim.h);
+    return batch*output_dim.tot() + n*input_dim.w*input_dim.h*(coords+classes+1) +
+           entry*input_dim.w*input_dim.h + loc;
+}
+
+dnnType* Region::infer(dataDim_t &dim, dnnType* srcData) {
+
+    checkCuda( cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+
+    for (int b = 0; b < dim.n; ++b){
+        for(int n = 0; n < num; ++n){
+            int index = entry_index(b, n*dim.w*dim.h, 0, coords, classes, input_dim, output_dim);
+            activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
+            
+            index = entry_index(b, n*dim.w*dim.h, coords, coords, classes, input_dim, output_dim);
+            activationLOGISTICForward(srcData + index, dstData + index, dim.w*dim.h);
+        }
+    }
+
+    //softmax start
+    int index = entry_index(0, 0, coords + 1, coords, classes, input_dim, output_dim);
+    softmaxForward(srcData + index, classes, output_dim.n*num, output_dim.tot()/num, 
+                   output_dim.w*output_dim.h, 1, output_dim.w*output_dim.h, 1, dstData + index);
+
+    dim = output_dim;
+    return dstData;
+}
+
+
+/* Intepret class */
+RegionInterpret::RegionInterpret(dataDim_t input_dim, dataDim_t output_dim, 
+    int classes, int coords, int num, float thresh, std::string fname_weights) {
+
+    this->input_dim = input_dim;
+    this->output_dim = output_dim;
+
+    this->classes = classes;
+    this->coords = coords;
+    this->num = num;
+    this->thresh = thresh;
+    this->res_boxes_n = 0;
+
+    int tot = output_dim.w*output_dim.h*num;
+    boxes = (box*)    malloc(tot*sizeof(box));
+    probs = (float**) malloc(tot*sizeof(float *));
+    for(int j = 0; j < tot; ++j) probs[j] = (float*) malloc((classes + 1)*sizeof(float *));
+    s = (sortable_bbox*) malloc(tot*sizeof(sortable_bbox));
+
+    //load anchors
+    readBinaryFile(fname_weights, 2*num, &bias_h, &bias_d);
+}
+
+RegionInterpret::~RegionInterpret() {
+
+    delete [] boxes;
+    for(int j = 0; j < output_dim.w*output_dim.h*num; ++j) 
+        delete [] probs[j];
+    delete [] probs;
+    delete [] s;
+
+    delete [] bias_h;
+    checkCuda( cudaFree(bias_d) );
+}
+
+box RegionInterpret::get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride)
+{
+    box b;
+    b.x = (i + x[index + 0*stride]) / w;
+    b.y = (j + x[index + 1*stride]) / h;
+    b.w = exp(x[index + 2*stride]) * biases[2*n]   / w;
+    b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
+    return b;
+}
+
+void RegionInterpret::get_region_boxes(  float *input, int w, int h, int netw, int neth, float thresh, 
+                                float **probs, box *boxes, int only_objectness, 
+                                int *map, float tree_thresh, int relative) {
+    int lh = output_dim.h;
+    int lw = output_dim.w;
+    float *predictions = input;
+    for (int i = 0; i < lw*lh; ++i){
+
+        int row = i / lw;
+        int col = i % lw;
+        for(int n = 0; n < num; ++n){
+
+            int index = n*lw*lh + i;
+            for(int j = 0; j < classes; ++j){
+                probs[index][j] = 0;
+            }
+            int obj_index  = entry_index(0, n*lw*lh + i, 
+                                         coords, coords, classes, output_dim, output_dim);
+            int box_index  = entry_index(0, n*lw*lh + i, 0, 
+                                         coords, classes, output_dim, output_dim);
+            float scale = predictions[obj_index];
+            boxes[index] = get_region_box(predictions, bias_h, n, box_index, col, row, lw, lh, lw*lh);
+
+            float max = 0;
+            for(int j = 0; j < classes; ++j){
+                int class_index = entry_index(0, n*lw*lh + i, coords + 1 + j,
+                                              coords, classes, output_dim, output_dim);
+                float prob = scale*predictions[class_index];
+                probs[index][j] = (prob > thresh) ? prob : 0;
+                if(prob > max) max = prob;
+            }
+            probs[index][classes] = max;
+        }
+    }
+    correct_region_boxes(boxes, lw*lh*num, w, h, netw, neth, relative);
+}
+
+
+void RegionInterpret::correct_region_boxes(box *boxes, int n, int w, int h, int netw, int neth, int relative) {
+    int i;
+    int new_w=0;
+    int new_h=0;
+    if (((float)netw/w) < ((float)neth/h)) {
+        new_w = netw;
+        new_h = (h * netw)/w;
+    } else {
+        new_h = neth;
+        new_w = (w * neth)/h;
+    }
+    for (i = 0; i < n; ++i){
+        box b = boxes[i];
+        b.x =  (b.x - (netw - new_w)/2./netw) / ((float)new_w/netw); 
+        b.y =  (b.y - (neth - new_h)/2./neth) / ((float)new_h/neth); 
+        b.w *= (float)netw/new_w;
+        b.h *= (float)neth/new_h;
+        if(!relative){
+            b.x *= w;
+            b.w *= w;
+            b.y *= h;
+            b.h *= h;
+        }
+        boxes[i] = b;
+    }
+}
+
+
+
+
+//############################ BOX PROBABILITY UTILS ############################
+int nms_comparator(const void *pa, const void *pb) {
+    sortable_bbox a = *(sortable_bbox *)pa;
+    sortable_bbox b = *(sortable_bbox *)pb;
+    float diff = a.probs[a.index][b.cl] - b.probs[b.index][b.cl];
+    if(diff < 0) return 1;
+    else if(diff > 0) return -1;
+    return 0;
+}
+float overlap(float x1, float w1, float x2, float w2) {
+    /*
+    //SLOW METHOD
+    float l1 = x1 - w1/2;
+    float l2 = x2 - w2/2;
+    float left = l1 > l2 ? l1 : l2;
+    float r1 = x1 + w1/2;
+    float r2 = x2 + w2/2;
+    float right = r1 < r2 ? r1 : r2;
+    return right - left;
+    */
+
+    //SPALLA METHOD
+    float l;
+    w1 < w2? l=w1 : l=w2;
+    float d = fabs(x1 - x2);
+    float k = fabs(w1 - w2)/2;
+    if      (d <= k)    return l;
+    else if (d <= k +l) return l - (d-k);
+    else                return 0;
+}
+float box_intersection(box a, box b) {
+    float w = overlap(a.x, a.w, b.x, b.w);
+    if(w <= 0) return 0;
+    float h = overlap(a.y, a.h, b.y, b.h);
+    if(h <= 0) return 0;
+    float area = w*h;
+    return area;
+}
+float box_union(box a, box b) {
+    float i = box_intersection(a, b);
+    float u = a.w*a.h + b.w*b.h - i;
+    return u;
+}
+int max_index(float *a, int n) {
+    if(n <= 0) return -1;
+    int i, max_i = 0;
+    float max = a[0];
+    for(i = 1; i < n; ++i){
+        if(a[i] > max){
+            max = a[i];
+            max_i = i;
+        }
+    }
+    return max_i;
+}
+//###############################################################################
+float RegionInterpret::box_iou(box a, box b) {
+    if(fabs(a.x - b.x) > (a.w+b.w)/2 || fabs(a.y - b.y) > (a.h+b.h)/2)
+        return 0;
+    return box_intersection(a, b)/box_union(a, b);
+}
+
+
+
+void RegionInterpret::interpretData(dnnType *data_h, int imageW, int imageH) {
+
+    int imW, imH;
+    if(imageW <= 0 || imageH <= 0) {
+        imW = input_dim.w;
+        imH = input_dim.h;
+    } else {
+        imW = imageW;
+        imH = imageH;
+    }
+
+    int tot = output_dim.w*output_dim.h*num;
+
+    get_region_boxes(data_h, imW, imH, output_dim.w, output_dim.h, thresh, probs, boxes, 0, 0, 0.5, 1);
+
+    //delete repeats
+    for(int i = 0; i < tot; ++i){
+        s[i].index = i;       
+        s[i].cl = classes;
+        s[i].probs = probs;
+    }
+    qsort(s, tot, sizeof(sortable_bbox), nms_comparator);
+
+    for(int i = 0; i < tot; ++i){
+        if(probs[s[i].index][classes] == 0) continue;
+        box a = boxes[s[i].index];
+        for(int j = i+1; j < tot; ++j){
+            box b = boxes[s[j].index];
+            if (box_iou(a, b) > 0.3f){
+                for(int k = 0; k < classes+1; ++k){
+                    probs[s[j].index][k] = 0;
+                }
+            }
+        }
+    }
+
+    res_boxes_n = 0;
+    //print results
+    for(int i = 0; i < tot; ++i){
+        int cl = max_index(probs[i], classes);
+        float prob = probs[i][cl];
+
+        if(prob > thresh) {
+            box b = boxes[i];
+            int x = (b.x)*imW;
+            int w = (b.w)*imW - b.x;
+            int y = (b.y)*imH;
+            int h = (b.h)*imH - b.y;
+
+            //if(x < 0) x = 0;
+            //if(y < 0) y = 0;
+            //if(w > imW) w = imW;
+            //if(h > imH) h = imH;
+
+            //printf("%d: %.0f%% box(x1, y1, x2, y2): %d %d %d %d\n", cl, prob*100, x, y, w, h);
+            b.x = x;
+            b.y = y;
+            b.h = h;
+            b.w = w;
+            b.cl = cl;
+            b.prob = prob;
+            res_boxes[res_boxes_n] = b;
+            res_boxes_n++;
+        }
+    }
+}
+
+void RegionInterpret::showImageResult(dnnType *input_h) {
+
+#ifdef OPENCV
+    dataDim_t dim = input_dim;
+    // read an image
+    cv::Mat r(dim.h, dim.w, CV_32F, input_h);
+    cv::Mat g(dim.h, dim.w, CV_32F, input_h + dim.h*dim.w);
+    cv::Mat b(dim.h, dim.w, CV_32F, input_h + dim.h*dim.w*2);
+    std::vector<cv::Mat> array_to_merge;
+    array_to_merge.push_back(b);
+    array_to_merge.push_back(g);
+    array_to_merge.push_back(r);
+    cv::Mat color;
+    cv::merge(array_to_merge, color);
+
+    for(int i=0; i<res_boxes_n; i++) {
+        box bx = res_boxes[i];
+        cv::rectangle(color, cv::Point(bx.x - bx.w/2, bx.y - bx.h/2), 
+                             cv::Point(bx.x + bx.w/2, bx.y + bx.h/2),
+        cv::Scalar( 0, 0, 255), 2);
+    }
+    cv::namedWindow("result");
+    // show the image on window
+    cv::imshow("result", color);
+    // wait key for 5000 ms
+    cv::waitKey(0);
+#else
+    std::cout<<"Visualization not supported, please recompile with OpenCV\n";
+#endif
+}
+
+}}

src/Reorg.cpp

@@ -0,0 +1,34 @@
+#include <iostream>
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Reorg::Reorg(Network *net, int stride) : Layer(net) {
+
+    this->stride = stride;
+    
+    output_dim.n = input_dim.n;
+    output_dim.c = input_dim.c*stride*stride;
+    output_dim.h = input_dim.h/stride;
+    output_dim.w = input_dim.w/stride;
+    output_dim.l = input_dim.l;
+    
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+}
+
+Reorg::~Reorg() {
+
+    checkCuda( cudaFree(dstData) );
+}
+
+dnnType* Reorg::infer(dataDim_t &dim, dnnType* srcData) {
+
+    reorgForward(srcData, dstData, dim.n, dim.c, dim.h, dim.w, stride);
+
+    dim = output_dim;
+    return dstData;
+}
+
+}}

src/Route.cpp

@@ -0,0 +1,56 @@
+#include <iostream>
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Route::Route(Network *net, Layer **layers, int layers_n) : Layer(net) {
+
+    this->layers = layers;
+    this->layers_n = layers_n;
+
+    //get dims
+    output_dim.l = 1;
+    output_dim.c = 0;
+    for(int i=0; i<layers_n; i++) {
+
+        if(i==0) {
+            output_dim.w = layers[i]->output_dim.w;
+            output_dim.h = layers[i]->output_dim.h;
+        } else {
+            if( layers[i]->output_dim.w != output_dim.w ||
+                layers[i]->output_dim.h != output_dim.h   )
+                FatalError("Route Output dim missmatch");
+        }
+        output_dim.c += layers[i]->output_dim.c;
+    }
+
+    input_dim = output_dim;
+
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+}
+
+Route::~Route() {
+
+    checkCuda( cudaFree(dstData) );
+}
+
+dnnType* Route::infer(dataDim_t &dim, dnnType* srcData) {
+
+
+    int offset = 0;
+    for(int i=0; i<layers_n; i++) {
+        dnnType *input = layers[i]->dstData;
+        int in_dim = layers[i]->output_dim.tot();
+        checkCuda( cudaMemcpy(dstData + offset, input, in_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+        offset += in_dim;
+    }
+
+    //update data dimensions    
+    dim = output_dim;
+
+    return dstData;
+}
+
+}}

src/Shortcut.cpp

@@ -0,0 +1,37 @@
+#include <iostream>
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) {
+
+    this->backLayer = backLayer;
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+
+    if( backLayer->output_dim.c != input_dim.c ||
+        backLayer->output_dim.w != input_dim.w ||
+        backLayer->output_dim.h != input_dim.h   )
+        FatalError("Shortcut dim missmatch");
+}
+
+Shortcut::~Shortcut() {
+
+    checkCuda( cudaFree(dstData) );
+}
+
+dnnType* Shortcut::infer(dataDim_t &dim, dnnType* srcData) {
+
+    dataDim_t bdim = this->backLayer->output_dim;
+
+    checkCuda(cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+    shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1);
+
+    //update data dimensions    
+    dim = output_dim;
+
+    return dstData;
+}
+
+}}

src/Softmax.cpp

@@ -3,12 +3,11 @@
 #include "Layer.h"
 #include "kernels.h"
 
-namespace tkDNN {
+namespace tk { namespace dnn {
 
-Softmax::Softmax(Network *net, dataDim_t input_dim) : 
+Softmax::Softmax(Network *net) : Layer(net) {
-    Layer(net, input_dim) {
 
-    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(value_type)) );
+    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 
     checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
                                         net->tensorFormat,
@@ -29,10 +28,10 @@ Softmax::~Softmax() {
     checkCuda( cudaFree(dstData) );
 }
 
-value_type* Softmax::infer(dataDim_t &dim, value_type* srcData) {
+dnnType* Softmax::infer(dataDim_t &dim, dnnType* srcData) {
 
-    value_type alpha = value_type(1);
+    dnnType alpha = dnnType(1);
-    value_type beta  = value_type(0);
+    dnnType beta  = dnnType(0);
     checkCUDNN( cudnnSoftmaxForward(net->cudnnHandle,
                                     CUDNN_SOFTMAX_ACCURATE ,
                                     CUDNN_SOFTMAX_MODE_CHANNEL,
@@ -45,4 +44,4 @@ value_type* Softmax::infer(dataDim_t &dim, value_type* srcData) {
     return dstData;
 }
 
-}
+}}

src/Upsample.cpp

@@ -0,0 +1,35 @@
+#include <iostream>
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Upsample::Upsample(Network *net, int stride) : Layer(net) {
+
+    this->stride = stride;
+
+    output_dim.n = input_dim.n;
+    output_dim.c = input_dim.c;
+    output_dim.h = input_dim.h*stride;
+    output_dim.w = input_dim.w*stride;
+    output_dim.l = input_dim.l;
+    
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+}
+
+Upsample::~Upsample() {
+
+    checkCuda( cudaFree(dstData) );
+}
+
+dnnType* Upsample::infer(dataDim_t &dim, dnnType* srcData) {
+
+    fill(dstData, output_dim.tot(), 0.0);
+    upsampleForward(srcData, dstData, input_dim.n, input_dim.c, input_dim.h, input_dim.w, stride, 1, 1);
+
+    dim = output_dim;
+    return dstData;
+}
+
+}}

src/Yolo.cpp

@@ -0,0 +1,251 @@
+#include <iostream>
+
+#ifdef OPENCV
+    #include <opencv2/core/core.hpp>
+    #include <opencv2/highgui/highgui.hpp>
+    #include <opencv2/imgproc/imgproc.hpp>
+#endif
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights) : 
+    Layer(net) {
+    
+    this->classes = classes;
+    this->num = num;
+
+    // load anchors
+    if(fname_weights != "") {
+        int seek = 0;
+        readBinaryFile(fname_weights, num, &mask_h, &mask_d, seek);
+        seek += num;
+        readBinaryFile(fname_weights, 3*num*2, &bias_h, &bias_d, seek);
+    }
+
+    // init default classes name
+    classesNames.clear();
+    for(int i=0; i<classes; i++) {
+        classesNames.push_back(std::to_string(i));
+    }
+
+    // same
+    output_dim.n = input_dim.n;
+    output_dim.c = input_dim.c;
+    output_dim.h = input_dim.h;
+    output_dim.w = input_dim.w;
+    output_dim.l = input_dim.l;
+
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+    predictions = nullptr;
+}
+
+Yolo::~Yolo() {
+    checkCuda( cudaFree(dstData) );
+}
+
+int entry_index(int batch, int location, int entry, 
+            int classes, dataDim_t &input_dim, dataDim_t &output_dim) {
+    int n =   location / (input_dim.w*input_dim.h);
+    int loc = location % (input_dim.w*input_dim.h);
+    return batch*output_dim.tot() + n*input_dim.w*input_dim.h*(4+classes+1) +
+           entry*input_dim.w*input_dim.h + loc;
+}
+
+Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride) {
+    Yolo::box b;
+    b.x = (i + x[index + 0*stride]) / lw;
+    b.y = (j + x[index + 1*stride]) / lh;
+    b.w = exp(x[index + 2*stride]) * biases[2*n]   / w;
+    b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
+    return b;
+}
+
+dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) {
+
+    checkCuda( cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+
+    for (int b = 0; b < dim.n; ++b){
+        for(int n = 0; n < num; ++n){
+            int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim);
+            activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
+            
+            index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim);
+            activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h);
+        }
+    }
+
+    dim = output_dim;
+    return dstData;
+}
+
+void correct_yolo_boxes(Yolo::detection *dets, int n, int w, int h, int netw, int neth, int relative)
+{
+    int i;
+    int new_w=0;
+    int new_h=0;
+    if (((float)netw/w) < ((float)neth/h)) {
+        new_w = netw;
+        new_h = (h * netw)/w;
+    } else {
+        new_h = neth;
+        new_w = (w * neth)/h;
+    }
+    for (i = 0; i < n; ++i){
+        Yolo::box b = dets[i].bbox;
+        b.x =  (b.x - (netw - new_w)/2./netw) / ((float)new_w/netw); 
+        b.y =  (b.y - (neth - new_h)/2./neth) / ((float)new_h/neth); 
+        b.w *= (float)netw/new_w;
+        b.h *= (float)neth/new_h;
+        if(!relative){
+            b.x *= w;
+            b.w *= w;
+            b.y *= h;
+            b.h *= h;
+        }
+        dets[i].bbox = b;
+    }
+}
+
+int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh) {
+
+    if(predictions == nullptr)
+        predictions = new dnnType[output_dim.tot()];
+    checkCuda( cudaMemcpy(predictions, dstData, output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost));
+
+    int lw = output_dim.w;
+    int lh = output_dim.h;
+
+    if (output_dim.n == 2) {
+        FatalError("BATCH of 2 not supported"); 
+        //avg_flipped_yolo(l);
+    }
+    int i,j,n;
+    int count = ndets;
+    for (i = 0; i < lw*lh; ++i){
+        int row = i / lw;
+        int col = i % lw;
+        for(n = 0; n < num; ++n){
+            int obj_index  = entry_index(0, n*lw*lh + i, 4, classes, input_dim, output_dim);
+            float objectness = predictions[obj_index];
+            if(objectness <= thresh) continue;
+            int box_index  = entry_index(0, n*lw*lh + i, 0, classes, input_dim, output_dim);
+            
+            dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh);
+            dets[count].objectness = objectness;
+            dets[count].classes = classes;
+            for(j = 0; j < classes; ++j){
+                int class_index = entry_index(0, n*lw*lh + i, 4 + 1 + j, classes, input_dim, output_dim);
+                float prob = objectness*predictions[class_index];
+                dets[count].prob[j] = (prob > thresh) ? prob : 0;
+            }
+            
+            ++count;
+            if(count >= MAX_DETECTIONS)
+                FatalError("reach max boxes");
+        }
+    }
+
+    correct_yolo_boxes(dets + ndets, count, netw, neth, netw, neth, 0);
+    ndets = count;
+    return count;
+}
+
+//////////////////////////////////////////////////////////////////
+float yolo_overlap(float x1, float w1, float x2, float w2)
+{
+    float l1 = x1 - w1/2;
+    float l2 = x2 - w2/2;
+    float left = l1 > l2 ? l1 : l2;
+    float r1 = x1 + w1/2;
+    float r2 = x2 + w2/2;
+    float right = r1 < r2 ? r1 : r2;
+    return right - left;
+}
+
+float yolo_box_intersection(Yolo::box a, Yolo::box b)
+{
+    float w = yolo_overlap(a.x, a.w, b.x, b.w);
+    float h = yolo_overlap(a.y, a.h, b.y, b.h);
+    if(w < 0 || h < 0) return 0;
+    float area = w*h;
+    return area;
+}
+
+float yolo_box_union(Yolo::box a, Yolo::box b)
+{
+    float i = yolo_box_intersection(a, b);
+    float u = a.w*a.h + b.w*b.h - i;
+    return u;
+}
+
+float yolo_box_iou(Yolo::box a, Yolo::box b)
+{
+    return yolo_box_intersection(a, b)/yolo_box_union(a, b);
+}
+
+int yolo_nms_comparator(const void *pa, const void *pb)
+{
+    Yolo::detection a = *(Yolo::detection *)pa;
+    Yolo::detection b = *(Yolo::detection *)pb;
+    float diff = 0;
+    if(b.sort_class >= 0){
+        diff = a.prob[b.sort_class] - b.prob[b.sort_class];
+    } else {
+        diff = a.objectness - b.objectness;
+    }
+    if(diff < 0) return 1;
+    else if(diff > 0) return -1;
+    return 0;
+}
+//////////////////////////////////////////////////////////////////7
+
+Yolo::detection *Yolo::allocateDetections(int nboxes, int classes) {
+    
+    int i;
+    Yolo::detection *dets = (Yolo::detection*) calloc(nboxes, sizeof(Yolo::detection));
+    for(i = 0; i < nboxes; ++i){
+        dets[i].prob = (float*) calloc(classes, sizeof(float));
+    }
+    return dets;
+}
+
+void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes) {
+    double nms_thresh = 0.45;
+    int total = ndets;
+
+    int i, j, k;
+    k = total-1;
+    for(i = 0; i <= k; ++i){
+        if(dets[i].objectness == 0){
+            detection swap = dets[i];
+            dets[i] = dets[k];
+            dets[k] = swap;
+            --k;
+            --i;
+        }
+    }
+    total = k+1;
+
+    for(k = 0; k < classes; ++k){
+        for(i = 0; i < total; ++i){
+            dets[i].sort_class = k;
+        }
+        qsort(dets, total, sizeof(detection), yolo_nms_comparator);
+        for(i = 0; i < total; ++i){
+            if(dets[i].prob[k] == 0) continue;
+            box a = dets[i].bbox;
+            for(j = i+1; j < total; ++j){
+                box b = dets[j].bbox;
+                if (yolo_box_iou(a, b) > nms_thresh){
+                    dets[j].prob[k] = 0;
+                }
+            }
+        }
+    }
+
+}
+
+}}

src/Yolo3Detection.cpp

@@ -0,0 +1,152 @@
+#include "Yolo3Detection.h"
+
+namespace tk { namespace dnn {
+
+float _colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} };
+float get_color(int c, int x, int max)
+{
+    float ratio = ((float)x/max)*5;
+    int i = floor(ratio);
+    int j = ceil(ratio);
+    ratio -= i;
+    float r = (1-ratio) * _colors[i % 6][c % 3] + ratio*_colors[j % 6][c % 3];
+    //printf("%f\n", r);
+    return r;
+}
+
+bool Yolo3Detection::init(std::string tensor_path) {
+
+    //const char *tensor_path = "../data/yolo3/yolo3_berkeley.rt";
+
+    //convert network to tensorRT
+    std::cout<<(tensor_path).c_str()<<"\n";
+    netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
+
+    if(netRT->pluginFactory->n_yolos != 3) {
+        FatalError("this is not yolo3");
+    }
+
+    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
+        YoloRT *yRT = netRT->pluginFactory->yolos[i];
+        classes = yRT->classes;
+        num = yRT->num;
+
+        // make a yolo layer for interpret predictions
+        yolo[i] = new tk::dnn::Yolo(nullptr, classes, num, ""); // yolo without input and bias
+        yolo[i]->mask_h = new dnnType[num];
+        yolo[i]->bias_h = new dnnType[num*3*2];
+        memcpy(yolo[i]->mask_h, yRT->mask, sizeof(dnnType)*num);
+        memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*3*2);
+        yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w);
+        yolo[i]->classesNames = yRT->classesNames;
+    }
+
+    dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
+    
+    checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()));
+    checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
+
+
+    // class colors precompute    
+    for(int c=0; c<classes; c++) {
+        int offset = c*123457 % classes;
+        float r = get_color(2, offset, classes);
+        float g = get_color(1, offset, classes);
+        float b = get_color(0, offset, classes);
+        colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
+    }
+    return true;
+} 
+
+
+void Yolo3Detection::update(cv::Mat &imageORIG) {
+
+    if(!imageORIG.data) {
+        std::cout<<"YOLO: NO IMAGE DATA\n";
+        return;
+    }     
+    float xRatio =  float(imageORIG.cols) / float(netRT->input_dim.w);
+    float yRatio =  float(imageORIG.rows) / float(netRT->input_dim.h);
+
+    resize(imageORIG, imageORIG, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+
+    imageORIG.convertTo(imageF, CV_32FC3, 1/255.0); 
+
+    //split channels
+    cv::split(imageF,bgr);//split source
+
+    //write channels
+    for(int i=0; i<netRT->input_dim.c; i++) {
+        int idx = i*imageF.rows*imageF.cols;
+        int ch = netRT->input_dim.c-1 -i;
+        memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
+    }
+
+
+    //DO INFERENCE
+    dnnType *rt_out[3]; 
+    tk::dnn::dataDim_t dim = netRT->input_dim;
+    checkCuda(cudaMemcpyAsync(input_d, input, dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
+
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim.print();
+        TIMER_START
+        netRT->infer(dim, input_d);
+        TIMER_STOP
+        dim.print();
+    
+        stats.push_back(t_ns);
+    }
+    
+    TIMER_START
+    // compute dets
+    ndets = 0;
+    for(int i=0; i<3; i++) {
+        rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
+        yolo[i]->dstData = rt_out[i];
+        yolo[i]->computeDetections(dets, ndets, netRT->input_dim.w, netRT->input_dim.h, thresh);
+    }
+    tk::dnn::Yolo::mergeDetections(dets, ndets, classes);
+    TIMER_STOP
+
+    // fill detected
+    detected.clear();
+    for(int j=0; j<ndets; j++) {
+        tk::dnn::Yolo::box b = dets[j].bbox;
+        int x0   = (b.x-b.w/2.);
+        int x1   = (b.x+b.w/2.);
+        int y0   = (b.y-b.h/2.);
+        int y1   = (b.y+b.h/2.);
+        int obj_class = -1;
+        float prob = 0;
+        for(int c=0; c<classes; c++) {
+            if(dets[j].prob[c] >= thresh) {
+                obj_class = c;
+                prob = dets[j].prob[c];
+            }
+        }
+
+        if(obj_class >= 0) {
+            //std::cout<<obj_class<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
+            //cv::rectangle(image, cv::Point(x0, y0), cv::Point(x1, y1), colors[obj_class], 2);
+            
+            // convert to image coords
+            x0 = xRatio*x0;
+            x1 = xRatio*x1;
+            y0 = yRatio*y0;
+            y1 = yRatio*y1;
+              
+            tk::dnn::box res;
+            res.cl = obj_class;
+            res.prob = prob;
+            res.x = x0;
+            res.y = y0;
+            res.w = x1 - x0;
+            res.h = y1 - y0;
+            detected.push_back(res);
+        }
+    }
+
+}
+
+}}

src/kernels/activation_elu.cu

@@ -7,12 +7,12 @@
         x > 0 :    y = x
 */
 __global__
-void activation_elu(value_type *input, value_type *output, int size) {
+void activation_elu(dnnType *input, dnnType *output, int size) {
 
     int i = blockDim.x*blockIdx.x + threadIdx.x;
 
     if(i<size) {    
-        value_type k0, k1;
+        dnnType k0, k1;
         
         if (input[i]>0)
             k0 = 1.0f;
@@ -28,11 +28,10 @@ void activation_elu(value_type *input, value_type *output, int size) {
 /**
     ELU activation function
 */
-void activationELUForward(value_type* srcData, value_type* dstData, int size)
+void activationELUForward(dnnType* srcData, dnnType* dstData, int size, const cudaStream_t stream)
 {
     int blocks = (size+255)/256;
     int threads = 256;
     
-    activation_elu<<<blocks, threads>>>(srcData, dstData, size);
+    activation_elu<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
-    checkCuda( cudaDeviceSynchronize() );
+}
-}

src/kernels/activation_leaky.cu

@@ -0,0 +1,28 @@
+#include "kernels.h"
+
+__global__
+void activation_leaky(dnnType *input, dnnType *output, int size) {
+
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+    if(i<size) {    
+        if (input[i]>0)
+            output[i] = input[i];
+        else
+            output[i] = 0.1f*input[i];
+    }
+ }
+
+
+/**
+    ELU activation function
+*/
+void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    activation_leaky<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
+}
+
+

src/kernels/activation_logistic.cu

@@ -0,0 +1,25 @@
+#include "kernels.h"
+
+__global__
+void activation_logistic(dnnType *input, dnnType *output, int size) {
+
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+    if(i<size) {    
+        output[i] =  1.0f/(1.0f + exp(-input[i]));;
+    }
+ }
+
+
+/**
+    LOGISTIC activation function
+*/
+void activationLOGISTICForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    activation_logistic<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
+}
+
+

src/kernels/convert.cu

@@ -0,0 +1,21 @@
+#include "kernels.h"
+
+__global__
+void float2half_device(float *input, __half *output, int size) {
+
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+    if(i<size) {    
+        output[i] = __float2half(input[i]);
+    }
+}
+
+
+void float2half(float* srcData, __half *dstData, int size, const cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    float2half_device<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
+    cudaDeviceSynchronize();
+}

src/kernels/fill.cu

@@ -0,0 +1,21 @@
+#include "kernels.h"
+
+__global__
+void fill_kernel(dnnType *data, int size, dnnType val) {
+
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+    if(i<size) {    
+        data[i] = val;
+    }
+ }
+
+
+void fill(dnnType* data, int size, dnnType val, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    fill_kernel<<<blocks, threads, 0, stream>>>(data, size, val);
+}
+
+

src/kernels/reorg.cu

@@ -0,0 +1,49 @@
+#include "kernels.h"
+
+__global__ void reorg_kernel(int N, float *x, int w, int h, int c, int batch, int stride, int forward, float *out)
+{
+    int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if(i >= N) return;
+    int in_index = i;
+    int in_w = i%w;
+    i = i/w;
+    int in_h = i%h;
+    i = i/h;
+    int in_c = i%c;
+    i = i/c;
+    int b = i%batch;
+
+    int out_c = c/(stride*stride);
+
+    int c2 = in_c % out_c;
+    int offset = in_c / out_c;
+    int w2 = in_w*stride + offset % stride;
+    int h2 = in_h*stride + offset / stride;
+    //printf("%d\n", offset);
+    int out_index = w2 + w*stride*(h2 + h*stride*(c2 + out_c*b));
+
+   // printf("%d %d %d\n", w2, h2, c2);
+    //printf("%d %d\n", in_index, out_index);
+    //if(out_index >= N || out_index < 0) printf("bad bad bad \n");
+
+    if(forward) out[out_index] = x[in_index];
+    else out[in_index] = x[out_index];
+    //if(forward) out[1] = x[1];
+    //else out[0] = x[0];
+}
+
+/**
+    reorg function function
+*/
+void reorgForward(dnnType* srcData, dnnType* dstData, 
+                  int n, int c, int h, int w, int stride, cudaStream_t stream) {
+
+    int size = n*c*h*w;
+
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    reorg_kernel<<<blocks, threads, 0, stream>>>(size, srcData, w, h, c, n, stride, false, dstData);
+}
+
+

src/kernels/shortcut.cu

@@ -0,0 +1,47 @@
+#include "kernels.h"
+#include "assert.h"
+
+__global__ void shortcut_kernel(int size, int minw, int minh, int minc, int stride, int sample, int batch, 
+                                int w1, int h1, int c1, dnnType *add, 
+                                int w2, int h2, int c2, float s1, float s2, dnnType *out)
+{
+    int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if (id >= size) return;
+    int i = id % minw;
+    id /= minw;
+    int j = id % minh;
+    id /= minh;
+    int k = id % minc;
+    id /= minc;
+    int b = id % batch;
+
+    int out_index = i*sample + w2*(j*sample + h2*(k + c2*b));
+    int add_index = i*stride + w1*(j*stride + h1*(k + c1*b));
+    out[out_index] = s1*out[out_index] + s2*add[add_index];
+    //out[out_index] += add[add_index];
+}
+
+void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1,
+                                                         int n2, int c2, int h2, int w2, int s2, 
+                     cudaStream_t stream)
+{
+    assert(n1 == n2);
+    int batch = n1;
+
+    int minw = (w1 < w2) ? w1 : w2;
+    int minh = (h1 < h2) ? h1 : h2;
+    int minc = (c1 < c2) ? c1 : c2;
+
+    int stride = w1/w2;
+    int sample = w2/w1;
+    assert(stride == h1/h2);
+    assert(sample == h2/h1);
+    if(stride < 1) stride = 1;
+    if(sample < 1) sample = 1;
+
+    int size = batch * minw * minh * minc;
+    int blocks = (size+255)/256;
+    int threads = 256;
+    shortcut_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, stride, sample, batch, 
+        w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
+}

src/kernels/softmax.cu

@@ -0,0 +1,42 @@
+#include "kernels.h"
+
+__device__ void softmax_device(float *input, int n, float temp, int stride, float *output)
+{
+    int i;
+    float sum = 0;
+    float largest = -INFINITY;
+    for(i = 0; i < n; ++i){
+        int val = input[i*stride];
+        largest = (val>largest) ? val : largest;
+    }
+    for(i = 0; i < n; ++i){
+        float e = exp(input[i*stride]/temp - largest/temp);
+        sum += e;
+        output[i*stride] = e;
+    }
+    for(i = 0; i < n; ++i){
+        output[i*stride] /= sum;
+    }
+}
+
+__global__ void softmax_kernel(float *input, int n, int batch, int batch_offset, int groups, int group_offset, int stride, float temp, float *output)
+{
+    int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if (id >= batch*groups) return;
+    int b = id / groups;
+    int g = id % groups;
+    softmax_device(input + b*batch_offset + g*group_offset, n, temp, stride, output + b*batch_offset + g*group_offset);
+}
+
+/**
+    softmax function
+*/
+void softmaxForward(float *input, int n, int batch, int batch_offset, 
+                    int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream)
+{
+    int size = groups*batch;
+    int blocks = (size+255)/256;
+    int threads = 256;
+
+    softmax_kernel<<<blocks, threads, 0, stream>>>(input, n, batch, batch_offset, groups, group_offset, stride, temp, output);
+}

src/kernels/upsample.cu

@@ -0,0 +1,35 @@
+#include "kernels.h"
+
+__global__ void upsample_kernel(size_t N, dnnType *x, int w, int h, int c, int batch, int stride, int forward, float scale, dnnType *out)
+{
+    size_t i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if(i >= N) return;
+    int out_index = i;
+    int out_w = i%(w*stride);
+    i = i/(w*stride);
+    int out_h = i%(h*stride);
+    i = i/(h*stride);
+    int out_c = i%c;
+    i = i/c;
+    int b = i%batch;
+
+    int in_w = out_w / stride;
+    int in_h = out_h / stride;
+    int in_c = out_c;
+
+    int in_index = b*w*h*c + in_c*w*h + in_h*w + in_w;
+
+
+    if(forward) out[out_index] += scale * x[in_index];
+    else atomicAdd(x+in_index, scale * out[out_index]);
+}
+
+void upsampleForward(dnnType* srcData, dnnType* dstData, 
+                     int n, int c, int h, int w, int s, int forward, float scale,                                   
+                     cudaStream_t stream) {
+
+    int size = w*h*c*n*s*s;
+    int blocks = (size+255)/256;
+    int threads = 256;
+    upsample_kernel<<<blocks, threads, 0, stream>>>(size, srcData, w, h, c, n, s, forward, scale, dstData);
+}

src/utils.cpp

@@ -1,6 +1,27 @@
 #include "utils.h"
+#include <string.h>
 
-void readBinaryFile(const char* fname, int size, value_type** data_h, value_type** data_d)
+void printCenteredTitle(const char *title, char fill, int dim) {
+
+    int len = strlen(title);
+    int first = dim/2 + len/2;
+    
+    if(len >0)
+        std::cout<<"\n";
+    std::cout.width(first); std::cout.fill(fill); std::cout<<std::right<<title;
+    std::cout.width(dim - first); std::cout<<"\n"; 
+    std::cout.fill(' ');
+}
+
+bool fileExist(const char *fname) {
+    std::ifstream dataFile (fname, std::ios::in | std::ios::binary);
+    if(!dataFile)
+        return false;
+    return true;
+}
+
+
+void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek)
 {
     std::ifstream dataFile (fname, std::ios::in | std::ios::binary);
     std::stringstream error_s;
@@ -9,8 +30,13 @@ void readBinaryFile(const char* fname, int size, value_type** data_h, value_type
         error_s << "Error opening file " << fname; 
         FatalError(error_s.str());
     }
-    int size_b = size*sizeof(value_type);
+
-    *data_h = new value_type[size];
+    if(seek != 0) {
+        dataFile.seekg(seek*sizeof(dnnType), dataFile.cur);
+    }
+
+    int size_b = size*sizeof(dnnType);
+    *data_h = new dnnType[size];
     if (!dataFile.read ((char*) *data_h, size_b)) 
     {
         error_s << "Error reading file " << fname; 
@@ -18,49 +44,97 @@ void readBinaryFile(const char* fname, int size, value_type** data_h, value_type
     }
     
     checkCuda( cudaMalloc(data_d, size_b) );
-    checkCuda( cudaMemcpy(*data_d, *data_h,
+    checkCuda( cudaMemcpy(*data_d, *data_h, size_b, cudaMemcpyHostToDevice) );
-                                size_b,
-                                cudaMemcpyHostToDevice) );
 }
 
-void printDeviceVector(int size, value_type* vec_d)
+void printDeviceVector(int size, dnnType* vec_d, bool device)
 {
-    value_type *vec;
+    dnnType *vec;
-    vec = new value_type[size];
+    if(device) {
-    cudaDeviceSynchronize();
+        vec = new dnnType[size];
-    cudaMemcpy(vec, vec_d, size*sizeof(value_type), cudaMemcpyDeviceToHost);
+        cudaDeviceSynchronize();
-    for (int i = 0; i < size; i++)
+        cudaMemcpy(vec, vec_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost);
-    {
+    } else {
+        vec = vec_d;
+    }
+    
+    for (int i = 0; i < size; i++) {
         std::cout << vec[i] << " ";
     }
     std::cout << std::endl;
-    delete [] vec;
+
+    if(device)
+        delete [] vec;
 }
 
-void resize(int size, value_type **data)
+int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device) {
+
+    dnnType *data_h, *correct_h;
+    const float eps = 0.02f;
+
+    if(device) {
+        data_h = new dnnType[size];
+        correct_h = new dnnType[size];
+        cudaDeviceSynchronize();
+        cudaMemcpy(data_h, data_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost);
+        cudaMemcpy(correct_h, correct_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost);
+    
+    } else {
+        data_h = data_d;
+        correct_h = correct_d;
+    }
+
+    int diffs = 0;
+    for(int i=0; i<size; i++) {
+        if(data_h[i] != data_h[i] || correct_h[i] != correct_h[i] || //nan control
+           fabs(data_h[i] - correct_h[i]) > eps) {
+            diffs += 1;
+            if(diffs == 1)
+                std::cout<<"\n";
+            if(diffs < 10)
+                std::cout<<" | [ "<<i<<" ]: "<<data_h[i]<<" "<<correct_h[i]<<"\n";
+        }
+    }
+
+    if(device) {
+        delete [] data_h;
+        delete [] correct_h;
+    }
+
+    std::cout<<" | ";
+    if(diffs == 0)
+        std::cout<<COL_GREENB<<"OK";
+    else
+        std::cout<<COL_REDB<<"Wrongs: "<<diffs;
+
+    std::cout<<COL_END<<" ~"<<eps<<"\n";
+    return diffs;
+}
+
+void resize(int size, dnnType **data)
 {
     if (*data != NULL)
         checkCuda( cudaFree(*data) );
-    checkCuda( cudaMalloc(data, size*sizeof(value_type)) );
+    checkCuda( cudaMalloc(data, size*sizeof(dnnType)) );
 }
 
-void matrixTranspose(cublasHandle_t handle, value_type* srcData, value_type* dstData, int rows, int cols) {
+void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols) {
 
-    value_type *A = srcData, *clone = dstData;
+    dnnType *A = srcData, *clone = dstData;
     int m = rows, n= cols;
-    checkCuda( cudaMemcpy(clone, A, m*n*sizeof(value_type), cudaMemcpyDeviceToDevice));
+    checkCuda( cudaMemcpy(clone, A, m*n*sizeof(dnnType), cudaMemcpyDeviceToDevice));
    
     float const alpha(1.0);
     float const beta(0.0);
     checkERROR( cublasSgeam( handle, CUBLAS_OP_T, CUBLAS_OP_N, m, n, &alpha, A, n, &beta, A, m, clone, m ));
 }
 
-void matrixMulAdd(  cublasHandle_t handle, value_type* srcData, value_type* dstData, 
+void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData, 
-                    value_type* add_vector, int dim, value_type mul) {
+                    dnnType* add_vector, int dim, dnnType mul) {
 
-    checkCuda( cudaMemcpy(dstData, add_vector, dim*sizeof(value_type), cudaMemcpyDeviceToDevice));
+    checkCuda( cudaMemcpy(dstData, add_vector, dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
         
-    value_type alpha = mul;
+    dnnType alpha = mul;
     checkERROR( cublasSaxpy(handle, dim, &alpha, srcData, 1, dstData, 1));
 
-}
+}

tests/build_models.sh

@@ -1,4 +1,11 @@
 #!/bin/bash
+if [ "$1" == "download" ]; then
+    wget https://github.com/ceccocats/tkDNN/releases/download/testData/tkDNN_testwg.tar.gz --no-check-certificate
+    tar -xf tkDNN_testwg.tar.gz
+    rm tkDNN_testwg.tar.gz
+    exit
+fi
+
 echo "build test Model"
 cd test
 python test_model.py

tests/mnist/test.cpp

@@ -1,58 +0,0 @@
-#include<iostream>
-#include "tkdnn.h"
-
-const char *input_bin   = "../tests/mnist/input.bin";
-const char *c0_bin      = "../tests/mnist/layers/Convolution0.bin";
-const char *c0_bias_bin = "../tests/mnist/layers/Convolution0.bias.bin";
-const char *c1_bin      = "../tests/mnist/layers/Convolution1.bin";
-const char *c1_bias_bin = "../tests/mnist/layers/Convolution1.bias.bin";
-const char *d2_bin      = "../tests/mnist/layers/InnerProduct2.bin";
-const char *d2_bias_bin = "../tests/mnist/layers/InnerProduct2.bias.bin";
-const char *d3_bin      = "../tests/mnist/layers/InnerProduct3.bin";
-const char *d3_bias_bin = "../tests/mnist/layers/InnerProduct3.bias.bin";
-const char *output_bin   = "../tests/mnist/output.bin";
-
-int main() {
-
-    // Network layout
-    tkDNN::Network net;
-    tkDNN::dataDim_t dim(1, 1, 28, 28, 1);
-    tkDNN::Layer *l;
-    l = new tkDNN::Conv2d     (&net, dim, 20, 5, 5, 1, 1, c0_bin, c0_bias_bin);
-    l = new tkDNN::Pooling    (&net, l->output_dim, 2, 2, 2, 2, tkDNN::POOLING_MAX);
-    l = new tkDNN::Conv2d     (&net, l->output_dim, 50, 5, 5, 1, 1, c1_bin, c1_bias_bin);
-    l = new tkDNN::Pooling    (&net, l->output_dim, 2, 2, 2, 2, tkDNN::POOLING_MAX);
-    l = new tkDNN::Dense      (&net, l->output_dim, 500, d2_bin, d2_bias_bin);
-    l = new tkDNN::Activation (&net, l->output_dim, CUDNN_ACTIVATION_RELU);
-    l = new tkDNN::Dense      (&net, l->output_dim, 10, d3_bin, d3_bias_bin);
-    l = new tkDNN::Softmax    (&net, l->output_dim);
- 
-    // Load input
-    value_type *data;
-    value_type *input_h;
-    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
-
-    printDeviceVector(dim.tot(), data);
-    dim.print(); //print initial dimension
-    
-    TIMER_START
-
-    // Inference
-    data = net.infer(dim, data);
-    
-    TIMER_STOP
-    dim.print();   
-
-    // Print result
-    std::cout<<"\n======= RESULT =======\n";
-    printDeviceVector(dim.tot(), data);
- 
-    // Print real test
-    std::cout<<"\n==== CHECK RESULT ====\n";
-    value_type *out;
-    value_type *out_h;
-    readBinaryFile(output_bin, dim.tot(), &out_h, &out);
-    printDeviceVector(dim.tot(), out);
- 
-    return 0;
-}

tests/mnist/test_mnist.cpp

@@ -0,0 +1,72 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin   = "../tests/mnist/input.bin";
+const char *c0_bin      = "../tests/mnist/layers/c0.bin";
+const char *c1_bin      = "../tests/mnist/layers/c1.bin";
+const char *d2_bin      = "../tests/mnist/layers/d2.bin";
+const char *d3_bin      = "../tests/mnist/layers/d3.bin";
+const char *output_bin   = "../tests/mnist/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 1, 28, 28, 1);
+    tk::dnn::Network net(dim);
+    tk::dnn::Conv2d     l0(&net, 20, 5, 5, 1, 1, 0, 0, c0_bin);
+    tk::dnn::Pooling    l1(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+    tk::dnn::Conv2d     l2(&net, 50, 5, 5, 1, 1, 0, 0, c1_bin);
+    tk::dnn::Pooling    l3(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+    tk::dnn::Dense      l4(&net, 500, d2_bin);
+    tk::dnn::Activation l5(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Dense      l6(&net, 10, d3_bin);
+    tk::dnn::Softmax    l7(&net);
+
+    tk::dnn::NetworkRT netRT(&net, "mnist.rt");
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+
+    dnnType *out_data, *out_data2;
+
+    std::cout<<"CUDNN inference:\n"; {
+        dim.print(); //print initial dimension  
+        TIMER_START
+        out_data = net.infer(dim, data);    
+        TIMER_STOP
+        dim.print();   
+    }
+
+    // Print result
+    //std::cout<<"\n======= CUDNN RESULT =======\n";
+    //printDeviceVector(10, out_data);
+ 
+    tk::dnn::dataDim_t dim2(1, 1, 28, 28, 1);
+
+    std::cout<<"TENSORRT inference:\n"; {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    // Print result
+    //std::cout<<"\n======= TENRT RESULT =======\n";
+    //printDeviceVector(10, out_data);
+
+    std::cout<<"\n======= CHECK RESULT =======\n";
+    checkResult(dim.tot(), out_data, out_data2);
+
+ /*
+    // Print real test
+    std::cout<<"\n==== CHECK RESULT ====\n";
+    dnnType *out;
+    dnnType *out_h;
+    readBinaryFile(output_bin, dim.tot(), &out_h, &out);
+    printDeviceVector(dim.tot(), out);
+*/ 
+    return 0;
+}

tests/mnist/test_mnistRT.cpp

@@ -0,0 +1,178 @@
+#include<iostream>
+#include<cassert>
+#include "tkdnn.h"
+#include "NvInfer.h"
+
+const char *input_bin   = "../tests/mnist/input.bin";
+const char *c0_bin      = "../tests/mnist/layers/c0.bin";
+const char *c1_bin      = "../tests/mnist/layers/c1.bin";
+const char *d2_bin      = "../tests/mnist/layers/d2.bin";
+const char *d3_bin      = "../tests/mnist/layers/d3.bin";
+const char *output_bin   = "../tests/mnist/output.bin";
+
+using namespace nvinfer1;
+
+// Logger for info/warning/errors
+class Logger : public ILogger			
+{
+	void log(Severity severity, const char* msg) override
+	{
+		// suppress info-level messages
+		if (severity != Severity::kINFO)
+			std::cout << msg << std::endl;
+	}
+} gLogger;
+
+int main() {
+
+    std::cout<<"\n==== CUDNN ====\n";
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 1, 28, 28, 1);
+	tk::dnn::Network net(dim);
+    tk::dnn::Conv2d     l0(&net, 20, 5, 5, 1, 1, 0, 0, c0_bin);
+    tk::dnn::Pooling    l1(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+    tk::dnn::Conv2d     l2(&net, 50, 5, 5, 1, 1, 0, 0, c1_bin);
+    tk::dnn::Pooling    l3(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+    tk::dnn::Dense      l4(&net, 500, d2_bin);
+    tk::dnn::Activation l5(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Dense      l6(&net, 10, d3_bin);
+    tk::dnn::Softmax    l7(&net);
+ 
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+
+    dim.print(); //print initial dimension
+    
+    // Inference
+    {
+        TIMER_START
+        data = net.infer(dim, data);
+        TIMER_STOP
+        dim.print();   
+    }
+
+    // Print real test
+    std::cout<<"\n==== CHECK CUDNN RESULT ====\n";
+    dnnType *out;
+    dnnType *out_h;
+    readBinaryFile(output_bin, dim.tot(), &out_h, &out);
+    std::cout<<"Diff: "<<checkResult(dim.tot(), out, data)<<"\n";
+ 
+
+    std::cout<<"\n==== TensorRT ====\n";
+	// create the builder
+	IBuilder* builder = nvinfer1::createInferBuilder(gLogger);
+	INetworkDefinition* network = builder->createNetwork();
+
+    DataType dt = DataType::kFLOAT;
+	//  Create input of shape { 1, 1, 28, 28 } with name referenced by "data"
+	auto input = network->addInput("data", dt, DimsCHW{ 1, 28, 28});
+	assert(input != nullptr);
+
+    tk::dnn::Conv2d *c0 = &l0; 
+    Weights w { dt, c0->data_h, c0->inputs*c0->outputs*c0->kernelH*c0->kernelW};
+    Weights b { dt, c0->bias_h, c0->outputs};
+	// Add a convolution layer with 20 outputs and a 5x5 filter.
+	auto conv1 = network->addConvolution(*input, 20, DimsHW{5, 5}, w, b);
+	assert(conv1 != nullptr);
+	conv1->setStride(DimsHW{1, 1});
+
+	// Add a max pooling layer with stride of 2x2 and kernel size of 2x2.
+	auto pool1 = network->addPooling(*conv1->getOutput(0), PoolingType::kMAX, DimsHW{2, 2});
+	assert(pool1 != nullptr);
+	pool1->setStride(DimsHW{2, 2});
+
+    tk::dnn::Conv2d *c1 = &l2; 
+    Weights w1 { dt, c1->data_h, c1->inputs*c1->outputs*c1->kernelH*c1->kernelW};
+    Weights b1 { dt, c1->bias_h, c1->outputs};
+	// Add a second convolution layer with 50 outputs and a 5x5 filter.
+	auto conv2 = network->addConvolution(*pool1->getOutput(0), 50, DimsHW{5, 5}, w1, b1);
+	assert(conv2 != nullptr);
+	conv2->setStride(DimsHW{1, 1});
+
+	// Add a second max pooling layer with stride of 2x2 and kernel size of 2x3>
+	auto pool2 = network->addPooling(*conv2->getOutput(0), PoolingType::kMAX, DimsHW{2, 2});
+	assert(pool2 != nullptr);
+	pool2->setStride(DimsHW{2, 2});
+
+    tk::dnn::Dense *d2 = &l4; 
+    Weights w2 { dt, d2->data_h, d2->inputs*d2->outputs};
+    Weights b2 { dt, d2->bias_h, d2->outputs};
+	// Add a fully connected layer with 500 outputs.
+	auto ip1 = network->addFullyConnected(*pool2->getOutput(0), 500, w2, b2);
+	assert(ip1 != nullptr);
+
+	// Add an activation layer using the ReLU algorithm.
+	auto relu1 = network->addActivation(*ip1->getOutput(0), ActivationType::kRELU);
+	assert(relu1 != nullptr);
+
+    tk::dnn::Dense *d3 = &l6; 
+    Weights w3 { dt, d3->data_h, d3->inputs*d3->outputs};
+    Weights b3 { dt, d3->bias_h, d3->outputs};
+	// Add a second fully connected layer with 20 outputs.
+	auto ip2 = network->addFullyConnected(*relu1->getOutput(0), 10, w3, b3);
+	assert(ip2 != nullptr);
+
+	// Add a softmax layer to determine the probability.
+	auto prob = network->addSoftMax(*ip2->getOutput(0));
+	assert(prob != nullptr);
+	prob->getOutput(0)->setName("out");
+
+	network->markOutput(*prob->getOutput(0));
+
+	// Build the engine
+	builder->setMaxBatchSize(1);
+	builder->setMaxWorkspaceSize(1 << 20);
+
+	auto engine = builder->buildCudaEngine(*network);
+	// we don't need the network any more
+	network->destroy();
+
+	IExecutionContext *context = engine->createExecutionContext();
+
+	// run inference
+	// input and output buffer pointers that we pass to the engine - the engine requires exactly IEngine::getNbBindings(),
+	// of these, but in this case we know that there is exactly one input and one output.
+	assert(engine->getNbBindings() == 2);
+	void* buffers[2];
+
+	// In order to bind the buffers, we need to know the names of the input and output tensors.
+	// note that indices are guaranteed to be less than IEngine::getNbBindings()
+	int inputIndex = engine->getBindingIndex("data"); 
+    int outputIndex = engine->getBindingIndex("out");
+
+    float output[10];
+	// create GPU buffers and a stream
+	checkCuda(cudaMalloc(&buffers[inputIndex], 28*28*sizeof(float)));
+	checkCuda(cudaMalloc(&buffers[outputIndex], 10*sizeof(float)));
+
+	cudaStream_t stream;
+	checkCuda(cudaStreamCreate(&stream));
+
+	// DMA the input to the GPU,  execute the batch asynchronously, and DMA it back:
+    {
+        checkCuda(cudaMemcpyAsync(buffers[inputIndex], input_h, 1 * 28*28* sizeof(float), cudaMemcpyHostToDevice, stream));
+        cudaStreamSynchronize(stream);  //want to test only the inference time
+        TIMER_START
+        context->enqueue(1, buffers, stream, nullptr);
+        TIMER_STOP
+        checkCuda(cudaMemcpyAsync(output, buffers[outputIndex],10*sizeof(float), cudaMemcpyDeviceToHost, stream));
+        cudaStreamSynchronize(stream);
+    }
+
+    std::cout<<"\n==== CHECK CUDNN RESULT ====\n";
+    std::cout<<"Diff: "<<checkResult(dim.tot(), (float*)buffers[outputIndex], data)<<"\n";
+
+	// release the stream and the buffers
+	cudaStreamDestroy(stream);
+	checkCuda(cudaFree(buffers[inputIndex]));
+	checkCuda(cudaFree(buffers[outputIndex]));
+
+	// destroy the engine
+	context->destroy();
+	engine->destroy();
+
+    return 0;
+}

tests/simple/test_simple.cpp

@@ -0,0 +1,47 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin   = "../tests/simple/input.bin";
+const char *c0_bin      = "../tests/simple/layers/c0.bin";
+const char *c1_bin      = "../tests/simple/layers/c1.bin";
+const char *d2_bin      = "../tests/simple/layers/d2.bin";
+const char *output_bin  = "../tests/simple/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 1, 10, 10, 1);
+    tk::dnn::Network net(dim);
+    tk::dnn::Conv2d     l0(&net, 2, 4, 4, 2, 2, 0, 0, c0_bin);
+    tk::dnn::Activation l1(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     l2(&net, 4, 2, 2, 1, 1, 0, 0, c1_bin);
+    tk::dnn::Activation l3(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Flatten    l4(&net);
+    tk::dnn::Dense      l5(&net, 4, d2_bin);
+    tk::dnn::Activation l6(&net, CUDNN_ACTIVATION_RELU);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+
+    printDeviceVector(dim.tot(), data);
+    dim.print(); //print initial dimension
+    
+    TIMER_START
+    // Inference
+    data = net.infer(dim, data); dim.print();
+    TIMER_STOP
+
+    // Print result
+    std::cout<<"\n======= RESULT =======\n";
+    printDeviceVector(dim.tot(), data);
+
+    // Print real test
+    std::cout<<"\n==== CHECK RESULT ====\n";
+    dnnType *out;
+    dnnType *out_h;
+    readBinaryFile(output_bin, dim.tot(), &out_h, &out);
+    printDeviceVector(dim.tot(), out);
+    return 0;
+}

tests/test/test.cpp

@@ -1,54 +0,0 @@
-#include<iostream>
-#include "tkdnn.h"
-
-const char *input_bin   = "../tests/test/input.bin";
-const char *c0_bin      = "../tests/test/layers/conv0.bin";
-const char *c0_bias_bin = "../tests/test/layers/conv0.bias.bin";
-const char *c1_bin      = "../tests/test/layers/conv1.bin";
-const char *c1_bias_bin = "../tests/test/layers/conv1.bias.bin";
-const char *d2_bin      = "../tests/test/layers/dense2.bin";
-const char *d2_bias_bin = "../tests/test/layers/dense2.bias.bin";
-const char *output_bin   = "../tests/test/output.bin";
-
-int main() {
-
-    // Network layout
-    tkDNN::Network net;
-    tkDNN::dataDim_t dim(1, 1, 10, 10, 1);
-    tkDNN::Layer *l;
-    l = new tkDNN::Conv2d     (&net, dim, 2, 4, 4, 2, 2, c0_bin, c0_bias_bin);
-    l = new tkDNN::Activation (&net, l->output_dim, CUDNN_ACTIVATION_RELU);
-    l = new tkDNN::Conv2d     (&net, l->output_dim, 4, 2, 2, 1, 1, c1_bin, c1_bias_bin);
-    l = new tkDNN::Activation (&net, l->output_dim, CUDNN_ACTIVATION_RELU);
-    l = new tkDNN::Flatten      (&net, l->output_dim);
-    l = new tkDNN::Dense      (&net, l->output_dim, 4, d2_bin, d2_bias_bin);
-    l = new tkDNN::Activation (&net, l->output_dim, CUDNN_ACTIVATION_RELU);
-
-    // Load input
-    value_type *data;
-    value_type *input_h;
-    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
-
-    printDeviceVector(dim.tot(), data);
-    dim.print(); //print initial dimension
-    
-    TIMER_START
-
-    // Inference
-    data = net.infer(dim, data); dim.print();
-    
-
-    TIMER_STOP
-
-    // Print result
-    std::cout<<"\n======= RESULT =======\n";
-    printDeviceVector(dim.tot(), data);
-
-    // Print real test
-    std::cout<<"\n==== CHECK RESULT ====\n";
-    value_type *out;
-    value_type *out_h;
-    readBinaryFile(output_bin, dim.tot(), &out_h, &out);
-    printDeviceVector(dim.tot(), out);
-    return 0;
-}

tests/test_rtinference/rtinference.cpp

@@ -0,0 +1,34 @@
+#include<iostream>
+#include "tkdnn.h"
+#include <stdlib.h>     /* srand, rand */
+
+int main(int argc, char *argv[]) {
+
+    if(argc < 2 || !fileExist(argv[1]))
+        FatalError("unable to read serialRT file");
+
+    //always same test 
+    srand (0); 
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(NULL, argv[1]);
+
+    dnnType *input = new float[netRT.input_dim.tot()];
+    dnnType *output = new float[netRT.input_dim.tot()];
+
+    printCenteredTitle(" TENSORRT inference ", '=', 30); 
+    for(int i=0; i<100; i++) {
+        for(int j=0; j<netRT.input_dim.tot(); j++)
+            input[j] = ((float) rand() / (RAND_MAX));
+        TIMER_START
+        checkCuda(  cudaMemcpyAsync(netRT.buffersRT[netRT.buf_input_idx], input, 
+                    netRT.input_dim.tot()*sizeof(float), cudaMemcpyHostToDevice, netRT.stream));
+        netRT.enqueue();
+        checkCuda(  cudaMemcpyAsync(output, netRT.buffersRT[netRT.buf_output_idx], 
+                                    netRT.output_dim.tot()*sizeof(float), cudaMemcpyDeviceToHost, netRT.stream));
+        cudaStreamSynchronize(netRT.stream);
+        TIMER_STOP
+    }
+    
+    return 0;
+}

tests/yolo/yolo.cfg

@@ -0,0 +1,258 @@
+[net]
+# Testing
+#batch=1
+#subdivisions=1
+# Training
+ batch=32
+ subdivisions=8
+width=608
+height=608
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+
+#######
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[route]
+layers=-9
+
+[convolutional]
+batch_normalize=1
+size=1
+stride=1
+pad=1
+filters=64
+activation=leaky
+
+[reorg]
+stride=2
+
+[route]
+layers=-1,-4
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=425
+activation=linear
+
+
+[region]
+anchors =  0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828
+bias_match=1
+classes=80
+coords=4
+num=5
+softmax=1
+jitter=.3
+rescore=1
+
+object_scale=5
+noobject_scale=1
+class_scale=1
+coord_scale=1
+
+absolute=1
+thresh = .6
+random=1

tests/yolo/yolo.cpp

@@ -0,0 +1,150 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo/layers/input.bin";
+const char *c0_bin     = "../tests/yolo/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo/layers/c4.bin";
+const char *c5_bin     = "../tests/yolo/layers/c5.bin";
+const char *c6_bin     = "../tests/yolo/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo/layers/c8.bin";
+const char *c9_bin     = "../tests/yolo/layers/c9.bin";
+const char *c10_bin    = "../tests/yolo/layers/c10.bin";
+const char *c12_bin    = "../tests/yolo/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo/layers/c13.bin";
+const char *c14_bin    = "../tests/yolo/layers/c14.bin";
+const char *c15_bin    = "../tests/yolo/layers/c15.bin";
+const char *c16_bin    = "../tests/yolo/layers/c16.bin";
+const char *c18_bin    = "../tests/yolo/layers/c18.bin";
+const char *c19_bin    = "../tests/yolo/layers/c19.bin";
+const char *c20_bin    = "../tests/yolo/layers/c20.bin";
+const char *c21_bin    = "../tests/yolo/layers/c21.bin";
+const char *c22_bin    = "../tests/yolo/layers/c22.bin";
+const char *c23_bin    = "../tests/yolo/layers/c23.bin";
+const char *c24_bin    = "../tests/yolo/layers/c24.bin";
+const char *c26_bin    = "../tests/yolo/layers/c26.bin";
+const char *c29_bin    = "../tests/yolo/layers/c29.bin";
+const char *c30_bin    = "../tests/yolo/layers/c30.bin";
+const char *g31_bin    = "../tests/yolo/layers/g31.bin";
+const char *output_bin = "../tests/yolo/layers/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 608, 608, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d     c0 (&net, 32, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 64, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 128, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c5 (&net, 64, 1, 1, 1, 1, 0, 0,   c5_bin, true);
+    tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p7 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8 (&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c9 (&net, 128, 1, 1, 1, 1, 0, 0,  c9_bin, true);
+    tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c10(&net, 256, 3, 3, 1, 1, 1, 1,  c10_bin, true);
+    tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p11(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c12(&net, 512, 3, 3, 1, 1, 1, 1,  c12_bin, true);
+    tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c13(&net, 256, 1, 1, 1, 1, 0, 0,  c13_bin, true);
+    tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c14(&net, 512, 3, 3, 1, 1, 1, 1,  c14_bin, true);
+    tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c15(&net, 256, 1, 1, 1, 1, 0, 0,  c15_bin, true);
+    tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c16(&net, 512, 3, 3, 1, 1, 1, 1,  c16_bin, true); 
+    tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p17(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true);
+    tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c19(&net, 512, 1, 1, 1, 1, 0, 0,  c19_bin, true);
+    tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true);
+    tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c21(&net, 512, 1, 1, 1, 1, 0, 0,  c21_bin, true);
+    tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true);
+    tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true);
+    tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true);
+    tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Layer *m25_layers[1] = { &a16 };
+    tk::dnn::Route      m25(&net, m25_layers, 1);
+    tk::dnn::Conv2d     c26(&net, 64, 1, 1, 1, 1, 0, 0,   c26_bin, true);
+    tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Reorg      r27(&net, 2);
+
+    tk::dnn::Layer *m28_layers[2] = { &r27, &a24 };
+    tk::dnn::Route      m28(&net, m28_layers, 2);
+
+    tk::dnn::Conv2d     c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true);
+    tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY);   
+    tk::dnn::Conv2d     c30(&net, 425, 1, 1, 1, 1, 0, 0,  c30_bin, false);
+    tk::dnn::Region     g31(&net, 80, 4, 5);
+    
+    tk::dnn::RegionInterpret rI(dim, g31.output_dim, 80, 4, 5, 0.6f, g31_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo.rt");
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+    
+    std::cout<<"\n\nDetected objects: \n";
+    dnnType *output_h = new dnnType[rI.output_dim.tot()];
+    checkCuda(cudaMemcpy(output_h, out_data2, 
+              rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost));
+    rI.interpretData(output_h);
+    rI.showImageResult(input_h);
+    return 0;
+}

tests/yolo3_berkeley/yolo3_berkeley.cfg

@@ -0,0 +1,785 @@
+[net]
+# Testing
+batch=1
+subdivisions=1
+# Training
+#batch=32
+#subdivisions=8
+width=544
+height=320
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 50200
+policy=steps
+steps=40000,45000
+scales=.1,.1
+
+
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+######################
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=45
+activation=linear
+
+[yolo]
+mask = 6,7,8
+anchors = 8.2087,8.5515, 18.4134,20.3391, 40.2194,29.2990, 31.6137,69.2240, 69.8497,48.3838, 108.8817,76.6316, 96.5753,145.5743, 165.9182,117.4493, 215.7497,198.4648
+classes=10
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=0
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 61
+
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=45
+activation=linear
+
+[yolo]
+mask = 3,4,5
+anchors = 8.2087,8.5515, 18.4134,20.3391, 40.2194,29.2990, 31.6137,69.2240, 69.8497,48.3838, 108.8817,76.6316, 96.5753,145.5743, 165.9182,117.4493, 215.7497,198.4648
+classes=10
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=0
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 36
+
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=45
+activation=linear
+
+[yolo]
+mask = 0,1,2
+anchors = 8.2087,8.5515, 18.4134,20.3391, 40.2194,29.2990, 31.6137,69.2240, 69.8497,48.3838, 108.8817,76.6316, 96.5753,145.5743, 165.9182,117.4493, 215.7497,198.4648
+classes=10
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=0
+

tests/yolo3_berkeley/yolo3_berkeley.cpp

@@ -0,0 +1,92 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 320, 544, 1);
+    tk::dnn::Network net(dim);
+
+    // create yolo3 model
+    std::string bin_path  = "../tests/yolo3_berkeley";
+    int classes = 10;
+    tk::dnn::Yolo *yolo [3];
+    #include "models/Yolo3.h"
+
+    // fill classes names
+    for(int i=0; i<3; i++) {
+        yolo[i]->classesNames = {"person", "car", "truck", "bus", "motor", "bike", "rider", "traffic light", "traffic sign", "train"};
+    }
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo3_berkeley.rt");
+
+    // the network have 3 outputs
+    tk::dnn::dataDim_t out_dim[3];
+    for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; 
+    dnnType *cudnn_out[3], *rt_out[3]; 
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData;
+    
+    printCenteredTitle(" compute detections ", '=', 30);
+    TIMER_START
+    int ndets = 0;
+    tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
+    for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5);
+    tk::dnn::Yolo::mergeDetections(dets, ndets, classes);
+
+    for(int j=0; j<ndets; j++) {
+        tk::dnn::Yolo::box b = dets[j].bbox;
+        int x0   = (b.x-b.w/2.);
+        int x1   = (b.x+b.w/2.);
+        int y0   = (b.y-b.h/2.);
+        int y1   = (b.y+b.h/2.);
+
+        int cl = 0;
+        for(int c = 0; c < classes; ++c){
+            float prob = dets[j].prob[c];
+            if(prob > 0)
+                cl = c;
+        }
+        std::cout<<cl<<": "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
+    }
+    TIMER_STOP
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+    for(int i=0; i<3; i++) rt_out[i] = (dnnType*)netRT.buffersRT[i+1];
+
+    for(int i=0; i<3; i++) {
+        printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30);
+        dnnType *out, *out_h;
+        int odim = out_dim[i].tot();
+        readBinaryFile(output_bins[i], odim, &out_h, &out);
+        std::cout<<"CUDNN vs correct"; checkResult(odim, cudnn_out[i], out);
+        std::cout<<"TRT   vs correct"; checkResult(odim, rt_out[i], out);
+        std::cout<<"CUDNN vs TRT    "; checkResult(odim, cudnn_out[i], rt_out[i]);
+    }
+    return 0;
+}

tests/yolo3_coco4/yolo3_coco4.cpp

@@ -0,0 +1,87 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 416, 416, 1);
+    tk::dnn::Network net(dim);
+
+    // create yolo3 model
+    std::string bin_path  = "../tests/yolo3_coco4";
+    int classes = 4;
+    tk::dnn::Yolo *yolo [3];
+    #include "models/Yolo3.h"
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo3_coco4.rt");
+
+    // the network have 3 outputs
+    tk::dnn::dataDim_t out_dim[3];
+    for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; 
+    dnnType *cudnn_out[3], *rt_out[3]; 
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData;
+    
+    printCenteredTitle(" compute detections ", '=', 30);
+    TIMER_START
+    int ndets = 0;
+    tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
+    for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5);
+    tk::dnn::Yolo::mergeDetections(dets, ndets, classes);
+
+    for(int j=0; j<ndets; j++) {
+        tk::dnn::Yolo::box b = dets[j].bbox;
+        int x0   = (b.x-b.w/2.);
+        int x1   = (b.x+b.w/2.);
+        int y0   = (b.y-b.h/2.);
+        int y1   = (b.y+b.h/2.);
+
+        int cl = 0;
+        for(int c = 0; c < classes; ++c){
+            float prob = dets[j].prob[c];
+            if(prob > 0)
+                cl = c;
+        }
+        std::cout<<cl<<": "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
+    }
+    TIMER_STOP
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+    for(int i=0; i<3; i++) rt_out[i] = (dnnType*)netRT.buffersRT[i+1];
+
+    for(int i=0; i<3; i++) {
+        printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30);
+        dnnType *out, *out_h;
+        int odim = out_dim[i].tot();
+        readBinaryFile(output_bins[i], odim, &out_h, &out);
+        std::cout<<"CUDNN vs correct"; checkResult(odim, cudnn_out[i], out);
+        std::cout<<"TRT   vs correct"; checkResult(odim, rt_out[i], out);
+        std::cout<<"CUDNN vs TRT    "; checkResult(odim, cudnn_out[i], rt_out[i]);
+    }
+    return 0;
+}

tests/yolo3_coco4/yolov3-coco4.cfg

@@ -0,0 +1,785 @@
+[net]
+# Testing
+batch=1
+subdivisions=1
+# Training
+#batch=32
+#subdivisions=8
+width=416
+height=416
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 50200
+policy=steps
+steps=40000,45000
+scales=.1,.1
+
+
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+######################
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=27
+activation=linear
+
+[yolo]
+mask = 6,7,8
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=4
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=1
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 61
+
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=27
+activation=linear
+
+[yolo]
+mask = 3,4,5
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=4
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=1
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 36
+
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=27
+activation=linear
+
+[yolo]
+mask = 0,1,2
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=4
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=1
+

tests/yolo3_flir/yolo3_flir.cfg

@@ -0,0 +1,785 @@
+[net]
+# Testing
+#batch=1
+#subdivisions=1
+# Training
+batch=32
+subdivisions=8
+width=544
+height=320
+channels=1
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 20000
+policy=steps
+steps=8000,9000
+scales=.1,.1
+
+
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+######################
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=24
+activation=linear
+
+[yolo]
+mask = 6,7,8
+anchors = 8.2087,8.5515, 18.4134,20.3391, 40.2194,29.2990, 31.6137,69.2240, 69.8497,48.3838, 108.8817,76.6316, 96.5753,145.5743, 165.9182,117.4493, 215.7497,198.4648
+classes=3
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=0
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 61
+
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=24
+activation=linear
+
+[yolo]
+mask = 3,4,5
+anchors = 8.2087,8.5515, 18.4134,20.3391, 40.2194,29.2990, 31.6137,69.2240, 69.8497,48.3838, 108.8817,76.6316, 96.5753,145.5743, 165.9182,117.4493, 215.7497,198.4648
+classes=3
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=0
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 36
+
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=24
+activation=linear
+
+[yolo]
+mask = 0,1,2
+anchors = 8.2087,8.5515, 18.4134,20.3391, 40.2194,29.2990, 31.6137,69.2240, 69.8497,48.3838, 108.8817,76.6316, 96.5753,145.5743, 165.9182,117.4493, 215.7497,198.4648
+classes=3
+num=9
+jitter=.3
+ignore_thresh = .5
+truth_thresh = 1
+random=0
+

tests/yolo3_flir/yolo3_flir.cpp

@@ -0,0 +1,93 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 1, 320, 544, 1);
+    tk::dnn::Network net(dim);
+
+    // create yolo3 model
+    std::string bin_path  = "../tests/yolo3_flir";
+    int classes = 3;
+    tk::dnn::Yolo *yolo [3];
+    #include "models/Yolo3.h"
+
+    // fill classes names
+    for(int i=0; i<3; i++) {
+        yolo[i]->classesNames = {"person", "bike", "car"};
+    }
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo3_flir.rt");
+
+    // the network have 3 outputs
+    tk::dnn::dataDim_t out_dim[3];
+    for(int i=0; i<3; i++) out_dim[i] = yolo[i]->output_dim; 
+    dnnType *cudnn_out[3], *rt_out[3]; 
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    for(int i=0; i<3; i++) cudnn_out[i] = yolo[i]->dstData;
+    
+    printCenteredTitle(" compute detections ", '=', 30);
+    TIMER_START
+    int ndets = 0;
+    tk::dnn::Yolo::detection *dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
+    for(int i=0; i<3; i++) yolo[i]->computeDetections(dets, ndets, net.input_dim.w, net.input_dim.h, 0.5);
+    tk::dnn::Yolo::mergeDetections(dets, ndets, classes);
+
+    for(int j=0; j<ndets; j++) {
+        tk::dnn::Yolo::box b = dets[j].bbox;
+        int x0   = (b.x-b.w/2.);
+        int x1   = (b.x+b.w/2.);
+        int y0   = (b.y-b.h/2.);
+        int y1   = (b.y+b.h/2.);
+
+        int cl = 0;
+        for(int c = 0; c < classes; ++c){
+            float prob = dets[j].prob[c];
+            if(prob > 0)
+                cl = c;
+        }
+        std::cout<<cl<<": "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
+    }
+    TIMER_STOP
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+    for(int i=0; i<3; i++) rt_out[i] = (dnnType*)netRT.buffersRT[i+1];
+
+    for(int i=0; i<3; i++) {
+        printCenteredTitle((std::string(" YOLO ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30);
+        dnnType *out, *out_h;
+        int odim = out_dim[i].tot();
+        readBinaryFile(output_bins[i], odim, &out_h, &out);
+        std::cout<<"CUDNN vs correct"; checkResult(odim, cudnn_out[i], out);
+        std::cout<<"TRT   vs correct"; checkResult(odim, rt_out[i], out);
+        std::cout<<"CUDNN vs TRT    "; checkResult(odim, cudnn_out[i], rt_out[i]);
+    }
+    return 0;
+}

tests/yolo_224/yolo_224.cfg

@@ -0,0 +1,258 @@
+[net]
+# Testing
+#batch=1
+#subdivisions=1
+# Training
+ batch=64
+ subdivisions=16
+width=224
+height=224
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+
+#######
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[route]
+layers=-9
+
+[convolutional]
+batch_normalize=1
+size=1
+stride=1
+pad=1
+filters=64
+activation=leaky
+
+[reorg]
+stride=2
+
+[route]
+layers=-1,-4
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=425
+activation=linear
+
+
+[region]
+anchors =  0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828
+bias_match=1
+classes=80
+coords=4
+num=5
+softmax=1
+jitter=.3
+rescore=1
+
+object_scale=5
+noobject_scale=1
+class_scale=1
+coord_scale=1
+
+absolute=1
+thresh = .6
+random=1

tests/yolo_224/yolo_224.cpp

@@ -0,0 +1,150 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo_224/layers/input.bin";
+const char *c0_bin     = "../tests/yolo_224/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo_224/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo_224/layers/c4.bin";
+const char *c5_bin     = "../tests/yolo_224/layers/c5.bin";
+const char *c6_bin     = "../tests/yolo_224/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo_224/layers/c8.bin";
+const char *c9_bin     = "../tests/yolo_224/layers/c9.bin";
+const char *c10_bin    = "../tests/yolo_224/layers/c10.bin";
+const char *c12_bin    = "../tests/yolo_224/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo_224/layers/c13.bin";
+const char *c14_bin    = "../tests/yolo_224/layers/c14.bin";
+const char *c15_bin    = "../tests/yolo_224/layers/c15.bin";
+const char *c16_bin    = "../tests/yolo_224/layers/c16.bin";
+const char *c18_bin    = "../tests/yolo_224/layers/c18.bin";
+const char *c19_bin    = "../tests/yolo_224/layers/c19.bin";
+const char *c20_bin    = "../tests/yolo_224/layers/c20.bin";
+const char *c21_bin    = "../tests/yolo_224/layers/c21.bin";
+const char *c22_bin    = "../tests/yolo_224/layers/c22.bin";
+const char *c23_bin    = "../tests/yolo_224/layers/c23.bin";
+const char *c24_bin    = "../tests/yolo_224/layers/c24.bin";
+const char *c26_bin    = "../tests/yolo_224/layers/c26.bin";
+const char *c29_bin    = "../tests/yolo_224/layers/c29.bin";
+const char *c30_bin    = "../tests/yolo_224/layers/c30.bin";
+const char *g31_bin    = "../tests/yolo_224/layers/g31.bin";
+const char *output_bin = "../tests/yolo_224/layers/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 224, 224, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d     c0 (&net, 32, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 64, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 128, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c5 (&net, 64, 1, 1, 1, 1, 0, 0,   c5_bin, true);
+    tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p7 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8 (&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c9 (&net, 128, 1, 1, 1, 1, 0, 0,  c9_bin, true);
+    tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c10(&net, 256, 3, 3, 1, 1, 1, 1,  c10_bin, true);
+    tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p11(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c12(&net, 512, 3, 3, 1, 1, 1, 1,  c12_bin, true);
+    tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c13(&net, 256, 1, 1, 1, 1, 0, 0,  c13_bin, true);
+    tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c14(&net, 512, 3, 3, 1, 1, 1, 1,  c14_bin, true);
+    tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c15(&net, 256, 1, 1, 1, 1, 0, 0,  c15_bin, true);
+    tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c16(&net, 512, 3, 3, 1, 1, 1, 1,  c16_bin, true); 
+    tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p17(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true);
+    tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c19(&net, 512, 1, 1, 1, 1, 0, 0,  c19_bin, true);
+    tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true);
+    tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c21(&net, 512, 1, 1, 1, 1, 0, 0,  c21_bin, true);
+    tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true);
+    tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true);
+    tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true);
+    tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Layer *m25_layers[1] = { &a16 };
+    tk::dnn::Route      m25(&net, m25_layers, 1);
+    tk::dnn::Conv2d     c26(&net, 64, 1, 1, 1, 1, 0, 0,   c26_bin, true);
+    tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Reorg      r27(&net, 2);
+
+    tk::dnn::Layer *m28_layers[2] = { &r27, &a24 };
+    tk::dnn::Route      m28(&net, m28_layers, 2);
+
+    tk::dnn::Conv2d     c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true);
+    tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY);   
+    tk::dnn::Conv2d     c30(&net, 425, 1, 1, 1, 1, 0, 0,  c30_bin, false);
+    tk::dnn::Region     g31(&net, 80, 4, 5);
+    
+    tk::dnn::RegionInterpret rI(dim, g31.output_dim, 80, 4, 5, 0.6f, g31_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo_224.rt");
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+    
+    std::cout<<"\n\nDetected objects: \n";
+    dnnType *output_h = new dnnType[rI.output_dim.tot()];
+    checkCuda(cudaMemcpy(output_h, out_data2, 
+              rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost));
+    rI.interpretData(output_h);
+    rI.showImageResult(input_h);
+    return 0;
+}

tests/yolo_berkeley/yolo_berkeley.cpp

@@ -0,0 +1,150 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo_berkeley/layers/input.bin";
+const char *c0_bin     = "../tests/yolo_berkeley/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo_berkeley/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo_berkeley/layers/c4.bin";
+const char *c5_bin     = "../tests/yolo_berkeley/layers/c5.bin";
+const char *c6_bin     = "../tests/yolo_berkeley/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo_berkeley/layers/c8.bin";
+const char *c9_bin     = "../tests/yolo_berkeley/layers/c9.bin";
+const char *c10_bin    = "../tests/yolo_berkeley/layers/c10.bin";
+const char *c12_bin    = "../tests/yolo_berkeley/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo_berkeley/layers/c13.bin";
+const char *c14_bin    = "../tests/yolo_berkeley/layers/c14.bin";
+const char *c15_bin    = "../tests/yolo_berkeley/layers/c15.bin";
+const char *c16_bin    = "../tests/yolo_berkeley/layers/c16.bin";
+const char *c18_bin    = "../tests/yolo_berkeley/layers/c18.bin";
+const char *c19_bin    = "../tests/yolo_berkeley/layers/c19.bin";
+const char *c20_bin    = "../tests/yolo_berkeley/layers/c20.bin";
+const char *c21_bin    = "../tests/yolo_berkeley/layers/c21.bin";
+const char *c22_bin    = "../tests/yolo_berkeley/layers/c22.bin";
+const char *c23_bin    = "../tests/yolo_berkeley/layers/c23.bin";
+const char *c24_bin    = "../tests/yolo_berkeley/layers/c24.bin";
+const char *c26_bin    = "../tests/yolo_berkeley/layers/c26.bin";
+const char *c29_bin    = "../tests/yolo_berkeley/layers/c29.bin";
+const char *c30_bin    = "../tests/yolo_berkeley/layers/c30.bin";
+const char *g31_bin    = "../tests/yolo_berkeley/layers/g31.bin";
+const char *output_bin = "../tests/yolo_berkeley/layers/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 416, 736, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d     c0 (&net, 32, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 64, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 128, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c5 (&net, 64, 1, 1, 1, 1, 0, 0,   c5_bin, true);
+    tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p7 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8 (&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c9 (&net, 128, 1, 1, 1, 1, 0, 0,  c9_bin, true);
+    tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c10(&net, 256, 3, 3, 1, 1, 1, 1,  c10_bin, true);
+    tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p11(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c12(&net, 512, 3, 3, 1, 1, 1, 1,  c12_bin, true);
+    tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c13(&net, 256, 1, 1, 1, 1, 0, 0,  c13_bin, true);
+    tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c14(&net, 512, 3, 3, 1, 1, 1, 1,  c14_bin, true);
+    tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c15(&net, 256, 1, 1, 1, 1, 0, 0,  c15_bin, true);
+    tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c16(&net, 512, 3, 3, 1, 1, 1, 1,  c16_bin, true); 
+    tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p17(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true);
+    tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c19(&net, 512, 1, 1, 1, 1, 0, 0,  c19_bin, true);
+    tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true);
+    tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c21(&net, 512, 1, 1, 1, 1, 0, 0,  c21_bin, true);
+    tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true);
+    tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true);
+    tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true);
+    tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Layer *m25_layers[1] = { &a16 };
+    tk::dnn::Route      m25(&net, m25_layers, 1);
+    tk::dnn::Conv2d     c26(&net, 64, 1, 1, 1, 1, 0, 0,   c26_bin, true);
+    tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Reorg      r27(&net, 2);
+
+    tk::dnn::Layer *m28_layers[2] = { &r27, &a24 };
+    tk::dnn::Route      m28(&net, m28_layers, 2);
+
+    tk::dnn::Conv2d     c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true);
+    tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY);   
+    tk::dnn::Conv2d     c30(&net, 75, 1, 1, 1, 1, 0, 0,  c30_bin, false);
+    tk::dnn::Region     g31(&net, 10, 4, 5);
+    
+    tk::dnn::RegionInterpret rI(dim, g31.output_dim, 10, 4, 5, 0.3f, g31_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo_berkeley.rt");
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+    
+    std::cout<<"\n\nDetected objects: \n";
+    dnnType *output_h = new dnnType[rI.output_dim.tot()];
+    checkCuda(cudaMemcpy(output_h, out_data2, 
+              rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost));
+    rI.interpretData(output_h);
+    rI.showImageResult(input_h);
+    return 0;
+}

tests/yolo_berkeley/yolov2-voc-10-resize-test.cfg

@@ -0,0 +1,259 @@
+[net]
+# Testing
+batch=1
+subdivisions=1
+# Training
+#batch=64
+#subdivisions=8
+height=416
+width=736
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 80200
+policy=steps
+steps=40000,60000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+
+#######
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[route]
+layers=-9
+
+[convolutional]
+batch_normalize=1
+size=1
+stride=1
+pad=1
+filters=64
+activation=leaky
+
+[reorg]
+stride=2
+
+[route]
+layers=-1,-4
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=75
+activation=linear
+
+
+[region]
+anchors =  0.4043,0.4167, 1.2109,1.1018, 2.7258,2.1215, 4.9477,3.9132, 7.9508,6.6806
+bias_match=1
+classes=10
+coords=4
+num=5
+softmax=1
+jitter=.3
+rescore=1
+
+object_scale=5
+noobject_scale=1
+class_scale=1
+coord_scale=1
+
+absolute=1
+thresh = .6
+random=0
+flip=1

tests/yolo_relu/yolo_relu.cfg

@@ -0,0 +1,258 @@
+[net]
+# Testing
+#batch=1
+#subdivisions=1
+# Training
+ batch=64
+ subdivisions=16
+width=608
+height=608
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=relu
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=relu
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=relu
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=relu
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=relu
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=relu
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=relu
+
+
+#######
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=relu
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=relu
+
+[route]
+layers=-9
+
+[convolutional]
+batch_normalize=1
+size=1
+stride=1
+pad=1
+filters=64
+activation=relu
+
+[reorg]
+stride=2
+
+[route]
+layers=-1,-4
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=relu
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=425
+activation=linear
+
+
+[region]
+anchors =  0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828
+bias_match=1
+classes=80
+coords=4
+num=5
+softmax=1
+jitter=.3
+rescore=1
+
+object_scale=5
+noobject_scale=1
+class_scale=1
+coord_scale=1
+
+absolute=1
+thresh = .6
+random=1

tests/yolo_relu/yolo_relu.cpp

@@ -0,0 +1,151 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo_relu/layers/input.bin";
+const char *c0_bin     = "../tests/yolo_relu/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo_relu/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo_relu/layers/c4.bin";
+const char *c5_bin     = "../tests/yolo_relu/layers/c5.bin";
+const char *c6_bin     = "../tests/yolo_relu/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo_relu/layers/c8.bin";
+const char *c9_bin     = "../tests/yolo_relu/layers/c9.bin";
+const char *c10_bin    = "../tests/yolo_relu/layers/c10.bin";
+const char *c12_bin    = "../tests/yolo_relu/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo_relu/layers/c13.bin";
+const char *c14_bin    = "../tests/yolo_relu/layers/c14.bin";
+const char *c15_bin    = "../tests/yolo_relu/layers/c15.bin";
+const char *c16_bin    = "../tests/yolo_relu/layers/c16.bin";
+const char *c18_bin    = "../tests/yolo_relu/layers/c18.bin";
+const char *c19_bin    = "../tests/yolo_relu/layers/c19.bin";
+const char *c20_bin    = "../tests/yolo_relu/layers/c20.bin";
+const char *c21_bin    = "../tests/yolo_relu/layers/c21.bin";
+const char *c22_bin    = "../tests/yolo_relu/layers/c22.bin";
+const char *c23_bin    = "../tests/yolo_relu/layers/c23.bin";
+const char *c24_bin    = "../tests/yolo_relu/layers/c24.bin";
+const char *c26_bin    = "../tests/yolo_relu/layers/c26.bin";
+const char *c29_bin    = "../tests/yolo_relu/layers/c29.bin";
+const char *c30_bin    = "../tests/yolo_relu/layers/c30.bin";
+const char *g31_bin    = "../tests/yolo_relu/layers/g31.bin";
+const char *output_bin = "../tests/yolo_relu/layers/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 608, 608, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d     c0 (&net, 32, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 64, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 128, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c5 (&net, 64, 1, 1, 1, 1, 0, 0,   c5_bin, true);
+    tk::dnn::Activation a5 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Pooling    p7 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8 (&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c9 (&net, 128, 1, 1, 1, 1, 0, 0,  c9_bin, true);
+    tk::dnn::Activation a9 (&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c10(&net, 256, 3, 3, 1, 1, 1, 1,  c10_bin, true);
+    tk::dnn::Activation a10(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Pooling    p11(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c12(&net, 512, 3, 3, 1, 1, 1, 1,  c12_bin, true);
+    tk::dnn::Activation a12(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c13(&net, 256, 1, 1, 1, 1, 0, 0,  c13_bin, true);
+    tk::dnn::Activation a13(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c14(&net, 512, 3, 3, 1, 1, 1, 1,  c14_bin, true);
+    tk::dnn::Activation a14(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c15(&net, 256, 1, 1, 1, 1, 0, 0,  c15_bin, true);
+    tk::dnn::Activation a15(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c16(&net, 512, 3, 3, 1, 1, 1, 1,  c16_bin, true); 
+    tk::dnn::Activation a16(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Pooling    p17(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true);
+    tk::dnn::Activation a18(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c19(&net, 512, 1, 1, 1, 1, 0, 0,  c19_bin, true);
+    tk::dnn::Activation a19(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true);
+    tk::dnn::Activation a20(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c21(&net, 512, 1, 1, 1, 1, 0, 0,  c21_bin, true);
+    tk::dnn::Activation a21(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true);
+    tk::dnn::Activation a22(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true);
+    tk::dnn::Activation a23(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true);
+    tk::dnn::Activation a24(&net, CUDNN_ACTIVATION_RELU);
+
+    tk::dnn::Layer *m25_layers[1] = { &a16 };
+    tk::dnn::Route      m25(&net, m25_layers, 1);
+    tk::dnn::Conv2d     c26(&net, 64, 1, 1, 1, 1, 0, 0,   c26_bin, true);
+    tk::dnn::Activation a26(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Reorg      r27(&net, 2);
+
+    tk::dnn::Layer *m28_layers[2] = { &r27, &a24 };
+    tk::dnn::Route      m28(&net, m28_layers, 2);
+
+    tk::dnn::Conv2d     c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true);
+    tk::dnn::Activation a29(&net, CUDNN_ACTIVATION_RELU);   
+    tk::dnn::Conv2d     c30(&net, 425, 1, 1, 1, 1, 0, 0,  c30_bin, false);
+    tk::dnn::Region     g31(&net, 80, 4, 5);
+    
+    tk::dnn::RegionInterpret rI(dim, g31.output_dim, 80, 4, 5, 0.3f, g31_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo_relu.rt");
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+  
+    std::cout<<"\n\nDetected objects: \n";
+    dnnType *output_h = new dnnType[rI.output_dim.tot()];
+    checkCuda(cudaMemcpy(output_h, out_data2, 
+              rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost));
+    rI.interpretData(output_h, 608, 608);
+    rI.showImageResult(input_h);
+
+    return 0;
+}

tests/yolo_tiny/tiny-yolo.cfg

@@ -0,0 +1,139 @@
+[net]
+ Training
+ batch=64
+ subdivisions=8
+# Testing
+# batch=1
+# subdivisions=1
+width=416
+height=416
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=16
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+#[maxpool]
+#size=2
+#stride=1
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+###########
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=425
+activation=linear
+
+[region]
+anchors =  0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828
+bias_match=1
+classes=80
+coords=4
+num=5
+softmax=1
+jitter=.2
+rescore=0
+
+object_scale=5
+noobject_scale=1
+class_scale=1
+coord_scale=1
+
+absolute=1
+thresh = .6
+random=1

tests/yolo_tiny/yolo_tiny.cpp

@@ -0,0 +1,93 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo_tiny/layers/input.bin";
+const char *c0_bin     = "../tests/yolo_tiny/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo_tiny/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo_tiny/layers/c4.bin";
+const char *c5_bin     = "../tests/yolo_tiny/layers/c5.bin";
+const char *c6_bin     = "../tests/yolo_tiny/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo_tiny/layers/c8.bin";
+const char *c10_bin    = "../tests/yolo_tiny/layers/c10.bin";
+const char *c11_bin    = "../tests/yolo_tiny/layers/c11.bin";
+const char *c12_bin    = "../tests/yolo_tiny/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo_tiny/layers/c13.bin";
+const char *g14_bin    = "../tests/yolo_tiny/layers/g14.bin";
+const char *output_bin = "../tests/yolo_tiny/layers/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 416, 416, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d     c0 (&net, 16, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 32, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 64, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p5 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p7(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8(&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p9(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c10(&net, 512, 3, 3, 1, 1, 1, 1, c10_bin, true);
+    tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Conv2d     c11(&net, 1024, 3, 3, 1, 1, 1, 1, c11_bin, true);
+    tk::dnn::Activation a11(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c12(&net, 512, 3, 3, 1, 1, 1, 1, c12_bin, true);
+    tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c13(&net, 425, 1, 1, 1, 1, 0, 0, c13_bin, false);
+    tk::dnn::Region     g14(&net, 80, 4, 5);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo_tiny.rt");
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+ 
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+    return 0;
+}

tests/yolo_voc/yolo_voc.cfg

@@ -0,0 +1,258 @@
+[net]
+# Testing
+batch=1
+subdivisions=1
+# Training
+# batch=64
+# subdivisions=8
+height=416
+width=416
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 80200
+policy=steps
+steps=40000,60000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+
+#######
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[route]
+layers=-9
+
+[convolutional]
+batch_normalize=1
+size=1
+stride=1
+pad=1
+filters=64
+activation=leaky
+
+[reorg]
+stride=2
+
+[route]
+layers=-1,-4
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=125
+activation=linear
+
+
+[region]
+anchors =  1.3221, 1.73145, 3.19275, 4.00944, 5.05587, 8.09892, 9.47112, 4.84053, 11.2364, 10.0071
+bias_match=1
+classes=20
+coords=4
+num=5
+softmax=1
+jitter=.3
+rescore=1
+
+object_scale=5
+noobject_scale=1
+class_scale=1
+coord_scale=1
+
+absolute=1
+thresh = .6
+random=1

tests/yolo_voc/yolo_voc.cpp

@@ -0,0 +1,150 @@
+#include<iostream>
+#include "tkdnn.h"
+
+const char *input_bin  = "../tests/yolo_voc/layers/input.bin";
+const char *c0_bin     = "../tests/yolo_voc/layers/c0.bin";
+const char *c2_bin     = "../tests/yolo_voc/layers/c2.bin";
+const char *c4_bin     = "../tests/yolo_voc/layers/c4.bin";
+const char *c5_bin     = "../tests/yolo_voc/layers/c5.bin";
+const char *c6_bin     = "../tests/yolo_voc/layers/c6.bin";
+const char *c8_bin     = "../tests/yolo_voc/layers/c8.bin";
+const char *c9_bin     = "../tests/yolo_voc/layers/c9.bin";
+const char *c10_bin    = "../tests/yolo_voc/layers/c10.bin";
+const char *c12_bin    = "../tests/yolo_voc/layers/c12.bin";
+const char *c13_bin    = "../tests/yolo_voc/layers/c13.bin";
+const char *c14_bin    = "../tests/yolo_voc/layers/c14.bin";
+const char *c15_bin    = "../tests/yolo_voc/layers/c15.bin";
+const char *c16_bin    = "../tests/yolo_voc/layers/c16.bin";
+const char *c18_bin    = "../tests/yolo_voc/layers/c18.bin";
+const char *c19_bin    = "../tests/yolo_voc/layers/c19.bin";
+const char *c20_bin    = "../tests/yolo_voc/layers/c20.bin";
+const char *c21_bin    = "../tests/yolo_voc/layers/c21.bin";
+const char *c22_bin    = "../tests/yolo_voc/layers/c22.bin";
+const char *c23_bin    = "../tests/yolo_voc/layers/c23.bin";
+const char *c24_bin    = "../tests/yolo_voc/layers/c24.bin";
+const char *c26_bin    = "../tests/yolo_voc/layers/c26.bin";
+const char *c29_bin    = "../tests/yolo_voc/layers/c29.bin";
+const char *c30_bin    = "../tests/yolo_voc/layers/c30.bin";
+const char *g31_bin    = "../tests/yolo_voc/layers/g31.bin";
+const char *output_bin = "../tests/yolo_voc/layers/output.bin";
+
+int main() {
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 416, 416, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d     c0 (&net, 32, 3, 3, 1, 1, 1, 1,   c0_bin, true);
+    tk::dnn::Activation a0 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p1 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c2 (&net, 64, 3, 3, 1, 1, 1, 1,   c2_bin, true);
+    tk::dnn::Activation a2 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p3 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c4 (&net, 128, 3, 3, 1, 1, 1, 1,  c4_bin, true);
+    tk::dnn::Activation a4 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c5 (&net, 64, 1, 1, 1, 1, 0, 0,   c5_bin, true);
+    tk::dnn::Activation a5 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c6 (&net, 128, 3, 3, 1, 1, 1, 1,  c6_bin, true);
+    tk::dnn::Activation a6 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p7 (&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c8 (&net, 256, 3, 3, 1, 1, 1, 1,  c8_bin, true);
+    tk::dnn::Activation a8 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c9 (&net, 128, 1, 1, 1, 1, 0, 0,  c9_bin, true);
+    tk::dnn::Activation a9 (&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c10(&net, 256, 3, 3, 1, 1, 1, 1,  c10_bin, true);
+    tk::dnn::Activation a10(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p11(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c12(&net, 512, 3, 3, 1, 1, 1, 1,  c12_bin, true);
+    tk::dnn::Activation a12(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c13(&net, 256, 1, 1, 1, 1, 0, 0,  c13_bin, true);
+    tk::dnn::Activation a13(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c14(&net, 512, 3, 3, 1, 1, 1, 1,  c14_bin, true);
+    tk::dnn::Activation a14(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c15(&net, 256, 1, 1, 1, 1, 0, 0,  c15_bin, true);
+    tk::dnn::Activation a15(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c16(&net, 512, 3, 3, 1, 1, 1, 1,  c16_bin, true); 
+    tk::dnn::Activation a16(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Pooling    p17(&net, 2, 2, 2, 2, tk::dnn::POOLING_MAX);
+
+    tk::dnn::Conv2d     c18(&net, 1024, 3, 3, 1, 1, 1, 1, c18_bin, true);
+    tk::dnn::Activation a18(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c19(&net, 512, 1, 1, 1, 1, 0, 0,  c19_bin, true);
+    tk::dnn::Activation a19(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c20(&net, 1024, 3, 3, 1, 1, 1, 1, c20_bin, true);
+    tk::dnn::Activation a20(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c21(&net, 512, 1, 1, 1, 1, 0, 0,  c21_bin, true);
+    tk::dnn::Activation a21(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c22(&net, 1024, 3, 3, 1, 1, 1, 1, c22_bin, true);
+    tk::dnn::Activation a22(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c23(&net, 1024, 3, 3, 1, 1, 1, 1, c23_bin, true);
+    tk::dnn::Activation a23(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Conv2d     c24(&net, 1024, 3, 3, 1, 1, 1, 1, c24_bin, true);
+    tk::dnn::Activation a24(&net, tk::dnn::ACTIVATION_LEAKY);
+
+    tk::dnn::Layer *m25_layers[1] = { &a16 };
+    tk::dnn::Route      m25(&net, m25_layers, 1);
+    tk::dnn::Conv2d     c26(&net, 64, 1, 1, 1, 1, 0, 0,   c26_bin, true);
+    tk::dnn::Activation a26(&net, tk::dnn::ACTIVATION_LEAKY);
+    tk::dnn::Reorg      r27(&net, 2);
+
+    tk::dnn::Layer *m28_layers[2] = { &r27, &a24 };
+    tk::dnn::Route      m28(&net, m28_layers, 2);
+
+    tk::dnn::Conv2d     c29(&net, 1024, 3, 3, 1, 1, 1, 1, c29_bin, true);
+    tk::dnn::Activation a29(&net, tk::dnn::ACTIVATION_LEAKY);   
+    tk::dnn::Conv2d     c30(&net, 125, 1, 1, 1, 1, 0, 0,  c30_bin, false);
+    tk::dnn::Region     g31(&net, 20, 4, 5);
+    
+    tk::dnn::RegionInterpret rI(dim, g31.output_dim, 20, 4, 5, 0.6f, g31_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, "yolo_voc.rt");
+
+    dnnType *out_data, *out_data2; // cudnn output, tensorRT output
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TIMER_START
+        out_data = net.infer(dim1, data);    
+        TIMER_STOP
+        dim1.print();   
+    }
+    
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30); {
+        dim2.print();
+        TIMER_START
+        out_data2 = netRT.infer(dim2, data);
+        TIMER_STOP
+        dim2.print();
+    }
+
+    printCenteredTitle(" CHECK RESULTS ", '=', 30);
+    dnnType *out, *out_h;
+    int out_dim = net.getOutputDim().tot();
+    readBinaryFile(output_bin, out_dim, &out_h, &out);
+    std::cout<<"CUDNN vs correct"; checkResult(out_dim, out_data, out);
+    std::cout<<"TRT   vs correct"; checkResult(out_dim, out_data2, out);
+    std::cout<<"CUDNN vs TRT    "; checkResult(out_dim, out_data, out_data2);
+    
+    std::cout<<"\n\nDetected objects: \n";
+    dnnType *output_h = new dnnType[rI.output_dim.tot()];
+    checkCuda(cudaMemcpy(output_h, out_data2, 
+              rI.output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost));
+    rI.interpretData(output_h);
+    rI.showImageResult(input_h);
+    return 0;
+}