memory release

Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>

.gitignore

@@ -9,4 +9,8 @@ build/
 *.tar.gz
 *.weights
 .idea/
-*.hdf5
+*.hdf5
+*.pk
+*.table
+demo/COCO_val2017
+demo/BDD100k_val

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5)
 
 project (tkDNN)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN)
 
 # project specific flags
@@ -17,11 +17,12 @@ endif()
 find_package(CUDA 9.0 REQUIRED)
 SET(CUDA_SEPARABLE_COMPILATION ON)
 #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -arch=sm_30 --compiler-options '-fPIC'")
+set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} --maxrregcount=32)
 
 find_package(CUDNN REQUIRED)
 
 # compile
-file(GLOB tkdnn_CUSRC "src/kernels/*.cu")
+file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu")
 cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
 cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
 
@@ -29,17 +30,22 @@ cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
 #-------------------------------------------------------------------------------
 # External Libraries
 #-------------------------------------------------------------------------------
+find_package(Eigen3 REQUIRED)
+include_directories(${EIGEN3_INCLUDE_DIR})
+
 find_package(OpenCV REQUIRED)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV")
 
+# gives problems in cross-compiling, probably malformed cmake config
+#find_package(yaml-cpp REQUIRED)
 
 #-------------------------------------------------------------------------------
 # Build Libraries
 #-------------------------------------------------------------------------------
 file(GLOB tkdnn_SRC "src/*.cpp")
-set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS})
+set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES})
 add_library(tkDNN SHARED ${tkdnn_SRC})
 target_link_libraries(tkDNN ${tkdnn_LIBS})
@@ -48,6 +54,7 @@ target_link_libraries(tkDNN ${tkdnn_LIBS})
 #add_library(tkDNN_static STATIC ${tkdnn_SRC})
 #target_link_libraries(tkDNN_static ${tkdnn_LIBS})
 
+# SMALL NETS
 add_executable(test_simple tests/simple/test_simple.cpp)
 target_link_libraries(test_simple tkDNN)
 
@@ -57,46 +64,51 @@ target_link_libraries(test_mnist tkDNN)
 add_executable(test_mnistRT tests/mnist/test_mnistRT.cpp)
 target_link_libraries(test_mnistRT tkDNN)
 
-## YOLO NETS
-add_executable(test_yolo tests/yolo/yolo.cpp)
-target_link_libraries(test_yolo tkDNN)
-
-add_executable(test_yolo_voc tests/yolo_voc/yolo_voc.cpp)
-target_link_libraries(test_yolo_voc tkDNN)
-
-add_executable(test_yolo_tiny tests/yolo_tiny/yolo_tiny.cpp)
-target_link_libraries(test_yolo_tiny tkDNN)
-
-add_executable(test_yolo_relu tests/yolo_relu/yolo_relu.cpp)
-target_link_libraries(test_yolo_relu tkDNN)
-
-
-add_executable(test_yolo_224 tests/yolo_224/yolo_224.cpp)
-target_link_libraries(test_yolo_224 tkDNN)
-
-add_executable(test_yolo_berkeley tests/yolo_berkeley/yolo_berkeley.cpp)
-target_link_libraries(test_yolo_berkeley tkDNN)
-
-add_executable(test_yolo3_coco4 tests/yolo3_coco4/yolo3_coco4.cpp)
-target_link_libraries(test_yolo3_coco4 tkDNN)
-
-add_executable(test_yolo3_berkeley tests/yolo3_berkeley/yolo3_berkeley.cpp)
-target_link_libraries(test_yolo3_berkeley tkDNN)
-
-add_executable(test_yolo3_flir tests/yolo3_flir/yolo3_flir.cpp)
-target_link_libraries(test_yolo3_flir tkDNN)
-
 add_executable(test_imuodom tests/imuodom/imuodom.cpp)
 target_link_libraries(test_imuodom tkDNN)
-################################################################################
 
+# DARKNET
+file(GLOB darknet_SRC "tests/darknet/*.cpp")
+foreach(test_SRC ${darknet_SRC})
+    get_filename_component(test_NAME "${test_SRC}" NAME_WE)
+    set(test_NAME test_${test_NAME})
+    add_executable(${test_NAME} ${test_SRC})
+    target_link_libraries(${test_NAME} tkDNN)
+endforeach()
 
+# MOBILENET
+add_executable(test_mobilenetv2ssd tests/mobilenet/mobilenetv2ssd/mobilenetv2ssd.cpp)
+target_link_libraries(test_mobilenetv2ssd tkDNN)
+
+add_executable(test_bdd-mobilenetv2ssd tests/mobilenet/bdd-mobilenetv2ssd/bdd-mobilenetv2ssd.cpp)
+target_link_libraries(test_bdd-mobilenetv2ssd tkDNN)
+
+add_executable(test_mobilenetv2ssd512 tests/mobilenet/mobilenetv2ssd512/mobilenetv2ssd512.cpp)
+target_link_libraries(test_mobilenetv2ssd512 tkDNN)
+
+# BACKBONES
+add_executable(test_resnet101 tests/backbones/resnet101/resnet101.cpp)
+target_link_libraries(test_resnet101 tkDNN)
+
+add_executable(test_dla34 tests/backbones/dla34/dla34.cpp)
+target_link_libraries(test_dla34 tkDNN)
+
+# CENTERNET
+add_executable(test_resnet101_cnet tests/centernet/resnet101_cnet/resnet101_cnet.cpp)
+target_link_libraries(test_resnet101_cnet tkDNN)
+
+add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp)
+target_link_libraries(test_dla34_cnet tkDNN)
+
+# DEMOS
 add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
 target_link_libraries(test_rtinference tkDNN)
 
-add_executable(yolo3_demo demo/demo/demo.cpp)
-target_link_libraries(yolo3_demo tkDNN)
+add_executable(map_demo demo/demo/map.cpp)
+target_link_libraries(map_demo tkDNN)
 
+add_executable(demo demo/demo/demo.cpp)
+target_link_libraries(demo tkDNN)
 
 #-------------------------------------------------------------------------------
 # Install
@@ -112,18 +124,3 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory
         DESTINATION "share/tkDNN/cmake/" # target directory
 )
 
-
-#-------------------------------------------------------------------------------
-# Prepare for test
-#-------------------------------------------------------------------------------
-set(TEST_DATA true CACHE BOOL "If true download deps")
-if( ${TEST_DATA} )
-	message("Launching pre-build dependency installer script...")
-
-    execute_process (COMMAND bash -c "bash build_models.sh download"
-        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests)
-
-	set(TEST_DATA false CACHE BOOL "If true download deps" FORCE)
-	message("Finished dowloading test weights")
-endif()
-

LICENSE

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    tkDNN 
+    Copyright (C) 2017 Francesco Gatti
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

README.md

@@ -1,52 +1,289 @@
 # tkDNN
-tkDNN is a Deep Neural Network library built with cuDNN primitives specifically thought to work on NVIDIA TK1(and all successive) board.<br>
-The main scope is to do high performance inference on already trained models.
+tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives, specifically thought to work on NVIDIA Jetson Boards. It has been tested on TK1(branch cudnn2), TX1, TX2, AGX Xavier and several discrete GPU.
+The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training. 
 
-this branch actually work on every NVIDIA GPU that support the dependencies:
+Accepted paper @ IRC 2020, will soon been published.
+M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars",  in proceedings in IEEE Robotic Computing (2020)
+
+## Index
+- [tkDNN](#tkdnn)
+  - [Index](#index)
+  - [Dependencies](#dependencies)
+  - [About OpenCV](#about-opencv)
+  - [How to compile this repo](#how-to-compile-this-repo)
+  - [Workflow](#workflow)
+  - [How to export weights](#how-to-export-weights)
+    - [1)Export weights from darknet](#1export-weights-from-darknet)
+    - [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
+    - [3)Export weights for CenterNet](#3export-weights-for-centernet)
+    - [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
+  - [Run the demo](#run-the-demo)
+    - [FP16 inference](#fp16-inference)
+    - [INT8 inference](#int8-inference)
+  - [mAP demo](#map-demo)
+  - [Existing tests and supported networks](#existing-tests-and-supported-networks)
+  - [References](#references)
+
+
+
+
+## Dependencies
+This branch works on every NVIDIA GPU that supports the dependencies:
 * CUDA 10.0
 * CUDNN 7.603
 * TENSORRT 6.01
-* OPENCV 4.1
+* OPENCV 3.4
+* yaml-cpp 0.5.2 (sudo apt install libyaml-cpp-dev)
 
-## Workflow
-The recommended workflow follow these step:
-* Build and train a model in Keras (on any PC)
-* Export weights and bias 
-* Define the model on tkDNN
-* Do inference (on TK1)
-
-## Compile the library
-Build with cmake
+## About OpenCV
+To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh```. It will download and compile OpenCV in Download folder.
 ```
+bash scripts/install_OpenCV4.sh
+```
+When using openCV not compiled with contrib, comment the definition of OPENCV_CUDACONTRIBCONTRIB in include/tkDNN/DetectionNN.h. When commented, the preprocessing of the networks is computed on the CPU, otherwise on the GPU. In the latter case some milliseconds are saved in the end-to-end latency. 
+
+## How to compile this repo
+Build with cmake. If using Ubuntu 18.04 a new version of cmake is needed (3.15 or above). 
+```
+git clone https://github.com/ceccocats/tkDNN
+cd tkDNN
 mkdir build
 cd build
-cmake ..
-# use -DTEST_DATA=False to skip dataset download
+cmake .. 
 make
 ```
-during the cmake configuration it will be dowloaded the weights needed for running
-the tests
 
-## Test
-Assumiung you have correctly builded the library these are the test ready to exec:
-* test_simple: a simple convolutional and dense network (CUDNN only)
-* test_mnist: the famous mnist netwok (CUDNN and TENSORRT)
-* test_mnistRT: the mnist network hardcoded in using tensorRT apis (TENSORRT only)
-* test_yolo: YOLO detection network (CUDNN and TENSORRT)
-* test_yolo_tiny: smaller version of YOLO (CUDNN and TENSRRT)
-* test_yolo3_berkeley: our yolo3 version trained with BDD100K dateset 
+## Workflow
+Steps needed to do inference on tkDNN with a custom neural network. 
+* Build and train a NN model with your favorite framework.
+* Export weights and bias for each layer and save them in a binary file (one for layer).
+* Export outputs for each layer and save them in a binary file (one for layer).
+* Create a new test and define the network, layer by layer using the weights extracted and the output to check the results. 
+* Do inference.
 
-## yolo3 berkeley demo detection
-For the live detection you need to precompile the tensorRT file by luncing the desidered network test, this is the recommended process:
+## How to export weights
+
+Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
 ```
-export TKDNN_MODE=FP16   # set the half floating point optimization
-rm yolo3_berkeley.rt		 # be sure to delete(or move) old tensorRT files
-./test_yolo3_berkeley              # run the yolo test (is slow)
-# with f16 inference the result will be a bit incorrect
+    test_nn
+        |---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
+        |---- debug/  (folder containing a binary file for each layer with the corresponding outputs)
 ```
-this will genereate a yolo3_berkeley.rt file that can be used for live detection:
+Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
+
+### 1)Export weights from darknet
+To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
+
 ```
-./yolo3_demo                               # launch detection on a demo video
-./yolo3_demo yolo3_berkeley.rt /dev/video0 # launch detection on device 0
+git clone https://git.hipert.unimore.it/fgatti/darknet.git
+cd darknet
+make
+mkdir layers debug
+./darknet export <path-to-cfg-file> <path-to-weights> layers
 ```
+N.b. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug. 
+
+### 2)Export weights for DLA34 and ResNet101 
+To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.   
+
+Create Anaconda environment and activate it:
+```
+conda env create -f file_name.yml
+source activate env_name
+python <script name>
+```
+### 3)Export weights for CenterNet
+To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet. 
+```
+git clone https://github.com/sapienzadavide/CenterNet.git
+```
+* follow the instruction in the README.md and INSTALL.md
+
+```
+python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
+python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
+```
+### 4)Export weights for MobileNetSSD
+To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network. 
+
+```
+git clone https://github.com/mive93/pytorch-ssd
+cd pytorch-ssd
+conda env create -f env_mobv2ssd.yml
+python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
+```
+
+## Darknet Parser
+tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
+```
+// example of parsing yolo4
+tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
+net->print();
+```
+All models from darknet are now parsed directly from cfg, you still need to export the weights with the descripted tools in the previus section.
+<details>
+  <summary>Supported layers</summary>
+  convolutional
+  maxpool
+  avgpool
+  shortcut
+  upsample
+  route
+  reorg
+  region
+  yolo
+</details>
+<details>
+  <summary>Supported activations</summary>
+  relu
+  leaky
+  mish
+</details>
+
+## Run the demo
+
+To run the an object detection demo follow these steps (example with yolov3):
+```
+rm yolo3_fp32.rt        # be sure to delete(or move) old tensorRT files
+./test_yolo3            # run the yolo test (is slow)
+./demo yolo3_fp32.rt ../demo/yolo_test.mp4 y
+```
+In general the demo program takes 4 parameters:
+```
+./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag>
+```
+where
+*  ```<network-rt-file>``` is the rt file generated by a test
+*  ```<<path-to-video>``` is the path to a video file or a camera input  
+*  ```<kind-of-network>``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
+*  ```<number-of-classes>```is the number of classes the network is trained on
+*  ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
+*  ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
+
+N.b. By default it is used FP32 inference
+
 ![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
+
+### FP16 inference
+
+To run the an object detection demo with FP16 inference follow these steps (example with yolov3):
+```
+export TKDNN_MODE=FP16  # set the half floating point optimization
+rm yolo3_fp16.rt        # be sure to delete(or move) old tensorRT files
+./test_yolo3            # run the yolo test (is slow)
+./demo yolo3_fp16.rt ../demo/yolo_test.mp4 y
+```
+N.b. Using FP16 inference will lead to some errors in the results (first or second decimal). 
+
+### INT8 inference
+
+To run the an object detection demo with INT8 inference follow these steps (example with yolov3):
+```
+export TKDNN_MODE=INT8  # set the 8-bit integer optimization
+
+# image_list.txt contains the list of the absolute paths to the calibration images
+export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt
+
+# label_list.txt contains the list of the absolute paths to the calibration labels
+export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt
+rm yolo3_int8.rt        # be sure to delete(or move) old tensorRT files
+./test_yolo3            # run the yolo test (is slow)
+./demo yolo3_int8.rt ../demo/yolo_test.mp4 y
+```
+N.b. Using INT8 inference will lead to some errors in the results. 
+
+N.b. The test will be slower: this is due to the INT8 calibration, which may take some time to complete. 
+
+N.b. INT8 calibration requires TensorRT version greater than or equal to 6.0
+
+### BatchSize bigger than 1
+```
+export TKDNN_BATCHSIZE=2
+# build tensorRT files
+```
+This will create a TensorRT file with the desidered **max** batch size.
+The test will still run with a batch of 1, but the created tensorRT can manage the desidered batch size.
+
+### Test batch Inference
+This will test the network with random input and check if the output of each batch is the same.
+```
+./test_rtinference <network-rt-file> <number-of-batches>
+# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
+
+# example
+export TKDNN_BATCHSIZE=4           # set max batch size
+rm yolo3_fp32.rt                   # be sure to delete(or move) old tensorRT files
+./test_yolo3                       # build RT file
+./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
+```
+
+## mAP demo
+
+To compute mAP, precision, recall and f1score, run the map_demo.
+
+A validation set is needed. 
+To download COCO_val2017 (80 classes) run (form the root folder): 
+```
+bash scripts/download_validation.sh COCO
+```
+To download Berkeley_val (10 classes) run (form the root folder): 
+```
+bash scripts/download_validation.sh BDD
+```
+
+To compute the map, the following parameters are needed:
+```
+./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
+```
+where 
+* ```<network rt>```: rt file of a chosen network on which compute the mAP.
+* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
+* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg. 
+* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
+
+Example:
+
+```
+cd build
+./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
+```
+
+This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
+
+## Existing tests and supported networks
+
+| Test Name         | Network                                       | Dataset                                                       | N Classes | Input size    | Weights                                                                   |
+| :---------------- | :-------------------------------------------- | :-----------------------------------------------------------: | :-------: | :-----------: | :------------------------------------------------------------------------ |
+| yolo              | YOLO v2<sup>1</sup>                           | [COCO 2014](http://cocodataset.org/)                          | 80        | 608x608       | [weights](https://cloud.hipert.unimore.it/s/nf4PJ3k8bxBETwL/download)                                                                   |
+| yolo_224          | YOLO v2<sup>1</sup>                           | [COCO 2014](http://cocodataset.org/)                          | 80        | 224x224       | weights                                                                   |
+| yolo_berkeley     | YOLO v2<sup>1</sup>                           | [BDD100K  ](https://bair.berkeley.edu/blog/2018/05/30/bdd/)   | 10        | 416x736       | weights                                                                   |
+| yolo_relu         | YOLO v2 (with ReLU, not Leaky)<sup>1</sup>    | [COCO 2014](http://cocodataset.org/)                          | 80        | 416x416       | weights                                                                   |
+| yolo_tiny         | YOLO v2 tiny<sup>1</sup>                      | [COCO 2014](http://cocodataset.org/)                          | 80        | 416x416       | [weights](https://cloud.hipert.unimore.it/s/m3orfJr8pGrN5mQ/download)                                                                   |
+| yolo_voc          | YOLO v2<sup>1</sup>                           | [VOC      ](http://host.robots.ox.ac.uk/pascal/VOC/)          | 21        | 416x416       | [weights](https://cloud.hipert.unimore.it/s/DJC5Fi2pEjfNDP9/download)                                                                   |
+| yolo3             | YOLO v3<sup>2</sup>                           | [COCO 2014](http://cocodataset.org/)                          | 80        | 416x416       | [weights](https://cloud.hipert.unimore.it/s/jPXmHyptpLoNdNR/download)     |
+| yolo3_512   | YOLO v3<sup>2</sup>                                 | [COCO 2017](http://cocodataset.org/)                          | 80        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/RGecMeGLD4cXEWL/download)     |
+| yolo3_berkeley    | YOLO v3<sup>2</sup>                           | [BDD100K  ](https://bair.berkeley.edu/blog/2018/05/30/bdd/)   | 10        | 320x544       | [weights](https://cloud.hipert.unimore.it/s/o5cHa4AjTKS64oD/download)                                                                   |
+| yolo3_coco4       | YOLO v3<sup>2</sup>                           | [COCO 2014](http://cocodataset.org/)                          | 4         | 416x416       | [weights](https://cloud.hipert.unimore.it/s/o27NDzSAartbyc4/download)                                                                   |
+| yolo3_flir        | YOLO v3<sup>2</sup>                           | [FREE FLIR](https://www.flir.com/oem/adas/adas-dataset-form/) | 3         | 320x544       | [weights](https://cloud.hipert.unimore.it/s/62DECncmF6bMMiH/download)                                                                   |
+| yolo3_tiny        | YOLO v3 tiny<sup>2</sup>                      | [COCO 2014](http://cocodataset.org/)                          | 80        | 416x416       | [weights](https://cloud.hipert.unimore.it/s/LMcSHtWaLeps8yN/download)     |
+| yolo3_tiny512     | YOLO v3 tiny<sup>2</sup>                      | [COCO 2017](http://cocodataset.org/)                          | 80        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/8Zt6bHwHADqP4JC/download)     |
+| dla34             | Deep Leayer Aggreagtion (DLA) 34<sup>3</sup>  | [COCO 2014](http://cocodataset.org/)                          | 80        | 224x224       | weights                                                                   |
+| dla34_cnet        | Centernet (DLA34 backend)<sup>4</sup>         | [COCO 2017](http://cocodataset.org/)                          | 80        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/KRZBbCQsKAtQwpZ/download)     |
+| mobilenetv2ssd    | Mobilnet v2 SSD Lite<sup>5</sup>              | [VOC      ](http://host.robots.ox.ac.uk/pascal/VOC/)          | 21        | 300x300       | [weights](https://cloud.hipert.unimore.it/s/x4ZfxBKN23zAJQp/download)     |
+| mobilenetv2ssd512 | Mobilnet v2 SSD Lite<sup>5</sup>              | [COCO 2017](http://cocodataset.org/)                          | 81        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/pdCw2dYyHMJrcEM/download)     |
+| resnet101         | Resnet 101<sup>6</sup>                        | [COCO 2014](http://cocodataset.org/)                          | 80        | 224x224       | weights                                                                   |
+| resnet101_cnet    | Centernet (Resnet101 backend)<sup>4</sup>     | [COCO 2017](http://cocodataset.org/)                          | 80        | 512x512       | [weights](https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download)     |
+| csresnext50-panet-spp    | Cross Stage Partial Network <sup>7</sup>     | [COCO 2014](http://cocodataset.org/)                          | 80        | 416x416       | [weights](https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download)     |
+| yolo4             | Yolov4 <sup>8</sup>                           | [COCO 2017](http://cocodataset.org/)                          | 80        | 416x416       | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download)     |
+
+
+## References
+
+1. Redmon, Joseph, and Ali Farhadi. "YOLO9000: better, faster, stronger." Proceedings of the IEEE conference on computer vision and pattern recognition. 2017.
+2. Redmon, Joseph, and Ali Farhadi. "Yolov3: An incremental improvement." arXiv preprint arXiv:1804.02767 (2018).
+3. Yu, Fisher, et al. "Deep layer aggregation." Proceedings of the IEEE conference on computer vision and pattern recognition. 2018.
+4. Zhou, Xingyi, Dequan Wang, and Philipp Krähenbühl. "Objects as points." arXiv preprint arXiv:1904.07850 (2019).
+5. Sandler, Mark, et al. "Mobilenetv2: Inverted residuals and linear bottlenecks." Proceedings of the IEEE conference on computer vision and pattern recognition. 2018.
+6. He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016.
+7. Wang, Chien-Yao, et al. "CSPNet: A New Backbone that can Enhance Learning Capability of CNN." arXiv preprint arXiv:1911.11929 (2019).
+8. Bochkovskiy, Alexey, Chien-Yao Wang, and Hong-Yuan Mark Liao. "YOLOv4: Optimal Speed and Accuracy of Object Detection." arXiv preprint arXiv:2004.10934 (2020).

demo/config.yaml

@@ -0,0 +1,7 @@
+classes     : 80    #number of classes
+map_points  : 101   #number of recall points (0 for all, 101 for COCO, 11 PascalVOC)
+map_levels  : 10    #number of IoU step for the AP
+map_step    : 0.05  #step of IoU
+IoU_thresh  : 0.5   #starting IoU threshold
+conf_thresh : 0.0   #threshold on the condifence of the bbox
+verbose     : false #print on screen information

demo/config_tetra.yaml

@@ -0,0 +1,7 @@
+classes     : 3    #number of classes
+map_points  : 101   #number of recall points (0 for all, 101 for COCO, 11 PascalVOC)
+map_levels  : 10    #number of IoU step for the AP
+map_step    : 0.05  #step of IoU
+IoU_thresh  : 0.5   #starting IoU threshold
+conf_thresh : 0.0   #threshold on the condifence of the bbox
+verbose     : false #print on screen information

demo/demo/demo.cpp

@@ -3,16 +3,12 @@
 #include <stdlib.h>     /* srand, rand */
 #include <unistd.h>
 #include <mutex>
-#include "utils.h"
-
-#include <opencv2/core/core.hpp>
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/videoio.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
 
+#include "CenternetDetection.h"
+#include "MobilenetDetection.h"
 #include "Yolo3Detection.h"
 
-bool 			gRun;
+bool gRun;
 bool SAVE_RESULT = false;
 
 void sig_handler(int signo) {
@@ -26,15 +22,54 @@ int main(int argc, char *argv[]) {
     signal(SIGINT, sig_handler);
 
 
-    char *net = "yolo3_berkeley.rt";
+    std::string net = "yolo3_berkeley.rt";
     if(argc > 1)
         net = argv[1]; 
-    char *input = "../demo/yolo_test.mp4";
+    std::string input = "../demo/yolo_test.mp4";
     if(argc > 2)
         input = argv[2]; 
+    char ntype = 'y';
+    if(argc > 3)
+        ntype = argv[3][0]; 
+    int n_classes = 80;
+    if(argc > 4)
+        n_classes = atoi(argv[4]); 
+    int n_batch = 1;
+    if(argc > 5)
+        n_batch = atoi(argv[5]); 
+    bool show = true;
+    if(argc > 6)
+        show = atoi(argv[6]); 
+
+    if(n_batch < 1 || n_batch > 64)
+        FatalError("Batch dim not supported");
+
+    if(!show)
+        SAVE_RESULT = true;
 
     tk::dnn::Yolo3Detection yolo;
-    yolo.init(net);
+    tk::dnn::CenternetDetection cnet;
+    tk::dnn::MobilenetDetection mbnet;  
+
+    tk::dnn::DetectionNN *detNN;  
+
+    switch(ntype)
+    {
+        case 'y':
+            detNN = &yolo;
+            break;
+        case 'c':
+            detNN = &cnet;
+            break;
+        case 'm':
+            detNN = &mbnet;
+            n_classes++;
+            break;
+        default:
+        FatalError("Network type not allowed (3rd parameter)\n");
+    }
+
+    detNN->init(net, n_classes, n_batch);
 
     gRun = true;
 
@@ -44,7 +79,6 @@ int main(int argc, char *argv[]) {
     else
         std::cout<<"camera started\n";
 
-
     cv::VideoWriter resultVideo;
     if(SAVE_RESULT) {
         int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
@@ -53,57 +87,53 @@ int main(int argc, char *argv[]) {
     }
 
     cv::Mat frame;
-    cv::Mat dnn_input;
-    cv::namedWindow("detection", cv::WINDOW_NORMAL);
-    
+    if(show)
+        cv::namedWindow("detection", cv::WINDOW_NORMAL);
+
+    std::vector<cv::Mat> batch_frame;
+    std::vector<cv::Mat> batch_dnn_input;
+
     while(gRun) {
-        cap >> frame; 
-        if(!frame.data) {
+        batch_dnn_input.clear();
+        batch_frame.clear();
+        
+        for(int bi=0; bi< n_batch; ++bi){
+            cap >> frame; 
+            if(!frame.data) 
+                break;
+            
+            batch_frame.push_back(frame);
+
+            // this will be resized to the net format
+            batch_dnn_input.push_back(frame.clone());
+        } 
+        if(!frame.data) 
             break;
-        }  
- 
-        // this will be resized to the net format
-        dnn_input = frame.clone();
-        // TODO: async infer
-        yolo.update(dnn_input);
-
-        // draw dets
-        for(int i=0; i<yolo.detected.size(); i++) {
-            tk::dnn::box b = yolo.detected[i];
-            int x0   				= b.x;
-            int x1   				= b.x + b.w;
-            int y0   				= b.y;
-            int y1   				= b.y + b.h;
-            std::string det_class 	= yolo.getYoloLayer()->classesNames[b.cl];
-            float prob 				= b.prob;
-
-            std::cout<<det_class<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
-			// draw rectangle
-            cv::rectangle(frame, cv::Point(x0, y0), cv::Point(x1, y1), yolo.colors[b.cl], 2); 
-
-	        // draw label
-            int baseline = 0;
-            float fontScale = 0.5;
-            int thickness = 2;
-            cv::Size textSize = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseline);
-            cv::rectangle(frame, cv::Point(x0, y0), cv::Point((x0 + textSize.width - 2), (y0 - textSize.height - 2)), yolo.colors[b.cl], -1);                      
-            cv::putText(frame, det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, fontScale, cv::Scalar(255, 255, 255), thickness);
-        }
     
-        cv::imshow("detection", frame);
-        cv::waitKey(1);
-        if(SAVE_RESULT)
+        //inference
+        detNN->update(batch_dnn_input, n_batch);
+        detNN->draw(batch_frame);
+
+        if(show){
+            for(int bi=0; bi< n_batch; ++bi){
+                cv::imshow("detection", batch_frame[bi]);
+                cv::waitKey(1);
+            }
+        }
+        if(n_batch == 1 && SAVE_RESULT)
             resultVideo << frame;
     }
 
     std::cout<<"detection end\n";   
-
-
+    double mean = 0; 
+    
     std::cout<<COL_GREENB<<"\n\nTime stats:\n";
-    std::cout<<"Min: "<<*std::min_element(yolo.stats.begin(), yolo.stats.end())<<" ms\n";    
-    std::cout<<"Max: "<<*std::max_element(yolo.stats.begin(), yolo.stats.end())<<" ms\n";    
-    double mean = 0; for(int i=0; i<yolo.stats.size(); i++) mean += yolo.stats[i]; mean /= yolo.stats.size();
-    std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;    
+    std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";    
+    std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";    
+    for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
+    std::cout<<"Avg: "<<mean/n_batch<<" ms\t"<<1000/(mean/n_batch)<<" FPS\n"<<COL_END;   
+    
+
     return 0;
 }
 

demo/demo/map.cpp

@@ -0,0 +1,235 @@
+
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>     /* srand, rand */
+#include <unistd.h>
+#include <mutex>
+#include "utils.h"
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "Yolo3Detection.h"
+#include "CenternetDetection.h"
+#include "MobilenetDetection.h"
+
+#include "evaluation.h"
+
+#include <map>
+
+void convertFilename(std::string &filename,const std::string l_folder, const std::string i_folder, const std::string l_ext,const std::string i_ext)
+{
+    filename.replace(filename.find(l_folder),l_folder.length(),i_folder);
+    filename.replace(filename.find(l_ext),l_ext.length(),i_ext);
+}
+
+int main(int argc, char *argv[]) 
+{
+    char ntype = 'y';
+    const char *config_filename = "../demo/config.yaml";
+    const char * net = "yolo3.rt";
+    const char * labels_path = "../demo/COCO_val2017/all_labels.txt";
+    bool show = false;
+    bool write_dets = false;
+    bool write_res_on_file = true;
+    bool write_coco_json = true;
+    int n_images = 5000;
+
+    bool verbose;
+    int classes, map_points, map_levels;
+    float map_step, IoU_thresh, conf_thresh;
+
+    double vm_total = 0, rss_total = 0;
+    double vm, rss;
+
+    //read args
+    if(argc > 1)
+        net = argv[1]; 
+    if(argc > 2)
+        ntype = argv[2][0];    
+    if(argc > 3)
+        labels_path = argv[3]; 
+    if(argc > 4)
+        config_filename = argv[4]; 
+
+    //check if files needed exist
+    if(!fileExist(config_filename))
+        FatalError("Wrong config file path.");
+    if(!fileExist(net))
+        FatalError("Wrong net file path.");
+    if(!fileExist(labels_path))
+        FatalError("Wrong labels file path.");
+
+    //read mAP parameters
+    tk::dnn::readmAPParams( config_filename, classes,  map_points, map_levels, map_step,
+                IoU_thresh, conf_thresh, verbose);
+
+    //extract network name from rt path
+    std::string net_name;
+    removePathAndExtension(net, net_name);
+    std::cout<<"Network: "<<net_name<<std::endl;
+
+    //open files (if needed)
+    std::ofstream times, memory, coco_json;
+
+    if(write_coco_json){
+        coco_json.open(net_name+"_COCO_res.json");
+        coco_json << "[\n";
+    }
+
+    if(write_res_on_file){
+        times.open("times_"+net_name+".csv");
+        memory.open("memory.csv", std::ios_base::app);
+        memory<<net<<";";
+    }
+
+    // instantiate detector
+    tk::dnn::Yolo3Detection yolo;
+    tk::dnn::CenternetDetection cnet;
+    tk::dnn::MobilenetDetection mbnet;
+    tk::dnn::DetectionNN *detNN;  
+    int n_classes = classes;   
+    switch(ntype){
+        case 'y':
+            detNN = &yolo;
+            break;
+        case 'c':
+            detNN = &cnet;
+            break;
+        case 'm':
+            detNN = &mbnet;
+            n_classes++;
+            break;
+        default:
+            FatalError("Network type not allowed (3rd parameter)\n");
+    }
+    detNN->init(net, n_classes);
+
+    //read images 
+    std::ifstream all_labels(labels_path);
+    std::string l_filename;
+    std::vector<tk::dnn::Frame> images;
+    std::vector<tk::dnn::box> detected_bbox;
+
+    std::cout<<"Reading groundtruth and generating detections"<<std::endl;
+
+    if(show)
+        cv::namedWindow("detection", cv::WINDOW_NORMAL);
+
+    int images_done;
+    for (images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) {
+        std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\n"<<COL_END;
+
+        tk::dnn::Frame f;
+        f.lFilename = l_filename;
+        f.iFilename = l_filename;
+        convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
+
+        // read frame
+        if(!fileExist(f.iFilename.c_str()))
+            FatalError("Wrong image file path.");
+
+        cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
+        std::vector<cv::Mat> batch_frames;
+        batch_frames.push_back(frame);
+        int height = frame.rows;
+        int width = frame.cols;
+
+        if(!frame.data) 
+            break;
+        std::vector<cv::Mat> batch_dnn_input;
+        batch_dnn_input.push_back(frame.clone());
+
+        //inference 
+        detected_bbox.clear();
+        detNN->update(batch_dnn_input,1,write_res_on_file, &times, write_coco_json);
+        detNN->draw(batch_frames);
+        detected_bbox = detNN->detected;
+
+        if(write_coco_json)
+            printJsonCOCOFormat(&coco_json, f.iFilename.c_str(), detected_bbox, classes,  width, height);        
+
+        std::ofstream myfile;
+        if(write_dets)
+            myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("000")));
+
+        // save detections labels
+        for(auto d:detected_bbox){
+            //convert detected bb in the same format as label
+            //<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
+            tk::dnn::BoundingBox b;
+            b.x = (d.x + d.w/2) / width;
+            b.y = (d.y + d.h/2) / height;
+            b.w = d.w / width;
+            b.h = d.h / height;
+            b.prob = d.prob;
+            b.cl = d.cl;
+            f.det.push_back(b);
+
+            if(write_dets)
+                myfile << d.cl << " "<< d.prob << " "<< d.x << " "<< d.y << " "<< d.w << " "<< d.h <<"\n";
+
+			if(show)// draw rectangle for detection
+                cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);             
+        }
+
+        if(write_dets)
+            myfile.close();
+
+        // read and save groundtruth labels
+        if(fileExist(f.lFilename.c_str()))
+        {
+            std::ifstream labels(l_filename);
+            for(std::string line; std::getline(labels, line); ){
+                std::istringstream in(line); 
+                tk::dnn::BoundingBox b;
+                in >> b.cl >> b.x >> b.y >> b.w >> b.h;  
+                b.prob = 1;
+                b.truthFlag = 1;
+                f.gt.push_back(b);
+
+                if(show)// draw rectangle for groundtruth
+                    cv::rectangle(batch_frames[0], cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2);             
+            }
+        }    
+      
+        images.push_back(f);
+        
+        if(show){
+            cv::imshow("detection", batch_frames[0]);
+            cv::waitKey(0);
+        }
+
+        getMemUsage(vm, rss);
+        vm_total += vm;
+        rss_total += rss;
+
+        
+    }
+
+    if(write_coco_json){
+        coco_json.seekp (coco_json.tellp() - std::streampos(2));
+        coco_json << "\n]\n";
+        coco_json.close();
+    }
+
+    std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl;
+
+    //compute mAP
+    double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,conf_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name);
+    std::cout<<"mAP "<<IoU_thresh<<":"<<IoU_thresh+map_step*(map_levels-1)<<" = "<<AP<<std::endl;
+
+    //compute average precision, recall and f1score
+    tk::dnn::computeTPFPFN(images,classes,IoU_thresh,conf_thresh, verbose, write_res_on_file, net_name);
+
+    if(write_res_on_file){
+        memory<<vm_total/images_done/1024.0<<";"<<rss_total/images_done/1024.0<<"\n";
+        times.close();
+        memory.close();
+    }
+
+    return 0;
+}
+

include/tkDNN/BoundingBox.h

@@ -0,0 +1,31 @@
+#ifndef BOUNDINGBOX_H
+#define BOUNDINGBOX_H
+
+#include <iostream>
+#include "tkdnn.h"
+
+namespace tk { namespace dnn {
+class BoundingBox : public tk::dnn::box
+{
+    float overlap(const float p1, const float l1, const float p2, const float l22);
+    float boxesIntersection(const BoundingBox &b);
+    float boxesUnion(const BoundingBox &b);
+
+    public:
+    
+    int uniqueTruthIndex = -1;
+    int truthFlag = 0;
+    float maxIoU = 0;
+
+    float IoU(const BoundingBox &b);
+    void clear();
+
+    friend std::ostream& operator<<(std::ostream& os, const BoundingBox& bb);
+};
+
+std::ostream& operator<<(std::ostream& os, const BoundingBox& bb);
+bool boxComparison (const BoundingBox& a,const BoundingBox& b) ;
+
+}}
+#endif /*BOUNDINGBOX_H*/
+

include/tkDNN/CenternetDetection.h

@@ -0,0 +1,86 @@
+#ifndef CENTERNETDETECTION_H
+#define CENTERNETDETECTION_H
+
+#include "kernels.h"
+#include <opencv2/videoio.hpp>
+#include "opencv2/opencv.hpp"
+#include <time.h>
+#include <vector>
+#include <numeric>      // std::iota
+#include <algorithm>    // std::sort
+
+#include "DetectionNN.h"
+
+#include "kernelsThrust.h"
+
+
+namespace tk { namespace dnn { 
+
+class CenternetDetection : public DetectionNN
+{
+private:
+    tk::dnn::dataDim_t dim;
+    tk::dnn::dataDim_t dim2;
+    tk::dnn::dataDim_t dim_hm; 
+    tk::dnn::dataDim_t dim_wh; 
+    tk::dnn::dataDim_t dim_reg;
+    float *topk_scores;
+    int *topk_inds_;
+    float *topk_ys_;
+    float *topk_xs_;
+    int *ids_d, *ids_, *ids_2, *ids_2d;
+
+    float *scores, *scores_d;
+    int *clses, *clses_d; 
+    int *topk_inds_d;
+    float *topk_ys_d; 
+    float *topk_xs_d;
+    int *inttopk_xs_d, *inttopk_ys_d;
+
+
+    float *bbx0, *bby0, *bbx1, *bby1;
+    float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
+    
+    float *target_coords;
+
+    #ifdef OPENCV_CUDACONTRIB
+        float *mean_d;
+        float *stddev_d;
+    #else
+        cv::Vec<float, 3> mean;
+        cv::Vec<float, 3> stddev;
+        dnnType *input;
+    #endif
+
+    float *d_ptrs;
+
+    cv::Mat src;
+    cv::Mat dst;
+    cv::Mat dst2;  
+    cv::Mat trans, trans2;
+    //processing
+    float toll = 0.000001;
+    int K = 100;
+    int width = 128;//56;        // TODO
+
+    // pointer used in the kernels
+    float *src_out;
+    int *ids_out;
+
+    struct threshold op;
+    
+public:
+    CenternetDetection() {};
+    ~CenternetDetection() {}; 
+
+    bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
+    void preprocess(cv::Mat &frame, const int bi=0);
+    void postprocess(const int bi=0,const bool mAP=false);
+};
+
+
+} // namespace dnn
+} // namespace tk
+
+
+#endif /*CENTERNETDETECTION_H*/

include/tkDNN/DarknetParser.h

@@ -0,0 +1,293 @@
+#pragma once
+#include <iostream>
+#include "tkdnn.h"
+
+namespace tk { namespace dnn {
+
+    struct darknetFields_t{
+        std::string type = "";
+        int width = 0;
+        int height = 0;
+        int channels = 3;
+        int batch_normalize=0;
+        int groups = 1;
+        int filters=1;
+        int size_x=1;
+        int size_y=1;
+        int stride_x=1;
+        int stride_y=1;
+        int padding_x = 0;
+        int padding_y = 0;
+        int n_mask = 0;
+        int classes = 20;
+        int num = 1;
+        int pad = 0;
+        int coords = 4;
+        float scale_xy = 1;
+        std::vector<int> layers;
+        std::string activation = "linear";
+
+    };
+
+    std::ostream& operator<<(std::ostream& os, const darknetFields_t& f){
+        os << f.width << " " << f.height << " " << f.channels << " " << f.batch_normalize<< " " << f.filters << " "  << f.activation<< " " << f.scale_xy;
+        return os;
+    }
+
+    std::string darknetParseType(const std::string& line){
+            size_t start = line.find("[");
+            size_t end  = line.find("]");
+            if( start == std::string::npos || end == std::string::npos)
+                return "";
+            start++;
+            std::string type = line.substr(start, end-start);
+            return type;
+    }
+
+    bool divideNameAndValue(const std::string& line, std::string&name, std::string& value){
+        size_t sep = line.find("=");
+        if(sep == std::string::npos)
+            return false;
+        
+        name   = line.substr(0, sep);   
+        value  = line.substr(sep+1, line.size() - (sep+1));   
+        return true;
+    }
+
+    std::vector<int> fromStringToIntVec(const std::string& line, const char delimiter){
+        std::stringstream linestream(line);
+        std::string value;
+        std::vector<int> values;
+
+        while(getline(linestream,value,delimiter))
+            values.push_back(std::stoi(value));
+        return values;
+    }
+
+    bool darknetParseFields(const std::string& line, darknetFields_t& fields){
+
+        std::string name,value;
+        if(!divideNameAndValue(line, name, value))
+            return false;
+        if(name.find("width") !=  std::string::npos)
+            fields.width = std::stoi(value);
+        else if(name.find("height") !=  std::string::npos)
+            fields.height = std::stoi(value);
+        else if(name.find("channels") !=  std::string::npos)
+            fields.channels = std::stoi(value);
+        else if(name.find("batch_normalize") !=  std::string::npos)
+            fields.batch_normalize = std::stoi(value);
+        else if(name.find("filters") !=  std::string::npos)
+            fields.filters = std::stoi(value);
+        else if(name.find("activation") !=  std::string::npos)
+            fields.activation = value;
+        else if(name.find("size") !=  std::string::npos){
+            fields.size_x = std::stoi(value);
+            fields.size_y = std::stoi(value);
+        }
+        else if(name.find("size_x") !=  std::string::npos)
+            fields.size_x = std::stoi(value);
+        else if(name.find("size_y") !=  std::string::npos)
+            fields.size_y = std::stoi(value);
+        else if(name.find("stride") !=  std::string::npos){
+            fields.stride_x = std::stoi(value);
+            fields.stride_y = std::stoi(value);
+        }
+        else if(name.find("stride_x") !=  std::string::npos)
+            fields.stride_x = std::stoi(value);
+        else if(name.find("stride_y") !=  std::string::npos)
+            fields.stride_y = std::stoi(value);
+        else if(name.find("pad") !=  std::string::npos)
+            fields.pad = std::stoi(value);
+        else if(name.find("classes") !=  std::string::npos)
+            fields.classes = std::stoi(value);
+        else if(name.find("num") !=  std::string::npos)
+            fields.num = std::stoi(value);
+        else if(name.find("coords") !=  std::string::npos)
+            fields.coords = std::stoi(value);
+        else if(name.find("groups") !=  std::string::npos)
+            fields.groups = std::stoi(value);
+        else if(name.find("scale_x_y") !=  std::string::npos)
+            fields.scale_xy = std::stof(value);
+        else if(name.find("from") !=  std::string::npos)
+            fields.layers.push_back(std::stof(value));
+        else if(name.find("mask") !=  std::string::npos){
+            auto vec = fromStringToIntVec(value, ',');
+            fields.n_mask = vec.size();
+        }
+        else if(name.find("layers") !=  std::string::npos)
+            fields.layers = fromStringToIntVec(value, ',');
+
+        else
+            std::cout<<"Not supported field: "<<line<<std::endl;
+        return true;
+    }
+
+    tk::dnn::Network *darknetAddNet(darknetFields_t &fields) {
+        //std::cout<<"Add Net: "<<fields.type<<"\n";
+        dataDim_t dim(1, fields.channels, fields.height, fields.width);
+        return new tk::dnn::Network(dim);
+    }
+
+
+    void darknetAddLayer(tk::dnn::Network *net, darknetFields_t &f, std::string wgs_path, std::vector<tk::dnn::Layer*> &netLayers, const std::vector<std::string>& names) {
+        if(net == nullptr)
+            FatalError("Cant add a layer without a Net\n");
+
+        // padding compute
+        if(f.pad == 1) {
+            f.padding_x = f.padding_y = f.size_x /2;
+        }
+        //std::cout<<"Add layer: "<<f.type<<"\n";
+        if(f.type == "convolutional") {
+            std::string wgs = wgs_path + "/c" + std::to_string(netLayers.size()) + ".bin";
+            //printf("%d (%d,%d) (%d,%d) (%d,%d) %s %d %d\n", f.filters, f.size_x, f.size_y, f.stride_x, f.stride_y, f.padding_x, f.padding_y, wgs.c_str(), f.batch_normalize, f.groups);
+            tk::dnn::Conv2d *l= new tk::dnn::Conv2d(net, f.filters, f.size_x, f.size_y, f.stride_x, 
+                                f.stride_y, f.padding_x, f.padding_y, wgs, f.batch_normalize, false, f.groups);
+            netLayers.push_back(l);
+        } else if(f.type == "maxpool") {
+            if(f.stride_x == 1 && f.stride_y == 1)
+                netLayers.push_back(new tk::dnn::Pooling(net, f.size_x, f.size_y, f.stride_x, f.stride_y, 
+                    f.padding_x, f.padding_y, tk::dnn::POOLING_MAX_FIXEDSIZE));
+            else
+                netLayers.push_back(new tk::dnn::Pooling(net, f.size_x, f.size_y, f.stride_x, f.stride_y, 
+                    f.padding_x, f.padding_y, tk::dnn::POOLING_MAX));
+
+        } else if(f.type == "avgpool") {
+            netLayers.push_back(new tk::dnn::Pooling(net, f.size_x, f.size_y, f.stride_x, f.stride_y, 
+                f.padding_x, f.padding_y, tk::dnn::POOLING_AVERAGE));
+
+        } else if(f.type == "shortcut") {
+            if(f.layers.size() != 1) FatalError("no layers to shortcut\n");
+            int layerIdx = f.layers[0];
+            if(layerIdx < 0)
+                layerIdx = netLayers.size() + layerIdx; 
+            if(layerIdx < 0 || layerIdx >= netLayers.size()) FatalError("impossible to shortcut\n");
+            //std::cout<<"shortcut to "<<layerIdx<<" "<<netLayers[layerIdx]->getLayerName()<<"\n";
+            netLayers.push_back(new tk::dnn::Shortcut(net, netLayers[layerIdx]));
+
+        } else if(f.type == "upsample") {
+            netLayers.push_back(new tk::dnn::Upsample(net, f.stride_x));
+
+        } else if(f.type == "route") {
+            if(f.layers.size() == 0) FatalError("no layers to Route\n");
+            std::vector<tk::dnn::Layer*> layers;
+            for(int i=0; i<f.layers.size(); i++) {
+                int layerIdx = f.layers[i];
+                if(layerIdx < 0)
+                    layerIdx = netLayers.size() + layerIdx; 
+                if(layerIdx < 0 || layerIdx >= netLayers.size()) FatalError("impossible to route\n");
+                //std::cout<<"Route to "<<layerIdx<<" "<<netLayers[layerIdx]->getLayerName()<<"\n";
+                layers.push_back(netLayers[layerIdx]);
+            }
+            netLayers.push_back(new tk::dnn::Route(net, layers.data(), layers.size()));
+
+        } else if(f.type == "reorg") {
+            netLayers.push_back(new tk::dnn::Reorg(net, f.stride_x));
+
+        } else if(f.type == "region") {
+            netLayers.push_back(new tk::dnn::Region(net, f.classes, f.coords, f.num));
+
+        } else if(f.type == "yolo") {
+            std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin";
+            //printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy);
+            tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy);
+            if(names.size() != f.classes)
+                FatalError("Mismatch between number of classes and names");
+            l->classesNames = names;
+            netLayers.push_back(l);
+
+        } else{
+            FatalError("layer not supported: " + f.type);
+        }
+
+        // add activation
+        if(netLayers.size() > 0 && f.activation != "linear") {
+            tkdnnActivationMode_t act;
+            if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU);
+            else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY;
+            else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH;
+            else { FatalError("activation not supported: " + f.activation); }
+            netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act);
+        };
+    }
+
+    std::vector<std::string> darknetReadNames(const std::string& names_file){
+        std::ifstream if_names(names_file);
+        if(!if_names.is_open())
+            FatalError("cloud not open names file: " + names_file);       
+
+        std::vector<std::string> names;
+        std::string line;
+        while(std::getline(if_names, line))
+            if(line != "")
+                names.push_back(line);
+        
+        if_names.close();
+        return names;
+    }
+
+    tk::dnn::Network* darknetParser(const std::string& cfg_file, const std::string& wgs_path, const std::string& names_file) {
+
+        tk::dnn::Network *net = nullptr;
+        
+        // layers without activations to retrive correct id number
+        std::vector<tk::dnn::Layer*> netLayers;
+
+        std::ifstream if_cfg(cfg_file);
+        if(!if_cfg.is_open())
+            FatalError("cloud not open cfg file: " + cfg_file);
+
+        std::vector<std::string> names = darknetReadNames(names_file);
+
+        darknetFields_t fields; // will be filled with layers fields
+        std::string line;
+        while(std::getline(if_cfg, line)) {
+            // remove comments
+            std::size_t found = line.find("#");
+            if ( found != std::string::npos ) {
+                line = line.substr(0, found);
+            }
+
+            // skip empty lines
+            if(line.size() == 0)
+                continue;
+            
+            std::string type = darknetParseType(line);
+            if(type.size() > 0) {
+                // end of filled type
+                if(fields.type != "") {
+                    if(fields.type == "net")
+                        net = darknetAddNet(fields);
+                    else
+                        darknetAddLayer(net, fields, wgs_path, netLayers, names);
+                }
+
+                // new type
+                //std::cout<<"type: "<<type<<"\n";
+                fields = darknetFields_t(); // reset to default
+                fields.type = type;
+                continue;
+            }
+
+            if(darknetParseFields(line, fields)) {
+                // already parsed do nothing
+            } else {
+                FatalError("could not parse line: " + line);
+            }
+        }
+
+        // end of filled type
+        if(fields.type != "") {
+            darknetAddLayer(net, fields, wgs_path, netLayers, names);
+        }
+
+        if(net == nullptr) {
+            FatalError("net not found\n");
+        }
+        return net;
+    }
+        
+    
+
+}}

include/tkDNN/DetectionNN.h

@@ -0,0 +1,183 @@
+#ifndef DETECTIONNN_H
+#define DETECTIONNN_H
+
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>    
+#include <unistd.h>
+#include <mutex>
+#include "utils.h"
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "tkdnn.h"
+
+// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
+
+#ifdef OPENCV_CUDACONTRIB
+#include <opencv2/cudawarping.hpp>
+#include <opencv2/cudaarithm.hpp>
+#endif
+
+
+namespace tk { namespace dnn {
+
+class DetectionNN {
+
+    protected:
+        tk::dnn::NetworkRT *netRT = nullptr;
+        dnnType *input_d;
+
+        std::vector<cv::Size> originalSize;
+
+        cv::Scalar colors[256];
+
+        int nBatches = 1;
+
+#ifdef OPENCV_CUDACONTRIB
+        cv::cuda::GpuMat bgr[3];
+        cv::cuda::GpuMat imagePreproc;
+#else
+        cv::Mat bgr[3];
+        cv::Mat imagePreproc;
+        dnnType *input;
+#endif
+
+        /**
+         * This method preprocess the image, before feeding it to the NN.
+         *
+         * @param frame original frame to adapt for inference.
+         * @param bi batch index
+         */
+        virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
+
+        /**
+         * This method postprocess the output of the NN to obtain the correct 
+         * boundig boxes. 
+         * 
+         * @param bi batch index
+         * @param mAP set to true only if all the probabilities for a bounding 
+         *            box are needed, as in some cases for the mAP calculation
+         */
+        virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
+
+    public:
+        int classes = 0;
+        float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
+
+        std::vector<tk::dnn::box> detected; /*bounding boxes in output*/
+        std::vector<std::vector<tk::dnn::box>> batchDetected; /*bounding boxes in output*/
+        std::vector<double> stats; /*keeps track of inference times (ms)*/
+        std::vector<std::string> classesNames;
+
+        DetectionNN() {};
+        ~DetectionNN(){};
+
+        /**
+         * Method used to inialize the class, allocate memory and compute 
+         * needed data.
+         * 
+         * @param tensor_path path to the rt file og the NN.
+         * @param n_classes number of classes for the given dataset.
+         * @param n_batches maximum number of batches to use in inference
+         * @return true if everything is correct, false otherwise.
+         */
+        virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0;
+        
+        /**
+         * This method performs the whole detection of the NN.
+         * 
+         * @param frames frames to run detection on.
+         * @param cur_batches number of batches to use in inference
+         * @param save_times if set to true, preprocess, inference and postprocess times 
+         *        are saved on a csv file, otherwise not.
+         * @param times pointer to the output stream where to write times
+         * @param mAP set to true only if all the probabilities for a bounding 
+         *            box are needed, as in some cases for the mAP calculation
+         */
+        void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false, std::ofstream *times=nullptr, const bool mAP=false){
+            if(save_times && times==nullptr)
+                FatalError("save_times set to true, but no valid ofstream given");
+            if(cur_batches > nBatches)
+                FatalError("A batch size greater than nBatches cannot be used");
+
+            originalSize.clear();
+            if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30); 
+            {
+                TKDNN_TSTART
+                for(int bi=0; bi<cur_batches;++bi){
+                    if(!frames[bi].data)
+                        FatalError("No image data feed to detection");
+                    originalSize.push_back(frames[bi].size());
+                    preprocess(frames[bi], bi);    
+                }
+                TKDNN_TSTOP
+                if(save_times) *times<<t_ns<<";";
+            }
+
+            //do inference
+            tk::dnn::dataDim_t dim = netRT->input_dim;
+            dim.n = cur_batches;
+            {
+                if(TKDNN_VERBOSE) dim.print();
+                TKDNN_TSTART
+                netRT->infer(dim, input_d);
+                TKDNN_TSTOP
+                if(TKDNN_VERBOSE) dim.print();
+                stats.push_back(t_ns);
+                if(save_times) *times<<t_ns<<";";
+            }
+
+            batchDetected.clear();
+            {
+                TKDNN_TSTART
+                for(int bi=0; bi<cur_batches;++bi)
+                    postprocess(bi, mAP);
+                TKDNN_TSTOP
+                if(save_times) *times<<t_ns<<"\n";
+            }
+        }      
+
+        /**
+         * Method to draw boundixg boxes and labels on a frame.
+         * 
+         * @param frames orginal frame to draw bounding box on.
+         */
+        void draw(std::vector<cv::Mat>& frames) {
+            tk::dnn::box b;
+            int x0, w, x1, y0, h, y1;
+            int objClass;
+            std::string det_class;
+
+            int baseline = 0;
+            float font_scale = 0.5;
+            int thickness = 2;   
+
+            for(int bi=0; bi<frames.size(); ++bi){
+                // draw dets
+                for(int i=0; i<batchDetected[bi].size(); i++) { 
+                    b           = batchDetected[bi][i];
+                    x0   		= b.x;
+                    x1   		= b.x + b.w;
+                    y0   		= b.y;
+                    y1   		= b.y + b.h;
+                    det_class 	= classesNames[b.cl];
+
+                    // draw rectangle
+                    cv::rectangle(frames[bi], cv::Point(x0, y0), cv::Point(x1, y1), colors[b.cl], 2); 
+
+                    // draw label
+                    cv::Size text_size = getTextSize(det_class, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
+                    cv::rectangle(frames[bi], cv::Point(x0, y0), cv::Point((x0 + text_size.width - 2), (y0 - text_size.height - 2)), colors[b.cl], -1);                      
+                    cv::putText(frames[bi], det_class, cv::Point(x0, (y0 - (baseline / 2))), cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
+                }
+            }
+        }
+
+};
+
+}}
+
+#endif /* DETECTIONNN_H*/

include/tkDNN/ImuOdom.h

@@ -0,0 +1,161 @@
+#include <iostream>
+#include <signal.h>
+#include <stdlib.h>     /* srand, rand */
+#include <unistd.h>
+#include <mutex>
+#include <Eigen/Dense>
+#include "utils.h"
+#include "tkdnn.h"
+
+namespace tk { namespace dnn {
+
+/**
+ * 
+ * @author Francesco Gatti
+ */
+class ImuOdom {
+        
+    public:
+        tk::dnn::Network *net = nullptr;
+
+        // Network input dim
+        tk::dnn::dataDim_t dim0;
+        tk::dnn::dataDim_t dim1;
+        tk::dnn::dataDim_t dim2;
+
+        // Network output dim
+        tk::dnn::dataDim_t odim0;
+        tk::dnn::dataDim_t odim1;
+
+        // input pointers
+        dnnType *i0_d, *i1_d, *i2_d;
+        // output pointers
+        dnnType *o0_d, *o1_d;
+
+        // output eigen CPU
+        Eigen::MatrixXf deltaP, deltaQ;
+
+        Eigen::MatrixXd odomPOS, odomEULER;
+        Eigen::Matrix3d odomROT;
+        Eigen::Isometry3f tf = Eigen::Isometry3f::Identity();
+
+        ImuOdom() {}
+
+        virtual ~ImuOdom() {}
+
+        /**
+         * Method used for inizialize the class
+         * 
+         * @return Success of the initialization
+         */
+        bool init(std::string layers_path) {
+
+            dim0 = tk::dnn::dataDim_t(1, 4, 1, 100);
+            dim1 = tk::dnn::dataDim_t(1, 3, 1, 100);
+            dim2 = tk::dnn::dataDim_t(1, 3, 1, 100);
+
+            checkCuda( cudaMalloc(&i0_d, dim0.tot()*sizeof(dnnType)) );
+            checkCuda( cudaMalloc(&i1_d, dim1.tot()*sizeof(dnnType)) );
+            checkCuda( cudaMalloc(&i2_d, dim2.tot()*sizeof(dnnType)) );
+
+            std::string c0_bin = layers_path + "/conv1d_7.bin";
+            std::string c1_bin = layers_path + "/conv1d_8.bin";
+            std::string c2_bin = layers_path + "/conv1d_9.bin";
+            std::string c3_bin = layers_path + "/conv1d_10.bin";
+            std::string c4_bin = layers_path + "/conv1d_11.bin";
+            std::string c5_bin = layers_path + "/conv1d_12.bin";
+            std::string l0_bin = layers_path + "/bidirectional_3.bin";
+            std::string l1_bin = layers_path + "/bidirectional_4.bin";
+            std::string d0_bin = layers_path + "/dense_3.bin";
+            std::string d1_bin = layers_path + "/dense_4.bin";
+
+            net = new tk::dnn::Network(dim0);
+            tk::dnn::Input   *x0   = new tk::dnn::Input  (net, dim0, i0_d);
+            tk::dnn::Conv2d  *x0_0 = new tk::dnn::Conv2d (net, 128, 1, 11, 1, 1, 0, 0, c0_bin);
+            tk::dnn::Conv2d  *x0_1 = new tk::dnn::Conv2d (net, 128, 1, 11, 1, 1, 0, 0, c1_bin);
+            tk::dnn::Pooling *x0_2 = new tk::dnn::Pooling(net, 1, 3, 1, 3 ,0, 0, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX);
+
+            tk::dnn::Input   *x1   = new tk::dnn::Input  (net, dim1, i1_d);
+            tk::dnn::Conv2d  *x1_0 = new tk::dnn::Conv2d (net, 128, 1, 11, 1, 1, 0, 0, c2_bin);
+            tk::dnn::Conv2d  *x1_1 = new tk::dnn::Conv2d (net, 128, 1, 11, 1, 1, 0, 0, c3_bin);
+            tk::dnn::Pooling *x1_2 = new tk::dnn::Pooling(net, 1, 3, 1, 3, 0, 0, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX);
+
+            tk::dnn::Input   *x2   = new tk::dnn::Input  (net, dim2, i2_d);
+            tk::dnn::Conv2d  *x2_0 = new tk::dnn::Conv2d (net, 128, 1, 11, 1, 1, 0, 0, c4_bin);
+            tk::dnn::Conv2d  *x2_1 = new tk::dnn::Conv2d (net, 128, 1, 11, 1, 1, 0, 0, c5_bin);
+            tk::dnn::Pooling *x2_2 = new tk::dnn::Pooling(net, 1, 3, 1, 3, 0, 0, tk::dnn::tkdnnPoolingMode_t::POOLING_MAX);
+
+            tk::dnn::Layer *concat_l[3] = { x0_2, x1_2, x2_2 };
+            tk::dnn::Route *concat = new tk::dnn::Route(net, concat_l, 3);
+
+            tk::dnn::LSTM *lstm0 = new tk::dnn::LSTM(net, 128, true, l0_bin);
+            tk::dnn::LSTM *lstm1 = new tk::dnn::LSTM(net, 128, false, l1_bin);
+
+            tk::dnn::Dense *d0 = new tk::dnn::Dense(net, 3, d0_bin);
+
+            tk::dnn::Layer *lstm1_l[1] = { lstm1 };
+            tk::dnn::Route *lstm1_link = new tk::dnn::Route(net, lstm1_l, 1);
+            tk::dnn::Dense *d1 = new tk::dnn::Dense(net, 4, d1_bin);
+            
+            net->print();
+
+            // output data
+            o0_d = d0->dstData;
+            o1_d = d1->dstData;
+            odim0 = d0->output_dim;
+            odim1 = d1->output_dim;
+
+            deltaP.resize(odim0.tot(), 1);
+            deltaQ.resize(odim1.tot(), 1);
+
+            odomPOS = Eigen::MatrixXd::Zero(3, 1); 
+            odomROT = Eigen::MatrixXd::Identity(3, 3);
+            odomEULER = Eigen::MatrixXd::Zero(3, 1); 
+            return true;
+        }
+
+        void close() {
+            // TODO: dealloc :)
+        }
+
+        void update(dnnType *x0, dnnType *x1, dnnType *x2) {
+
+            checkCuda( cudaMemcpy(i0_d, x0, dim0.tot()*sizeof(dnnType), cudaMemcpyHostToDevice) );
+            checkCuda( cudaMemcpy(i1_d, x1, dim1.tot()*sizeof(dnnType), cudaMemcpyHostToDevice) );
+            checkCuda( cudaMemcpy(i2_d, x2, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice) );
+
+            // Inference
+            tk::dnn::dataDim_t dim;
+            net->infer(dim, nullptr);
+
+            checkCuda( cudaMemcpy(deltaP.data(), o0_d, odim0.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) );
+            checkCuda( cudaMemcpy(deltaQ.data(), o1_d, odim1.tot()*sizeof(dnnType), cudaMemcpyDeviceToHost) ); 
+
+            // compute odom
+            Eigen::Quaterniond q;
+            q.w() = deltaQ(0);
+            q.x() = deltaQ(1);
+            q.y() = deltaQ(2);
+            q.z() = deltaQ(3);    
+            odomPOS = odomPOS + odomROT*deltaP.cast<double>(); // V1
+            //odomPOS = odomPOS + deltaP.cast<double>(); // V2
+            odomROT = odomROT * q.normalized().toRotationMatrix();
+            
+            // compute euler
+            auto newEULER = odomROT.eulerAngles(0, 1, 2);
+            for(int i=0; i<3; i++) {
+                while( fabs(newEULER(i) - odomEULER(i)) > M_PI_2 )  {
+                    newEULER(i) += newEULER(i) - odomEULER(i) > 0 ? -M_PI : +M_PI;
+                    //std::cout<<newEULER(i)<<" "<<odomEULER(i)<<"\n";
+                }
+            }
+            odomEULER = newEULER;
+
+            // compose tf
+            tf.matrix().block(0, 0, 3, 3) = odomROT.cast<float>();
+            tf.matrix().block(0, 3, 3, 1) = odomPOS.cast<float>();
+        }
+
+};
+
+}}

include/tkDNN/Int8BatchStream.h

@@ -0,0 +1,69 @@
+#ifndef INT8BATCHSTREAM_H
+#define INT8BATCHSTREAM_H
+
+#include <vector>
+#include <assert.h>
+#include <algorithm>
+#include <iterator>
+#include <stdint.h>
+#include <iostream>
+#include <string>
+#include <fstream>
+#include <iomanip>
+#include <signal.h>
+#include <stdlib.h>    
+#include <unistd.h>
+#include <mutex>
+
+#include "NvInfer.h"
+#include "utils.h"
+#include "tkdnn.h"
+
+/*
+ * BatchStream implements the stream for the INT8 calibrator. 
+ * It reads the two files .txt with the list of image file names 
+ * and the list of label file names. 
+ * It then iterates on images and labels.
+ */
+class BatchStream {
+public:
+	BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches, const std::string& fileimglist, const std::string& filelabellist);
+	virtual ~BatchStream() { }
+	void reset(int firstBatch);
+	bool next();
+	void skip(int skipCount);
+	float *getBatch() { return mBatch.data(); }
+	float *getLabels() { return mLabels.data(); }
+	int getBatchesRead() const { return mBatchCount; }
+	int getBatchSize() const { return mBatchSize; }
+	nvinfer1::DimsNCHW getDims() const { return mDims; }
+	float* getFileBatch() { return &mFileBatch[0]; }
+	float* getFileLabels() { return &mFileLabels[0]; }
+	void readInListFile(const std::string& dataFilePath, std::vector<std::string>& mListIn);
+	void readCVimage(std::string inputFileName, std::vector<float>& res, bool fixshape = true);
+	void readLabels(std::string inputFileName ,std::vector<float>& ris);
+	bool update();
+
+private:
+	int mBatchSize{ 0 };
+	int mMaxBatches{ 0 };
+	int mBatchCount{ 0 };
+	int mFileCount{ 0 };
+	int mFileBatchPos{ 0 };
+	int mImageSize{ 0 };
+
+	nvinfer1::DimsNCHW mDims;
+	std::vector<float> mBatch;
+	std::vector<float> mLabels;
+	std::vector<float> mFileBatch;
+	std::vector<float> mFileLabels;
+
+	int mHeight;
+	int mWidth;
+	std::string mFileImgList;
+	std::vector<std::string> mListImg;
+	std::string mFileLabelList;
+	std::vector<std::string> mListLabel;
+};
+
+#endif //INT8BATCHSTREAM

include/tkDNN/Int8Calibrator.h

@@ -0,0 +1,49 @@
+#ifndef INT8CALIBRATOR_H
+#define INT8CALIBRATOR_H
+
+#include <vector>
+#include <assert.h>
+#include <algorithm>
+#include <iterator>
+#include <stdint.h>
+#include <iostream>
+#include <string>
+#include "NvInfer.h"
+
+#include <fstream>
+#include <iomanip>
+
+#include "Int8BatchStream.h"
+
+#include "tkdnn.h"
+#include "utils.h"
+
+/*
+ * Int8EntropyCalibrator implements the INT8 calibrator to achieve the
+ * INT8 quantization. It uses a BatchStream stream to scroll through 
+ * images data. It also implements the calibration cache, a way to 
+ * save the calibration process results to reduce the running time: 
+ * the calibration process takes a long time.
+ */
+class Int8EntropyCalibrator : public nvinfer1::IInt8EntropyCalibrator {
+public:
+	Int8EntropyCalibrator(BatchStream& stream, int firstBatch, const std::string& calibTableFilePath, 
+							const std::string& inputBlobName, bool readCache = true);
+	virtual ~Int8EntropyCalibrator() { checkCuda(cudaFree(mDeviceInput)); }
+	int getBatchSize() const override { return mStream.getBatchSize(); }
+	bool getBatch(void* bindings[], const char* names[], int nbBindings) override;
+	const void* readCalibrationCache(size_t& length) override;
+	void writeCalibrationCache(const void* cache, size_t length) override;
+
+private:
+	BatchStream mStream;
+	const std::string mCalibTableFilePath{ nullptr };
+	const std::string mInputBlobName;
+	bool mReadCache{ true };
+
+	size_t mInputCount;
+	void* mDeviceInput{ nullptr };
+	std::vector<char> mCalibrationCache;
+};
+
+#endif //INT8CALIBRATOR_H

include/tkDNN/Layer.h

@@ -12,9 +12,15 @@ enum layerType_t {
     LAYER_INPUT,
     LAYER_DENSE,
     LAYER_CONV2D,
+    LAYER_DECONV2D,
+    LAYER_DEFORMCONV2D,
     LAYER_LSTM,
     LAYER_ACTIVATION,
+    LAYER_ACTIVATION_CRELU,
+    LAYER_ACTIVATION_LEAKY,
+    LAYER_ACTIVATION_MISH,
     LAYER_FLATTEN,
+    LAYER_RESHAPE,
     LAYER_MULADD,
     LAYER_POOLING,
     LAYER_SOFTMAX,
@@ -42,29 +48,38 @@ public:
         std::cout<<"No infer action for this layer\n";
         return NULL;
     }
-
+    void setFinal() { this->final = true; }
     dataDim_t input_dim, output_dim;
-    dnnType *dstData;  //where results will be putted
+    dnnType *dstData = nullptr;  //where results will be putted
+
+    int id = 0;
+    bool final;        //if the layer is the final one
 
     std::string getLayerName() {
         layerType_t type = getLayerType();
         switch(type) {
-            case LAYER_INPUT:       return "Input";
-            case LAYER_DENSE:       return "Dense";
-            case LAYER_CONV2D:      return "Conv2d";
-            case LAYER_LSTM:        return "LSTM";
-            case LAYER_ACTIVATION:  return "Activation";
-            case LAYER_FLATTEN:     return "Flatten";
-            case LAYER_MULADD:      return "MulAdd";
-            case LAYER_POOLING:     return "Pooling";
-            case LAYER_SOFTMAX:     return "Softmax";
-            case LAYER_ROUTE:       return "Route";            
-            case LAYER_REORG:       return "Reorg";
-            case LAYER_SHORTCUT:    return "Shortcut";
-            case LAYER_UPSAMPLE:    return "Upsample";
-            case LAYER_REGION:      return "Region";
-            case LAYER_YOLO:        return "Yolo";
-            default:                return "unknown";
+            case LAYER_INPUT:               return "Input";
+            case LAYER_DENSE:               return "Dense";
+            case LAYER_CONV2D:              return "Conv2d";
+            case LAYER_DECONV2D:            return "DeConv2d";
+            case LAYER_DEFORMCONV2D:        return "DeformConv2d";
+            case LAYER_LSTM:                return "LSTM";
+            case LAYER_ACTIVATION:          return "Activation";
+            case LAYER_ACTIVATION_CRELU:    return "ActivationCReLU";
+            case LAYER_ACTIVATION_LEAKY:    return "ActivationLeaky";
+            case LAYER_ACTIVATION_MISH:     return "ActivationMish";
+            case LAYER_FLATTEN:             return "Flatten";
+            case LAYER_RESHAPE:             return "Reshape";
+            case LAYER_MULADD:              return "MulAdd";
+            case LAYER_POOLING:             return "Pooling";
+            case LAYER_SOFTMAX:             return "Softmax";
+            case LAYER_ROUTE:               return "Route";            
+            case LAYER_REORG:               return "Reorg";
+            case LAYER_SHORTCUT:            return "Shortcut";
+            case LAYER_UPSAMPLE:            return "Upsample";
+            case LAYER_REGION:              return "Region";
+            case LAYER_YOLO:                return "Yolo";
+            default:                        return "unknown";
         }
     }
 
@@ -81,8 +96,8 @@ protected:
 class LayerWgs : public Layer {
 
 public:
-    LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt, 
-             std::string fname_weights, bool batchnorm = false); 
+    LayerWgs(Network *net, int inputs, int outputs, int kh, int kw, int kt,
+             std::string fname_weights, bool batchnorm = false, bool additional_bias = false, bool deConv = false, int groups = 1); 
     virtual ~LayerWgs();
 
     int inputs, outputs;
@@ -91,21 +106,67 @@ public:
     dnnType *data_h, *data_d;
     dnnType *bias_h, *bias_d;
 
+    // additional bias for DCN
+    bool additional_bias;
+    dnnType *bias2_h = nullptr, *bias2_d = nullptr;
+
     //batchnorm
     bool batchnorm;
-    dnnType *power_h;
-    dnnType *scales_h,   *scales_d;
-    dnnType *mean_h,     *mean_d;
-    dnnType *variance_h, *variance_d;
+    dnnType *power_h    = nullptr;
+    dnnType *scales_h   = nullptr,   *scales_d = nullptr;
+    dnnType *mean_h     = nullptr,     *mean_d = nullptr;
+    dnnType *variance_h = nullptr, *variance_d = nullptr;
 
     //fp16
-    __half *data16_h, *bias16_h;
-    __half *data16_d, *bias16_d;
+    __half *data16_h  = nullptr, *bias16_h  = nullptr;
+    __half *data16_d  = nullptr, *bias16_d  = nullptr;
+    __half *bias216_h = nullptr, *bias216_d = nullptr;
 
-    __half *power16_h,    *power16_d;
-    __half *scales16_h,   *scales16_d;
-    __half *mean16_h,     *mean16_d;
-    __half *variance16_h, *variance16_d;
+    __half *power16_h    = nullptr,    *power16_d = nullptr;
+    __half *scales16_h   = nullptr,   *scales16_d = nullptr;
+    __half *mean16_h     = nullptr,     *mean16_d = nullptr;
+    __half *variance16_h = nullptr, *variance16_d = nullptr;
+
+    void releaseHost(bool release32 = true, bool release16 = true) {
+        if(release32) {
+            if(    data_h != nullptr) { delete []     data_h;     data_h = nullptr; }
+            if(    bias_h != nullptr) { delete []     bias_h;     bias_h = nullptr; }
+            if(   bias2_h != nullptr) { delete []    bias2_h;    bias2_h = nullptr; }
+            if(  scales_h != nullptr) { delete []   scales_h;   scales_h = nullptr; }
+            if(    mean_h != nullptr) { delete []     mean_h;     mean_h = nullptr; }
+            if(variance_h != nullptr) { delete [] variance_h; variance_h = nullptr; }
+            if(   power_h != nullptr) { delete []    power_h;    power_h = nullptr; }
+        }
+        if(net->fp16 && release16) {
+            if(    data16_h != nullptr) { delete []     data16_h;     data16_h = nullptr; }
+            if(    bias16_h != nullptr) { delete []     bias16_h;     bias16_h = nullptr; }
+            if(   bias216_h != nullptr) { delete []    bias216_h;    bias216_h = nullptr; }
+            if(  scales16_h != nullptr) { delete []   scales16_h;   scales16_h = nullptr; }
+            if(    mean16_h != nullptr) { delete []     mean16_h;     mean16_h = nullptr; }
+            if(variance16_h != nullptr) { delete [] variance16_h; variance16_h = nullptr; } 
+            if(   power16_h != nullptr) { delete []    power16_h;    power16_h = nullptr; }
+
+        }
+    }
+    void releaseDevice(bool release32 = true, bool release16 = true) {
+        if(release32) {
+            if(    data_d != nullptr) { cudaFree(    data_d);     data_d = nullptr; }
+            if(    bias_d != nullptr) { cudaFree(    bias_d);     bias_d = nullptr; }
+            if(   bias2_d != nullptr) { cudaFree(   bias2_d);    bias2_d = nullptr; }
+            if(  scales_d != nullptr) { cudaFree(  scales_d);   scales_d = nullptr; }
+            if(    mean_d != nullptr) { cudaFree(    mean_d);     mean_d = nullptr; }
+            if(variance_d != nullptr) { cudaFree(variance_d); variance_d = nullptr; }
+        }
+        if(net->fp16 && release16) {
+            if(    data16_d != nullptr) { cudaFree(    data16_d);     data16_d = nullptr; }
+            if(    bias16_d != nullptr) { cudaFree(    bias16_d);     bias16_d = nullptr; }
+            if(   bias216_d != nullptr) { cudaFree(   bias216_d);    bias216_d = nullptr; }
+            if(  scales16_d != nullptr) { cudaFree(  scales16_d);   scales16_d = nullptr; }
+            if(    mean16_d != nullptr) { cudaFree(    mean16_d);     mean16_d = nullptr; }
+            if(variance16_d != nullptr) { cudaFree(variance16_d); variance16_d = nullptr; } 
+            if(   power16_d != nullptr) { cudaFree(   power16_d);    power16_d = nullptr; }
+        }
+    }
 };
 
 
@@ -125,6 +186,7 @@ public:
     virtual layerType_t getLayerType() { return LAYER_INPUT; };
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData) {
+        dim = output_dim;
         return dstData;
     }
 };
@@ -149,7 +211,8 @@ public:
 */
 typedef enum {
     ACTIVATION_ELU     = 100,
-    ACTIVATION_LEAKY   = 101
+    ACTIVATION_LEAKY   = 101,
+    ACTIVATION_MISH   = 102
 } tkdnnActivationMode_t;
 
 /**
@@ -159,10 +222,20 @@ class Activation : public Layer {
 
 public:
     int act_mode;
+    float ceiling;
 
-    Activation(Network *net, int act_mode); 
+    Activation(Network *net, int act_mode, const float ceiling=0.0); 
     virtual ~Activation();
-    virtual layerType_t getLayerType() { return LAYER_ACTIVATION; };
+    virtual layerType_t getLayerType() { 
+        if(act_mode == CUDNN_ACTIVATION_CLIPPED_RELU)
+            return LAYER_ACTIVATION_CRELU;
+        else if (act_mode == ACTIVATION_LEAKY)
+            return LAYER_ACTIVATION_LEAKY;
+        else if (act_mode == ACTIVATION_MISH)
+            return LAYER_ACTIVATION_MISH;
+        else
+            return LAYER_ACTIVATION;
+         };
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 
@@ -187,20 +260,25 @@ class Conv2d : public LayerWgs {
 public:
     Conv2d( Network *net, int out_ch, int kernelH, int kernelW, 
                 int strideH, int strideW, int paddingH, int paddingW,
-                std::string fname_weights, bool batchnorm = false); 
+                std::string fname_weights, bool batchnorm = false, bool deConv = false, int groups = 1, bool additional_bias=false);
     virtual ~Conv2d();
     virtual layerType_t getLayerType() { return LAYER_CONV2D; };
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 
     int kernelH, kernelW, strideH, strideW, paddingH, paddingW;
+    bool deConv, additional_bias;
+    int groups;
 
 protected:
     cudnnFilterDescriptor_t filterDesc;
     cudnnConvolutionDescriptor_t convDesc;
-    cudnnConvolutionFwdAlgo_t algo;
+    cudnnConvolutionFwdAlgo_t     algo;
+    cudnnConvolutionBwdDataAlgo_t bwAlgo;
     cudnnTensorDescriptor_t biasTensorDesc;
 
+    void initCUDNN(bool back = false);
+    void inferCUDNN(dnnType* srcData, bool back = false);
     void*  workSpace;
     size_t ws_sizeInBytes;
 };
@@ -271,6 +349,56 @@ protected:
 };
 
 
+/**
+    Convolutional 2D layer
+*/
+class DeConv2d : public Conv2d {
+
+public:
+    DeConv2d( Network *net, int out_ch, int kernelH, int kernelW,
+            int strideH, int strideW, int paddingH, int paddingW,
+            std::string fname_weights, bool batchnorm = false, int groups = 1) :
+            Conv2d(net, out_ch, kernelH, kernelW, strideH, strideW, paddingH, paddingW, fname_weights, batchnorm, true, groups) {}
+    virtual ~DeConv2d() {}
+    virtual layerType_t getLayerType() { return LAYER_DECONV2D; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+};
+
+
+/**
+    Deformable Convolutionl 2d layer
+*/  
+class DeformConv2d : public LayerWgs {
+
+public:
+    DeformConv2d( Network *net, int out_ch, int deformable_group, int kernelH, int kernelW,
+                int strideH, int strideW, int paddingH, int paddingW,
+                std::string d_fname_weights, std::string fname_weights, bool batchnorm);
+    virtual ~DeformConv2d();
+    virtual layerType_t getLayerType() { return LAYER_DEFORMCONV2D; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    tk::dnn::Conv2d *preconv; 
+    int out_ch;
+    int deformableGroup;
+    int kernelH, kernelW, strideH, strideW, paddingH, paddingW;
+    dnnType *ones_d1;
+    dnnType *ones_d2;
+    int chunk_dim;
+    dnnType *offset, *mask;
+    dnnType *output_conv;
+
+    cublasStatus_t stat;
+    cublasHandle_t handle;
+
+protected:
+
+    cudnnTensorDescriptor_t biasTensorDesc;
+    void initCUDNN();
+
+};
+
 /**
     Flatten layer
     is actually a matrix transposition
@@ -285,6 +413,20 @@ public:
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 };
 
+/**
+    Reshape layer
+*/
+class Reshape : public Layer {
+
+public:
+    Reshape(Network *net, dataDim_t new_dim); 
+    virtual ~Reshape();
+    virtual layerType_t getLayerType() { return LAYER_RESHAPE; };
+
+    virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+
+};
+
 
 /**
     MulAdd layer
@@ -311,8 +453,9 @@ protected:
 */
 typedef enum {
     POOLING_MAX     = 0,
-    POOLING_AVERAGE = 1,                  // count for average includes padded values
-    POOLING_AVERAGE_EXCLUDE_PADDING = 2   // count for average does not include padded values
+    POOLING_AVERAGE = 1,                    // count for average includes padded values
+    POOLING_AVERAGE_EXCLUDE_PADDING = 2,    // count for average does not include padded values
+    POOLING_MAX_FIXEDSIZE = 100             // max pool darknet fashion
 } tkdnnPoolingMode_t;
 
 /**
@@ -325,9 +468,13 @@ public:
     int winH, winW;
     int strideH, strideW;
     int paddingH, paddingW;
+    bool size;
+    tkdnnPoolingMode_t pool_mode;
 
     Pooling(Network *net, int winH, int winW, 
-            int strideH, int strideW, tkdnnPoolingMode_t pool_mode); 
+            int strideH, int strideW, 
+            int paddingH, int paddingW,
+            tkdnnPoolingMode_t pool_mode);
     virtual ~Pooling();
     virtual layerType_t getLayerType() { return LAYER_POOLING; };
 
@@ -336,7 +483,6 @@ public:
 protected:
 
     cudnnPoolingDescriptor_t poolingDesc;
-    tkdnnPoolingMode_t pool_mode;
     dnnType *tmpInputData, *tmpOutputData;
     bool poolOn3d;
 };
@@ -347,11 +493,13 @@ protected:
 class Softmax : public Layer {
 
 public:
-    Softmax(Network *net); 
+    Softmax(Network *net, const tk::dnn::dataDim_t* dim=nullptr, const cudnnSoftmaxMode_t mode=CUDNN_SOFTMAX_MODE_CHANNEL); 
     virtual ~Softmax();
     virtual layerType_t getLayerType() { return LAYER_SOFTMAX; };
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
+    dataDim_t dim;
+    cudnnSoftmaxMode_t mode;
 };
 
 /**
@@ -368,7 +516,8 @@ public:
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
 
 public:
-    Layer **layers;  //ids of layers to be merged
+    static const int MAX_LAYERS = 32;
+    Layer *layers[MAX_LAYERS];  //ids of layers to be merged
     int layers_n; //number of layers
 };
 
@@ -427,6 +576,12 @@ struct box {
     int cl;
     float x, y, w, h;
     float prob;
+    std::vector<float> probs;
+
+    void print() 
+    {
+        std::cout<<"x: "<<x<<"\ty: "<<y<<"\tw: "<<w<<"\th: "<<h<<"\tcl: "<<cl<<"\tprob: "<<prob<<std::endl;
+    }
 };
 struct sortable_bbox {
     int index;
@@ -453,13 +608,14 @@ public:
         int sort_class;
     };
 
-    Yolo(Network *net, int classes, int num, std::string fname_weights);
+    Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1);
     virtual ~Yolo();
     virtual layerType_t getLayerType() { return LAYER_YOLO; };
 
-    int classes, num;
+    int classes, num, n_masks;
     dnnType *mask_h, *mask_d; //anchors
     dnnType *bias_h, *bias_d; //anchors
+    float scaleXY;
     std::vector<std::string> classesNames;
 
     virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
@@ -467,7 +623,7 @@ public:
 
     dnnType *predictions;
 
-    static const int MAX_DETECTIONS = 256;
+    static const int MAX_DETECTIONS = 8192;
     static Yolo::detection *allocateDetections(int nboxes, int classes);
     static void             mergeDetections(Yolo::detection *dets, int ndets, int classes);
 };

include/tkDNN/MobilenetDetection.h

@@ -0,0 +1,77 @@
+#ifndef MOBILENETDETECTION_H
+#define MOBILENETDETECTION_H
+
+#include <opencv2/videoio.hpp>
+#include "opencv2/opencv.hpp"
+
+#include "DetectionNN.h"
+
+#define N_COORDS 4
+#define N_SSDSPEC 6
+
+namespace tk { namespace dnn { 
+
+struct SSDSpec
+{
+    int featureSize = 0;
+    int shrinkage = 0;
+    int boxWidth = 0;
+    int boxHeight = 0;
+    int ratio1 = 0;
+    int ratio2 = 0;
+
+    SSDSpec() {}
+    SSDSpec(int feature_size, int shrinkage, int box_width, int box_height, int ratio1, int ratio2) : 
+                    featureSize(feature_size), shrinkage(shrinkage), boxWidth(box_width), 
+                    boxHeight(box_height), ratio1(ratio1), ratio2(ratio2) {}
+    void setAll(int feature_size, int shrinkage, int box_width, int box_height, int ratio1, int ratio2)
+    {
+        this->featureSize = feature_size;
+        this->shrinkage = shrinkage;
+        this->boxWidth = box_width;
+        this->boxHeight = box_height;
+        this->ratio1 = ratio1;
+        this->ratio2 = ratio2;
+    }
+    void print()
+    {
+        std::cout << "fsize: " << featureSize << "\tshrinkage: " << shrinkage << 
+                    "\t box W:" << boxWidth << "\tbox H: " << boxHeight << 
+                    "\t x ratio:" << ratio1 << "\t y ratio:" << ratio2 << std::endl;
+    }
+};
+
+class MobilenetDetection : public DetectionNN
+{
+private:
+    float IoUThreshold = 0.45;
+    float centerVariance = 0.1;
+    float sizeVariance = 0.2;
+    int imageSize;
+
+    float *priors = nullptr;
+    int nPriors = 0;
+    float *locations_h, *confidences_h;
+
+    
+
+    void generate_ssd_priors(const SSDSpec *specs, const int n_specs, bool clamp = true);
+    void convert_locatios_to_boxes_and_center();
+    float iou(const tk::dnn::box &a, const tk::dnn::box &b);
+
+    
+
+public:
+    MobilenetDetection() {};
+    ~MobilenetDetection() {}; 
+
+    bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1);
+    void preprocess(cv::Mat &frame, const int bi=0);
+    void postprocess(const int bi=0,const bool mAP=false);
+};
+
+
+} // namespace dnn
+} // namespace tk
+
+#endif /*MOBILENETDETECTION_H*/

include/tkDNN/Network.h

@@ -1,6 +1,7 @@
 #ifndef NETWORK_H
 #define NETWORK_H
 
+#include <string>
 #include "utils.h"
 
 namespace tk { namespace dnn {
@@ -32,13 +33,14 @@ struct dataDim_t {
 };
 
 class Layer;
-const int MAX_LAYERS = 256;
+const int MAX_LAYERS = 512;
 
 class Network {
 
 public:
     Network(dataDim_t input_dim);
     virtual ~Network();
+    void releaseLayers();
 
     /**
         Do inferece for every added layer
@@ -47,6 +49,7 @@ public:
 
     bool addLayer(Layer *l);
     void print();
+    const char *getNetworkRTName(const char *network_name);
 
     cudnnDataType_t dataType;
     cudnnTensorFormat_t tensorFormat;
@@ -59,7 +62,14 @@ public:
     dataDim_t input_dim;
     dataDim_t getOutputDim();
 
-    bool fp16, dla;
+    bool fp16, dla, int8;
+    int maxBatchSize;
+    bool dontLoadWeights;
+    std::string fileImgList;
+    std::string fileLabelList;
+    std::string networkName;
+    std::string networkNameRT;
+    
 };
 
 }}

include/tkDNN/NetworkRT.h

@@ -24,13 +24,19 @@ template<typename T> T readBUF(const char*& buffer)
 
 using namespace nvinfer1;
 #include "pluginsRT/ActivationLeakyRT.h"
+#include "pluginsRT/ActivationReLUCeilingRT.h"
+#include "pluginsRT/ActivationMishRT.h"
 #include "pluginsRT/ReorgRT.h"
 #include "pluginsRT/RegionRT.h"
 //#include "pluginsRT/RouteRT.h"
 #include "pluginsRT/ShortcutRT.h"
 #include "pluginsRT/YoloRT.h"
 #include "pluginsRT/UpsampleRT.h"
-//#include "pluginsRT/Int8Calibrator.h"
+#include "pluginsRT/ResizeLayerRT.h"
+#include "pluginsRT/DeformableConvRT.h"
+#include "pluginsRT/FlattenConcatRT.h"
+#include "pluginsRT/ReshapeRT.h"
+#include "pluginsRT/MaxPoolingFixedSizeRT.h"
 
 class PluginFactory : IPluginFactory
 {
@@ -50,12 +56,15 @@ public:
     nvinfer1::IBuilder *builderRT;
     nvinfer1::IRuntime *runtimeRT;
     nvinfer1::INetworkDefinition *networkRT; 
-    
+#if NV_TENSORRT_MAJOR >= 6  
+    nvinfer1::IBuilderConfig *configRT;
+#endif
     nvinfer1::ICudaEngine *engineRT;
     nvinfer1::IExecutionContext *contextRT;
 
     const static int MAX_BUFFERS_RT = 10;
     void* buffersRT[MAX_BUFFERS_RT];
+    dataDim_t buffersDIM[MAX_BUFFERS_RT];
     int buf_input_idx, buf_output_idx;
 
     dataDim_t input_dim, output_dim;
@@ -67,11 +76,25 @@ public:
     NetworkRT(Network *net, const char *name);
     virtual ~NetworkRT();
 
+    int getMaxBatchSize() {
+        if(engineRT != nullptr)
+            return engineRT->getMaxBatchSize();
+        else
+            return 0;
+    }
+
+    int getBuffersN() {
+        if(engineRT != nullptr)
+            return engineRT->getNbBindings();
+        else 
+            return 0;
+    }
+
     /**
         Do inferece
     */
     dnnType* infer(dataDim_t &dim, dnnType* data);
-    void enqueue();    
+    void enqueue(int batchSize = 1);    
 
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Layer *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Conv2d *l);
@@ -80,11 +103,14 @@ public:
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Pooling *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Softmax *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Yolo *l);
     nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Upsample *l);
+    nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, DeformConv2d *l);
 
     bool serialize(const char *filename);
     bool deserialize(const char *filename);

include/tkDNN/Yolo3Detection.h

@@ -1,67 +1,36 @@
-#include <iostream>
-#include <signal.h>
-#include <stdlib.h>     /* srand, rand */
-#include <unistd.h>
-#include <mutex>
-#include "utils.h"
+#ifndef Yolo3Detection_H
+#define Yolo3Detection_H
+#include <opencv2/videoio.hpp>
+#include "opencv2/opencv.hpp"
 
-#include <opencv2/core/core.hpp>
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
+#include "DetectionNN.h"
 
-#include "tkdnn.h"
+namespace tk { namespace dnn { 
 
-namespace tk { namespace dnn {
+class Yolo3Detection : public DetectionNN
+{
+private:
+    int num = 0;
+    int nMasks = 0;
+    int nDets = 0;
+    tk::dnn::Yolo::detection *dets = nullptr;
+    tk::dnn::Yolo* yolo[3];
 
-/**
- * 
- * @author Francesco Gatti
- */
-class Yolo3Detection {
+    tk::dnn::Yolo* getYoloLayer(int n=0);
 
-    private:
-        tk::dnn::NetworkRT *netRT = nullptr;
-        tk::dnn::Yolo* yolo[3];
-        dnnType *input, *input_d;
-
-        int ndets = 0;
-        tk::dnn::Yolo::detection *dets = nullptr;
-
-        cv::Mat imageF;
-        cv::Mat bgr[3]; 
-
-    public:
-        int classes = 0;
-        int num = 0;
-        float thresh = 0.3;
-        cv::Scalar colors[256];
-
-        // this is filled with results
-        std::vector<tk::dnn::box> detected;
-
-        // keep track of inference times (ms)
-        std::vector<double> stats;
-
-        Yolo3Detection() {}
-
-        virtual ~Yolo3Detection() {}
-
-        /**
-         * Method used for inizialize the class
-         * 
-         * @return Success of the initialization
-         */
-        bool init(std::string tensor_path);
-
-        void update(cv::Mat &frame);
-
-        tk::dnn::Yolo* getYoloLayer(int n=0) {
-            if(n<3)
-                return yolo[n];
-            else 
-                return nullptr;
-        }
+    cv::Mat bgr_h;
+    
+public:
+    Yolo3Detection() {};
+    ~Yolo3Detection() {}; 
 
+    bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
+    void preprocess(cv::Mat &frame, const int bi=0);
+    void postprocess(const int bi=0,const bool mAP=false);
 };
 
-}}
+
+} // namespace dnn
+} // namespace tk
+
+#endif /* Yolo3Detection_H*/

include/tkDNN/evaluation.h

@@ -0,0 +1,116 @@
+#ifndef EVALUATION_H
+#define EVALUATION_H
+
+#include <iostream>
+#include <vector>
+#include <algorithm>
+
+#include <yaml-cpp/yaml.h>
+
+#include "tkdnn.h"
+#include "BoundingBox.h"
+
+namespace tk { namespace dnn {
+
+struct Frame
+{
+    std::string lFilename;
+    std::string iFilename;
+    std::vector<BoundingBox> gt;
+    std::vector<BoundingBox> det;
+
+    void print() const;
+};
+
+struct PR 
+{
+    double precision = 0;
+    double recall = 0;
+    int tp = 0, fp = 0, fn = 0;
+
+    void print();
+};
+
+void readmAPParams( const char* config_filename, int& classes, int& map_points, 
+                    int& map_levels, float& map_step, float& IoU_thresh, 
+                    float& conf_thresh, bool& verbose);
+
+/**
+ * This method computes the mean Average Precision for a set of detections and 
+ * groundtruths. It returns the mAP for a given IoU threshold, and a given 
+ * confidence threshold over all the classes.
+ *
+ * @param images collection of frames on which to compute the metrics
+ * @param classes number of classes of the considered dataset
+ * @param IoU_thresh threshold used to compute Intersection over Union
+ * @param conf_thresh threshold used to filter bounding boxes based on their 
+ *                    confidence (or probability)
+ * @param map_points number of point used to compute the mAP. if 0 is given, 
+ *                  all the recall levels are evaluated, otherwise only 
+ *                  map_point recall levels are used. For COCO evaluation 
+ *                  101 points are used.
+ * @param verbose is set to true, prints on screen additional info
+ *
+ * @return mAP computed
+ */
+double computeMap(  std::vector<Frame> &images,const int classes,
+                    const float IoU_thresh, const float conf_thresh=0.3, 
+                    const int map_points=101, const bool verbose=false);
+
+
+/**
+ * This method computes the mean Average Precision for a set of detections and 
+ * groundtruths on several IoU thresholds. It is used to compute, for example, 
+ * the most used metric in Object Detection, namely the mAP 0.5:0.95, which is 
+ * the average among the mAP for IoU level from 0.5 to 0.95 with a step of 0.05.
+ *
+ * @param images collection of frames on which to compute the metrics
+ * @param classes number of classes of the considered dataset
+ * @param IoU_thresh starting threshold used to compute Intersection over Union
+ * @param conf_thresh threshold used to filter bounding boxes based on their 
+ *                    confidence (or probability)
+ * @param map_points number of point used to compute the mAP. if 0 is given, 
+ *                  all the recall levels are evaluated, otherwise only 
+ *                  map_point recall levels are used. For COCO evaluation 
+ *                  101 points are used.
+ * @param map_step step used to increment IoU theshold
+ * @param map_levels number of IoU step to perform
+ * @param verbose is set to true, prints on screen additional info
+ * @param write_on_file if set to true, the results produced by this function 
+ *                      are written on file
+ * @param net name of the considerd neural network 
+ *
+ * @return  mAP IoU_tresh:IoU_tresh+map_step*map_levels (e.g. mAP 0.5:0.95 when 
+ *          map_step=0.05 and map_levels=10)
+ */
+double computeMapNIoULevels(std::vector<Frame> &images,const int classes,
+                const float i_IoU_thresh=0.5, const float conf_thresh=0.3, 
+                const int map_points=101, const float map_step=0.05, 
+                const int map_levels=10, const bool verbose=false, 
+                const bool write_on_file = false, std::string net = "");
+/**
+ * This method computes the numper of True Positive (TP), False Positive (FP),
+ * False Negative (FN), precision, recall and f1-score.
+ * Those values are computer over all the detections, over all the classes.
+ *
+ * @param images collection of frames on which to compute the metrics
+ * @param classes number of classes of the considered dataset
+ * @param IoU_thresh threshold used to compute Intersection over Union
+ * @param conf_thresh threshold used to filter bounding boxes based on their 
+ *                    confidence (or probability)
+ * @param verbose is set to true, prints on screen additional info
+ * @param write_on_file if set to true, the results produced by this function 
+ *                      are written on file
+ * @param net name of the considerd neural network 
+ */
+void computeTPFPFN( std::vector<Frame> &images,const int classes,
+                    const float IoU_thresh=0.5, const float conf_thresh=0.3, 
+                    bool verbose=false, const bool write_on_file=false,    
+                    std::string net="");
+
+
+void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path, std::vector<tk::dnn::box> bbox, const int classes, const int w, const int h);
+
+}}
+#endif /*EVALUATION_H*/
+

include/tkDNN/kernels.h

@@ -3,25 +3,49 @@
 
 #include "utils.h"
 
-void activationELUForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream = cudaStream_t(0));
-void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream = cudaStream_t(0));
-void activationLOGISTICForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationELUForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationReLUCeilingForward(dnnType *srcData, dnnType *dstData, int size, const float ceiling, cudaStream_t stream = cudaStream_t(0));
+void activationLOGISTICForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationSIGMOIDForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
+void activationMishForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream= cudaStream_t(0));
 
-void fill(dnnType* data, int size, dnnType val, cudaStream_t stream = cudaStream_t(0));
+void fill(dnnType *data, int size, dnnType val, cudaStream_t stream = cudaStream_t(0));
 
-void reorgForward(  dnnType* srcData, dnnType* dstData, 
-                    int n, int c, int h, int w, int stride, cudaStream_t stream = cudaStream_t(0));
-void softmaxForward(float *input, int n, int batch, int batch_offset, 
+void resizeForward(dnnType *srcData, dnnType *dstData, int n, int i_c, int i_h, int i_w,
+                   int o_c, int o_h, int o_w, cudaStream_t stream = cudaStream_t(0));
+
+void reorgForward(dnnType *srcData, dnnType *dstData,
+                  int n, int c, int h, int w, int stride, cudaStream_t stream = cudaStream_t(0));
+
+void MaxPoolingForward(dnnType *srcData, dnnType *dstData, int n, int c, int h, int w, int stride_x, int stride_y, int size, int padding, cudaStream_t stream = cudaStream_t(0));
+
+void softmaxForward(float *input, int n, int batch, int batch_offset,
                     int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream = cudaStream_t(0));
 
-
-void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1,
-                                                         int n2, int c2, int h2, int w2, int s2, 
+void shortcutForward(dnnType *srcData, dnnType *dstData, int n1, int c1, int h1, int w1, int s1,
+                     int n2, int c2, int h2, int w2, int s2,
                      cudaStream_t stream = cudaStream_t(0));
 
-void upsampleForward(dnnType* srcData, dnnType* dstData, 
-                     int n, int c, int h, int w, int s, int forward, float scale,                                   
+void upsampleForward(dnnType *srcData, dnnType *dstData,
+                     int n, int c, int h, int w, int s, int forward, float scale,
                      cudaStream_t stream = cudaStream_t(0));
 
-void float2half(float* srcData, __half* dstData, int size, const cudaStream_t stream = cudaStream_t(0));
+void float2half(float *srcData, __half *dstData, int size, const cudaStream_t stream = cudaStream_t(0));
+
+void dcnV2CudaForward(cublasStatus_t stat, cublasHandle_t handle,
+                         float *input, float *weight,
+                         float *bias, float *ones,
+                         float *offset, float *mask,
+                         float *output, float *columns,
+                         int kernel_h, int kernel_w,
+                         const int stride_h, const int stride_w,
+                         const int pad_h, const int pad_w,
+                         const int dilation_h, const int dilation_w,
+                         const int deformable_group, const int batch_id,
+                         const int in_n, const int in_c, const int in_h, const int in_w,
+                         const int out_n, const int out_c, const int out_h, const int out_w,
+                         const int dst_dim, cudaStream_t stream = cudaStream_t(0));
+
+void scalAdd(dnnType* dstData, int size, float alpha, float beta, int inc, cudaStream_t stream = cudaStream_t(0));
 #endif //KERNELS_H

include/tkDNN/kernelsThrust.h

@@ -0,0 +1,39 @@
+#ifndef KERNELSTHRUST_H
+#define KERNELSTHRUST_H
+
+
+#include <thrust/sort.h>
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/transform.h>
+#include <thrust/iterator/constant_iterator.h>
+#include <thrust/gather.h>
+#include <thrust/copy.h>
+
+#include "tkdnn.h"
+
+struct threshold : public thrust::binary_function<float,float,float>
+{
+  __host__ __device__
+  float operator()(float x, float y) { 
+    double toll = 1e-6; 
+    if(fabsf(x-y)>toll) 
+        return 0.0f; 
+    else
+        return x; 
+    }
+};
+
+void sort(dnnType *src_begin, dnnType *src_end, int *idsrc);
+void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
+            int *topk_inds, float *topk_ys, float *topk_xs);
+// void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes);            
+void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev);
+void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op);
+void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys);
+void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin, 
+                     float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out);
+void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, 
+            dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out);
+
+#endif //KERNELSTHRUST_H

include/tkDNN/models/Yolo3.h

@@ -1,289 +0,0 @@
-int preYoloFilters = (classes+5)*3; 
-
-std::string input_bin = bin_path + "/layers/input.bin";
-std::vector<std::string> output_bins = {
-    bin_path + "/debug/layer82_out.bin",
-    bin_path + "/debug/layer94_out.bin",
-    bin_path + "/debug/layer106_out.bin"
-};
-std::string c0_bin    = bin_path + "/layers/c0.bin";
-std::string c1_bin    = bin_path + "/layers/c1.bin";
-std::string c2_bin    = bin_path + "/layers/c2.bin";
-std::string c3_bin    = bin_path + "/layers/c3.bin";
-std::string c5_bin    = bin_path + "/layers/c5.bin";
-std::string c6_bin    = bin_path + "/layers/c6.bin";
-std::string c7_bin    = bin_path + "/layers/c7.bin";
-std::string c9_bin    = bin_path + "/layers/c9.bin";
-std::string c10_bin   = bin_path + "/layers/c10.bin";
-std::string c12_bin   = bin_path + "/layers/c12.bin";
-std::string c13_bin   = bin_path + "/layers/c13.bin";
-std::string c14_bin   = bin_path + "/layers/c14.bin";
-std::string c16_bin   = bin_path + "/layers/c16.bin";
-std::string c17_bin   = bin_path + "/layers/c17.bin";
-std::string c19_bin   = bin_path + "/layers/c19.bin";
-std::string c20_bin   = bin_path + "/layers/c20.bin";
-std::string c22_bin   = bin_path + "/layers/c22.bin";
-std::string c23_bin   = bin_path + "/layers/c23.bin";
-std::string c25_bin   = bin_path + "/layers/c25.bin";
-std::string c26_bin   = bin_path + "/layers/c26.bin";
-std::string c28_bin   = bin_path + "/layers/c28.bin";
-std::string c29_bin   = bin_path + "/layers/c29.bin";
-std::string c31_bin   = bin_path + "/layers/c31.bin";
-std::string c32_bin   = bin_path + "/layers/c32.bin";
-std::string c34_bin   = bin_path + "/layers/c34.bin";
-std::string c35_bin   = bin_path + "/layers/c35.bin";
-std::string c37_bin   = bin_path + "/layers/c37.bin";
-std::string c38_bin   = bin_path + "/layers/c38.bin";
-std::string c39_bin   = bin_path + "/layers/c39.bin";
-std::string c41_bin   = bin_path + "/layers/c41.bin";
-std::string c42_bin   = bin_path + "/layers/c42.bin";
-std::string c44_bin   = bin_path + "/layers/c44.bin";
-std::string c45_bin   = bin_path + "/layers/c45.bin";
-std::string c47_bin   = bin_path + "/layers/c47.bin";
-std::string c48_bin   = bin_path + "/layers/c48.bin";
-std::string c50_bin   = bin_path + "/layers/c50.bin";
-std::string c51_bin   = bin_path + "/layers/c51.bin";
-std::string c53_bin   = bin_path + "/layers/c53.bin";
-std::string c54_bin   = bin_path + "/layers/c54.bin";
-std::string c56_bin   = bin_path + "/layers/c56.bin";
-std::string c57_bin   = bin_path + "/layers/c57.bin";
-std::string c59_bin   = bin_path + "/layers/c59.bin";
-std::string c60_bin   = bin_path + "/layers/c60.bin";
-std::string c62_bin   = bin_path + "/layers/c62.bin";
-std::string c63_bin   = bin_path + "/layers/c63.bin";
-std::string c64_bin   = bin_path + "/layers/c64.bin";
-std::string c66_bin   = bin_path + "/layers/c66.bin";
-std::string c67_bin   = bin_path + "/layers/c67.bin";
-std::string c69_bin   = bin_path + "/layers/c69.bin";
-std::string c70_bin   = bin_path + "/layers/c70.bin";
-std::string c72_bin   = bin_path + "/layers/c72.bin";
-std::string c73_bin   = bin_path + "/layers/c73.bin";
-std::string c75_bin   = bin_path + "/layers/c75.bin";
-std::string c76_bin   = bin_path + "/layers/c76.bin";
-std::string c77_bin   = bin_path + "/layers/c77.bin";
-std::string c78_bin   = bin_path + "/layers/c78.bin";
-std::string c79_bin   = bin_path + "/layers/c79.bin";
-std::string c80_bin   = bin_path + "/layers/c80.bin";
-std::string c81_bin   = bin_path + "/layers/c81.bin";
-std::string g82_bin   = bin_path + "/layers/g82.bin";
-std::string c84_bin   = bin_path + "/layers/c84.bin";
-std::string c87_bin   = bin_path + "/layers/c87.bin";
-std::string c88_bin   = bin_path + "/layers/c88.bin";
-std::string c89_bin   = bin_path + "/layers/c89.bin";
-std::string c90_bin   = bin_path + "/layers/c90.bin";
-std::string c91_bin   = bin_path + "/layers/c91.bin";
-std::string c92_bin   = bin_path + "/layers/c92.bin";
-std::string c93_bin   = bin_path + "/layers/c93.bin";
-std::string g94_bin   = bin_path + "/layers/g94.bin";
-std::string c96_bin   = bin_path + "/layers/c96.bin";
-std::string c99_bin   = bin_path + "/layers/c99.bin";
-std::string c100_bin  = bin_path + "/layers/c100.bin";
-std::string c101_bin  = bin_path + "/layers/c101.bin";
-std::string c102_bin  = bin_path + "/layers/c102.bin";
-std::string c103_bin  = bin_path + "/layers/c103.bin";
-std::string c104_bin  = bin_path + "/layers/c104.bin";
-std::string c105_bin  = bin_path + "/layers/c105.bin";
-std::string g106_bin  = bin_path + "/layers/g106.bin";
-
-tk::dnn::Conv2d     c0   (&net,  32, 3, 3, 1, 1, 1, 1,  c0_bin, true);
-tk::dnn::Activation a0   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c1   (&net,  64, 3, 3, 2, 2, 1, 1,  c1_bin, true);
-tk::dnn::Activation a1   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c2   (&net,  32, 1, 1, 1, 1, 0, 0,  c2_bin, true);
-tk::dnn::Activation a2   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c3   (&net,  64, 3, 3, 1, 1, 1, 1,  c3_bin, true);
-tk::dnn::Activation a3   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s4   (&net, &a1);
-tk::dnn::Conv2d     c5   (&net, 128, 3, 3, 2, 2, 1, 1,  c5_bin, true);
-tk::dnn::Activation a5   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c6   (&net,  64, 1, 1, 1, 1, 0, 0,  c6_bin, true);
-tk::dnn::Activation a6   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c7   (&net, 128, 3, 3, 1, 1, 1, 1,  c7_bin, true);
-tk::dnn::Activation a7   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s8   (&net, &a5);
-tk::dnn::Conv2d     c9   (&net,  64, 1, 1, 1, 1, 0, 0,  c9_bin, true);
-tk::dnn::Activation a9   (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c10  (&net, 128, 3, 3, 1, 1, 1, 1, c10_bin, true);
-tk::dnn::Activation a10  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s11  (&net, &s8);
-
-tk::dnn::Conv2d     c12  (&net, 256, 3, 3, 2, 2, 1, 1, c12_bin, true);
-tk::dnn::Activation a12  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c13  (&net, 128, 1, 1, 1, 1, 0, 0, c13_bin, true);
-tk::dnn::Activation a13  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c14  (&net, 256, 3, 3, 1, 1, 1, 1, c14_bin, true);
-tk::dnn::Activation a14  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s15  (&net, &a12);
-
-tk::dnn::Conv2d     c16  (&net, 128, 1, 1, 1, 1, 0, 0, c16_bin, true);
-tk::dnn::Activation a16  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c17  (&net, 256, 3, 3, 1, 1, 1, 1, c17_bin, true);
-tk::dnn::Activation a17  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s18  (&net, &s15);
-tk::dnn::Conv2d     c19  (&net, 128, 1, 1, 1, 1, 0, 0, c19_bin, true);
-tk::dnn::Activation a19  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c20  (&net, 256, 3, 3, 1, 1, 1, 1, c20_bin, true);
-tk::dnn::Activation a20  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s21  (&net, &s18);
-tk::dnn::Conv2d     c22  (&net, 128, 1, 1, 1, 1, 0, 0, c22_bin, true);
-tk::dnn::Activation a22  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c23  (&net, 256, 3, 3, 1, 1, 1, 1, c23_bin, true);
-tk::dnn::Activation a23  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s24  (&net, &s21);
-tk::dnn::Conv2d     c25  (&net, 128, 1, 1, 1, 1, 0, 0, c25_bin, true);
-tk::dnn::Activation a25  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c26  (&net, 256, 3, 3, 1, 1, 1, 1, c26_bin, true);
-tk::dnn::Activation a26  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s27  (&net, &s24);
-tk::dnn::Conv2d     c28  (&net, 128, 1, 1, 1, 1, 0, 0, c28_bin, true);
-tk::dnn::Activation a28  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c29  (&net, 256, 3, 3, 1, 1, 1, 1, c29_bin, true);
-tk::dnn::Activation a29  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s30  (&net, &s27);
-tk::dnn::Conv2d     c31  (&net, 128, 1, 1, 1, 1, 0, 0, c31_bin, true);
-tk::dnn::Activation a31  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c32  (&net, 256, 3, 3, 1, 1, 1, 1, c32_bin, true);
-tk::dnn::Activation a32  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s33  (&net, &s30);
-tk::dnn::Conv2d     c34  (&net, 128, 1, 1, 1, 1, 0, 0, c34_bin, true);
-tk::dnn::Activation a34  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c35  (&net, 256, 3, 3, 1, 1, 1, 1, c35_bin, true);
-tk::dnn::Activation a35  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s36  (&net, &s33);
-
-tk::dnn::Conv2d     c37  (&net, 512, 3, 3, 2, 2, 1, 1, c37_bin, true);
-tk::dnn::Activation a37  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c38  (&net, 256, 1, 1, 1, 1, 0, 0, c38_bin, true);
-tk::dnn::Activation a38  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c39  (&net, 512, 3, 3, 1, 1, 1, 1, c39_bin, true);
-tk::dnn::Activation a39  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s40  (&net, &a37);
-
-tk::dnn::Conv2d     c41  (&net, 256, 1, 1, 1, 1, 0, 0, c41_bin, true);
-tk::dnn::Activation a41  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c42  (&net, 512, 3, 3, 1, 1, 1, 1, c42_bin, true);
-tk::dnn::Activation a42  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s43  (&net, &s40);
-tk::dnn::Conv2d     c44  (&net, 256, 1, 1, 1, 1, 0, 0, c44_bin, true);
-tk::dnn::Activation a44  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c45  (&net, 512, 3, 3, 1, 1, 1, 1, c45_bin, true);
-tk::dnn::Activation a45  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s46  (&net, &s43);
-tk::dnn::Conv2d     c47  (&net, 256, 1, 1, 1, 1, 0, 0, c47_bin, true);
-tk::dnn::Activation a47  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c48  (&net, 512, 3, 3, 1, 1, 1, 1, c48_bin, true);
-tk::dnn::Activation a48  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s49  (&net, &s46);
-tk::dnn::Conv2d     c50  (&net, 256, 1, 1, 1, 1, 0, 0, c50_bin, true);
-tk::dnn::Activation a50  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c51  (&net, 512, 3, 3, 1, 1, 1, 1, c51_bin, true);
-tk::dnn::Activation a51  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s52  (&net, &s49);
-tk::dnn::Conv2d     c53  (&net, 256, 1, 1, 1, 1, 0, 0, c53_bin, true);
-tk::dnn::Activation a53  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c54  (&net, 512, 3, 3, 1, 1, 1, 1, c54_bin, true);
-tk::dnn::Activation a54  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s55  (&net, &s52);
-tk::dnn::Conv2d     c56  (&net, 256, 1, 1, 1, 1, 0, 0, c56_bin, true);
-tk::dnn::Activation a56  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c57  (&net, 512, 3, 3, 1, 1, 1, 1, c57_bin, true);
-tk::dnn::Activation a57  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s58  (&net, &s55);
-tk::dnn::Conv2d     c59  (&net, 256, 1, 1, 1, 1, 0, 0, c59_bin, true);
-tk::dnn::Activation a59  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c60  (&net, 512, 3, 3, 1, 1, 1, 1, c60_bin, true);
-tk::dnn::Activation a60  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s61  (&net, &s58);
-
-tk::dnn::Conv2d     c62  (&net,1024, 3, 3, 2, 2, 1, 1, c62_bin, true);
-tk::dnn::Activation a62  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c63  (&net, 512, 1, 1, 1, 1, 0, 0, c63_bin, true);
-tk::dnn::Activation a63  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c64  (&net,1024, 3, 3, 1, 1, 1, 1, c64_bin, true);
-tk::dnn::Activation a64  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s65  (&net, &a62);
-
-tk::dnn::Conv2d     c66  (&net, 512, 1, 1, 1, 1, 0, 0, c66_bin, true);
-tk::dnn::Activation a66  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c67  (&net,1024, 3, 3, 1, 1, 1, 1, c67_bin, true);
-tk::dnn::Activation a67  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s68  (&net, &s65);
-
-tk::dnn::Conv2d     c69  (&net, 512, 1, 1, 1, 1, 0, 0, c69_bin, true);
-tk::dnn::Activation a69  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c70  (&net,1024, 3, 3, 1, 1, 1, 1, c70_bin, true);
-tk::dnn::Activation a70  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s71  (&net, &s68);
-
-tk::dnn::Conv2d     c72  (&net, 512, 1, 1, 1, 1, 0, 0, c72_bin, true);
-tk::dnn::Activation a72  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c73  (&net,1024, 3, 3, 1, 1, 1, 1, c73_bin, true);
-tk::dnn::Activation a73  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Shortcut   s74  (&net, &s71);
-
-tk::dnn::Conv2d     c75  (&net, 512, 1, 1, 1, 1, 0, 0, c75_bin, true);
-tk::dnn::Activation a75  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c76  (&net,1024, 3, 3, 1, 1, 1, 1, c76_bin, true);
-tk::dnn::Activation a76  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c77  (&net, 512, 1, 1, 1, 1, 0, 0, c77_bin, true);
-tk::dnn::Activation a77  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c78  (&net,1024, 3, 3, 1, 1, 1, 1, c78_bin, true);
-tk::dnn::Activation a78  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c79  (&net, 512, 1, 1, 1, 1, 0, 0, c79_bin, true);
-tk::dnn::Activation a79  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c80  (&net,1024, 3, 3, 1, 1, 1, 1, c80_bin, true);
-tk::dnn::Activation a80  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c81  (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c81_bin, false);
-tk::dnn::Yolo     yolo0  (&net, classes, 3, g82_bin);
-
-tk::dnn::Layer *m83_layers[1] = { &a79 };
-tk::dnn::Route      m83  (&net, m83_layers, 1);
-tk::dnn::Conv2d     c84  (&net, 256, 1, 1, 1, 1, 0, 0, c84_bin, true);
-tk::dnn::Activation a84  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Upsample   u85  (&net, 2);
-
-tk::dnn::Layer *m86_layers[2] = { &u85, &s61 };
-tk::dnn::Route      m86  (&net, m86_layers, 2);
-tk::dnn::Conv2d     c87  (&net, 256, 1, 1, 1, 1, 0, 0, c87_bin, true);
-tk::dnn::Activation a87  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c88  (&net, 512, 3, 3, 1, 1, 1, 1, c88_bin, true);
-tk::dnn::Activation a88  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c89  (&net, 256, 1, 1, 1, 1, 0, 0, c89_bin, true);
-tk::dnn::Activation a89  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c90  (&net, 512, 3, 3, 1, 1, 1, 1, c90_bin, true);
-tk::dnn::Activation a90  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c91  (&net, 256, 1, 1, 1, 1, 0, 0, c91_bin, true);
-tk::dnn::Activation a91  (&net, tk::dnn::ACTIVATION_LEAKY);
-
-tk::dnn::Conv2d     c92  (&net, 512, 3, 3, 1, 1, 1, 1, c92_bin, true);
-tk::dnn::Activation a92  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c93  (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c93_bin, false);
-tk::dnn::Yolo     yolo1  (&net, classes, 3, g94_bin);
-
-tk::dnn::Layer *m95_layers[1] = { &a91 };
-tk::dnn::Route      m95  (&net, m95_layers, 1);
-tk::dnn::Conv2d     c96  (&net, 128, 1, 1, 1, 1, 0, 0, c96_bin, true);
-tk::dnn::Activation a96  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Upsample   u97  (&net, 2);
-
-tk::dnn::Layer *m98_layers[2] = { &u97, &s36 };
-tk::dnn::Route      m98  (&net, m98_layers, 2);
-tk::dnn::Conv2d     c99  (&net, 128, 1, 1, 1, 1, 0, 0, c99_bin, true);
-tk::dnn::Activation a99  (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c100 (&net, 256, 3, 3, 1, 1, 1, 1, c100_bin, true);
-tk::dnn::Activation a100 (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c101 (&net, 128, 1, 1, 1, 1, 0, 0, c101_bin, true);
-tk::dnn::Activation a101 (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c102 (&net, 256, 3, 3, 1, 1, 1, 1, c102_bin, true);
-tk::dnn::Activation a102 (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c103 (&net, 128, 1, 1, 1, 1, 0, 0, c103_bin, true);
-tk::dnn::Activation a103 (&net, tk::dnn::ACTIVATION_LEAKY);
-
-tk::dnn::Conv2d     c104 (&net, 256, 3, 3, 1, 1, 1, 1, c104_bin, true);
-tk::dnn::Activation a104 (&net, tk::dnn::ACTIVATION_LEAKY);
-tk::dnn::Conv2d     c105 (&net, preYoloFilters, 1, 1, 1, 1, 0, 0, c105_bin, false);
-tk::dnn::Yolo      yolo2 (&net, classes, 3, g106_bin);
-
-yolo[0] = &yolo0;
-yolo[1] = &yolo1;
-yolo[2] = &yolo2;

include/tkDNN/pluginsRT/ActivationLeakyRT.h

@@ -42,7 +42,7 @@ public:
 	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
 
 		activationLEAKYForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
-											reinterpret_cast<dnnType*>(outputs[0]), size, stream);
+											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
 		return 0;
 	}
 

include/tkDNN/pluginsRT/ActivationMishRT.h

@@ -0,0 +1,60 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ActivationMishRT : public IPlugin {
+
+public:
+	ActivationMishRT() {
+
+
+	}
+
+	~ActivationMishRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return inputs[0];
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		size = 1;
+		for(int i=0; i<outputDims[0].nbDims; i++)
+			size *= outputDims[0].d[i];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		activationMishForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 1*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, size);
+	}
+
+	int size;
+};

include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h

@@ -0,0 +1,62 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ActivationReLUCeiling : public IPlugin {
+
+public:
+	ActivationReLUCeiling(const float ceiling) {
+		this->ceiling = ceiling;
+	}
+
+	~ActivationReLUCeiling(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return inputs[0];
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		size = 1;
+		for(int i=0; i<outputDims[0].nbDims; i++)
+			size *= outputDims[0].d[i];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		activationReLUCeilingForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, ceiling, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 1*sizeof(int) +  1*sizeof(float);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, ceiling);
+		tk::dnn::writeBUF(buf, size);
+		
+	}
+
+	int size;
+	float ceiling;
+};

include/tkDNN/pluginsRT/ActivationSigmoidRT.h

@@ -0,0 +1,60 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ActivationSigmoidRT : public IPlugin {
+
+public:
+	ActivationSigmoidRT() {
+
+
+	}
+
+	~ActivationSigmoidRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return inputs[0];
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		size = 1;
+		for(int i=0; i<outputDims[0].nbDims; i++)
+			size *= outputDims[0].d[i];
+	}
+
+	int initialize() override {
+
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		activationSIGMOIDForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+											reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 1*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, size);
+	}
+
+	int size;
+};

include/tkDNN/pluginsRT/DeformableConvRT.h

@@ -0,0 +1,195 @@
+#include<cassert>
+#include "../kernels.h"
+
+
+class DeformableConvRT : public IPlugin {
+
+
+
+public:
+	DeformableConvRT(int chunk_dim, int kh, int kw, int sh, int sw, int ph, int pw, 
+					int deformableGroup, int i_n, int i_c, int i_h, int i_w, 
+					int o_n, int o_c, int o_h, int o_w, 
+					tk::dnn::DeformConv2d *deformable = nullptr) {
+		this->chunk_dim = chunk_dim;
+		this->kh = kh;
+		this->kw = kw;
+		this->sh = sh;
+		this->sw = sw;
+		this->ph = ph;
+		this->pw = pw;
+		this->deformableGroup = deformableGroup;
+		this->i_n = i_n;
+		this->i_c = i_c;
+		this->i_h = i_h;
+		this->i_w = i_w;
+		this->o_n = o_n;
+		this->o_c = o_c;
+		this->o_h = o_h;
+		this->o_w = o_w;
+		height_ones = (i_h + 2 * ph - (1 * (kh - 1) + 1)) / sh + 1;
+		width_ones = (i_w + 2 * pw - (1 * (kw - 1) + 1)) / sw + 1;
+		dim_ones = i_c * kh * kw * 1 * height_ones * width_ones;
+		std::cout<<i_c * o_c * kh * kw * 1<<"\n";
+		checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
+		checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
+		checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));
+		checkCuda( cudaMalloc(&offset, 2*chunk_dim*sizeof(dnnType)));
+		checkCuda( cudaMalloc(&mask, chunk_dim*sizeof(dnnType)));
+		checkCuda( cudaMalloc(&ones_d2, dim_ones*sizeof(dnnType)));
+		if(deformable != nullptr) {
+			this->defRT = deformable;
+			checkCuda( cudaMemcpy(data_d, deformable->data_d, sizeof(dnnType)*i_c * o_c * kh * kw * 1, cudaMemcpyDeviceToDevice) );
+			checkCuda( cudaMemcpy(bias2_d, deformable->bias2_d, sizeof(dnnType)*o_c, cudaMemcpyDeviceToDevice) );
+            checkCuda( cudaMemcpy(ones_d1, deformable->ones_d1, sizeof(dnnType)*height_ones*width_ones, cudaMemcpyDeviceToDevice) );
+            checkCuda( cudaMemcpy(offset, deformable->offset, sizeof(dnnType)*2*chunk_dim, cudaMemcpyDeviceToDevice) );
+            checkCuda( cudaMemcpy(mask, deformable->mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToDevice) );
+            checkCuda( cudaMemcpy(ones_d2, deformable->ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToDevice) );
+		}
+		stat = cublasCreate(&handle);
+		if (stat != CUBLAS_STATUS_SUCCESS)
+			FatalError("CUBLAS initialization failed\n");
+	}
+
+	~DeformableConvRT() {
+		checkCuda( cudaFree(data_d) );
+		checkCuda( cudaFree(bias2_d) );
+		checkCuda( cudaFree(ones_d1) );
+		checkCuda( cudaFree(offset) );
+		checkCuda( cudaFree(mask) );
+		checkCuda( cudaFree(ones_d2) );
+		cublasDestroy(handle);
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{defRT->output_dim.c, defRT->output_dim.h, defRT->output_dim.w};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override { }
+
+	int initialize() override {
+		return 0;
+	}
+
+	virtual void terminate() override { }
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *output_conv = (dnnType*)reinterpret_cast<const dnnType*>(inputs[1]);
+
+		// split conv2d outputs into offset to mask
+		for(int b=0; b<batchSize; b++) {
+			checkCuda(cudaMemcpy(offset, output_conv + b * 3 * chunk_dim, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); 
+			checkCuda(cudaMemcpy(mask, output_conv + b * 3 * chunk_dim + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); 
+			// kernel sigmoide
+			activationSIGMOIDForward(mask, mask, chunk_dim);
+			// deformable convolution
+			dcnV2CudaForward(stat, handle, 
+								srcData, data_d,
+								bias2_d, ones_d1,
+								offset, mask,
+								reinterpret_cast<dnnType*>(outputs[0]), ones_d2,
+								kh, kw,
+								sh, sw,
+								ph, pw,
+								1, 1,
+								deformableGroup, b,
+								i_n, i_c, i_h, i_w,
+								o_n, o_c, o_h, o_w,
+								chunk_dim);
+		}
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 16 * sizeof(int) + chunk_dim * 3 * sizeof(dnnType) + (i_c * o_c * kh * kw * 1 ) * sizeof(dnnType) +
+			   o_c * sizeof(dnnType) + height_ones * width_ones * sizeof(dnnType) + dim_ones * sizeof(dnnType);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, chunk_dim);
+		tk::dnn::writeBUF(buf, kh);
+		tk::dnn::writeBUF(buf, kw);
+		tk::dnn::writeBUF(buf, sh);
+		tk::dnn::writeBUF(buf, sw);
+		tk::dnn::writeBUF(buf, ph);
+		tk::dnn::writeBUF(buf, pw);
+		tk::dnn::writeBUF(buf, deformableGroup);
+		tk::dnn::writeBUF(buf, i_n);
+		tk::dnn::writeBUF(buf, i_c);
+		tk::dnn::writeBUF(buf, i_h);
+		tk::dnn::writeBUF(buf, i_w);
+		tk::dnn::writeBUF(buf, o_n);
+		tk::dnn::writeBUF(buf, o_c);
+		tk::dnn::writeBUF(buf, o_h);
+		tk::dnn::writeBUF(buf, o_w);
+		dnnType *aus = new dnnType[chunk_dim*2];
+		checkCuda( cudaMemcpy(aus, offset, sizeof(dnnType)*2*chunk_dim, cudaMemcpyDeviceToHost) );
+        for(int i=0; i<chunk_dim*2; i++)
+    		tk::dnn::writeBUF(buf, aus[i]);
+		free(aus);
+		aus = new dnnType[chunk_dim];
+		checkCuda( cudaMemcpy(aus, mask, sizeof(dnnType)*chunk_dim, cudaMemcpyDeviceToHost) );
+        for(int i=0; i<chunk_dim; i++)
+    		tk::dnn::writeBUF(buf, aus[i]);
+		free(aus);
+		aus = new dnnType[(i_c * o_c * kh * kw * 1 )];
+		checkCuda( cudaMemcpy(aus, data_d, sizeof(dnnType)*(i_c * o_c * kh * kw * 1 ), cudaMemcpyDeviceToHost) );
+        for(int i=0; i<(i_c * o_c * kh * kw * 1 ); i++)
+    		tk::dnn::writeBUF(buf, aus[i]);
+		free(aus);
+		aus = new dnnType[o_c];
+		checkCuda( cudaMemcpy(aus, bias2_d, sizeof(dnnType)*o_c, cudaMemcpyDeviceToHost) );
+        for(int i=0; i < o_c; i++)
+    		tk::dnn::writeBUF(buf, aus[i]);
+		free(aus);
+		aus = new dnnType[height_ones * width_ones];
+		checkCuda( cudaMemcpy(aus, ones_d1, sizeof(dnnType)*height_ones * width_ones, cudaMemcpyDeviceToHost) );
+        for(int i=0; i<height_ones * width_ones; i++)
+    		tk::dnn::writeBUF(buf, aus[i]);
+		free(aus);
+		aus = new dnnType[dim_ones];
+		checkCuda( cudaMemcpy(aus, ones_d2, sizeof(dnnType)*dim_ones, cudaMemcpyDeviceToHost) );
+        for(int i=0; i<dim_ones; i++)
+    		tk::dnn::writeBUF(buf, aus[i]);
+		free(aus);
+	}
+
+	cublasStatus_t stat; 
+	cublasHandle_t handle; 
+	int i_n, i_c, i_h, i_w;
+	int o_n, o_c, o_h, o_w;
+	int size;
+	int chunk_dim;
+	int kh, kw;
+	int sh, sw;
+	int ph, pw;
+	int deformableGroup;
+	int height_ones;
+	int width_ones;
+	int dim_ones;
+		
+	dnnType *data_d;
+    dnnType *bias2_d;
+	dnnType *ones_d1;
+	dnnType * offset;
+	dnnType * mask;
+	dnnType *ones_d2;
+	// dnnType *input_n;
+	// dnnType *offset_n;
+	// dnnType *mask_n;
+	// dnnType *output_n;
+	
+	
+	tk::dnn::DeformConv2d *defRT;
+};

include/tkDNN/pluginsRT/FlattenConcatRT.h

@@ -0,0 +1,80 @@
+#include<cassert>
+
+class FlattenConcatRT : public IPlugin {
+
+public:
+	FlattenConcatRT() {
+		stat = cublasCreate(&handle);
+		if (stat != CUBLAS_STATUS_SUCCESS) {
+			printf ("CUBLAS initialization failed\n");
+			return;
+  		}
+	}
+
+	~FlattenConcatRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{ inputs[0].d[0] * inputs[0].d[1] * inputs[0].d[2], 1, 1};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		assert(nbOutputs == 1 && nbInputs ==1);
+		rows = inputDims[0].d[0];
+		cols = inputDims[0].d[1] * inputDims[0].d[2];
+		c = inputDims[0].d[0] * inputDims[0].d[1] * inputDims[0].d[2];
+		h = 1;
+		w = 1;
+	}
+
+	int initialize() override {
+		return 0;
+	}
+
+	virtual void terminate() override {
+		checkERROR(cublasDestroy(handle));
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*rows*cols*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+
+		checkERROR( cublasSetStream(handle, stream) );	
+		for(int i=0; i<batchSize; i++) {
+			float const alpha(1.0);
+			float const beta(0.0);
+			int offset = i*rows*cols;
+			checkERROR( cublasSgeam( handle, CUBLAS_OP_T, CUBLAS_OP_N, rows, cols, &alpha, srcData + offset, cols, &beta, srcData + offset, rows, dstData + offset, rows ));
+		}
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 5*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+		tk::dnn::writeBUF(buf, rows);
+		tk::dnn::writeBUF(buf, cols);
+	}
+
+	int c, h, w;
+	int rows, cols;
+	cublasStatus_t stat; 
+	cublasHandle_t handle; 
+};

include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h

@@ -0,0 +1,74 @@
+#include<cassert>
+#include "../kernels.h"
+
+class MaxPoolFixedSizeRT : public IPlugin {
+
+public:
+	MaxPoolFixedSizeRT(int c, int h, int w, int n, int strideH, int strideW, int winSize, int padding) {
+		this->c = c;	
+		this->h = h;
+		this->w = w;
+		this->n = n;
+		this->stride_H = strideH;
+		this->stride_W = strideW;
+		this->winSize = winSize;
+		this->padding = padding;
+	}
+
+	~MaxPoolFixedSizeRT(){
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{this->c, this->h, this->w};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {        
+	}
+
+	int initialize() override {
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+
+		//std::cout<<this->n<<"  "<<this->c<<"  "<<this->h<<"  "<<this->w<<"  "<<this->stride_H<<"  "<<this->stride_W<<"  "<<this->winSize<<"  "<<this->padding<<std::endl;
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+		MaxPoolingForward(srcData, dstData, batchSize, this->c, this->h, this->w, this->stride_H, this->stride_W, this->winSize, this->padding, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 8*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+
+		tk::dnn::writeBUF(buf, this->c);
+		tk::dnn::writeBUF(buf, this->h);
+		tk::dnn::writeBUF(buf, this->w);
+		tk::dnn::writeBUF(buf, this->n);
+		tk::dnn::writeBUF(buf, this->stride_H);
+		tk::dnn::writeBUF(buf, this->stride_W);
+		tk::dnn::writeBUF(buf, this->winSize);
+		tk::dnn::writeBUF(buf, this->padding);
+	}
+
+	int n, c, h, w;
+	int stride_H, stride_W;
+	int winSize;
+	int padding;
+};

include/tkDNN/pluginsRT/RegionRT.h

@@ -50,18 +50,18 @@ public:
 
 		for (int b = 0; b < batchSize; ++b){
 			for(int n = 0; n < num; ++n){
-				int index = entry_index(b, n*w*h, 0, batchSize);
+				int index = entry_index(b, n*w*h, 0);
 				activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
 				
-				index = entry_index(b, n*w*h, coords, batchSize);
+				index = entry_index(b, n*w*h, coords);
 				activationLOGISTICForward(srcData + index, dstData + index, w*h, stream);
 			}
 		}
 
 		//softmax start
-		int index = entry_index(0, 0, coords + 1, batchSize);
+		int index = entry_index(0, 0, coords + 1);
 		softmaxForward(	srcData + index, classes, batchSize*num, 
-						(batchSize*c*h*w)/num, 
+						(c*h*w)/num, 
 						w*h, 1, w*h, 1, dstData + index, stream);
 
 		return 0;
@@ -85,10 +85,10 @@ public:
 	int c, h, w;
     int classes, coords, num;
 
-	int entry_index(int batch, int location, int entry, int batchSize) {
+	int entry_index(int batch, int location, int entry) {
 		int n =   location / (w*h);
 		int loc = location % (w*h);
-		return batch*c*h*w*batchSize + n*w*h*(coords+classes+1) + entry*w*h + loc;
+		return batch*c*h*w + n*w*h*(coords+classes+1) + entry*w*h + loc;
 	}
 
 };

include/tkDNN/pluginsRT/ReshapeRT.h

@@ -0,0 +1,61 @@
+#include<cassert>
+
+class ReshapeRT : public IPlugin {
+
+public:
+	ReshapeRT(dataDim_t new_dim) {
+		n = new_dim.n;
+		c = new_dim.c;
+		h = new_dim.h;
+		w = new_dim.w;
+	}
+
+	~ReshapeRT(){
+
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{ c,h,w};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+	}
+
+	int initialize() override {
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+		dnnType *srcData = (dnnType*)reinterpret_cast<const dnnType*>(inputs[0]);
+		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
+
+		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 4*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, n);
+		tk::dnn::writeBUF(buf, c);
+		tk::dnn::writeBUF(buf, h);
+		tk::dnn::writeBUF(buf, w);
+	}
+
+	int n, c, h, w;
+};

include/tkDNN/pluginsRT/ResizeLayerRT.h

@@ -0,0 +1,67 @@
+#include<cassert>
+#include "../kernels.h"
+
+class ResizeLayerRT : public IPlugin {
+
+public:
+	ResizeLayerRT(int c, int h, int w) {
+		o_c = c;
+		o_h = h;
+		o_w = w;	
+	}
+
+	~ResizeLayerRT(){
+	}
+
+	int getNbOutputs() const override {
+		return 1;
+	}
+
+	Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
+		return DimsCHW{o_c, o_h, o_w};
+	}
+
+	void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
+		i_c = inputDims[0].d[0];
+		i_h = inputDims[0].d[1];
+		i_w = inputDims[0].d[2];        
+	}
+
+	int initialize() override {
+		return 0;
+	}
+
+	virtual void terminate() override {
+	}
+
+	virtual size_t getWorkspaceSize(int maxBatchSize) const override {
+		return 0;
+	}
+
+	virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
+    	// printf("%d %d %d %d %d %d\n", i_c, i_w, i_h, o_c, o_w, o_h);
+        resizeForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]), 
+					  reinterpret_cast<dnnType*>(outputs[0]), 
+					  batchSize, i_c, i_h, i_w, o_c, o_h, o_w, stream);
+		return 0;
+	}
+
+
+	virtual size_t getSerializationSize() override {
+		return 6*sizeof(int);
+	}
+
+	virtual void serialize(void* buffer) override {
+		char *buf = reinterpret_cast<char*>(buffer);
+
+		tk::dnn::writeBUF(buf, o_c);
+		tk::dnn::writeBUF(buf, o_h);
+		tk::dnn::writeBUF(buf, o_w);
+
+		tk::dnn::writeBUF(buf, i_c);
+		tk::dnn::writeBUF(buf, i_h);
+		tk::dnn::writeBUF(buf, i_w);
+	}
+
+	int i_c, i_h, i_w, o_c, o_h, o_w;
+};

include/tkDNN/pluginsRT/RouteRT.h

@@ -3,6 +3,10 @@
 
 class RouteRT : public IPlugin {
 
+	/**
+		THIS IS NOT USED ANYMORE
+	*/
+
 public:
 	RouteRT() {
 	}

include/tkDNN/pluginsRT/ShortcutRT.h

@@ -4,7 +4,10 @@
 class ShortcutRT : public IPlugin {
 
 public:
-	ShortcutRT() {
+	ShortcutRT(tk::dnn::dataDim_t bdim) {
+		this->bc = bdim.c;
+		this->bh = bdim.h;
+		this->bw = bdim.w;
 	}
 
 	~ShortcutRT(){
@@ -44,22 +47,28 @@ public:
 		dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
 
 		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
-		shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, c, h, w, 1, stream);
+		for(int b=0; b < batchSize; ++b)
+			shortcutForward(srcDataBack + b*bc*bh*bw, dstData + b*c*h*w, 1, c, h, w, 1, 1, bc, bh, bw, 1, stream);
 
 		return 0;
 	}
 
 
 	virtual size_t getSerializationSize() override {
-		return 3*sizeof(int);
+		return 6*sizeof(int);
 	}
 
 	virtual void serialize(void* buffer) override {
 		char *buf = reinterpret_cast<char*>(buffer);
+		tk::dnn::writeBUF(buf, bc);
+		tk::dnn::writeBUF(buf, bh);
+		tk::dnn::writeBUF(buf, bw);
 		tk::dnn::writeBUF(buf, c);
 		tk::dnn::writeBUF(buf, h);
 		tk::dnn::writeBUF(buf, w);
+		
 	}
 
 	int c, h, w;
+	int bc, bh, bw;
 };

include/tkDNN/pluginsRT/YoloRT.h

@@ -8,16 +8,18 @@ class YoloRT : public IPlugin {
 
 
 public:
-	YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr) {
+	YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1) {
 
 		this->classes = classes;
 		this->num = num;
+		this->n_masks = n_masks;
+		this->scaleXY = scale_xy;
 
-        mask = new dnnType[num];
-        bias = new dnnType[num*3*2];
+        mask = new dnnType[n_masks];
+        bias = new dnnType[num*n_masks*2];
         if(yolo != nullptr) {
-            memcpy(mask, yolo->mask_h, sizeof(dnnType)*num);
-            memcpy(bias, yolo->bias_h, sizeof(dnnType)*num*3*2);
+            memcpy(mask, yolo->mask_h, sizeof(dnnType)*n_masks);
+            memcpy(bias, yolo->bias_h, sizeof(dnnType)*num*n_masks*2);
 			classesNames = yolo->classesNames;
         }
 	}
@@ -60,11 +62,13 @@ public:
 		checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
 
 		for (int b = 0; b < batchSize; ++b){
-			for(int n = 0; n < num; ++n){
-				int index = entry_index(b, n*w*h, 0, batchSize);
+			for(int n = 0; n < n_masks; ++n){
+				int index = entry_index(b, n*w*h, 0);
 				activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
+
+				if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
 				
-				index = entry_index(b, n*w*h, 4, batchSize);
+				index = entry_index(b, n*w*h, 4);
 				activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream);
 			}
 		}
@@ -75,19 +79,21 @@ public:
 
 
 	virtual size_t getSerializationSize() override {
-		return 5*sizeof(int) + num*sizeof(dnnType) + num*3*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
+		return 6*sizeof(int) + sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
 	}
 
 	virtual void serialize(void* buffer) override {
 		char *buf = reinterpret_cast<char*>(buffer);
 		tk::dnn::writeBUF(buf, classes);
 		tk::dnn::writeBUF(buf, num);
+		tk::dnn::writeBUF(buf, n_masks);
 		tk::dnn::writeBUF(buf, c);
 		tk::dnn::writeBUF(buf, h);
 		tk::dnn::writeBUF(buf, w);
-        for(int i=0; i<num; i++)
+		tk::dnn::writeBUF(buf, scaleXY);
+        for(int i=0; i<n_masks; i++)
     		tk::dnn::writeBUF(buf, mask[i]);
-        for(int i=0; i<3*2*num; i++)
+        for(int i=0; i<n_masks*2*num; i++)
     		tk::dnn::writeBUF(buf, bias[i]);
 
 		// save classes names
@@ -101,16 +107,17 @@ public:
 	}
 
 	int c, h, w;
-    int classes, num;
+    int classes, num, n_masks;
+	float scaleXY;
 	std::vector<std::string> classesNames;
 
     dnnType *mask;
     dnnType *bias;
 
-	int entry_index(int batch, int location, int entry, int batchSize) {
+	int entry_index(int batch, int location, int entry) {
 		int n =   location / (w*h);
 		int loc = location % (w*h);
-		return batch*c*h*w*batchSize + n*w*h*(4+classes+1) + entry*w*h + loc;
+		return batch*c*h*w + n*w*h*(4+classes+1) + entry*w*h + loc;
 	}
 
 };

include/tkDNN/test.h

@@ -0,0 +1,73 @@
+
+#include <tkdnn.h>
+int testInference(std::vector<std::string> input_bins, std::vector<std::string> output_bins, 
+  tk::dnn::Network *net, tk::dnn::NetworkRT *netRT = nullptr) {
+
+    std::vector<tk::dnn::Layer*> outputs;
+    for(int i=0; i<net->num_layers; i++) {
+        if(net->layers[i]->final)
+            outputs.push_back(net->layers[i]);
+    }
+    // no final layers, set last as output
+    if(outputs.size() == 0) {
+        outputs.push_back(net->layers[net->num_layers-1]);
+    }
+
+
+    // check input
+    if(input_bins.size() != 1) {
+        FatalError("currently support only 1 input");
+    }
+    if(output_bins.size() != outputs.size()) {
+        std::cout<<output_bins.size()<<" "<<outputs.size()<<"\n";
+        FatalError("outputs size missmatch");
+    }
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bins[0], net->input_dim.tot(), &input_h, &data);
+
+    // outputs
+    dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()]; 
+
+    tk::dnn::dataDim_t dim1 =  net->input_dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30); {
+        dim1.print();
+        TKDNN_TSTART
+        net->infer(dim1, data);    
+        TKDNN_TSTOP
+        dim1.print();   
+    }
+    for(int i=0; i<outputs.size(); i++) cudnn_out[i] = outputs[i]->dstData;
+
+    if(netRT != nullptr) {
+        tk::dnn::dataDim_t dim2 = net->input_dim;
+        printCenteredTitle(" TENSORRT inference ", '=', 30); {
+            dim2.print();
+            TKDNN_TSTART
+            netRT->infer(dim2, data);
+            TKDNN_TSTOP
+            dim2.print();
+        }
+        for(int i=0; i<outputs.size(); i++) rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
+    }
+
+    int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; 
+    for(int i=0; i<outputs.size(); i++) {
+        printCenteredTitle((std::string(" OUTPUT ") + std::to_string(i) + " CHECK RESULTS ").c_str(), '=', 30);
+        dnnType *out, *out_h;
+        int odim = outputs[i]->output_dim.tot();
+        readBinaryFile(output_bins[i], odim, &out_h, &out);
+        std::cout<<"CUDNN vs correct"; 
+        ret_cudnn |= checkResult(odim, cudnn_out[i], out) == 0 ? 0: ERROR_CUDNN;
+        if(netRT != nullptr) {
+            std::cout<<"TRT   vs correct"; 
+            ret_tensorrt |= checkResult(odim, rt_out[i], out) == 0 ? 0 : ERROR_TENSORRT;
+            std::cout<<"CUDNN vs TRT    "; 
+            ret_cudnn_tensorrt |= checkResult(odim, cudnn_out[i], rt_out[i]) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
+        }
+    }
+    return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
+
+  }

include/tkDNN/tkdnn.h

@@ -5,4 +5,4 @@
 #include "Layer.h"
 #include "NetworkRT.h"
 
-#define TKDNN_VERSION 400
+#define TKDNN_VERSION 500

include/tkDNN/utils.h

@@ -12,8 +12,13 @@
 #include <cublas_v2.h>
 #include <cudnn.h>
 
+#include <unistd.h>
+#include <ios>
+
+
 #define dnnType float
 
+
 // Colored output
 #define COL_END "\033[0m"
 
@@ -31,16 +36,18 @@
 #define COL_PURPLEB "\033[1;35m"
 #define COL_CYANB "\033[1;36m"
 
+#define TKDNN_VERBOSE 0
+
 // Simple Timer 
-#define TIMER_START timespec start, end;                               \
+#define TKDNN_TSTART timespec start, end;                               \
                     clock_gettime(CLOCK_MONOTONIC, &start);            
 
-#define TIMER_STOP_C(col)  clock_gettime(CLOCK_MONOTONIC, &end);       \
+#define TKDNN_TSTOP_C(col, show)  clock_gettime(CLOCK_MONOTONIC, &end);       \
     double t_ns = ((double)(end.tv_sec - start.tv_sec) * 1.0e9 +       \
                   (double)(end.tv_nsec - start.tv_nsec))/1.0e6;        \
-    std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END; 
+    if(show) std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END; 
 
-#define TIMER_STOP TIMER_STOP_C(COL_CYANB)
+#define TKDNN_TSTOP TKDNN_TSTOP_C(COL_CYANB, TKDNN_VERBOSE)
 
 /********************************************************
  * Prints the error message, and exits
@@ -88,15 +95,27 @@
     }                                                                  \
 }
 
-void printCenteredTitle(const char *title, char fill, int dim);
+typedef enum {
+  ERROR_CUDNN = 2,
+  ERROR_TENSORRT = 4,
+  ERROR_CUDNNvsTENSORRT = 8    
+} resultError_t;
+
+void printCenteredTitle(const char *title, char fill, int dim = 30);
 bool fileExist(const char *fname);
+void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url);
 void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
 int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10);
 void printDeviceVector(int size, dnnType* vec_d, bool device = true);
+float getColor(const int c, const int x, const int max);
 void resize(int size, dnnType **data);
 
 void matrixTranspose(cublasHandle_t handle, dnnType* srcData, dnnType* dstData, int rows, int cols);
 
 void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData, 
                     dnnType* add_vector, int dim, dnnType mul);
+
+void getMemUsage(double& vm_usage_kb, double& resident_set_kb);
+void printCudaMemUsage();
+void removePathAndExtension(const std::string &full_string, std::string &name);
 #endif //UTILS_H

scripts/download_validation.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+
+function elaborate_testset {
+    wget $1 -O $2.zip
+    unzip -d $2 $2.zip
+    rm $2.zip
+    cd $2/
+    realpath labels/* > all_labels.txt
+    realpath images/* > all_images.txt
+    cd ..
+} 
+
+cd demo 
+for valset in $@
+do
+    
+    if [ $valset = "COCO" ]; then
+        echo "Downloading $valset validation set in demo"
+        elaborate_testset "https://cloud.hipert.unimore.it/s/LNxBDk4wzqXPL8c/download" "COCO_val2017"
+    elif [ $valset = "BDD" ]; then
+        echo "Downloading $valset validation set in demo"
+        elaborate_testset "https://cloud.hipert.unimore.it/s/bikqk3FzCq2tg4D/download" "BDD100K_val"
+    fi
+
+done

scripts/install_OpenCV4.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+#based on https://devtalk.nvidia.com/default/topic/1042035/installing-opencv4-on-xavier/ & https://github.com/markste-in/OpenCV4XAVIER/blob/master/buildOpenCV4.sh
+
+# Compute Capabilities can be found here https://developer.nvidia.com/cuda-gpus#compute
+ARCH_BIN=7.2 # AGX Xavier
+#ARCH_BIN=6.2 # Tx2
+
+cd ~/Downloads
+sudo apt-get install -y build-essential \
+    unzip \
+    pkg-config \
+    libjpeg-dev \
+    libpng-dev \
+    libtiff-dev \
+    libavcodec-dev \
+    libavformat-dev \
+    libswscale-dev \
+    libv4l-dev \
+    libxvidcore-dev \
+    libx264-dev \
+    libgtk-3-dev \
+    libatlas-base-dev \
+    gfortran \
+    python3-dev \
+    python3-venv \
+    libgstreamer1.0-dev \
+    libgstreamer-plugins-base1.0-dev \
+    libdc1394-22-dev \
+    libavresample-dev
+
+git clone https://github.com/opencv/opencv.git
+git clone https://github.com/opencv/opencv_contrib.git
+
+python3 -m venv opencv4
+source opencv4/bin/activate
+pip install wheel
+pip install numpy
+
+cd opencv && mkdir build && cd build
+
+cmake -D CMAKE_BUILD_TYPE=RELEASE \
+    -D CMAKE_INSTALL_PREFIX=/usr/local \
+    -D INSTALL_PYTHON_EXAMPLES=ON \
+    -D INSTALL_C_EXAMPLES=OFF \
+    -D OPENCV_EXTRA_MODULES_PATH='~/Downloads/opencv_contrib/modules' \
+    -D PYTHON_EXECUTABLE='~/Downloads/opencv4/bin/python' \
+    -D BUILD_EXAMPLES=ON \
+    -D WITH_CUDA=ON \
+    -D CUDA_ARCH_BIN=${ARCH_BIN} \
+    -D CUDA_ARCH_PTX="" \
+    -D ENABLE_FAST_MATH=ON \
+    -D CUDA_FAST_MATH=ON \
+    -D WITH_CUBLAS=ON \
+    -D WITH_LIBV4L=ON \
+    -D WITH_GSTREAMER=ON \
+    -D WITH_GSTREAMER_0_10=OFF \
+    -D WITH_TBB=ON \
+    ../
+
+make -j4
+sudo make install
+sudo ldconfig
+
+cd '~/Downloads/opencv4/lib/python3.6/site-packages'
+ln -s /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-aarch64-linux-gnu.so cv2.so

scripts/test_all_tests.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+cd build
+
+RED='\033[1;31m'
+GREEN='\033[1;32m'
+ORANGE='\033[1;33m'
+PINK='\033[1;95m'
+NC='\033[0m' # No Color
+
+function print_output {
+    if [ $1 -eq 0 ]; then
+        echo -e "$2 ${GREEN}OK${NC}"
+    elif [ $1 -eq 1 ]; then
+        echo -e "$2 ${RED}FATAL ERROR${NC}"
+    elif [ $1 -eq 2 ] || [ $1 -eq 10 ]; then
+        echo -e "$2 ${PINK}CUDNN ERROR${NC}"
+    elif [ $1 -eq 4 ] || [ $1 -eq 12 ]; then
+        echo -e "$2 ${PINK}TENSORRT ERROR${NC}"
+    elif [ $1 -eq 8 ]; then
+        echo -e "$2 ${PINK}CUDNN vs TENSORRT ERROR${NC}"
+    elif [ $1 -eq 6 ]; then
+        echo -e "$2 ${PINK}CUDNN & TENSORTRT ERROR${NC}"
+    elif [ $1 -eq 14 ]; then
+        echo -e "$2 ${PINK}ERROR FOR EVERY CHECK${NC}"
+    else
+        echo -e "$2 ${RED}NOT OKAY (OPENCV maybe)${NC}"
+    fi 
+
+} 
+
+out_file=results.log
+rm $out_file
+
+function test_net {
+    ./test_$1 &>> $out_file
+    print_output $? $1
+    ./test_rtinference $1*.rt $TKDNN_BATCHSIZE &>> $out_file
+    print_output $? "batched $1"
+}
+
+
+modes=( 1 ) # only FP32
+# modes=( 1 2 ) # FP32 and FP16
+# modes=( 1 2 3 ) # FP32, FP16 and INT8
+
+for i in "${modes[@]}"
+do
+    rm *rt
+    if [ $i -eq 1 ]
+    then
+        export TKDNN_MODE=FP32
+        echo -e "${ORANGE}Test FP32${NC}"
+    fi
+    if [ $i -eq 2 ]
+    then
+        export TKDNN_MODE=FP16
+        echo -e "${ORANGE}Test FP16${NC}"
+    fi
+    if [ $i -eq 3 ]
+    then
+        export TKDNN_MODE=INT8
+        export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
+        export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
+        echo -e "${ORANGE}Test INT8${NC}"
+	fi
+
+    export TKDNN_BATCHSIZE=2
+    echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
+    
+    test_net mnist    
+    ./test_imuodom &>> $out_file
+    print_output $? imuodom
+
+    test_net yolo4
+    test_net yolo4_berkeley
+    test_net yolo3
+    test_net yolo3_berkeley
+    test_net yolo3_coco4
+    test_net yolo3_flir
+    test_net yolo3_512
+    test_net yolo3tiny
+    test_net yolo3tiny_512
+    test_net yolo2
+    test_net yolo2_voc
+    #test_net yolo2tiny
+    test_net csresnext50-panet-spp
+    #test_net csresnext50-panet-spp_berkeley
+    test_net resnet101_cnet
+    test_net dla34_cnet
+    test_net mobilenetv2ssd
+    test_net mobilenetv2ssd512
+    test_net bdd-mobilenetv2ssd
+done
+
+echo "If errors occured, check logfile $out_file" 

src/Activation.cpp

@@ -5,10 +5,11 @@
 
 namespace tk { namespace dnn {
 
-Activation::Activation(Network *net, int act_mode) : 
+Activation::Activation(Network *net, int act_mode, const float ceiling) : 
     Layer(net) {
 
     this->act_mode = act_mode;
+    this->ceiling = ceiling;
     checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 
     if(int(act_mode) < 100) {
@@ -31,7 +32,7 @@ Activation::Activation(Network *net, int act_mode) :
         checkCUDNN( cudnnSetActivationDescriptor(activDesc,
                                                 (cudnnActivationMode_t) act_mode,
                                                 CUDNN_PROPAGATE_NAN,
-                                                0.0) );
+                                                ceiling) );
     }
 }
 
@@ -44,10 +45,13 @@ Activation::~Activation() {
 }
 
 dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
-
     if(act_mode == ACTIVATION_LEAKY) {
         activationLEAKYForward(srcData, dstData, dim.tot());
 
+    }
+    else if(act_mode == ACTIVATION_MISH) {
+        activationMishForward(srcData, dstData, dim.tot());
+
     } else {
         dnnType alpha = dnnType(1);
         dnnType beta  = dnnType(0);

src/BoundingBox.cpp

@@ -0,0 +1,56 @@
+#include "BoundingBox.h"
+
+namespace tk { namespace dnn {
+
+float BoundingBox::overlap(const float p1, const float d1, const float p2, const float d2){
+    float l1 = p1 - d1/2;
+    float l2 = p2 - d2/2;
+    float left = l1 > l2 ? l1 : l2;
+    float r1 = p1 + d1/2;
+    float r2 = p2 + d2/2;
+    float right = r1 < r2 ? r1 : r2;
+    return right - left;
+}
+
+float BoundingBox::boxesIntersection(const BoundingBox &b){
+    float width = this->overlap(x, w, b.x, b.w);
+    float height = this->overlap(y, h, b.y, b.h);
+    if(width < 0 || height < 0) 
+        return 0;
+    float area = width*height;
+    return area;
+}
+
+float BoundingBox::boxesUnion(const BoundingBox &b){
+    float i = this->boxesIntersection(b);
+    float u = w*h + b.w*b.h - i;
+    return u;
+}
+
+float BoundingBox::IoU(const BoundingBox &b){
+    float I = this->boxesIntersection(b);
+    float U = this->boxesUnion(b);
+    if (I == 0 || U == 0) 
+        return 0;
+    return I / U;
+}
+
+void BoundingBox::clear(){
+    uniqueTruthIndex = -1;
+    truthFlag = 0;
+    maxIoU = 0;
+}
+
+std::ostream& operator<<(std::ostream& os, const BoundingBox& bb){
+    os <<"w: "<< bb.w << ", h: "<< bb.h << ", x: "<< bb.x << ", y: "<< bb.y <<
+         ", cat: "<< bb.cl << ", conf: "<< bb.prob<< ", truth: "<< 
+         bb.truthFlag<< ", assignedGT: "<< bb.uniqueTruthIndex<< 
+         ", maxIoU: "<< bb.maxIoU<<"\n";
+    return os;
+}
+
+bool boxComparison (const BoundingBox& a,const BoundingBox& b) { 
+    return (a.prob>b.prob); 
+}
+
+}}

src/CenternetDetection.cpp

@@ -0,0 +1,402 @@
+#include "CenternetDetection.h"
+
+
+namespace tk { namespace dnn {
+
+bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
+    std::cout<<(tensor_path).c_str()<<"\n";
+    netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
+    classes = n_classes;
+    nBatches = n_batches;
+
+    dim = netRT->input_dim;
+
+    const char *coco_class_name[] = {
+            "person", "bicycle", "car", "motorcycle", "airplane", 
+            "bus", "train", "truck", "boat", "traffic light", "fire hydrant",
+            "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse",
+            "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
+            "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis",
+            "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
+            "skateboard", "surfboard", "tennis racket", "bottle", "wine glass",
+            "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich",
+            "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
+            "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv",
+            "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
+            "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
+            "scissors", "teddy bear", "hair drier", "toothbrush"
+            };
+    classesNames = std::vector<std::string>(coco_class_name, std::end( coco_class_name));
+    
+    for(int c=0; c<classes; c++) {
+        int offset = c*123457 % classes;
+        float r = getColor(2, offset, classes);
+        float g = getColor(1, offset, classes);
+        float b = getColor(0, offset, classes);
+        colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
+    }
+    
+    src = cv::Mat(cv::Size(2,3), CV_32F);
+    dst = cv::Mat(cv::Size(2,3), CV_32F);
+    dst2 = cv::Mat(cv::Size(2,3), CV_32F);
+    trans = cv::Mat(cv::Size(3,2), CV_32F);
+    trans2 = cv::Mat(cv::Size(3,2), CV_32F);
+
+    checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
+
+    dim_hm = tk::dnn::dataDim_t(1, 80, 128, 128, 1);
+    dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
+    dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
+
+    checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
+    checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );      
+    checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );      
+    checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );    
+    checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
+    checkCuda( cudaMalloc(&ids_2d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
+    checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
+    checkCuda( cudaMallocHost(&ids_2, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
+    for(int i =0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
+        ids_[i] = i;
+    }
+    int val = 0;
+    for(int i =0; i <dim_hm.c *  dim_hm.h * dim_hm.w; i++){
+        ids_2[i] = val;
+        if(i%dim_hm.c == 0)
+            val = 0;
+    }
+
+    checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
+    checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
+
+    checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
+    checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
+
+    checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
+    checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );     
+    checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) ); 
+    checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
+    checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
+
+    checkCuda( cudaMallocHost(&bbx0, K * sizeof(float)) ); 
+    checkCuda( cudaMallocHost(&bby0, K * sizeof(float)) ); 
+    checkCuda( cudaMallocHost(&bbx1, K * sizeof(float)) ); 
+    checkCuda( cudaMallocHost(&bby1, K * sizeof(float)) ); 
+    checkCuda( cudaMalloc(&bbx0_d, K * sizeof(float)) ); 
+    checkCuda( cudaMalloc(&bby0_d, K * sizeof(float)) ); 
+    checkCuda( cudaMalloc(&bbx1_d, K * sizeof(float)) ); 
+    checkCuda( cudaMalloc(&bby1_d, K * sizeof(float)) ); 
+
+    checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
+
+#ifdef OPENCV_CUDACONTRIB
+
+    checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
+    checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
+    float mean[3] = {0.408, 0.447, 0.47};
+    float stddev[3] = {0.289, 0.274, 0.278};
+    
+    checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
+    checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
+#else
+    checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()* nBatches));
+    mean << 0.408, 0.447, 0.47;
+    stddev << 0.289, 0.274, 0.278;
+#endif
+
+    checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) );
+
+    // Alloc array used in the kernel 
+    checkCuda( cudaMalloc(&src_out, K *sizeof(float)) );
+    checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) );
+
+    dst2.at<float>(0,0)=width * 0.5;
+    dst2.at<float>(0,1)=width * 0.5;
+    dst2.at<float>(1,0)=width * 0.5;
+    dst2.at<float>(1,1)=width * 0.5 +  width * -0.5; 
+    
+    dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
+    dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
+    
+}
+
+
+void CenternetDetection::preprocess(cv::Mat &frame, const int bi){
+     // -----------------------------------pre-process ------------------------------------------
+    
+    // auto start_t = std::chrono::steady_clock::now();
+    // auto step_t = std::chrono::steady_clock::now();
+    // auto end_t = std::chrono::steady_clock::now();
+    cv::Size sz = originalSize[bi];
+    // std::cout<<"image: "<<sz.width<<", "<<sz.height<<std::endl;
+    cv::Size sz_old;
+    float scale = 1.0;
+    float new_height = sz.height * scale;
+    float new_width = sz.width * scale;
+    if(sz.height != sz_old.height && sz.width != sz_old.width){
+        float c[] = {new_width / 2.0f, new_height /2.0f};
+        float s[2];
+
+        if(sz.width > sz.height){
+            s[0] = sz.width * 1.0;
+            s[1] = sz.width * 1.0;
+        }
+        else{
+            s[0] = sz.height * 1.0;    
+            s[1] = sz.height * 1.0;    
+        }
+
+        // ----------- get_affine_transform
+        // rot_rad = pi * 0 / 100 --> 0
+        
+        src.at<float>(0,0)=c[0];
+        src.at<float>(0,1)=c[1];
+        src.at<float>(1,0)=c[0];
+        src.at<float>(1,1)=c[1] + s[0] * -0.5;
+        dst.at<float>(0,0)=netRT->input_dim.w * 0.5;
+        dst.at<float>(0,1)=netRT->input_dim.h * 0.5;
+        dst.at<float>(1,0)=netRT->input_dim.w * 0.5;
+        dst.at<float>(1,1)=netRT->input_dim.h * 0.5 +  netRT->input_dim.w * -0.5; 
+        
+        src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
+        src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
+        dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
+        dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
+
+        trans = cv::getAffineTransform( src, dst );
+        // end_t = std::chrono::steady_clock::now();
+        // std::cout << " TIME gett affine trans: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+        // step_t = end_t;
+
+        trans2 = cv::getAffineTransform( dst2, src );
+        // end_t = std::chrono::steady_clock::now();
+        // std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+        // step_t = end_t;
+    }
+    sz_old = sz;
+#ifdef OPENCV_CUDACONTRIB
+    cv::cuda::GpuMat im_Orig; 
+    cv::cuda::GpuMat imageF1_d, imageF2_d;
+        
+    im_Orig = cv::cuda::GpuMat(frame);
+    cv::cuda::resize (im_Orig, imageF1_d, cv::Size(new_width, new_height)); 
+    checkCuda( cudaDeviceSynchronize() );
+    
+    sz = imageF1_d.size();
+    // std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+    
+    cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
+    checkCuda( cudaDeviceSynchronize() );
+    
+    imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0); 
+    checkCuda( cudaDeviceSynchronize() );
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME convert: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+    
+    dim2 = dim;
+    cv::cuda::GpuMat bgr[3]; 
+    cv::cuda::split(imageF1_d,bgr);//split source
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME split: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    for(int i=0; i<dim.c; i++)
+        checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
+        
+    normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
+    
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME normalize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+    
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+#else
+ 
+    cv::Mat imageF;
+    resize(frame, imageF, cv::Size(new_width, new_height));
+    sz = imageF.size();
+    // std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    cv::Mat trans = cv::getAffineTransform( src, dst );
+    cv::warpAffine(imageF, imageF, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME warpAffine: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    sz = imageF.size();
+    // std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
+    imageF.convertTo(imageF, CV_32FC3, 1/255.0); 
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME convertto: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    dim2 = dim;
+    //split channels
+    cv::Mat bgr[3]; 
+    cv::split(imageF,bgr);//split source
+    for(int i=0; i<3; i++){
+        bgr[i] = bgr[i] - mean[i];
+        bgr[i] = bgr[i] / stddev[i];
+    }
+
+    //write channels
+    for(int i=0; i<dim2.c; i++) {
+        int idx = i*imageF.rows*imageF.cols;
+        int ch = dim2.c-3 +i;
+        // std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
+        memcpy((void*)&input[idx+ netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
+    }
+    checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
+#endif
+}
+
+void CenternetDetection::postprocess(const int bi, const bool mAP){
+    dnnType *rt_out[4];
+    rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
+    rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
+    rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi; 
+    rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi; 
+
+    // auto start_t = std::chrono::steady_clock::now();
+    // auto step_t = std::chrono::steady_clock::now();
+    // auto end_t = std::chrono::steady_clock::now();
+    // ------------------------------------ process --------------------------------------------
+    activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
+    checkCuda( cudaDeviceSynchronize() );
+
+    subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME threshold: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+    // ----------- nms end
+    // ----------- topk  
+      
+    if(K > dim_hm.h * dim_hm.w){
+        printf ("Error topk (K is too large)\n");
+        return;
+    }
+    
+    checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );    
+
+    sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
+    checkCuda( cudaDeviceSynchronize() );   
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME sort: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
+    checkCuda( cudaDeviceSynchronize() );    
+
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME topk: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+        
+    checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
+
+    topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME topk x y clses 2: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
+    checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
+    
+    checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
+    
+    // ----------- topk end 
+    
+    topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out);
+    // checkCuda( cudaDeviceSynchronize() );
+    
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME add offset: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+    
+    bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out);
+    // checkCuda( cudaDeviceSynchronize() );
+    
+    checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); 
+    checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); 
+    checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); 
+    checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) ); 
+    
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME bboxes: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+
+    // ---------------------------------- post-process -----------------------------------------
+    
+    // --------- ctdet_post_process
+    // --------- transform_preds 
+    cv::Mat new_pt1(cv::Size(1,2), CV_32F);
+    cv::Mat new_pt2(cv::Size(1,2), CV_32F); 
+
+    for(int i = 0; i<K; i++){
+        new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx0[i] +
+                                static_cast<float>(trans2.at<double>(0,1))*bby0[i] +
+                                static_cast<float>(trans2.at<double>(0,2))*1.0;
+        new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx0[i] +
+                                static_cast<float>(trans2.at<double>(1,1))*bby0[i] +
+                                static_cast<float>(trans2.at<double>(1,2))*1.0;
+
+        new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*bbx1[i] +
+                                static_cast<float>(trans2.at<double>(0,1))*bby1[i] +
+                                static_cast<float>(trans2.at<double>(0,2))*1.0;
+        new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*bbx1[i] +
+                                static_cast<float>(trans2.at<double>(1,1))*bby1[i] +
+                                static_cast<float>(trans2.at<double>(1,2))*1.0;
+
+        target_coords[i*4] = new_pt1.at<float>(0,0);
+        target_coords[i*4+1] = new_pt1.at<float>(0,1);
+        target_coords[i*4+2] = new_pt2.at<float>(0,0);
+        target_coords[i*4+3] = new_pt2.at<float>(0,1);
+    }
+       
+    detected.clear();
+    for(int i = 0; i<classes; i++){
+        for(int j=0; j<K; j++)
+            if(clses[j] == i){
+                if(scores[j] > confThreshold){
+                    // std::cout<<"th: "<<scores[j]<<" - cl: "<<clses[j]<<" i: "<<i<<std::endl;
+                    //add coco bbox
+                    //det[0:4], i, det[4]
+                    int x0   = target_coords[j*4];
+                    int y0   = target_coords[j*4+1];
+                    int x1   = target_coords[j*4+2];
+                    int y1   = target_coords[j*4+3];
+                    int obj_class = clses[j];
+                    float prob = scores[j];
+                    // std::cout<<"("<<x0<<", "<<y0<<"),("<<x1<<", "<<y1<<")"<<std::endl;
+                    tk::dnn::box res;
+                    res.cl = obj_class;
+                    res.prob = prob;
+                    res.x = x0;
+                    res.y = y0;
+                    res.w = x1 - x0;
+                    res.h = y1 - y0;
+                    detected.push_back(res);
+                }
+            }
+    }
+    
+    batchDetected.push_back(detected);
+    // end_t = std::chrono::steady_clock::now();
+    // std::cout << " TIME detections: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << "  us" << std::endl;
+    // step_t = end_t;
+}
+
+
+}}
+
+

src/Conv2d.cpp

@@ -4,83 +4,174 @@
 
 namespace tk { namespace dnn {
 
-Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW, 
+void Conv2d::initCUDNN(bool back) {
+
+    cudnnTensorDescriptor_t srcTensor = srcTensorDesc;
+    cudnnTensorDescriptor_t dstTensor = dstTensorDesc;
+
+    dataDim_t idim, odim;
+    if(!back) {
+        idim = input_dim;
+        odim = output_dim;
+    } else {
+        idim = output_dim;
+        odim = input_dim;
+    }
+
+    checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) );
+    checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) );
+    checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) );
+
+    // input tensor dim
+    checkCUDNN( cudnnSetTensor4dDescriptor(srcTensor,
+                                           net->tensorFormat, net->dataType, idim.n, idim.c, idim.h, idim.w) );
+
+    checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc,
+                                           net->dataType, net->tensorFormat, odim.c, idim.c/groups,
+                                           kernelH, kernelW) );
+
+    checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc,
+                                                paddingH, paddingW, // padding
+                                                strideH, strideW, // stride
+                                                1,1, // upscale
+                                                CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) );
+
+    checkCUDNN(  cudnnSetConvolutionGroupCount(convDesc,
+                                                groups) );
+
+    // check dimension of convolution output
+    dataDim_t tmpdim;
+    checkCUDNN( cudnnGetConvolution2dForwardOutputDim(
+            convDesc, srcTensor, filterDesc,
+            &tmpdim.n, &tmpdim.c, &tmpdim.h, &tmpdim.w) );
+
+    if(odim.n != tmpdim.n || odim.c != tmpdim.c || odim.h != tmpdim.h || odim.w != tmpdim.w) {
+        std::cout<<"tkdim input: "; idim.print();
+        std::cout<<"tkdim output: "; odim.print();
+        std::cout<<"cudnndim: "; tmpdim.print();
+        FatalError("Error conv dimension mismatch");
+    }
+
+    checkCUDNN( cudnnSetTensor4dDescriptor(dstTensor,
+                                           net->tensorFormat, net->dataType, odim.n, odim.c, odim.h, odim.w) );
+
+    checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc,
+                                           net->tensorFormat, net->dataType,
+                                           1, output_dim.c, 1, 1) );
+
+    // init workspace
+    workSpace = NULL;
+    ws_sizeInBytes = 0;
+    if(back) {
+        checkCUDNN( cudnnGetConvolutionBackwardDataAlgorithm(net->cudnnHandle,
+                                                        filterDesc, dstTensor, convDesc, srcTensor,
+                                                        CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST, 0, &bwAlgo) );
+        checkCUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(net->cudnnHandle,
+                                                           filterDesc, dstTensor, convDesc, srcTensor,
+                                                           bwAlgo, &ws_sizeInBytes));
+
+        // invert tensors
+        srcTensorDesc = dstTensor;
+        dstTensorDesc = srcTensor;
+    } else {
+        checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle,
+                                                        srcTensor, filterDesc, convDesc, dstTensor,
+                                                        CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) );
+        checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
+                                                           srcTensor, filterDesc, convDesc, dstTensor,
+                                                           algo, &ws_sizeInBytes));
+    }
+}
+
+void Conv2d::inferCUDNN(dnnType* srcData, bool back) {
+
+    dnnType alpha = dnnType(1);
+    dnnType beta  = dnnType(0);
+    if(back) {
+        checkCUDNN(cudnnConvolutionBackwardData(net->cudnnHandle,
+                                                &alpha, filterDesc, data_d,
+                                                srcTensorDesc, srcData,
+                                                convDesc, bwAlgo, workSpace, ws_sizeInBytes,
+                                                &beta, dstTensorDesc, dstData));
+    } else {
+        checkCUDNN(cudnnConvolutionForward(net->cudnnHandle,
+                                           &alpha, srcTensorDesc, srcData, filterDesc,
+                                           data_d, convDesc, algo, workSpace, ws_sizeInBytes,
+                                           &beta, dstTensorDesc, dstData));
+    }
+
+    if(!batchnorm && !additional_bias) { //CHECK WITH IF CORRECT
+        // bias
+        alpha = dnnType(1);
+        beta  = dnnType(1);
+        checkCUDNN( cudnnAddTensor(net->cudnnHandle,
+                                   &alpha, biasTensorDesc, bias_d,
+                                   &beta, dstTensorDesc, dstData) );
+    } else {
+        if(additional_bias)
+        {
+            alpha = dnnType(1);
+            beta  = dnnType(1);
+            checkCUDNN( cudnnAddTensor(net->cudnnHandle,
+                                        &alpha, biasTensorDesc, bias2_d,
+                                        &beta, dstTensorDesc, dstData) );
+        }
+        if(batchnorm)
+        {
+            alpha = dnnType(1);
+            beta  = dnnType(0);
+            checkCUDNN( cudnnBatchNormalizationForwardInference(net->cudnnHandle,
+                                                    CUDNN_BATCHNORM_SPATIAL, &alpha, &beta,
+                                                    dstTensorDesc, dstData, dstTensorDesc,
+                                                    dstData, biasTensorDesc, //same tensor descriptor as bias
+                                                    scales_d, bias_d, mean_d, variance_d,
+                                                    TKDNN_BN_MIN_EPSILON) );
+        }
+    }
+}
+
+Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
                 int strideH, int strideW, int paddingH, int paddingW,
-                std::string fname_weights, bool batchnorm) : 
+                std::string fname_weights, bool batchnorm, bool deConv, int groups, bool additional_bias) :
     
     LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, 
-             fname_weights, batchnorm) {
-
+             fname_weights, batchnorm, additional_bias, deConv, groups) {
     this->kernelH = kernelH;
     this->kernelW = kernelW;
     this->strideH = strideH;
     this->strideW = strideW;
     this->paddingH = paddingH;
     this->paddingW = paddingW;
+    this->deConv = deConv;
+    this->groups = groups;
+    this->additional_bias = additional_bias;
 
-    checkCUDNN( cudnnCreateFilterDescriptor(&filterDesc) );
-    checkCUDNN( cudnnCreateConvolutionDescriptor(&convDesc) );
-    checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) );
-
-    int n = input_dim.n;
-    int c = input_dim.c;
-    int h = input_dim.h;
-    int w = input_dim.w;
-
-    checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
-                net->tensorFormat, net->dataType, n, c, h, w) );
-
-    checkCUDNN( cudnnSetFilter4dDescriptor(filterDesc,
-                net->dataType, net->tensorFormat, out_ch, input_dim.c, 
-                kernelH, kernelW) );
-
-    checkCUDNN( cudnnSetConvolution2dDescriptor(convDesc,
-                paddingH, paddingW, // padding
-                strideH, strideW, // stride
-                1,1, // upscale
-                CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT) );
-
-    // find dimension of convolution output
-    checkCUDNN( cudnnGetConvolution2dForwardOutputDim(
-                convDesc, srcTensorDesc, filterDesc,
-                &n, &c, &h, &w) );
-
-    checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
-                net->tensorFormat, net->dataType, n, c, h, w) );
-                
-    checkCUDNN( cudnnGetConvolutionForwardAlgorithm(net->cudnnHandle,
-                srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
-                CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, 0, &algo) );
- 
-    workSpace = NULL;
-    ws_sizeInBytes = 0;
-
-    checkCUDNN( cudnnGetConvolutionForwardWorkspaceSize(net->cudnnHandle,
-                srcTensorDesc, filterDesc, convDesc, dstTensorDesc,
-                algo, &ws_sizeInBytes) );
+    if(!deConv) {
+        output_dim.n = input_dim.n;
+        output_dim.c = out_ch;
+        output_dim.h = (input_dim.h + 2 * paddingH - kernelH) / strideH + 1;
+        output_dim.w = (input_dim.w + 2 * paddingW - kernelW) / strideW + 1;
+        output_dim.l = 1;
+    } else {
+        output_dim.n = input_dim.n;
+        output_dim.c = out_ch;
+        output_dim.h = ((input_dim.h-1) * strideH) - 2*paddingH + kernelH;
+        output_dim.w = ((input_dim.w-1) * strideW) - 2*paddingW + kernelW;
+        output_dim.l = 1;
+    }
+    initCUDNN(deConv);
 
+    // allocate warkspace
     if (ws_sizeInBytes!=0) {
         checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) );
     }
 
-
-    checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc,
-                net->tensorFormat, net->dataType,
-                1, out_ch, 1, 1) );
-
-
-    output_dim.n = n;
-    output_dim.c = c;
-    output_dim.h = h;
-    output_dim.w = w;
-    output_dim.l = 1;
-
     //allocate data for infer result
     checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 }
 
 Conv2d::~Conv2d() {
-    
+
     checkCUDNN( cudnnDestroyFilterDescriptor(filterDesc) );
     checkCUDNN( cudnnDestroyConvolutionDescriptor(convDesc) );
     checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) );
@@ -93,35 +184,25 @@ Conv2d::~Conv2d() {
 
 dnnType* Conv2d::infer(dataDim_t &dim, dnnType* srcData) {
 
+    if(deConv) {
+        FatalError("you must use DeConv class for Deconvolutional layers");
+    }
 
     // convolution
-    dnnType alpha = dnnType(1);
-    dnnType beta  = dnnType(0);
-    checkCUDNN( cudnnConvolutionForward(net->cudnnHandle,
-                &alpha, srcTensorDesc, srcData, filterDesc,
-                data_d, convDesc, algo, workSpace, ws_sizeInBytes,
-                &beta, dstTensorDesc, dstData) );
+    inferCUDNN(srcData, false);
 
-    if(!batchnorm) {
-        // bias
-        alpha = dnnType(1);
-        beta  = dnnType(1);
-        checkCUDNN( cudnnAddTensor(net->cudnnHandle,
-                    &alpha, biasTensorDesc, bias_d,
-                    &beta, dstTensorDesc, dstData) );
-    } else {
-        float one = 1;
-        float zero = 0;
-        cudnnBatchNormalizationForwardInference(net->cudnnHandle,
-            CUDNN_BATCHNORM_SPATIAL, &one, &zero, 
-            dstTensorDesc, dstData, dstTensorDesc, 
-            dstData, biasTensorDesc, //same tensor descriptor as bias 
-            scales_d, bias_d, mean_d, variance_d, 
-            TKDNN_BN_MIN_EPSILON);
-    }
     //update data dimensions    
     dim = output_dim;
+    return dstData;
+}
 
+dnnType* DeConv2d::infer(dataDim_t &dim, dnnType* srcData) {
+
+    // convolution
+    inferCUDNN(srcData, true);
+
+    //update data dimensions
+    dim = output_dim;
     return dstData;
 }
 

src/DeformConv2d.cpp

@@ -0,0 +1,142 @@
+#include <iostream>
+
+#include "Layer.h"
+#include "kernels.h"
+#include <math.h>
+
+
+namespace tk { namespace dnn {
+
+void DeformConv2d::initCUDNN() {
+
+    stat = cublasCreate(&handle);
+    if (stat != CUBLAS_STATUS_SUCCESS) 
+        FatalError("CUBLAS initialization failed\n");
+    
+    checkCUDNN( cudnnCreateTensorDescriptor(&biasTensorDesc) );
+    checkCUDNN( cudnnSetTensor4dDescriptor(biasTensorDesc,
+                                           net->tensorFormat, net->dataType,
+                                           1, output_dim.c, 1, 1) );
+
+    checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
+                                    net->tensorFormat, net->dataType, output_dim.n, output_dim.c, output_dim.h, output_dim.w));
+
+    const int height_ones = (preconv->input_dim.h + 2 * this->paddingH - (1 * (this->kernelH - 1) + 1)) / this->strideH + 1;
+    const int width_ones = (preconv->input_dim.w + 2 * this->paddingW - (1 * (this->kernelW - 1) + 1)) / this->strideW + 1;
+    const int dim_ones = preconv->input_dim.c * this->kernelH * this->kernelW * 1 * height_ones * width_ones;
+
+    int dst_dim = preconv->output_dim.tot();
+    if( dst_dim % 3 != 0 )
+        FatalError("DeformConv2d: the Conv2d output is not divisible by three");
+    chunk_dim = dst_dim/3;
+    checkCuda( cudaMalloc(&offset, 2*chunk_dim*sizeof(dnnType)));
+    checkCuda( cudaMalloc(&mask, chunk_dim*sizeof(dnnType)));
+    
+    // kernel ones
+    checkCuda( cudaMalloc(&ones_d1, (height_ones*width_ones)*sizeof(dnnType)) );
+    dnnType *ones_h1;
+    checkCuda( cudaMallocHost(&ones_h1, (height_ones*width_ones)*sizeof(dnnType)) );
+    for(int i=0; i<height_ones*width_ones; i++)
+        ones_h1[i]=1.0f;
+    checkCuda( cudaMemcpy(ones_d1, ones_h1, (height_ones*width_ones)*sizeof(dnnType), cudaMemcpyHostToDevice) );
+    checkCuda( cudaFreeHost(ones_h1) );
+    checkCuda( cudaMalloc(&ones_d2, dim_ones*sizeof(dnnType)) );
+    dnnType *ones_h2;
+    checkCuda( cudaMallocHost(&ones_h2, dim_ones*sizeof(dnnType)) );
+    for(int i=0; i<dim_ones; i++)
+        ones_h2[i]=1.0f;
+    checkCuda( cudaMemcpy(ones_d2, ones_h2, (dim_ones)*sizeof(dnnType), cudaMemcpyHostToDevice) );
+    checkCuda( cudaFreeHost(ones_h2) );
+    checkCuda( cudaDeviceSynchronize() );
+}
+
+DeformConv2d::DeformConv2d( Network *net, int out_ch, int deformable_group, int kernelH, int kernelW,
+                int strideH, int strideW, int paddingH, int paddingW,
+                std::string d_fname_weights, std::string fname_weights, bool batchnorm) : 
+
+    LayerWgs(net, net->getOutputDim().c, out_ch, kernelH, kernelW, 1, 
+             d_fname_weights, batchnorm, true) {
+    this->out_ch = out_ch;
+    this->deformableGroup = deformable_group;
+    this->kernelH = kernelH;
+    this->kernelW = kernelW;
+    this->strideH = strideH;
+    this->strideW = strideW;
+    this->paddingH = paddingH;
+    this->paddingW = paddingW;
+    
+    preconv = new tk::dnn::Conv2d(net, deformable_group * 3 * kernelH * kernelW, kernelH, kernelW,
+                strideH, strideW, paddingH, paddingW, fname_weights, false);
+    net->num_layers--;
+    
+    output_dim = preconv->output_dim;
+    
+    output_dim.c = out_ch;
+    initCUDNN();
+    //allocate data for infer result
+    checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
+}
+
+DeformConv2d::~DeformConv2d() {
+    checkCUDNN( cudnnDestroyTensorDescriptor(biasTensorDesc) );
+    checkCuda( cudaFree(dstData) );
+    checkCuda( cudaFree(ones_d1) );
+    checkCuda( cudaFree(ones_d2) );
+    checkCuda( cudaFree(offset) );
+    checkCuda( cudaFree(mask) );
+    checkCuda( cudaFree(output_conv) );
+    cublasDestroy(handle);
+}
+
+dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) {
+
+    // conv2d 
+    output_conv = preconv->infer(dim, srcData);
+    // split conv2d outputs into offset and mask
+    checkCuda(cudaMemcpy(offset, output_conv, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); 
+    checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice)); 
+    // kernel sigmoide
+    activationSIGMOIDForward(mask, mask, chunk_dim);
+
+    // deformable convolution
+    dcnV2CudaForward(stat, handle, 
+                         srcData, this->data_d,
+                         this->bias2_d, ones_d1,
+                         offset, mask,                      
+                         dstData, ones_d2,
+                         this->kernelH, this->kernelW,
+                         this->strideH, this->strideW,
+                         this->paddingH, this->paddingW,
+                         1, 1,
+                         this->deformableGroup, 0, //batch_id for cudnn is set to 0 (no batch)
+                         preconv->input_dim.n, preconv->input_dim.c, preconv->input_dim.h, preconv->input_dim.w,
+                         this->output_dim.n, this->output_dim.c, this->output_dim.h, this->output_dim.w,
+                         chunk_dim);
+
+    dnnType alpha = dnnType(1);
+    dnnType beta  = dnnType(0);
+    if(!batchnorm) {
+        // bias
+        alpha = dnnType(1);
+        beta  = dnnType(1);
+        checkCUDNN( cudnnAddTensor(net->cudnnHandle,
+                                   &alpha, biasTensorDesc, bias_d,
+                                   &beta, dstTensorDesc, dstData) );
+    } else {
+        alpha = dnnType(1);
+        beta  = dnnType(0);
+        checkCUDNN( cudnnBatchNormalizationForwardInference(net->cudnnHandle,
+                                                CUDNN_BATCHNORM_SPATIAL, &alpha, &beta,
+                                                dstTensorDesc, dstData, dstTensorDesc,
+                                                dstData, biasTensorDesc, //same tensor descriptor as bias
+                                                scales_d, bias_d, mean_d, variance_d,
+                                                TKDNN_BN_MIN_EPSILON) );
+    }
+
+    //update data dimensions
+    dim = output_dim;
+    return dstData;
+}
+
+
+}}

src/Int8BatchStream.cpp

@@ -0,0 +1,172 @@
+#include "Int8BatchStream.h"
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/dnn/dnn.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+BatchStream::BatchStream(tk::dnn::dataDim_t dim, int batchSize, int maxBatches, const std::string& fileimglist, const std::string& filelabellist) {
+    mBatchSize = batchSize;
+    mMaxBatches = maxBatches;
+    mDims = nvinfer1::DimsNCHW{ dim.n, dim.c, dim.h, dim.w };
+    mHeight = dim.h;
+    mWidth = dim.w;
+    mImageSize = mDims.c()*mDims.h()*mDims.w();
+    mBatch.resize(mBatchSize*mImageSize, 0);
+    mLabels.resize(mBatchSize, 0);
+    mFileBatch.resize(mDims.n()*mImageSize, 0);
+    mFileLabels.resize(mDims.n(), 0);
+    mFileImgList = fileimglist;
+    readInListFile(fileimglist, mListImg);
+    mFileLabelList = filelabellist;
+    readInListFile(filelabellist, mListLabel);
+
+    reset(0);
+}
+
+void BatchStream::reset(int firstBatch) {
+    mBatchCount = 0;
+    mFileCount = 0;
+    mFileBatchPos = mDims.n();
+    skip(firstBatch);
+}
+
+bool BatchStream::next() {
+    std::cout<<"Next batch: "<<mBatchCount<<" of "<<mMaxBatches<<"\n";
+    if (mBatchCount == mMaxBatches-1)
+        return false;
+
+    for (int csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize) {
+        assert(mFileBatchPos > 0 && mFileBatchPos <= mDims.n());
+        if (mFileBatchPos == mDims.n() && !update())
+            return false;
+
+        csize = std::min(mBatchSize - batchPos, mDims.n() - mFileBatchPos);
+        std::copy_n(getFileBatch() + mFileBatchPos * mImageSize, csize * mImageSize, getBatch() + batchPos * mImageSize);
+        std::copy_n(getFileLabels() + mFileBatchPos, csize, getLabels() + batchPos);
+    }
+    mBatchCount++;
+    return true;
+}
+
+void BatchStream::skip(int skipCount) {
+    if (mBatchSize >= mDims.n() && mBatchSize%mDims.n() == 0 && mFileBatchPos == mDims.n()) {
+        mFileCount += skipCount * mBatchSize / mDims.n();
+        return;
+    }
+
+    int x = mBatchCount;
+    for (int i = 0; i < skipCount; i++)
+        next();
+    mBatchCount = x;
+}
+
+void BatchStream::readInListFile(const std::string& dataFilePath, std::vector<std::string>& mListIn) {
+    // dataFilePath contains the list of image paths
+    int count = 0;
+    FILE* f = fopen(dataFilePath.c_str(), "r");
+    if (!f)
+        FatalError("failed to open " + dataFilePath);
+    
+    char str[512];
+    while (fgets(str, 512, f) != NULL) {
+        for (int i = 0; str[i] != '\0'; ++i) {
+            if (str[i] == '\n'){
+                str[i] = '\0';
+                break;
+            }
+        }
+        count ++;
+        mListIn.push_back(str);
+        if(count == mMaxBatches)
+            break;
+    }
+    fclose(f);
+}
+
+void BatchStream::readCVimage(std::string inputFileName, std::vector<float>& res, bool fixshape) {
+    // unaltered original DsImage
+    cv::Mat m_OrigImage;
+    // letterboxed DsImage given to the network as input
+    cv::Mat m_LetterboxImage;
+    m_OrigImage = cv::imread(inputFileName, cv::IMREAD_COLOR);
+
+    if (!m_OrigImage.data || m_OrigImage.cols <= 0 || m_OrigImage.rows <= 0)
+        FatalError("Unable to open " + inputFileName);
+
+    int m_Height = m_OrigImage.rows;
+    int m_Width = m_OrigImage.cols;
+    if(fixshape) {
+        m_Height = mHeight;
+        m_Width = mWidth;
+    }
+    std::cout<<"image is "<<inputFileName<<": "<<m_Height<<" * "<<m_Width<<std::endl;
+    // resize the DsImage with scale
+    float dim = std::max(m_Height, m_Width);
+    int resizeH = ((m_Height / dim) * m_Height);
+    int resizeW = ((m_Width / dim) * m_Width);
+    float m_ScalingFactor = static_cast<float>(resizeH) / static_cast<float>(m_Height);
+
+    // Additional checks for images with non even dims
+    if ((m_Width - resizeW) % 2) resizeW--;
+    if ((m_Height - resizeH) % 2) resizeH--;
+    assert((m_Width - resizeW) % 2 == 0);
+    assert((m_Height - resizeH) % 2 == 0);
+
+    int m_XOffset = (m_Width - resizeW) / 2;
+    int m_YOffset = (m_Height - resizeH) / 2;
+
+    assert(2 * m_XOffset + resizeW == m_Width);
+    assert(2 * m_YOffset + resizeH == m_Height);
+
+    // resizing
+    cv::resize(m_OrigImage, m_LetterboxImage, cv::Size(resizeW, resizeH), 0, 0, cv::INTER_CUBIC);
+    // letterboxing
+    cv::copyMakeBorder(m_LetterboxImage, m_LetterboxImage, m_YOffset, m_YOffset, m_XOffset,
+                    m_XOffset, cv::BORDER_CONSTANT, cv::Scalar(128, 128, 128));
+    m_LetterboxImage.convertTo(m_LetterboxImage, CV_32FC3, 1 / 255.0);
+    // converting to RGB and NCHW format
+    m_LetterboxImage = cv::dnn::blobFromImage(m_LetterboxImage);
+    res.assign(m_LetterboxImage.begin<float>(), m_LetterboxImage.end<float>());
+}
+
+void BatchStream::readLabels(std::string inputFileName, std::vector<float>& ris) {
+    std::ifstream is(inputFileName.c_str());
+    //read only the first number: the image sub-portion class
+    while (true) {
+        float val;
+		is >> val;
+        if (!is) {
+            break;
+        }
+        // insert the first number and skip all others
+        ris.push_back(val);
+        while( true ) {
+            char c;
+            is >> c;
+            if (is.peek() == '\n') //detect "\n"
+                break;
+        }   
+    }
+}
+
+bool BatchStream::update() {
+    std::string imgFileName = mListImg[mFileCount];
+    std::string labelFileName = mListLabel[mFileCount];
+    mFileCount++;
+
+    //read image
+    mFileBatch.clear();
+    readCVimage(imgFileName, mFileBatch);
+    // std::transform(
+    //     singleImg_rawData.begin(), singleImg_rawData.end(), mFileBatch.begin(), [](uint8_t val) { return static_cast<float>(val); });
+    
+    //read label
+    mFileLabels.clear();
+    readLabels(labelFileName, mFileLabels);
+    // std::transform(
+    //     singleLabels_rawData.begin(), singleLabels_rawData.end(), mFileLabels.begin(), [](uint8_t val) { return static_cast<float>(val); });
+    
+    mFileBatchPos = 0;
+    return true;
+}

src/Int8Calibrator.cpp

@@ -0,0 +1,46 @@
+#include "Int8Calibrator.h"
+
+Int8EntropyCalibrator::Int8EntropyCalibrator(BatchStream& stream, int firstBatch, 
+                                             const std::string& calibTableFilePath,
+                                             const std::string& inputBlobName,
+                                             bool readCache): 
+    mStream(stream), 
+    mCalibTableFilePath(calibTableFilePath),
+    mInputBlobName(inputBlobName.c_str()),
+    mReadCache(readCache) {
+    nvinfer1::DimsNCHW dims = mStream.getDims();
+    mInputCount = mStream.getBatchSize() * dims.c() * dims.h() * dims.w();
+    checkCuda(cudaMalloc(&mDeviceInput, mInputCount * sizeof(float)));
+    mStream.reset(firstBatch);
+}
+
+bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int nbBindings) {
+    if (!mStream.next())
+        return false;
+
+    checkCuda(cudaMemcpy(mDeviceInput, mStream.getBatch(), mInputCount * sizeof(float), cudaMemcpyHostToDevice));
+    assert(!strcmp(names[0], mInputBlobName.c_str()));
+    bindings[0] = mDeviceInput;
+    return true;
+}
+
+const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) {
+    mCalibrationCache.clear();
+    assert(!mCalibTableFilePath.empty());
+    std::ifstream input(mCalibTableFilePath, std::ios::binary);
+    input >> std::noskipws;
+    input >> std::noskipws;
+    if (mReadCache && input.good())
+        std::copy(std::istream_iterator<char>(input), std::istream_iterator<char>(),
+                  std::back_inserter(mCalibrationCache));
+
+    length = mCalibrationCache.size();
+    return length ? &mCalibrationCache[0] : nullptr;
+}
+
+void Int8EntropyCalibrator::writeCalibrationCache(const void* cache, size_t length) {
+    assert(!mCalibTableFilePath.empty());
+    std::ofstream output(mCalibTableFilePath, std::ios::binary);
+    output.write(reinterpret_cast<const char*>(cache), length);
+    output.close();
+}

src/Layer.cpp

@@ -7,7 +7,7 @@ namespace tk { namespace dnn {
 Layer::Layer(Network *net) {
 
     this->net = net;
-
+    this->final = false;
     if(net != nullptr) {
         this->input_dim = net->getOutputDim();
         this->output_dim = input_dim;
@@ -24,6 +24,11 @@ Layer::~Layer() {
 
     checkCUDNN( cudnnDestroyTensorDescriptor(srcTensorDesc) );
     checkCUDNN( cudnnDestroyTensorDescriptor(dstTensorDesc) );
+
+    if(dstData != nullptr) {
+        cudaFree(dstData);
+        dstData = nullptr;
+    }
 }
 
 }}

src/LayerWgs.cpp

@@ -8,18 +8,25 @@ namespace tk { namespace dnn {
 
 LayerWgs::LayerWgs(Network *net, int inputs, int outputs, 
                    int kh, int kw, int kl, 
-                   std::string fname_weights, bool batchnorm) : Layer(net) {
-
+                   std::string fname_weights, bool batchnorm, bool additional_bias, bool deConv, int groups) : Layer(net) {
+    inputs = inputs/groups;
+        
     this->inputs  = inputs;
     this->outputs = outputs;    
     this->weights_path  = std::string(fname_weights);
-   
+
     std::cout<<"Reading weights: I="<<inputs<<" O="<<outputs<<" KERNEL="<<kh<<"x"<<kw<<"x"<<kl<<"\n";
     int seek = 0;
     readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
     seek += inputs*outputs*kh*kw*kl;
-    readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
+    this->additional_bias = additional_bias;
+    if(additional_bias) {
+        readBinaryFile(weights_path.c_str(), outputs, &bias2_h, &bias2_d, seek);
+        seek += outputs;   
+    }
     
+    readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
+
     this->batchnorm = batchnorm;
     if(batchnorm) {
         seek += outputs;
@@ -52,6 +59,14 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
     float2half(data_d, data16_d, w_size);
     cudaMemcpy(data16_h, data16_d, w_size*sizeof(__half), cudaMemcpyDeviceToHost);
 
+    if(additional_bias){
+        int b2_size = outputs;
+        bias216_h = new __half[b2_size];
+        cudaMalloc(&bias216_d, w_size*sizeof(__half));
+        float2half(bias2_d, bias216_d, b2_size);
+        cudaMemcpy(bias216_h, bias216_d, b2_size*sizeof(__half), cudaMemcpyDeviceToHost);
+    }
+
     int b_size = outputs;
     bias16_h = new __half[b_size];
     cudaMalloc(&bias16_d, w_size*sizeof(__half));
@@ -80,7 +95,6 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
         cudaMemcpy(power16_h, power16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
 
         //mean array
-
         cudaMemcpy(tmp_d, mean_h, b_size*sizeof(float), cudaMemcpyHostToDevice);
         float2half(tmp_d, mean16_d, b_size);
         cudaMemcpy(mean16_h, mean16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
@@ -94,24 +108,14 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
         //conver scales
         float2half(scales_d, scales16_d, b_size);
         cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
+    
+        cudaFree(tmp_d);
     }
 }
 
 LayerWgs::~LayerWgs() {
-
-    delete [] data_h;
-    delete [] bias_h;
-    checkCuda( cudaFree(data_d) );
-    checkCuda( cudaFree(bias_d) );
-
-    if(batchnorm) {
-        delete [] scales_h;
-        delete [] mean_h;
-        delete [] variance_h;
-        checkCuda( cudaFree(scales_d) );
-        checkCuda( cudaFree(mean_d) );
-        checkCuda( cudaFree(variance_d) );
-    }
+    releaseHost();
+    releaseDevice();
 }
 
 }}

src/MobilenetDetection.cpp

@@ -0,0 +1,311 @@
+#include "MobilenetDetection.h"
+
+bool boxProbCmp(const tk::dnn::box &a, const tk::dnn::box &b){
+    return (a.prob > b.prob);
+}
+
+namespace tk{ namespace dnn{
+
+void MobilenetDetection::generate_ssd_priors(const SSDSpec *specs, const int n_specs, bool clamp){
+    nPriors = 0;
+    for (int i = 0; i < n_specs; i++){
+        nPriors += specs[i].featureSize * specs[i].featureSize * 6;
+    }
+
+    priors = (float *)malloc(N_COORDS * nPriors * sizeof(float));
+
+    int i_prio = 0;
+    float scale, x_center, y_center, h, w, size, ratio;
+    int min, max;
+    for (int i = 0; i < n_specs; i++){
+        scale = (float)imageSize / (float)specs[i].shrinkage;
+        min = specs[i].boxHeight > specs[i].boxWidth ? specs[i].boxWidth : specs[i].boxHeight;
+        max = specs[i].boxHeight < specs[i].boxWidth ? specs[i].boxWidth : specs[i].boxHeight;
+        for (int j = 0; j < specs[i].featureSize; j++){
+            for (int k = 0; k < specs[i].featureSize; k++){
+                //small sized square box
+                size = min;
+                x_center = (k + 0.5f) / scale;
+                y_center = (j + 0.5f) / scale;
+                h = w = (float)size / (float)imageSize;
+
+                priors[i_prio * N_COORDS + 0] = x_center;
+                priors[i_prio * N_COORDS + 1] = y_center;
+                priors[i_prio * N_COORDS + 2] = w;
+                priors[i_prio * N_COORDS + 3] = h;
+                ++i_prio;
+
+                //big sized square box
+                size = sqrt(max * min);
+                h = w = (float)size / (float)imageSize;
+
+                priors[i_prio * N_COORDS + 0] = x_center;
+                priors[i_prio * N_COORDS + 1] = y_center;
+                priors[i_prio * N_COORDS + 2] = w;
+                priors[i_prio * N_COORDS + 3] = h;
+                ++i_prio;
+
+                //change h/w ratio of the small sized box
+                size = min;
+                h = w = size / (float)imageSize;
+                ratio = sqrt(specs[i].ratio1);
+                priors[i_prio * N_COORDS + 0] = x_center;
+                priors[i_prio * N_COORDS + 1] = y_center;
+                priors[i_prio * N_COORDS + 2] = w * ratio;
+                priors[i_prio * N_COORDS + 3] = h / ratio;
+                ++i_prio;
+
+                priors[i_prio * N_COORDS + 0] = x_center;
+                priors[i_prio * N_COORDS + 1] = y_center;
+                priors[i_prio * N_COORDS + 2] = w / ratio;
+                priors[i_prio * N_COORDS + 3] = h * ratio;
+                ++i_prio;
+
+                ratio = sqrt(specs[i].ratio2);
+                priors[i_prio * N_COORDS + 0] = x_center;
+                priors[i_prio * N_COORDS + 1] = y_center;
+                priors[i_prio * N_COORDS + 2] = w * ratio;
+                priors[i_prio * N_COORDS + 3] = h / ratio;
+                ++i_prio;
+
+                priors[i_prio * N_COORDS + 0] = x_center;
+                priors[i_prio * N_COORDS + 1] = y_center;
+                priors[i_prio * N_COORDS + 2] = w / ratio;
+                priors[i_prio * N_COORDS + 3] = h * ratio;
+                ++i_prio;
+            }
+        }
+    }
+
+    if (clamp){
+        for (int i = 0; i < nPriors * N_COORDS; i++){
+            priors[i] = priors[i] > 1.0f ? 1.0f : priors[i];
+            priors[i] = priors[i] < 0.0f ? 0.0f : priors[i];
+        }
+    }
+}
+
+void MobilenetDetection::convert_locatios_to_boxes_and_center(){
+    float cur_x, cur_y;
+    for (int i = 0; i < nPriors; i++){
+        locations_h[i * N_COORDS + 0] = locations_h[i * N_COORDS + 0] * centerVariance * priors[i * N_COORDS + 2] + priors[i * N_COORDS + 0];
+        locations_h[i * N_COORDS + 1] = locations_h[i * N_COORDS + 1] * centerVariance * priors[i * N_COORDS + 3] + priors[i * N_COORDS + 1];
+        locations_h[i * N_COORDS + 2] = exp(locations_h[i * N_COORDS + 2] * sizeVariance) * priors[i * N_COORDS + 2];
+        locations_h[i * N_COORDS + 3] = exp(locations_h[i * N_COORDS + 3] * sizeVariance) * priors[i * N_COORDS + 3];
+
+        cur_x = locations_h[i * N_COORDS + 0];
+        cur_y = locations_h[i * N_COORDS + 1];
+
+        locations_h[i * N_COORDS + 0] = cur_x - locations_h[i * N_COORDS + 2] / 2;
+        locations_h[i * N_COORDS + 1] = cur_y - locations_h[i * N_COORDS + 3] / 2;
+        locations_h[i * N_COORDS + 2] = cur_x + locations_h[i * N_COORDS + 2] / 2;
+        locations_h[i * N_COORDS + 3] = cur_y + locations_h[i * N_COORDS + 3] / 2;
+    }
+}
+
+float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){
+    float max_x = a.x > b.x ? a.x : b.x;
+    float max_y = a.y > b.y ? a.y : b.y;
+    float min_w = a.w < b.w ? a.w : b.w;
+    float min_h = a.h < b.h ? a.h : b.h;
+
+    float ao_w = min_w - max_x > 0 ? min_w - max_x : 0;
+    float ao_h = min_h - max_y > 0 ? min_h - max_y : 0;
+
+    float area_overlap = ao_w * ao_h;
+    float area_0_w = a.w - a.x > 0 ? a.w - a.x : 0;
+    float area_0_h = a.h - a.y > 0 ? a.h - a.y : 0;
+
+    float area_1_w = b.w - b.x > 0 ? b.w - b.x : 0;
+    float area_1_h = b.h - b.y > 0 ? b.h - b.y : 0;
+
+    float area_0 = area_0_h * area_0_w;
+    float area_1 = area_1_h * area_1_w;
+
+    float iou = area_overlap / (area_0 + area_1 - area_overlap + 1e-5);
+    return iou;
+}
+
+bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
+    std::cout<<(tensor_path).c_str()<<"\n";
+    netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
+    imageSize = netRT->input_dim.h;
+    classes = n_classes;
+    nBatches = n_batches;
+
+    SSDSpec specs[N_SSDSPEC];
+
+    if(imageSize == 300){
+        specs[0].setAll(19, 16, 60, 105, 2, 3);
+        specs[1].setAll(10, 32, 105, 150, 2, 3);
+        specs[2].setAll(5, 64, 150, 195, 2, 3);
+        specs[3].setAll(3, 100, 195, 240, 2, 3);
+        specs[4].setAll(2, 150, 240, 285, 2, 3);
+        specs[5].setAll(1, 300, 285, 330, 2, 3);
+    }
+    else if(imageSize == 512){
+        specs[0].setAll(32, 16, 60, 105, 2, 3);
+        specs[1].setAll(16, 32, 105, 150, 2, 3);
+        specs[2].setAll(8, 64, 150, 195, 2, 3);
+        specs[3].setAll(4, 100, 195, 240, 2, 3);
+        specs[4].setAll(2, 150, 240, 285, 2, 3);
+        specs[5].setAll(1, 300, 285, 330, 2, 3);
+    }  
+    else{
+        FatalError("Input size for mobilenet not supported");
+    }
+
+    generate_ssd_priors(specs, N_SSDSPEC);
+
+#ifndef OPENCV_CUDACONTRIB
+    checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
+#endif
+    checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
+
+    locations_h = (float *)malloc(N_COORDS * nPriors * sizeof(float));
+    confidences_h = (float *)malloc(nPriors * classes * sizeof(float));
+
+    for (int c = 0; c < classes; c++){
+        int offset = c * 123457 % classes;
+        float r = getColor(2, offset, classes);
+        float g = getColor(1, offset, classes);
+        float b = getColor(0, offset, classes);
+        colors[c] = cv::Scalar(int(255.0 * b), int(255.0 * g), int(255.0 * r));
+    }
+
+    if(classes == 11){ //BDD
+        const char *classes_names_[] = {
+        "person","car","truck","bus","motor","bike","rider","traffic light","traffic sign","train"};
+        classesNames = std::vector<std::string>(classes_names_, std::end(classes_names_));
+    }
+    else if(classes == 21){ //VOC
+        const char *classes_names_[] = {
+        "aeroplane", "bicycle", "bird", "boat", "bottle", "bus",
+        "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike",
+        "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"};
+        classesNames = std::vector<std::string>(classes_names_, std::end(classes_names_));
+
+    }
+    else if (classes == 81){ //COCO
+        const char *classes_names_[] = {
+        "person" , "bicycle" , "car" , "motorbike" , "aeroplane" , "bus" ,
+        "train" , "truck" , "boat" , "traffic light" , "fire hydrant" , "stop sign" , 
+        "parking meter" , "bench" , "bird" , "cat" , "dog" , "horse" , "sheep" , "cow" , 
+        "elephant" , "bear" , "zebra" , "giraffe" , "backpack" , "umbrella" , "handbag" , 
+        "tie" , "suitcase" , "frisbee" , "skis" , "snowboard" , "sports ball" , "kite" , 
+        "baseball bat" , "baseball glove" , "skateboard" , "surfboard" , "tennis racket" , 
+        "bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" , 
+        "apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" , 
+        "donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" , 
+        "toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" , 
+        "cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" , 
+        "book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"};
+        classesNames = std::vector<std::string>(classes_names_, std::end(classes_names_));
+
+    }
+    else{
+        FatalError("Number of classes not supported for mobilenet");
+    }
+    return 1;
+}
+
+void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){
+#ifdef OPENCV_CUDACONTRIB
+        //move original image on GPU
+        cv::cuda::GpuMat orig_img, frame_nomean;
+        orig_img = cv::cuda::GpuMat(frame);
+
+        //resize image, remove mean, divide by std
+        cv::cuda::resize (orig_img, orig_img, cv::Size(netRT->input_dim.w, netRT->input_dim.h)); 
+        orig_img.convertTo(frame_nomean, CV_32FC3, 1, -127);
+        frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 128.0, 0);
+
+        //copy image into tensors
+        cv::cuda::split(imagePreproc, bgr);
+
+        for(int i=0; i < netRT->input_dim.c; i++){
+            int idx = i * imagePreproc.rows * imagePreproc.cols;
+            checkCuda( cudaMemcpy((void *)&input_d[idx + netRT->input_dim.tot()*bi], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols* sizeof(float), cudaMemcpyDeviceToDevice) );
+        }
+#else
+        //resize image, remove mean, divide by std
+        cv::Mat frame_nomean;
+        resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+        frame.convertTo(frame_nomean, CV_32FC3, 1, -127);
+        frame_nomean.convertTo(imagePreproc, CV_32FC3, 1 / 128.0, 0);
+
+        //copy image into tensor and copy it into GPU
+        cv::split(imagePreproc, bgr);
+        for (int i = 0; i < netRT->input_dim.c; i++){
+            int idx = i * imagePreproc.rows * imagePreproc.cols;
+            memcpy((void *)&input[idx + netRT->input_dim.tot()*bi], (void *)bgr[i].data, imagePreproc.rows * imagePreproc.cols * sizeof(dnnType));
+        }
+        checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
+#endif
+}
+
+void MobilenetDetection::postprocess(const int bi, const bool mAP){
+    //get confidences and locations_h
+    dnnType *rt_out[2];
+    rt_out[0] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
+    rt_out[1] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
+
+    detected.clear();
+
+    checkCuda(cudaMemcpy(confidences_h, rt_out[0], nPriors * classes * sizeof(float), cudaMemcpyDeviceToHost));
+    checkCuda(cudaMemcpy(locations_h, rt_out[1], N_COORDS * nPriors * sizeof(float), cudaMemcpyDeviceToHost));
+    convert_locatios_to_boxes_and_center();
+
+    int width =  originalSize[bi].width;
+    int height =  originalSize[bi].height;
+
+    float *conf_per_class;
+    for (int i = 1; i < classes; i++){
+        conf_per_class = &confidences_h[i * nPriors];
+        std::vector<tk::dnn::box> boxes;
+        for (int j = 0; j < nPriors; j++){
+
+            if (conf_per_class[j] > confThreshold){
+                tk::dnn::box b;
+                b.cl = i;
+                b.prob = conf_per_class[j];
+                b.x = locations_h[j * N_COORDS + 0];
+                b.y = locations_h[j * N_COORDS + 1];
+                b.w = locations_h[j * N_COORDS + 2];
+                b.h = locations_h[j * N_COORDS + 3];
+
+                if(mAP)
+                    for(int c=1; c<classes; c++) 
+                        b.probs.push_back(confidences_h[c * nPriors + j]);
+
+                boxes.push_back(b);
+            }
+        }
+        std::sort(boxes.begin(), boxes.end(), boxProbCmp);
+
+        std::vector<tk::dnn::box> remaining;        
+        while (boxes.size() > 0){
+            remaining.clear();
+
+            tk::dnn::box b;
+            b.cl = boxes[0].cl -1 ;             //remove background class
+            b.prob = boxes[0].prob;
+            b.x = boxes[0].x * width; 
+            b.y = boxes[0].y * height;
+            b.w = boxes[0].w * width - b.x;     //convert from x1 to width
+            b.h = boxes[0].h * height - b.y;    //convert from y1 to height
+            detected.push_back(b);
+            for (size_t j = 1; j < boxes.size(); j++){
+                if (iou(boxes[0], boxes[j]) <= IoUThreshold){
+                    remaining.push_back(boxes[j]);
+                }
+            }
+            boxes = remaining;
+        }
+    }
+    batchDetected.push_back(detected);
+}
+
+
+} // namespace dnn
+} // namespace tk

src/Network.cpp

@@ -17,23 +17,40 @@ Network::Network(dataDim_t input_dim) {
              <<", CUDNN v"<<cu_ver<<")\n";
     dataType = CUDNN_DATA_FLOAT;
     tensorFormat = CUDNN_TENSOR_NCHW;
+    dontLoadWeights = false;
     num_layers = 0;
 
     fp16 = false;
     dla = false;
+    int8 = false;
     if(const char* env_p = std::getenv("TKDNN_MODE")) {
         if(strcmp(env_p, "FP16") == 0)
             fp16 = true;
-	else if(strcmp(env_p, "DLA") == 0) {
-    	    dla = true;
-	    fp16 = true;	
-	}
+        else if(strcmp(env_p, "DLA") == 0) {
+            dla = true;
+            fp16 = true;	
+        }
+        else if(strcmp(env_p, "INT8") == 0) {
+            int8 = true;
+        }
     }
+    maxBatchSize = 1;
+    if(const char* env_p = std::getenv("TKDNN_BATCHSIZE")) {
+        maxBatchSize = atoi(env_p);
+    }
+    if(const char* env_p = std::getenv("TKDNN_CALIB_IMG_PATH"))
+        fileImgList = env_p;
+    
+    if(const char* env_p = std::getenv("TKDNN_CALIB_LABEL_PATH"))
+        fileLabelList = env_p;
+    
    
     if(fp16)
-        std::cout<<COL_REDB<<"!! FP16 INERENCE ENABLED !!"<<COL_END<<"\n";
+        std::cout<<COL_REDB<<"!! FP16 INFERENCE ENABLED !!"<<COL_END<<"\n";
     if(dla)
-        std::cout<<COL_GREENB<<"!! DLA INERENCE ENABLED !!"<<COL_END<<"\n";
+        std::cout<<COL_GREENB<<"!! DLA INFERENCE ENABLED !!"<<COL_END<<"\n";
+    if(int8)
+        std::cout<<COL_ORANGEB<<"!! INT8 INFERENCE ENABLED !!"<<COL_END<<"\n";
 
 
     checkCUDNN( cudnnCreate(&cudnnHandle) );
@@ -42,11 +59,16 @@ Network::Network(dataDim_t input_dim) {
 }
 
 Network::~Network() {
-
     checkCUDNN( cudnnDestroy(cudnnHandle) );
     checkERROR( cublasDestroy(cublasHandle) );
 }
 
+void Network::releaseLayers() {
+    for(int i=0; i<num_layers; i++)
+        delete layers[i];
+    num_layers = 0;
+}
+
 dnnType* Network::infer(dataDim_t &dim, dnnType* data) {
 
     //do infer for every layer
@@ -61,6 +83,7 @@ bool Network::addLayer(Layer *l) {
     if(num_layers == MAX_LAYERS)
         return false;
     
+    l->id = num_layers;
     layers[num_layers++] = l;
     return true;
 }
@@ -105,6 +128,36 @@ void Network::print() {
     }
     printCenteredTitle("", '=', 60);
     std::cout<<"\n";
+    printCudaMemUsage();
+}
+const char *Network::getNetworkRTName(const char *network_name){
+    networkName = network_name;
+    int network_name_len = strlen(network_name);
+    char *RTName = (char *)malloc((network_name_len + 9)*sizeof(char));
+    if (fp16){
+        strcpy(RTName, network_name);
+        strcat(RTName, "_fp16.rt");
+        RTName[network_name_len + 8] = '\0';
+    }
+    else if (dla){
+        strcpy(RTName, network_name);
+        strcat(RTName, "_dla.rt");
+        RTName[network_name_len + 7] = '\0';
+    }
+        
+    else if (int8){
+        strcpy(RTName, network_name);
+        strcat(RTName, "_int8.rt");
+        RTName[network_name_len + 8] = '\0';
+    }
+        
+    else{
+        strcpy(RTName, network_name);
+        strcat(RTName, "_fp32.rt");
+        RTName[network_name_len + 8] = '\0';
+    }
+    networkNameRT = RTName;
+    return RTName;
 }
 
 

src/NetworkRT.cpp

@@ -9,6 +9,7 @@
 #include "utils.h"
 #include "NvInfer.h"
 #include "NetworkRT.h"
+#include "Int8Calibrator.h"
 
 using namespace nvinfer1;
 
@@ -35,22 +36,39 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
     builderRT = createInferBuilder(loggerRT);
     std::cout<<"Float16 support: "<<builderRT->platformHasFastFp16()<<"\n";
     std::cout<<"Int8 support: "<<builderRT->platformHasFastInt8()<<"\n";
+#if NV_TENSORRT_MAJOR >= 5
     std::cout<<"DLAs: "<<builderRT->getNbDLACores()<<"\n";
+#endif
     networkRT = builderRT->createNetwork();
-
+#if NV_TENSORRT_MAJOR >= 6                
+        configRT = builderRT->createBuilderConfig();
+#endif
+    
     if(!fileExist(name)) {
+#if NV_TENSORRT_MAJOR >= 6                
+        // Calibrator life time needs to last until after the engine is built.
+        std::unique_ptr<IInt8EntropyCalibrator> calibrator;
 
+        configRT->setAvgTimingIterations(1);
+        configRT->setMinTimingIterations(1);
+        configRT->setMaxWorkspaceSize(1 << 30);
+        configRT->setFlag(BuilderFlag::kDEBUG);
+#endif
         //input and dataType
         dataDim_t dim = net->layers[0]->input_dim;
         dtRT = DataType::kFLOAT;
 
-        builderRT->setMaxBatchSize(1);
+        builderRT->setMaxBatchSize(net->maxBatchSize);
         builderRT->setMaxWorkspaceSize(1 << 30);
 
         if(net->fp16 && builderRT->platformHasFastFp16()) {
             dtRT = DataType::kHALF;
             builderRT->setHalf2Mode(true);
+#if NV_TENSORRT_MAJOR >= 6                
+            configRT->setFlag(BuilderFlag::kFP16);
+#endif
         }
+#if NV_TENSORRT_MAJOR >= 5
         if(net->dla && builderRT->getNbDLACores() > 0) {
             dtRT = DataType::kHALF;
             builderRT->setFp16Mode(true);
@@ -58,8 +76,33 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
             builderRT->setDefaultDeviceType(DeviceType::kDLA);
             builderRT->setDLACore(0);
         }
-       
-        //add input layer
+#endif
+#if NV_TENSORRT_MAJOR >= 6                
+        if(net->int8 && builderRT->platformHasFastInt8()){
+            // dtRT = DataType::kINT8;
+            // builderRT->setInt8Mode(true);
+            configRT->setFlag(BuilderFlag::kINT8);
+            BatchStream calibrationStream(dim, 1, 100,      //TODO: check if 100 images are sufficient to the calibration (or 4951) 
+                                            net->fileImgList, net->fileLabelList);
+            
+            /* The calibTableFilePath contains the path+filename of the calibration table.
+             * Each calibration table can be found in the corresponding network folder (../Test/*).
+             * Each network is located in a folder with the same name as the network.
+             * If the folder has a different name, the calibration table is saved in build/ folder.
+             */
+            std::string calib_table_name = net->networkName + "/" + net->networkNameRT.substr(0, net->networkNameRT.find('.')) + "-calibration.table";
+            std::string calib_table_path = net->networkName;
+            if(!fileExist((const char *)calib_table_path.c_str()))
+                calib_table_name = "./" + net->networkNameRT.substr(0, net->networkNameRT.find('.')) + "-calibration.table";
+
+            calibrator.reset(new Int8EntropyCalibrator(calibrationStream, 1, 
+                                            calib_table_name, 
+                                            "data"));
+            configRT->setInt8Calibrator(calibrator.get());
+        }
+#endif
+        
+        // add input layer
         ITensor *input = networkRT->addInput("data", DataType::kFLOAT, 
                         DimsCHW{ dim.c, dim.h, dim.w});
         checkNULL(input);
@@ -68,12 +111,18 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
         for(int i=0; i<net->num_layers; i++) {
             Layer *l = net->layers[i];
             ILayer *Ilay = convert_layer(input, l);
+#if NV_TENSORRT_MAJOR >= 6                
+            if(net->int8 && builderRT->platformHasFastInt8())
+            {
+                Ilay->setPrecision(DataType::kINT8);
+            }
+#endif
             Ilay->setName( (l->getLayerName() + std::to_string(i)).c_str() );
             
             input = Ilay->getOutput(0);
             input->setName( (l->getLayerName() + std::to_string(i) + "_out").c_str() );
             
-            if(l->getLayerType() == LAYER_YOLO)
+            if(l->final)
                 networkRT->markOutput(*input);
             tensors[l] = input;
         }
@@ -84,10 +133,19 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
         input->setName("out");
         networkRT->markOutput(*input);
 
+        std::cout<<"Selected maxBatchSize: "<<builderRT->getMaxBatchSize()<<"\n";
+        printCudaMemUsage();
         std::cout<<"Building tensorRT cuda engine...\n";
+#if NV_TENSORRT_MAJOR >= 6                
+        engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT);
+#else 
         engineRT = builderRT->buildCudaEngine(*networkRT);
+#endif
+        if(engineRT == nullptr)
+            FatalError("cloud not build cuda engine")
         // we don't need the network any more
         //networkRT->destroy();
+        std::cout<<"serialize net\n";
         serialize(name);
     } else {
         deserialize(name);
@@ -125,9 +183,11 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
     // create GPU buffers and a stream
     for(int i=0; i<engineRT->getNbBindings(); i++) {
         Dims dim = engineRT->getBindingDimensions(i);
-        checkCuda(cudaMalloc(&buffersRT[i], dim.d[0]*dim.d[1]*dim.d[2]*sizeof(dnnType)));
+        buffersDIM[i] = dataDim_t(1, dim.d[0], dim.d[1], dim.d[2]);
+        std::cout<<"RtBuffer "<<i<<"   dim: "; buffersDIM[i].print();
+        checkCuda(cudaMalloc(&buffersRT[i], engineRT->getMaxBatchSize()*dim.d[0]*dim.d[1]*dim.d[2]*sizeof(dnnType)));
     }
-    checkCuda(cudaMalloc(&output, output_dim.tot()*sizeof(dnnType)));
+    checkCuda(cudaMalloc(&output, engineRT->getMaxBatchSize()*output_dim.tot()*sizeof(dnnType)));
 	checkCuda(cudaStreamCreate(&stream));
 }
 
@@ -136,19 +196,24 @@ NetworkRT::~NetworkRT() {
 }
 
 dnnType* NetworkRT::infer(dataDim_t &dim, dnnType* data) {
+    int batches = dim.n;
+    if(batches > getMaxBatchSize()) {
+        FatalError("input batch size too large");
+    }
 
-    checkCuda(cudaMemcpyAsync(buffersRT[buf_input_idx], data, input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
-    contextRT->enqueue(1, buffersRT, stream, nullptr);
-    checkCuda(cudaMemcpyAsync(output, buffersRT[buf_output_idx], output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
-    cudaStreamSynchronize(stream);
+    checkCuda(cudaMemcpyAsync(buffersRT[buf_input_idx], data, batches*input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    contextRT->enqueue(batches, buffersRT, stream, nullptr);
+    checkCuda(cudaMemcpyAsync(output, buffersRT[buf_output_idx], batches*output_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
+    checkCuda(cudaStreamSynchronize(stream));
 
     dim = output_dim;
+    dim.n = batches;
 
     return output;
 }
 
-void NetworkRT::enqueue() {
-    contextRT->enqueue(1, buffersRT, stream, nullptr);
+void NetworkRT::enqueue(int batchSize) {
+    contextRT->enqueue(batchSize, buffersRT, stream, nullptr);
 }
 
 ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
@@ -157,16 +222,20 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
 
     if(type == LAYER_DENSE)
         return convert_layer(input, (Dense*) l);
-    if(type == LAYER_CONV2D)
+    if(type == LAYER_CONV2D || type == LAYER_DECONV2D)
         return convert_layer(input, (Conv2d*) l);
     if(type == LAYER_POOLING)
         return convert_layer(input, (Pooling*) l);
-    if(type == LAYER_ACTIVATION)
+    if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH)
         return convert_layer(input, (Activation*) l);
     if(type == LAYER_SOFTMAX)
         return convert_layer(input, (Softmax*) l);
     if(type == LAYER_ROUTE)
         return convert_layer(input, (Route*) l);
+    if(type == LAYER_FLATTEN)
+        return convert_layer(input, (Flatten*) l);
+    if(type == LAYER_RESHAPE)
+        return convert_layer(input, (Reshape*) l);
     if(type == LAYER_REORG)
         return convert_layer(input, (Reorg*) l);
     if(type == LAYER_REGION)
@@ -177,7 +246,10 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
         return convert_layer(input, (Yolo*) l);
     if(type == LAYER_UPSAMPLE)
         return convert_layer(input, (Upsample*) l);
+    if(type == LAYER_DEFORMCONV2D)
+        return convert_layer(input, (DeformConv2d*) l);
 
+    std::cout<<l->getLayerName()<<"\n";
     FatalError("Layer not implemented in tensorRT");
     return NULL;
 }
@@ -203,12 +275,15 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Dense *l) {
 
 
 ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
-    //std::cout<<"convert conv2D\n";
+    // std::cout<<"convert conv2D\n";
+    // printf("%d %d %d %d %d\n", l->kernelH, l->kernelW, l->inputs, l->outputs, l->batchnorm);
 
-    void *data_b, *bias_b, *power_b, *mean_b, *variance_b, *scales_b;
+
+    void *data_b, *bias_b, *bias2_b, *power_b, *mean_b, *variance_b, *scales_b;
     if(dtRT == DataType::kHALF) {
         data_b     = l->data16_h;    
         bias_b     = l->bias16_h;
+        bias2_b    = l->bias216_h;
         power_b    = l->power16_h;
         mean_b     = l->mean16_h;
         variance_b = l->variance16_h;
@@ -216,6 +291,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
     } else {
         data_b     = l->data_h;    
         bias_b     = l->bias_h;
+        bias2_b    = l->bias2_h;
         power_b    = l->power_h;
         mean_b     = l->mean_h;
         variance_b = l->variance_h;
@@ -227,23 +303,44 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
     Weights b;
     if(!l->batchnorm)
         b = { dtRT, bias_b, l->outputs};
-    else
-        b = { dtRT, nullptr, 0}; //on batchnorm bias are added later
+    else{
+        if (l->additional_bias)
+            b = { dtRT, bias2_b, l->outputs}; 
+        else
+            b = { dtRT, nullptr, 0}; //on batchnorm bias are added later
+    }
+
+    ILayer *lRT = nullptr;
+    if(!l->deConv) {
+        IConvolutionLayer *lRTconv = networkRT->addConvolution(*input, 
+            l->outputs, DimsHW{l->kernelH, l->kernelW}, w, b);
+        checkNULL(lRTconv);
+        lRTconv->setStride(DimsHW{l->strideH, l->strideW});
+        lRTconv->setPadding(DimsHW{l->paddingH, l->paddingW});
+        lRTconv->setNbGroups(l->groups);
+        lRT = (ILayer*) lRTconv;
+    } else {
+        IDeconvolutionLayer *lRTconv = networkRT->addDeconvolution(*input, 
+            l->outputs, DimsHW{l->kernelH, l->kernelW}, w, b);
+        checkNULL(lRTconv);
+        lRTconv->setStride(DimsHW{l->strideH, l->strideW});
+        lRTconv->setPadding(DimsHW{l->paddingH, l->paddingW});
+        lRTconv->setNbGroups(l->groups);
+        lRT = (ILayer*) lRTconv;
+        
+        Dims d = lRTconv->getOutput(0)->getDimensions();
+        //std::cout<<"DECONV: "<<d.d[0]<<" "<<d.d[1]<<" "<<d.d[2]<<" "<<d.d[3]<<"\n";
+    }
 
-    // Add a convolution layer with 20 outputs and a 5x5 filter.
-    IConvolutionLayer *lRT = networkRT->addConvolution(*input, 
-               l->outputs, DimsHW{l->kernelH, l->kernelW}, w, b);
     checkNULL(lRT);
-
-    lRT->setStride(DimsHW{l->strideH, l->strideW});
-    lRT->setPadding(DimsHW{l->paddingH, l->paddingW});
-
     if(l->batchnorm) {
         Weights power{dtRT, power_b, l->outputs};
         Weights shift{dtRT, mean_b, l->outputs};
         Weights scale{dtRT, variance_b, l->outputs};
+        // std::cout<<lRT->getNbOutputs()<<std::endl;
         IScaleLayer *lRT2 = networkRT->addScale(*lRT->getOutput(0), ScaleMode::kCHANNEL, 
                     shift, scale, power);
+        
         checkNULL(lRT2);
 
         Weights shift2{dtRT, bias_b, l->outputs};
@@ -259,14 +356,29 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Conv2d *l) {
 }
 
 ILayer* NetworkRT::convert_layer(ITensor *input, Pooling *l) {
-    //std::cout<<"convert Pooling\n";
+    // std::cout<<"convert Pooling\n";
 
-    IPoolingLayer *lRT = networkRT->addPooling(*input, 
-        PoolingType::kMAX, DimsHW{l->winH, l->winW});
-    checkNULL(lRT);
-    lRT->setStride(DimsHW{l->strideH, l->strideW});
+    PoolingType ptype;
+    if(l->pool_mode == tkdnnPoolingMode_t::POOLING_MAX) ptype = PoolingType::kMAX;
+    if(l->pool_mode == tkdnnPoolingMode_t::POOLING_AVERAGE) ptype = PoolingType::kAVERAGE;
+    if(l->pool_mode == tkdnnPoolingMode_t::POOLING_AVERAGE_EXCLUDE_PADDING) ptype = PoolingType::kMAX_AVERAGE_BLEND;
 
-    return lRT;
+    if(l->pool_mode == tkdnnPoolingMode_t::POOLING_MAX_FIXEDSIZE)
+    {
+        IPlugin *plugin = new MaxPoolFixedSizeRT(l->output_dim.c, l->output_dim.h, l->output_dim.w, l->output_dim.n, l->strideH, l->strideW, l->winH, l->winH-1);        
+        IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+        checkNULL(lRT);
+        return lRT;
+    }
+    else
+    {
+        IPoolingLayer *lRT = networkRT->addPooling(*input, ptype, DimsHW{l->winH, l->winW});
+        checkNULL(lRT);
+
+        lRT->setPadding(DimsHW{l->paddingH, l->paddingW});
+        lRT->setStride(DimsHW{l->strideH, l->strideW});
+        return lRT;
+    }  
 }
 
 ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
@@ -292,8 +404,24 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
         IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kRELU);
         checkNULL(lRT);
         return lRT;
-    
-    } else {
+    } else if(l->act_mode == CUDNN_ACTIVATION_SIGMOID) {
+        IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kSIGMOID);
+        checkNULL(lRT);
+        return lRT;
+    }
+    else if(l->act_mode == CUDNN_ACTIVATION_CLIPPED_RELU) {
+        IPlugin *plugin = new ActivationReLUCeiling(l->ceiling);
+        IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+        checkNULL(lRT);
+        return lRT;
+    } 
+    else if(l->act_mode == ACTIVATION_MISH) {
+        IPlugin *plugin = new ActivationMishRT();
+        IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+        checkNULL(lRT);
+        return lRT;
+    }
+    else {
         FatalError("this Activation mode is not yet implemented");
         return NULL;
     }
@@ -309,12 +437,19 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Softmax *l) {
 }
 
 ILayer* NetworkRT::convert_layer(ITensor *input, Route *l) {
-    //std::cout<<"convert route\n";
+    // std::cout<<"convert route\n";
+    
+
 
     ITensor **tens = new ITensor*[l->layers_n];
     for(int i=0; i<l->layers_n; i++) {
         tens[i] = tensors[l->layers[i]];
+        // for(int j=0; j<tens[i]->getDimensions().nbDims; j++) {
+        //     std::cout<<tens[i]->getDimensions().d[j]<<" ";
+        // }
+        // std::cout<<"\n";
     }
+    
     IConcatenationLayer *lRT = networkRT->addConcatenation(tens, l->layers_n);
     //IPlugin *plugin = new RouteRT();
     //IPluginLayer *lRT = networkRT->addPlugin(tens, l->layers_n, *plugin);
@@ -323,6 +458,24 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Route *l) {
     return lRT;
 }
 
+ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) {
+
+    IPlugin *plugin = new FlattenConcatRT();
+    IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+    checkNULL(lRT);
+    return lRT;
+}
+
+ILayer* NetworkRT::convert_layer(ITensor *input, Reshape *l) {
+    // std::cout<<"convert Reshape\n";
+
+    l->output_dim.print();
+    IPlugin *plugin = new ReshapeRT(l->output_dim);
+    IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
+    checkNULL(lRT);
+    return lRT;
+}
+
 ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
     //std::cout<<"convert Reorg\n";
 
@@ -349,27 +502,31 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
     //std::cout<<"New plugin Shortcut\n";
     
     ITensor *back_tens = tensors[l->backLayer];
-    /*
-    // plugin version
-    IPlugin *plugin = new ShortcutRT();
-    ITensor **inputs = new ITensor*[2];
-    inputs[0] = input;
-    inputs[1] = back_tens; 
-    IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin);
-    checkNULL(lRT);
-    */
 
-    IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM);
-    checkNULL(lRT);
-    
-    return lRT;
+    if(l->backLayer->output_dim.c == l->output_dim.c)
+    {
+        IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM);
+        checkNULL(lRT);
+        return lRT;
+    }
+    else
+    {
+        // plugin version
+        IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim);
+        ITensor **inputs = new ITensor*[2];
+        inputs[0] = input;
+        inputs[1] = back_tens; 
+        IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin);
+        checkNULL(lRT);
+        return lRT;
+    }
 }
 
 ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
     //std::cout<<"convert Yolo\n";
 
     //std::cout<<"New plugin YOLO\n";
-    IPlugin *plugin = new YoloRT(l->classes, l->num, l);
+    IPlugin *plugin = new YoloRT(l->classes, l->num, l, l->n_masks, l->scaleXY);
     IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
     checkNULL(lRT);
     return lRT;
@@ -385,6 +542,57 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Upsample *l) {
     return lRT;
 }
 
+ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) {
+    //std::cout<<"convert DEFORMABLE\n";
+    ILayer *preconv = convert_layer(input, l->preconv);
+    checkNULL(preconv);
+
+    ITensor **inputs = new ITensor*[2];
+    inputs[0] = input;
+    inputs[1] = preconv->getOutput(0);
+
+    //std::cout<<"New plugin DEFORMABLE\n";
+    IPlugin *plugin = new DeformableConvRT(l->chunk_dim, l->kernelH, l->kernelW, l->strideH, l->strideW, l->paddingH, l->paddingW, 
+                                            l->deformableGroup, l->input_dim.n, l->input_dim.c, l->input_dim.h, l->input_dim.w, 
+                                            l->output_dim.n, l->output_dim.c, l->output_dim.h, l->output_dim.w, l);
+    IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin);
+    checkNULL(lRT);
+    lRT->setName( ("Deformable" + std::to_string(l->id)).c_str() );
+    delete(inputs);
+    // batchnorm
+    void *bias_b, *power_b, *mean_b, *variance_b, *scales_b;
+    if(dtRT == DataType::kHALF) {
+        bias_b     = l->bias16_h;
+        power_b    = l->power16_h;
+        mean_b     = l->mean16_h;
+        variance_b = l->variance16_h;
+        scales_b   = l->scales16_h;
+    } else {
+        bias_b     = l->bias_h;
+        power_b    = l->power_h;
+        mean_b     = l->mean_h;
+        variance_b = l->variance_h;
+        scales_b   = l->scales_h;
+    }
+
+    Weights power{dtRT, power_b, l->outputs};
+    Weights shift{dtRT, mean_b, l->outputs};
+    Weights scale{dtRT, variance_b, l->outputs};
+    //std::cout<<lRT->getNbOutputs()<<std::endl;
+    IScaleLayer *lRT2 = networkRT->addScale(*lRT->getOutput(0), ScaleMode::kCHANNEL, 
+                shift, scale, power);
+    
+    checkNULL(lRT2);
+
+    Weights shift2{dtRT, bias_b, l->outputs};
+    Weights scale2{dtRT, scales_b, l->outputs};
+    IScaleLayer *lRT3 = networkRT->addScale(*lRT2->getOutput(0), ScaleMode::kCHANNEL, 
+                shift2, scale2, power);
+    checkNULL(lRT3);
+
+    return lRT3;
+}
+
 bool NetworkRT::serialize(const char *filename) {
 
     std::ofstream p(filename);
@@ -428,14 +636,25 @@ bool NetworkRT::deserialize(const char *filename) {
 
 IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, size_t serialLength) {
     const char * buf = reinterpret_cast<const char*>(serialData);
-    
-    std::string name(layerName);
 
-    if(name.find("Activation") == 0) {
+    std::string name(layerName);
+    //std::cout<<name<<std::endl;
+
+    if(name.find("ActivationLeaky") == 0) {
         ActivationLeakyRT *a = new ActivationLeakyRT();
         a->size = readBUF<int>(buf);
         return a;
     }
+    if(name.find("ActivationMish") == 0) {
+        ActivationMishRT *a = new ActivationMishRT();
+        a->size = readBUF<int>(buf);
+        return a;
+    }
+    if(name.find("ActivationCReLU") == 0) {
+        ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF<float>(buf));
+        a->size = readBUF<int>(buf);
+        return a;
+    }
 
     if(name.find("Region") == 0) {
         RegionRT *r = new RegionRT(readBUF<int>(buf),    //classes
@@ -457,22 +676,75 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
     } 
 
     if(name.find("Shortcut") == 0) {
-        ShortcutRT *r = new ShortcutRT();
+        tk::dnn::dataDim_t bdim;
+        bdim.c = readBUF<int>(buf);
+        bdim.h = readBUF<int>(buf);
+        bdim.w = readBUF<int>(buf);
+        bdim.l = 1;
+
+        ShortcutRT *r = new ShortcutRT(bdim);
         r->c = readBUF<int>(buf);
         r->h = readBUF<int>(buf);
         r->w = readBUF<int>(buf);
         return r;
     } 
 
-    if(name.find("Yolo") == 0) {
-        YoloRT *r = new YoloRT(readBUF<int>(buf),    //classes
-                                readBUF<int>(buf));   //num
+    if(name.find("Pooling") == 0) {
+        MaxPoolFixedSizeRT *r = new MaxPoolFixedSizeRT( readBUF<int>(buf), //c
+                                                        readBUF<int>(buf), //h
+                                                        readBUF<int>(buf), //w
+                                                        readBUF<int>(buf), //n
+                                                        readBUF<int>(buf), //strideH
+                                                        readBUF<int>(buf), //strideW
+                                                        readBUF<int>(buf), //winSize
+                                                        readBUF<int>(buf)); //padding
+        return r;
+    }
+
+    if(name.find("Resize") == 0) {
+        ResizeLayerRT *r = new ResizeLayerRT(readBUF<int>(buf), //o_c
+                                            readBUF<int>(buf), //o_h
+                                            readBUF<int>(buf)); //o_w
+        r->i_c = readBUF<int>(buf);
+        r->i_h = readBUF<int>(buf);
+        r->i_w = readBUF<int>(buf);
+        return r;
+    } 
+
+    if(name.find("Flatten") == 0) {
+        FlattenConcatRT *r = new FlattenConcatRT(); 
         r->c = readBUF<int>(buf);
         r->h = readBUF<int>(buf);
         r->w = readBUF<int>(buf);
-        for(int i=0; i<r->num; i++)
+        r->rows = readBUF<int>(buf);
+        r->cols = readBUF<int>(buf);
+        return r;
+    } 
+
+    if(name.find("Reshape") == 0) {
+
+        dataDim_t new_dim;
+        new_dim.n = readBUF<int>(buf);
+        new_dim.c = readBUF<int>(buf);
+        new_dim.h = readBUF<int>(buf);
+        new_dim.w = readBUF<int>(buf);
+        ReshapeRT *r = new ReshapeRT(new_dim); 
+        
+        return r;
+    } 
+
+    if(name.find("Yolo") == 0) {
+        YoloRT *r = new YoloRT(readBUF<int>(buf),    //classes
+                                readBUF<int>(buf),   //num
+                                nullptr,
+                                readBUF<int>(buf));   //n_masks
+        r->c = readBUF<int>(buf);
+        r->h = readBUF<int>(buf);
+        r->w = readBUF<int>(buf);
+        r->scaleXY = readBUF<float>(buf);
+        for(int i=0; i<r->n_masks; i++)
             r->mask[i] = readBUF<dnnType>(buf);
-        for(int i=0; i<3*2*r->num; i++)
+        for(int i=0; i<r->n_masks*2*r->num; i++)
             r->bias[i] = readBUF<dnnType>(buf);
 
 		// save classes names
@@ -487,7 +759,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         yolos[n_yolos++] = r;
         return r;
     } 
-
     if(name.find("Upsample") == 0) {
         UpsampleRT *r = new UpsampleRT(readBUF<int>(buf)); //stride
         r->c = readBUF<int>(buf);
@@ -507,6 +778,46 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
         return r;
     } 
 */
+    if(name.find("Deformable") == 0) {
+        DeformableConvRT *r = new DeformableConvRT(readBUF<int>(buf), readBUF<int>(buf), readBUF<int>(buf),
+                                                    readBUF<int>(buf), readBUF<int>(buf), readBUF<int>(buf),
+                                                    readBUF<int>(buf), readBUF<int>(buf),
+                                                    readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),
+                                                    readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),
+                                                    nullptr); 
+        dnnType *aus = new dnnType[r->chunk_dim*2];
+        for(int i=0; i<r->chunk_dim*2; i++)
+    		aus[i] = readBUF<dnnType>(buf);
+		checkCuda( cudaMemcpy(r->offset, aus, sizeof(dnnType)*2*r->chunk_dim, cudaMemcpyHostToDevice) );
+        free(aus);
+		aus = new dnnType[r->chunk_dim];
+		for(int i=0; i<r->chunk_dim; i++)
+            aus[i] = readBUF<dnnType>(buf);
+		checkCuda( cudaMemcpy(r->mask, aus, sizeof(dnnType)*r->chunk_dim, cudaMemcpyHostToDevice) );
+        free(aus);
+		aus = new dnnType[(r->i_c * r->o_c * r->kh * r->kw * 1 )];
+		for(int i=0; i<(r->i_c * r->o_c * r->kh * r->kw * 1 ); i++)
+    		aus[i] = readBUF<dnnType>(buf);
+		checkCuda( cudaMemcpy(r->data_d, aus, sizeof(dnnType)*(r->i_c * r->o_c * r->kh * r->kw * 1 ), cudaMemcpyHostToDevice) );
+        free(aus);
+		aus = new dnnType[r->o_c];
+		for(int i=0; i < r->o_c; i++)
+    		aus[i] = readBUF<dnnType>(buf);
+		checkCuda( cudaMemcpy(r->bias2_d, aus, sizeof(dnnType)*r->o_c, cudaMemcpyHostToDevice) );
+        free(aus);
+		aus = new dnnType[r->height_ones * r->width_ones];
+		for(int i=0; i<r->height_ones * r->width_ones; i++)
+    		aus[i] = readBUF<dnnType>(buf);
+		checkCuda( cudaMemcpy(r->ones_d1, aus, sizeof(dnnType)*r->height_ones * r->width_ones, cudaMemcpyHostToDevice) );
+        free(aus);
+		aus = new dnnType[r->dim_ones];
+		for(int i=0; i<r->dim_ones; i++)
+    		aus[i] = readBUF<dnnType>(buf);
+		checkCuda( cudaMemcpy(r->ones_d2, aus, sizeof(dnnType)*r->dim_ones, cudaMemcpyHostToDevice) );
+        free(aus);
+        return r;
+    } 
+
     FatalError("Cant deserialize Plugin");
     return NULL;
 }

src/Pooling.cpp

@@ -6,6 +6,7 @@
 namespace tk { namespace dnn {
 
 Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
+                  int paddingH, int paddingW,
                   tkdnnPoolingMode_t pool_mode) : 
     Layer(net) {
 
@@ -14,8 +15,8 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
     this->strideH = strideH;
     this->strideW = strideW;
     this->pool_mode = pool_mode;
-    this->paddingH = 0;
-    this->paddingW = 0;
+    this->paddingH = paddingH;
+    this->paddingW = paddingW;
 
     checkCUDNN( cudnnCreatePoolingDescriptor(&poolingDesc) );
 
@@ -24,6 +25,7 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
     int h = input_dim.h;
     int w = input_dim.w;
     int l = input_dim.l;
+    
 
     poolOn3d = false;
     
@@ -37,16 +39,32 @@ Pooling::Pooling( Network *net, int winH, int winW, int strideH, int strideW,
         n = l;
     }
 
-    checkCUDNN( cudnnSetPooling2dDescriptor(poolingDesc, cudnnPoolingMode_t(pool_mode),
-                CUDNN_NOT_PROPAGATE_NAN, winH, winW, 0, 0, strideH, strideW) );
+    cudnnPoolingMode_t cudnn_pool_mode = cudnnPoolingMode_t(pool_mode);
+    if(pool_mode == POOLING_MAX_FIXEDSIZE) cudnn_pool_mode = cudnnPoolingMode_t(tkdnnPoolingMode_t::POOLING_MAX);
+
+    checkCUDNN( cudnnSetPooling2dDescriptor(poolingDesc, cudnn_pool_mode,
+                CUDNN_NOT_PROPAGATE_NAN, winH, winW, paddingH, paddingW, strideH, strideW) );
 
     checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc, 
                 net->tensorFormat, net->dataType, n, c, h, w) );
 
     //get out dim
-    checkCUDNN( cudnnGetPooling2dForwardOutputDim(poolingDesc, srcTensorDesc, &n, &c, &h, &w)); 
-    //h = (h + winH*this->paddingH)/strideH;
-    //w = (w + winW*this->paddingW)/strideW;
+    // checkCUDNN( cudnnGetPooling2dForwardOutputDim(poolingDesc, srcTensorDesc, &n, &c, &h, &w)); 
+
+    //compute w and h as in darknet
+    if(pool_mode == tkdnnPoolingMode_t::POOLING_MAX_FIXEDSIZE){
+        int padH = paddingH == 0? winH -1 : paddingH;
+        int padW = paddingW == 0? winW -1 : paddingW;
+        h = (h + padH - winH)/strideH +1;
+        w =  (w + padW - winW)/strideW +1;
+    }
+    else{
+        h = (h + 2*paddingH - winH)/strideH +1 ;
+        w =  (w + 2*paddingW - winW)/strideW +1;
+    }
+    
+    // h = (h + winH*this->paddingH)/strideH;
+    // w = (w + winW*this->paddingW)/strideW;
 
     checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
                 net->tensorFormat, net->dataType, n, c, h, w) );
@@ -92,11 +110,16 @@ dnnType* Pooling::infer(dataDim_t &dim, dnnType* srcData) {
         poolDst = tmpOutputData;
     }
 
-    dnnType alpha = dnnType(1);
-    dnnType beta = dnnType(0);
-    checkCUDNN( cudnnPoolingForward(net->cudnnHandle, poolingDesc,
-                &alpha, srcTensorDesc, poolSrc,
-                &beta, dstTensorDesc, poolDst) );
+    if(pool_mode == tkdnnPoolingMode_t::POOLING_MAX_FIXEDSIZE){
+        MaxPoolingForward(poolSrc, poolDst, dim.n, dim.c, dim.h, dim.w, this->strideH, this->strideW, this->winH, this->winH-1);
+    }
+    else{
+        dnnType alpha = dnnType(1);
+        dnnType beta = dnnType(0);
+        checkCUDNN( cudnnPoolingForward(net->cudnnHandle, poolingDesc,
+                    &alpha, srcTensorDesc, poolSrc,
+                    &beta, dstTensorDesc, poolDst) );
+    }
 
     //update dim
     dim = output_dim;

src/Region.cpp

@@ -12,8 +12,7 @@
 namespace tk { namespace dnn {
 
 Region::Region(Network *net, int classes, int coords, int num) : 
-    Layer(net) {
-
+    Layer(net) {    
     this->classes = classes;
     this->coords = coords;
     this->num = num;

src/Reshape.cpp

@@ -0,0 +1,34 @@
+#include <iostream>
+
+#include "Layer.h"
+#include "kernels.h"
+
+namespace tk { namespace dnn {
+
+Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) {
+
+    checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
+
+    output_dim.n = new_dim.n;
+    output_dim.c = new_dim.c;
+    output_dim.h = new_dim.h;
+    output_dim.w = new_dim.w;
+    output_dim.l = new_dim.l;
+
+}
+
+Reshape::~Reshape() {
+
+    checkCuda( cudaFree(dstData) );
+}
+
+dnnType* Reshape::infer(dataDim_t &dim, dnnType* srcData) {
+
+    //just copies the data and changes the output dim
+    checkCuda( cudaMemcpy(dstData, srcData, dim.n*dim.c*dim.h*dim.w*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+    dim = output_dim;
+
+    return dstData;
+}
+
+}}

src/Route.cpp

@@ -7,9 +7,15 @@ namespace tk { namespace dnn {
 
 Route::Route(Network *net, Layer **layers, int layers_n) : Layer(net) {
 
-    this->layers = layers;
+    // copy input layers
+    if(layers_n > MAX_LAYERS) {
+        FatalError("ROUTE: reached max number of input layers");
+    }
+    for(int i=0; i<layers_n; i++) {
+        this->layers[i] = layers[i];
+    }
     this->layers_n = layers_n;
-
+    
     //get dims
     output_dim.l = 1;
     output_dim.c = 0;

src/Shortcut.cpp

@@ -10,7 +10,7 @@ Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) {
     this->backLayer = backLayer;
     checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
 
-    if( backLayer->output_dim.c != input_dim.c ||
+    if( /*backLayer->output_dim.c != input_dim.c ||*/
         backLayer->output_dim.w != input_dim.w ||
         backLayer->output_dim.h != input_dim.h   )
         FatalError("Shortcut dim missmatch");

src/Softmax.cpp

@@ -5,22 +5,40 @@
 
 namespace tk { namespace dnn {
 
-Softmax::Softmax(Network *net) : Layer(net) {
+Softmax::Softmax(Network *net, const tk::dnn::dataDim_t* dim, const cudnnSoftmaxMode_t mode) : Layer(net) {
 
     checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
 
+    this->mode = mode;
+    if(dim == nullptr)
+    {
+        this->dim.n= input_dim.n;
+        this->dim.c= input_dim.c;
+        this->dim.h= input_dim.h;
+        this->dim.w= input_dim.w;
+        this->dim.l= input_dim.l;
+    }
+    else
+    {
+        this->dim.n= dim->n;
+        this->dim.c= dim->c;
+        this->dim.h= dim->h;
+        this->dim.w= dim->w;
+        this->dim.l= dim->l;
+    }
+
     checkCUDNN( cudnnSetTensor4dDescriptor(srcTensorDesc,
                                         net->tensorFormat,
                                         net->dataType,
-                                        input_dim.n*input_dim.l, 
-                                        input_dim.c,
-                                        input_dim.h, input_dim.w) );
+                                        this->dim.n*this->dim.l, 
+                                        this->dim.c,
+                                        this->dim.h, this->dim.w) );
     checkCUDNN( cudnnSetTensor4dDescriptor(dstTensorDesc,
                                             net->tensorFormat,
                                             net->dataType,
-                                            input_dim.n*input_dim.l, 
-                                            input_dim.c,
-                                            input_dim.h, input_dim.w) );
+                                            this->dim.n*this->dim.l, 
+                                            this->dim.c,
+                                            this->dim.h, this->dim.w) );
 }
 
 Softmax::~Softmax() {
@@ -34,7 +52,7 @@ dnnType* Softmax::infer(dataDim_t &dim, dnnType* srcData) {
     dnnType beta  = dnnType(0);
     checkCUDNN( cudnnSoftmaxForward(net->cudnnHandle,
                                     CUDNN_SOFTMAX_ACCURATE ,
-                                    CUDNN_SOFTMAX_MODE_CHANNEL,
+                                    this->mode,
                                     &alpha,
                                     srcTensorDesc,
                                     srcData,

src/Yolo.cpp

@@ -11,18 +11,23 @@
 
 namespace tk { namespace dnn {
 
-Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights) : 
+Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy) : 
     Layer(net) {
-    
+    this->final = true;
+
     this->classes = classes;
     this->num = num;
+    this->n_masks = n_masks;
+    this->scaleXY = scale_xy;
 
     // load anchors
     if(fname_weights != "") {
         int seek = 0;
-        readBinaryFile(fname_weights, num, &mask_h, &mask_d, seek);
-        seek += num;
-        readBinaryFile(fname_weights, 3*num*2, &bias_h, &bias_d, seek);
+        readBinaryFile(fname_weights, n_masks, &mask_h, &mask_d, seek);
+        seek += n_masks;
+        readBinaryFile(fname_weights, n_masks*num*2, &bias_h, &bias_d, seek);
+        //for(int i=0; i<n_masks*num*2; i++)
+            //printf("%f\n", bias_h[i]);
     }
 
     // init default classes name
@@ -68,9 +73,11 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) {
     checkCuda( cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
 
     for (int b = 0; b < dim.n; ++b){
-        for(int n = 0; n < num; ++n){
+        for(int n = 0; n < n_masks; ++n){
             int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim);
             activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
+
+            if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
             
             index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim);
             activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h);
@@ -127,7 +134,7 @@ int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int net
     for (i = 0; i < lw*lh; ++i){
         int row = i / lw;
         int col = i % lw;
-        for(n = 0; n < num; ++n){
+        for(n = 0; n < n_masks; ++n){
             int obj_index  = entry_index(0, n*lw*lh + i, 4, classes, input_dim, output_dim);
             float objectness = predictions[obj_index];
             if(objectness <= thresh) continue;

src/Yolo3Detection.cpp

@@ -1,28 +1,19 @@
 #include "Yolo3Detection.h"
 
+
 namespace tk { namespace dnn {
 
-float _colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} };
-float get_color(int c, int x, int max)
-{
-    float ratio = ((float)x/max)*5;
-    int i = floor(ratio);
-    int j = ceil(ratio);
-    ratio -= i;
-    float r = (1-ratio) * _colors[i % 6][c % 3] + ratio*_colors[j % 6][c % 3];
-    //printf("%f\n", r);
-    return r;
-}
-
-bool Yolo3Detection::init(std::string tensor_path) {
-
-    //const char *tensor_path = "../data/yolo3/yolo3_berkeley.rt";
+bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) {
 
     //convert network to tensorRT
     std::cout<<(tensor_path).c_str()<<"\n";
     netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
 
-    if(netRT->pluginFactory->n_yolos != 3) {
+    nBatches = n_batches;
+    tk::dnn::dataDim_t idim = netRT->input_dim;    
+    idim.n = nBatches;
+
+    if(netRT->pluginFactory->n_yolos < 2 ) {
         FatalError("this is not yolo3");
     }
 
@@ -30,88 +21,93 @@ bool Yolo3Detection::init(std::string tensor_path) {
         YoloRT *yRT = netRT->pluginFactory->yolos[i];
         classes = yRT->classes;
         num = yRT->num;
+        nMasks = yRT->n_masks;
 
-        // make a yolo layer for interpret predictions
-        yolo[i] = new tk::dnn::Yolo(nullptr, classes, num, ""); // yolo without input and bias
-        yolo[i]->mask_h = new dnnType[num];
-        yolo[i]->bias_h = new dnnType[num*3*2];
-        memcpy(yolo[i]->mask_h, yRT->mask, sizeof(dnnType)*num);
-        memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*3*2);
+        // make a yolo layer to interpret predictions
+        yolo[i] = new tk::dnn::Yolo(nullptr, classes, nMasks, ""); // yolo without input and bias
+        yolo[i]->mask_h = new dnnType[nMasks];
+        yolo[i]->bias_h = new dnnType[num*nMasks*2];
+        memcpy(yolo[i]->mask_h, yRT->mask, sizeof(dnnType)*nMasks);
+        memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*nMasks*2);
         yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w);
         yolo[i]->classesNames = yRT->classesNames;
     }
 
     dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
-    
-    checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot()));
-    checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot()));
-
+#ifndef OPENCV_CUDACONTRIB
+    checkCuda(cudaMallocHost(&input, sizeof(dnnType)*idim.tot()));
+#endif
+    checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*idim.tot()));
 
     // class colors precompute    
     for(int c=0; c<classes; c++) {
         int offset = c*123457 % classes;
-        float r = get_color(2, offset, classes);
-        float g = get_color(1, offset, classes);
-        float b = get_color(0, offset, classes);
+        float r = getColor(2, offset, classes);
+        float g = getColor(1, offset, classes);
+        float b = getColor(0, offset, classes);
         colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
     }
+
+    classesNames = getYoloLayer()->classesNames;
     return true;
 } 
 
+void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
+#ifdef OPENCV_CUDACONTRIB
+    cv::cuda::GpuMat orig_img, img_resized;
+    orig_img = cv::cuda::GpuMat(frame);
+    cv::cuda::resize(orig_img, img_resized, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
 
-void Yolo3Detection::update(cv::Mat &imageORIG) {
-
-    if(!imageORIG.data) {
-        std::cout<<"YOLO: NO IMAGE DATA\n";
-        return;
-    }     
-    float xRatio =  float(imageORIG.cols) / float(netRT->input_dim.w);
-    float yRatio =  float(imageORIG.rows) / float(netRT->input_dim.h);
-
-    resize(imageORIG, imageORIG, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
-
-    imageORIG.convertTo(imageF, CV_32FC3, 1/255.0); 
+    img_resized.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
 
     //split channels
-    cv::split(imageF,bgr);//split source
+    cv::cuda::split(imagePreproc,bgr);//split source
 
     //write channels
     for(int i=0; i<netRT->input_dim.c; i++) {
-        int idx = i*imageF.rows*imageF.cols;
+        int size = imagePreproc.rows * imagePreproc.cols;
         int ch = netRT->input_dim.c-1 -i;
-        memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
+        bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
+        checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
     }
+#else
+    cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
+    frame.convertTo(imagePreproc, CV_32FC3, 1/255.0); 
 
+    //split channels
+    cv::split(imagePreproc,bgr);//split source
 
-    //DO INFERENCE
-    dnnType *rt_out[3]; 
-    tk::dnn::dataDim_t dim = netRT->input_dim;
-    checkCuda(cudaMemcpyAsync(input_d, input, dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
-
-    printCenteredTitle(" TENSORRT inference ", '=', 30); {
-        dim.print();
-        TIMER_START
-        netRT->infer(dim, input_d);
-        TIMER_STOP
-        dim.print();
-    
-        stats.push_back(t_ns);
+    //write channels
+    for(int i=0; i<netRT->input_dim.c; i++) {
+        int idx = i*imagePreproc.rows*imagePreproc.cols;
+        int ch = netRT->input_dim.c-1 -i;
+        memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));     
     }
-    
-    TIMER_START
+    checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
+#endif
+}
+
+void Yolo3Detection::postprocess(const int bi, const bool mAP){
+
+    //get yolo outputs
+    dnnType *rt_out[netRT->pluginFactory->n_yolos]; 
+    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) 
+        rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi;
+
+    float x_ratio =  float(originalSize[bi].width) / float(netRT->input_dim.w);
+    float y_ratio =  float(originalSize[bi].height) / float(netRT->input_dim.h);
+
     // compute dets
-    ndets = 0;
-    for(int i=0; i<3; i++) {
-        rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
+    nDets = 0;
+    for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
         yolo[i]->dstData = rt_out[i];
-        yolo[i]->computeDetections(dets, ndets, netRT->input_dim.w, netRT->input_dim.h, thresh);
+        yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold);
     }
-    tk::dnn::Yolo::mergeDetections(dets, ndets, classes);
-    TIMER_STOP
+    tk::dnn::Yolo::mergeDetections(dets, nDets, classes);
 
     // fill detected
     detected.clear();
-    for(int j=0; j<ndets; j++) {
+    for(int j=0; j<nDets; j++) {
         tk::dnn::Yolo::box b = dets[j].bbox;
         int x0   = (b.x-b.w/2.);
         int x1   = (b.x+b.w/2.);
@@ -120,21 +116,18 @@ void Yolo3Detection::update(cv::Mat &imageORIG) {
         int obj_class = -1;
         float prob = 0;
         for(int c=0; c<classes; c++) {
-            if(dets[j].prob[c] >= thresh) {
+            if(dets[j].prob[c] >= confThreshold) {
                 obj_class = c;
                 prob = dets[j].prob[c];
             }
         }
 
         if(obj_class >= 0) {
-            //std::cout<<obj_class<<" ("<<prob<<"): "<<x0<<" "<<y0<<" "<<x1<<" "<<y1<<"\n";
-            //cv::rectangle(image, cv::Point(x0, y0), cv::Point(x1, y1), colors[obj_class], 2);
-            
             // convert to image coords
-            x0 = xRatio*x0;
-            x1 = xRatio*x1;
-            y0 = yRatio*y0;
-            y1 = yRatio*y1;
+            x0 = x_ratio*x0;
+            x1 = x_ratio*x1;
+            y0 = y_ratio*y0;
+            y1 = y_ratio*y1;
               
             tk::dnn::box res;
             res.cl = obj_class;
@@ -143,10 +136,21 @@ void Yolo3Detection::update(cv::Mat &imageORIG) {
             res.y = y0;
             res.w = x1 - x0;
             res.h = y1 - y0;
+            if(mAP)
+                for(int c=0; c<classes; c++) 
+                    res.probs.push_back(dets[j].prob[c]);
             detected.push_back(res);
         }
     }
+    batchDetected.push_back(detected);
+}
 
+
+tk::dnn::Yolo* Yolo3Detection::getYoloLayer(int n) {
+    if(n<3)
+        return yolo[n];
+    else 
+        return nullptr;
 }
 
 }}

src/evaluation.cpp

@@ -0,0 +1,359 @@
+#include "evaluation.h"
+#include <fstream>
+
+namespace tk { namespace dnn {
+
+void Frame::print() const{
+    std::cout<<"labels filename: "<<lFilename<<std::endl;
+    std::cout<<"image filename: "<<iFilename<<std::endl;
+    std::cout<<"GT: "<<std::endl;
+    for(auto g: gt) std::cout<<g;
+    std::cout<<"DET: "<<std::endl;
+    for(auto d: det) std::cout<<d;
+}
+
+void PR::print(){
+    std::cout<<"precision: "<<precision<<" recall: "<<recall<<" tp: "<<tp<<" fp:"<<fp<<" fn:"<<fn<<std::endl;
+}
+
+void readmAPParams( const char* config_filename, int& classes, int& map_points, 
+                    int& map_levels, float& map_step, float& IoU_thresh, 
+                    float& conf_thresh, bool& verbose) {
+    YAML::Node config   = YAML::LoadFile(config_filename);
+    classes     = config["classes"].as<int>();
+    map_points  = config["map_points"].as<int>();
+    map_levels  = config["map_levels"].as<int>();
+    map_step    = config["map_step"].as<float>();
+    IoU_thresh  = config["IoU_thresh"].as<float>();
+    conf_thresh = config["conf_thresh"].as<float>();
+    verbose     = config["verbose"].as<bool>();
+}
+
+/* Credits to https://github.com/AlexeyAB/darknet/blob/master/src/detector.c*/
+double computeMap(  std::vector<Frame> &images,const int classes, 
+                    const float IoU_thresh, const float conf_thresh, 
+                    const int map_points, const bool verbose) {
+    if(verbose)
+        for(auto img:images)
+            img.print();
+
+    int detections_count = 0;
+    int groundtruths_count = 0;
+    int unique_truth_count = 0;
+    std::vector<int> truth_classes_count(classes,0);
+    std::vector<int> dets_classes_count(classes,0);
+
+    //count groundtruth and detections in total and for each class
+    for(auto i:images){
+        for(auto gt:i.gt)
+            truth_classes_count[gt.cl]++;
+        for(auto det:i.det)
+            dets_classes_count[det.cl]++;
+        detections_count += i.det.size();
+        groundtruths_count += i.gt.size();
+    }
+
+    if(verbose){
+        std::cout<<"gt_count: "<<groundtruths_count<<std::endl;
+        std::cout<<"det_count: "<<detections_count<<std::endl;
+    }
+
+    std::vector<BoundingBox> all_dets;
+    std::vector<BoundingBox> all_gts;
+    
+    int gt_checked = 0;
+
+    // for each detection comput IoU with groundtruth and match detetcion and 
+    // groundtruth with IoU greater than IoU_thresh
+    for(auto &img:images){
+        for(size_t i=0; i<img.det.size(); i++){
+            if(img.det[i].prob > conf_thresh){
+                float maxIoU = 0;
+                int truth_index = -1;
+                for(size_t j=0; j<img.gt.size(); j++){
+                    float currentIoU = img.det[i].IoU(img.gt[j]);
+                    if(currentIoU > maxIoU && img.det[i].cl == img.gt[j].cl){
+                        maxIoU = currentIoU;                    
+                        truth_index = j;
+                    }
+                }
+                if(truth_index > -1 && maxIoU > IoU_thresh){
+                    img.det[i].uniqueTruthIndex = truth_index + gt_checked;
+                    img.det[i].truthFlag = 1;
+                    img.det[i].maxIoU = maxIoU;
+                }
+            }
+
+            all_dets.push_back(img.det[i]);
+        }
+        gt_checked += img.gt.size();
+    }
+
+    if(verbose){
+        for(auto img:images)
+            img.print();
+        std::cout<<"\n\n\n\n";
+    }
+
+    //sort all detections by descending value of confidence
+    std::sort(all_dets.begin(), all_dets.end(), boxComparison);
+    std::vector<int> truth_flags(groundtruths_count,0);
+
+    if(verbose)
+        for(auto d:all_dets)
+            std::cout<<d;
+
+    //compute precision-recall curve
+    std::vector<std::vector<PR>> pr( classes, std::vector<PR>(detections_count));
+    for(int rank = 0; rank< detections_count; ++rank){
+        if (rank > 0) {
+            for (int class_id = 0; class_id < classes; ++class_id) {
+                pr[class_id][rank].tp = pr[class_id][rank - 1].tp;
+                pr[class_id][rank].fp = pr[class_id][rank - 1].fp;
+            }
+        }        
+
+        //if it was detected and never detected before
+        if (all_dets[rank].truthFlag == 1 && truth_flags[all_dets[rank].uniqueTruthIndex] == 0) {
+            truth_flags[all_dets[rank].uniqueTruthIndex] = 1;
+            pr[all_dets[rank].cl][rank].tp++;    // true-positive
+        }
+        else {
+            pr[all_dets[rank].cl][rank].fp++;    // false-positive
+        }
+
+        for (int i = 0; i < classes; ++i){
+            const int tp = pr[i][rank].tp;
+            const int fp = pr[i][rank].fp;
+            const int fn = truth_classes_count[i] - tp;    // false-negative = objects - true-positive
+            pr[i][rank].fn = fn;
+
+            if ((tp + fp) > 0) 
+                pr[i][rank].precision = (double)tp / (double)(tp + fp);
+            else 
+                pr[i][rank].precision = 0;
+
+            if ((tp + fn) > 0) 
+                pr[i][rank].recall = (double)tp / (double)(tp + fn);
+            else 
+                pr[i][rank].recall = 0;
+
+            if (rank == (detections_count - 1) && dets_classes_count[i] != (tp + fp)) {    
+                // check for last rank
+                printf(" class_id: %d - detections = %d, tp+fp = %d, tp = %d, fp = %d \n", i, dets_classes_count[i], tp+fp, tp, fp);
+            }
+        }
+    }
+
+    if(verbose){
+        for(int i=0; i < pr.size(); i++)   {
+            std::cout<<"---------Class "<<i<<std::endl;
+            for(auto r:pr[i])
+                r.print();
+        }
+    }
+
+    //compute average precision for each class. Two methods are avaible, 
+    //based on map_points required
+    double mean_average_precision = 0;
+    double last_recall, last_precision, delta_recall;
+    double cur_recall, cur_precision;
+    double avg_precision = 0;
+    for (int i = 0; i < classes; ++i) {
+        avg_precision = 0;
+        
+        if (map_points == 0){ //mAP calculation: ImageNet, PascalVOC 2010-2012
+            last_recall = pr[i][detections_count - 1].recall;
+            last_precision = pr[i][detections_count - 1].precision;
+            for (int rank = detections_count - 2; rank >= 0; --rank){
+                delta_recall = last_recall - pr[i][rank].recall;
+                last_recall = pr[i][rank].recall;
+
+                if (pr[i][rank].precision > last_precision) 
+                    last_precision = pr[i][rank].precision;
+
+                avg_precision += delta_recall * last_precision;
+            }
+        }
+        else {//MSCOCO - 101 Recall-points, PascalVOC - 11 Recall-points
+            for (int point = 0; point < map_points; ++point) {
+                cur_recall = point * 1.0 / ( map_points - 1 );
+                cur_precision = 0;
+                for (int rank = 0; rank < detections_count; ++rank)
+                    if (pr[i][rank].recall >= cur_recall && pr[i][rank].precision > cur_precision) 
+                        cur_precision = pr[i][rank].precision;
+
+                avg_precision += cur_precision;
+            }
+            avg_precision = avg_precision / map_points;
+        }
+
+        if(verbose)
+            std::cout<<"Class: "<<i<<" AP: "<< avg_precision<<std::endl;
+        mean_average_precision += avg_precision;
+    }
+
+    mean_average_precision = mean_average_precision / classes;
+
+    std::cout<<"Classes: "<<classes<<" mAP " <<IoU_thresh<<":\t"<< mean_average_precision<<std::endl;
+    return mean_average_precision;
+}
+
+double computeMapNIoULevels(std::vector<Frame> &images,const int classes, 
+                            const float i_IoU_thresh, const float conf_thresh, 
+                            const int map_points, const float map_step, 
+                            const int map_levels, const bool verbose, 
+                            const bool write_on_file, std::string net) {
+    std::ofstream out_file;
+    if(write_on_file){
+	out_file.open("map.csv", std::ios_base::app);
+        out_file<<net<<";";
+    }
+
+    double AP = 0, cur_AP = 0;
+    float IoU_thresh = i_IoU_thresh;
+
+    for(int i=0; i<map_levels; ++i){
+        //clear detection-grounthuth matching
+        for(auto& img:images)
+            for(auto & d:img.det)
+                d.clear();
+        //compute mAP for the new IoU threshold
+        cur_AP = computeMap(images,classes,IoU_thresh,conf_thresh,map_points, verbose);
+	    if(write_on_file)
+	        out_file<<cur_AP<<";";
+	    AP += cur_AP;
+            IoU_thresh +=map_step;
+    }
+    AP/=map_levels;
+
+    if(write_on_file){
+	    out_file<<AP<<"\n";
+	    out_file.close();
+    }
+    return AP;
+}
+
+void computeTPFPFN( std::vector<Frame> &images,const int classes,    
+                    const float IoU_thresh, const float conf_thresh, 
+                    bool verbose, const bool write_on_file, std::string net) {
+
+    std::ofstream out_file;
+    if(write_on_file){
+	    out_file.open("pr.csv", std::ios_base::app);
+        out_file<<net<<";";
+    }
+
+    std::vector<int> truth_classes_count(classes,0);
+    std::vector<int> dets_classes_count(classes,0);
+    std::vector<PR> pr(classes);
+    
+    //compute TP, FP, FN for each image, for each class
+    for(auto &img:images){
+        for(auto& tc: truth_classes_count) tc = 0;
+        for(auto& dc: dets_classes_count) dc = 0;
+
+        std::vector<bool> det_assigned(img.det.size(), false);
+        for(size_t j=0; j<img.gt.size(); j++){
+            truth_classes_count[img.gt[j].cl]++;
+            float maxIoU = 0;
+            int det_index = -1;
+            for(size_t i=0; i<img.det.size(); i++){
+                if(img.det[i].prob > conf_thresh){               
+                    float currentIoU = img.det[i].IoU(img.gt[j]);
+                    if(currentIoU > maxIoU && img.det[i].cl == img.gt[j].cl && !det_assigned[i]){
+                        maxIoU = currentIoU;                    
+                        det_index = i;
+                    }
+                }
+            }
+            if(det_index > -1 && maxIoU > IoU_thresh && !det_assigned[det_index]){
+                img.det[det_index].uniqueTruthIndex = j;
+                img.det[det_index].truthFlag = 1;
+                img.det[det_index].maxIoU = maxIoU;
+                det_assigned[det_index] = true;
+                dets_classes_count[img.det[det_index].cl]++;
+            }
+        }
+
+        for(size_t i=0; i<img.det.size(); i++){
+            if(img.det[i].truthFlag)
+                pr[img.det[i].cl].tp ++;
+            else
+                pr[img.det[i].cl].fp ++;
+        }
+        for(size_t i=0; i<classes; i++){
+            pr[i].fn += truth_classes_count[i] - dets_classes_count[i];
+        }
+    }
+
+    //count all TP, FP, FN and compute precsion, recall and f1-score
+    double avg_precision = 0, avg_recall = 0, f1_score = 0;
+    int TP = 0, FP = 0, FN = 0;
+    for(size_t i=0; i<classes; i++){
+        pr[i].precision = (pr[i].tp + pr[i].fp) > 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fp) : 0;
+        pr[i].recall = (pr[i].tp + pr[i].fn) > 0 ? (double)pr[i].tp / (double)(pr[i].tp +pr[i].fn) : 0;
+        if(verbose)
+            std::cout<<"Class "<<i<<"\tTP: "<<pr[i].tp<<"\tFP: "<<pr[i].fp<<"\tFN: "<<pr[i].fn<<"\tprecision: "<<pr[i].precision<<"\trecall: "<<pr[i].recall<<std::endl;
+        avg_precision += pr[i].precision;
+        avg_recall += pr[i].recall;
+
+        TP += pr[i].tp;
+        FP += pr[i].fp;
+        FN += pr[i].fn;
+    }
+    avg_precision /= classes;
+    avg_recall /= classes;
+
+    f1_score = avg_precision + avg_recall > 0 ? 2 * ( avg_precision * avg_recall ) / ( avg_precision + avg_recall ) : 0;
+
+    if(write_on_file){
+        out_file<<TP<<";"<<FP<<";"<<FN<<";"<<avg_precision<<";"<<avg_recall<<";"<<f1_score<<"\n";
+        out_file.close();
+    }
+
+    std::cout<<"avg precision: "<<avg_precision<<"\tavg recall: "<<avg_recall<<"\tavg f1 score:"<<f1_score<<std::endl;
+}
+
+void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path, std::vector<tk::dnn::box> bbox, const int classes, const int w, const int h)
+{
+    int coco_ids[] = { 1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90 };
+    std::string id = image_path.substr(image_path.find("images/")+7, image_path.find(".jpg") - image_path.find("images/") -7);
+    int image_id = std::stoi(id);
+    for (int i = 0; i < bbox.size(); ++i) {
+        float xmin = bbox[i].x ;
+        float xmax = bbox[i].x + float(bbox[i].w);
+        float ymin = bbox[i].y;
+        float ymax = bbox[i].y + float(bbox[i].h);
+
+        //limit to image borders
+        if (xmin < 0) xmin = 0;
+        if (ymin < 0) ymin = 0;
+        if (xmax > w) xmax = w;
+        if (ymax > h) ymax = h;
+
+        float bx = xmin;
+        float by = ymin;
+        float bw = xmax - xmin;
+        float bh = ymax - ymin;
+
+        if(bbox[i].probs.size() == classes)
+            for (int j = 0; j < classes; ++j) {
+                //min threshold confidence is set in DetectionNN.h
+                if (bbox[i].probs[j] > 0) {
+
+                    *out_file <<    "{\"image_id\":" << image_id <<
+                                    ", \"category_id\":" << coco_ids[j] << 
+                                    ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
+                                    "], \"score\":" << bbox[i].probs[j] << "},\n";
+                }
+            }
+        else
+            *out_file <<    "{\"image_id\":" << image_id <<
+                            ", \"category_id\":" << coco_ids[bbox[i].cl] << 
+                            ", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh << 
+                            "], \"score\":" << bbox[i].prob << "},\n";
+    }
+}
+
+}}

src/kernels/activation_mish.cu

@@ -0,0 +1,50 @@
+#include "kernels.h"
+#include <math.h>
+
+#define MISH_THRESHOLD 20
+
+__device__ 
+float tanh_activate_kernel(float x){return (2/(1 + expf(-2*x)) - 1);}
+
+__device__ 
+float softplus_kernel(float x, float threshold = 20) {
+    if (x > threshold) return x;                // too large
+    else if (x < -threshold) return expf(x);    // too small
+    return logf(expf(x) + 1);
+}
+
+
+
+__device__ 
+float mish_yashas(float x) { 
+    float e = __expf(x); 
+    if (x <= -18.0f) 
+        return x * e; 
+ 
+    float n = e * e + 2 * e; 
+    if (x <= -5.0f) 
+        return x * __fdividef(n, n + 2); 
+ 
+    return x - 2 * __fdividef(x, n + 2); 
+} 
+
+// https://github.com/digantamisra98/Mish
+// https://github.com/AlexeyAB/darknet/blob/master/src/activation_kernels.cu
+__global__
+void activation_mish(dnnType *input, dnnType *output, int size) {
+    int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if (i < size) 
+        // output[i] = input[i] * tanh_activate_kernel( softplus_kernel(input[i], MISH_THRESHOLD));    
+        output[i] = mish_yashas(input[i]);
+}
+
+/**
+    Mish activation function
+*/
+void activationMishForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    activation_mish<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
+}

src/kernels/activation_relu_ceiling.cu

@@ -0,0 +1,33 @@
+#include "kernels.h"
+
+__global__
+void activation_relu_ceiling(dnnType *input, dnnType *output, int size, const float ceiling) {
+
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+    if(i<size) {    
+        if (input[i]>0)
+        {
+            if (input[i]>ceiling)
+                output[i] = ceiling;
+            else
+                output[i] = input[i];
+        }
+        else
+            output[i] = 0.0f;
+    }
+ }
+
+
+/**
+    Relu ceiling activation function
+*/
+void activationReLUCeilingForward(dnnType* srcData, dnnType* dstData, int size, const float ceiling, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    activation_relu_ceiling<<<blocks, threads, 0, stream>>>(srcData, dstData, size, ceiling);
+}
+
+

src/kernels/activation_sigmoid.cu

@@ -0,0 +1,23 @@
+#include "kernels.h"
+#include <math.h>
+
+
+__global__
+void activation_sigmoid(dnnType *input, dnnType *output, int size) {
+
+    int i = blockDim.x * blockIdx.x + threadIdx.x;
+    if(i < size)
+        output[i] = 1.0f / (1.0f + exp (-input[i]));
+ }
+
+
+/**
+    ELU activation function
+*/
+void activationSIGMOIDForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    activation_sigmoid<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
+}

src/kernels/deformable_conv.cu

@@ -0,0 +1,300 @@
+#include <cstdio>
+#include <algorithm>
+#include <cstring>
+#include <string>
+#include <iostream> 
+#include "kernels.h"
+#include <errno.h>
+
+#define CUDA_KERNEL_LOOP(i, n)                          \
+  for (int i = blockIdx.x * blockDim.x + threadIdx.x;   \
+      i < (n);                                          \
+      i += blockDim.x * gridDim.x)
+
+const int CUDA_NUM_THREADS = 512;
+inline int GET_BLOCKS(const int N)
+{
+  return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS;
+}
+
+
+__device__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width,
+  const int height, const int width, float h, float w) {
+int h_low = floor(h);
+int w_low = floor(w);
+int h_high = h_low + 1;
+int w_high = w_low + 1;
+
+float lh = h - h_low;
+float lw = w - w_low;
+float hh = 1 - lh, hw = 1 - lw;
+
+float v1 = ( (h_low >= 0 && w_low >= 0) ? bottom_data[h_low * data_width + w_low]:0);
+float v2 = ( (h_low >= 0 && w_high <= width - 1) ? bottom_data[h_low * data_width + w_high]:0);
+float v3 = ( (h_high <= height - 1 && w_low >= 0) ? bottom_data[h_high * data_width + w_low]:0);
+float v4 = ( (h_high <= height - 1 && w_high <= width - 1) ? bottom_data[h_high * data_width + w_high]:0);
+
+float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw;
+
+float val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4);
+return val;
+}
+
+__global__ void modulated_deformable_im2col_gpu_kernel(const int n,
+  const float *data_im, const float *data_offset, const float *data_mask,
+  const int height, const int width,
+  const int batch_size, const int num_channels, const int deformable_group,
+  const int height_col, const int width_col,
+  float *data_col) {
+  CUDA_KERNEL_LOOP(index, n)
+  {
+    //If n is a power of 2, ( i / n ) is equivalent to ( i ≫ log2 n ) and ( i % n ) is equivalent to ( i & n - 1 ).
+    const int ind_on_w = index / width_col;
+    const int ind_on_w_on_h = ind_on_w / height_col; 
+    const int kk = 3 * 3;
+    // index index of output matrix
+    const int w_col = index % width_col;
+    const int h_col = (ind_on_w) % height_col;
+    const int b_col = (ind_on_w_on_h) % batch_size;
+    const int c_im = (ind_on_w_on_h) / batch_size;
+    const int c_col = c_im * kk;
+
+    // compute deformable group index
+    const int deformable_group_index = c_im / (int)(num_channels / deformable_group);
+
+    const int h_in = h_col - 1;
+    const int w_in = w_col - 1;
+    const int s_col = height_col * width_col;
+    const int s_col2 = 2 * s_col;
+
+    const int first_member = w_col + width_col * h_col;
+    // float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col;
+    float *data_col_ptr = data_col + first_member + s_col * (c_col * batch_size + b_col);
+    //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in;
+    const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width;
+    const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col;
+    const float *data_offset_ptr = data_offset + add_ptr + add_ptr;
+    const float *data_mask_ptr = data_mask + add_ptr;
+
+    #pragma unroll
+    for (int i = 0; i < 3; ++i) {
+      #pragma unroll
+      for (int j = 0; j < 3; ++j) {
+        const int iter_member = (i * 3 + j);
+        // const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col;
+        const int data_offset_h_ptr = first_member + s_col2 * iter_member;
+        
+        // const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col;
+        const int data_offset_w_ptr = s_col + first_member + s_col2 * iter_member;
+        
+        // const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col;
+        const int data_mask_hw_ptr = first_member + s_col * iter_member;
+
+        const float offset_h = data_offset_ptr[data_offset_h_ptr];
+        const float offset_w = data_offset_ptr[data_offset_w_ptr];
+        const float mask = data_mask_ptr[data_mask_hw_ptr];
+        const float h_im = offset_h + h_in + i;
+        const float w_im = offset_w + w_in + j;
+        //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) {
+        float val = static_cast<float>(0);
+        if (h_im < height && w_im < width && h_im > -1 && w_im > -1) {
+          //const float map_h = i * dilation_h + offset_h;
+          //const float map_w = j * dilation_w + offset_w;
+          //const int cur_height = height - h_in;
+          //const int cur_width = width - w_in;
+          //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w);
+          val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im);
+        }
+        *data_col_ptr = val * mask;
+        data_col_ptr += batch_size * s_col;
+        //data_col_ptr += height_col * width_col;
+      }
+    }
+  }
+}
+
+__global__ void modulated_deformable_im2col_gpu_kernel_general_version(const int n,
+  const float *data_im, const float *data_offset, const float *data_mask,
+  const int height, const int width, const int kernel_h, const int kernel_w,
+  const int pad_h, const int pad_w,
+  const int stride_h, const int stride_w,
+  const int dilation_h, const int dilation_w,
+  const int channel_per_deformable_group,
+  const int batch_size, const int num_channels, const int deformable_group,
+  const int height_col, const int width_col,
+  float *data_col) {
+  CUDA_KERNEL_LOOP(index, n)
+  {
+    //If n is a power of 2, ( i / n ) is equivalent to ( i ≫ log2 n ) and ( i % n ) is equivalent to ( i & n - 1 ).
+    const int ind_on_w = index / width_col;
+    const int ind_on_w_on_h = ind_on_w / height_col; 
+    const int kk = kernel_h * kernel_w;
+    // index index of output matrix
+    const int w_col = index % width_col;
+    const int h_col = (ind_on_w) % height_col;
+    const int b_col = (ind_on_w_on_h) % batch_size;
+    const int c_im = (ind_on_w_on_h) / batch_size;
+    const int c_col = c_im * kk;
+
+    // compute deformable group index
+    const int deformable_group_index = c_im / channel_per_deformable_group;
+
+    const int h_in = h_col * stride_h - pad_h;
+    const int w_in = w_col * stride_w - pad_w;
+    const int s_col = height_col * width_col;
+    const int s_col2 = 2 * s_col;
+
+    const int first_member = w_col + width_col * h_col;
+    // float *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col;
+    float *data_col_ptr = data_col + first_member + s_col * (c_col * batch_size + b_col);
+    //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in;
+    const float *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width;
+    const int add_ptr = (b_col * deformable_group + deformable_group_index) * kk * s_col;
+    const float *data_offset_ptr = data_offset + add_ptr + add_ptr;
+
+    const float *data_mask_ptr = data_mask + add_ptr;
+    
+    #pragma unroll
+    for (int i = 0; i < kernel_h; ++i) {
+      #pragma unroll
+      for (int j = 0; j < kernel_w; ++j) {
+        const int iter_member = (i * kernel_w + j);
+        // const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col;
+        const int data_offset_h_ptr = first_member + s_col2 * iter_member;
+        
+        // const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col;
+        const int data_offset_w_ptr = s_col + first_member + s_col2 * iter_member;
+        
+        // const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col;
+        const int data_mask_hw_ptr = first_member + s_col * iter_member;
+
+        const float offset_h = data_offset_ptr[data_offset_h_ptr];
+        const float offset_w = data_offset_ptr[data_offset_w_ptr];
+        const float mask = data_mask_ptr[data_mask_hw_ptr];
+        const float h_im = offset_h + h_in + i * dilation_h;
+        const float w_im = offset_w + w_in + j * dilation_w;
+        //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) {
+        float val = static_cast<float>(0);
+        if (h_im < height && w_im < width && h_im > -1 && w_im > -1) {
+          //const float map_h = i * dilation_h + offset_h;
+          //const float map_w = j * dilation_w + offset_w;
+          //const int cur_height = height - h_in;
+          //const int cur_width = width - w_in;
+          //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w);
+          val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im);
+        }
+        *data_col_ptr = val * mask;
+        data_col_ptr += batch_size * s_col;
+        //data_col_ptr += height_col * width_col;
+      }
+    }
+  }
+}
+
+void modulatedDeformableIm2colCuda(cudaStream_t stream,
+  const float* data_im, const float* data_offset, const float* data_mask,
+  const int batch_size, const int channels, const int height_im, const int width_im, 
+  const int height_col, const int width_col,
+  const int deformable_group, float* data_col) {
+  // num_axes should be smaller than block size
+  // const int channel_per_deformable_group = channels / deformable_group;
+  const int num_kernels = channels * batch_size * height_col * width_col;
+  modulated_deformable_im2col_gpu_kernel
+      <<<GET_BLOCKS(num_kernels), CUDA_NUM_THREADS,
+          0, stream>>>(
+      num_kernels, data_im, data_offset, data_mask, height_im, width_im, 
+      batch_size, channels, deformable_group, height_col, width_col, data_col);
+  
+  cudaError_t err = cudaGetLastError();
+  if (err != cudaSuccess)
+    FatalError("error in modulatedDeformableIm2colCuda: " + std::string(cudaGetErrorString(err)) + "\n");
+}
+
+void modulatedDeformableIm2colCudaGeneralVersion(cudaStream_t stream,
+  const float* data_im, const float* data_offset, const float* data_mask,
+  const int batch_size, const int channels, const int height_im, const int width_im, 
+  const int height_col, const int width_col, const int kernel_h, const int kenerl_w,
+  const int pad_h, const int pad_w, const int stride_h, const int stride_w, 
+  const int dilation_h, const int dilation_w,
+  const int deformable_group, float* data_col) {
+  // num_axes should be smaller than block size
+  const int channel_per_deformable_group = channels / deformable_group;
+  const int num_kernels = channels * batch_size * height_col * width_col;
+  modulated_deformable_im2col_gpu_kernel_general_version
+      <<<GET_BLOCKS(num_kernels), CUDA_NUM_THREADS,
+          0, stream>>>(
+      num_kernels, data_im, data_offset, data_mask, height_im, width_im, kernel_h, kenerl_w,
+      pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group,
+      batch_size, channels, deformable_group, height_col, width_col, data_col);
+  
+  cudaError_t err = cudaGetLastError();
+  if (err != cudaSuccess)
+    FatalError("error in modulatedDeformableIm2colCudaGeneralVersion: " + std::string(cudaGetErrorString(err)) + "\n");
+}
+
+void dcnV2CudaForward(cublasStatus_t stat, cublasHandle_t handle, 
+                         float *input, float *weight,
+                         float *bias, float *ones,
+                         float *offset, float *mask,
+                         float *output, float *columns,
+                         int kernel_h, int kernel_w,
+                         const int stride_h, const int stride_w,
+                         const int pad_h, const int pad_w,
+                         const int dilation_h, const int dilation_w,
+                         const int deformable_group, const int batch_id,
+                         const int in_n, const int in_c, const int in_h, const int in_w, 
+                         const int out_n, const int out_c, const int out_h, const int out_w,
+                         const int chunk_dim, cudaStream_t stream)
+{  
+  // stat and handle have be moved out to preserve 2 - 6 milliseconds every 100. 
+  const int batch = batch_id;
+  const int channels = in_c;
+  const int height = in_h;
+  const int width = in_w;
+
+  const int channels_out = out_c;
+
+  const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1;
+  const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1;
+  
+  long m = channels_out;
+  long n = height_out * width_out;
+  long k = 1;
+  float alpha = 1.0;
+  float beta = 0.0;
+  
+  stat = cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, 
+              n, m, k, &alpha, 
+              ones, k, bias, k, 
+              &beta, output + batch * out_c * out_h * out_w, n);
+  if (stat != CUBLAS_STATUS_SUCCESS)
+    FatalError("CUBLAS initialization failed\n");
+
+  modulatedDeformableIm2colCuda(stream,
+                                input + batch * channels * height * width,  
+                                offset,// + b * 2 * int((float)chunk_dim / batch),
+                                mask,// + b * int((float)chunk_dim / batch),
+                                    1, channels, height, width,
+                                    height_out, width_out, deformable_group, columns);
+  // modulatedDeformableIm2colCudaGeneralVersion(stream,
+  //                                   input, offset,
+  //                                   mask,
+  //                                   1, channels, height, width,
+  //                                   height_out, width_out, kernel_h, kernel_w,
+  //                                   pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w,
+  //                                   deformable_group, columns);
+  
+  //(k * m)  x  (m * n)
+  // Y = WC
+  k = channels * kernel_h * kernel_w;
+  beta = 1.0;
+  
+  stat = cublasSgemm(handle, CUBLAS_OP_N, CUBLAS_OP_N, 
+              n, m, k, &alpha, 
+              columns, n, weight, k, 
+              &beta, output + batch * out_c * out_h * out_w, n);
+
+  if (stat != CUBLAS_STATUS_SUCCESS)
+    FatalError("CUBLAS initialization failed\n");
+
+}

src/kernels/normalize.cu

@@ -0,0 +1,16 @@
+#include "kernelsThrust.h"
+
+__global__
+void normalize_kernel(float *bgr, const int dim, const float *mean, const float *stddev){
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+    int j = blockIdx.y;
+    bgr[j*(dim)+i] = bgr[j*(dim)+i] - mean[j];
+    bgr[j*(dim)+i] = bgr[j*(dim)+i] / stddev[j];
+    
+}
+
+void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev){
+    int num_thread = 256;
+    dim3 dimBlock(h*w/num_thread, ch);
+    normalize_kernel<<<dimBlock, num_thread, 0>>>(bgr, h*w, mean, stddev);
+}

src/kernels/pooling.cu

@@ -0,0 +1,52 @@
+#include "kernels.h"
+
+__global__ void forward_maxpool_layer_kernel(int n, int in_h, int in_w, int in_c, int stride_x, int stride_y, int size, int pad, float *input, float *output)
+{
+    int h = (in_h + pad - size) / stride_y + 1;
+    int w = (in_w + pad - size) / stride_x + 1;
+    int c = in_c;
+
+    int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if(id >= n) return;
+
+    int j = id % w;
+    id /= w;
+    int i = id % h;
+    id /= h;
+    int k = id % c;
+    id /= c;
+    int b = id;
+
+    int w_offset = -pad / 2;
+    int h_offset = -pad / 2;
+
+    int out_index = j + w*(i + h*(k + c*b));
+    float max = -9999999;
+    int max_i = -1;
+    int l, m;
+    for(l = 0; l < size; ++l){
+        for(m = 0; m < size; ++m){
+            int cur_h = h_offset + i*stride_y + l;
+            int cur_w = w_offset + j*stride_x + m;
+            int index = cur_w + in_w*(cur_h + in_h*(k + b*in_c));
+            int valid = (cur_h >= 0 && cur_h < in_h &&
+                    cur_w >= 0 && cur_w < in_w);
+            float val = (valid != 0) ? input[index] : -9999999;
+            max_i = (val > max) ? index : max_i;
+            max   = (val > max) ? val   : max;
+        }
+    }
+    output[out_index] = max;
+}
+
+void MaxPoolingForward(dnnType* srcData, dnnType* dstData, int n, int c, int h, int w, int stride_x, int stride_y, int size, int padding, cudaStream_t stream) 
+{
+
+    int tot_size = n*c*h*w;
+
+    int blocks = (tot_size+255)/256;
+    int threads = 256;
+    
+    forward_maxpool_layer_kernel<<<blocks, threads, 0, stream>>>(tot_size, h, w, c, stride_x, stride_y, size, padding, srcData, dstData);
+}
+

src/kernels/postprocessing.cu

@@ -0,0 +1,71 @@
+#include "kernelsThrust.h"
+
+
+void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){
+    thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op);
+}
+
+void sort(dnnType *src_begin, dnnType *src_end, int *idsrc){
+    thrust::sort_by_key(thrust::device,
+        src_begin, src_end, idsrc,
+        thrust::greater<float>());
+    // thrust::stable_sort_by_key(thrust::device,
+    //     src_begin, src_end, idsrc,
+    //     thrust::greater<float>());
+}
+
+void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
+            int *topk_inds, float *topk_ys, float *topk_xs){
+    checkCuda( cudaMemcpy(topk_scores, (float *)src_begin, K*sizeof(float), cudaMemcpyDeviceToDevice) );
+    checkCuda( cudaMemcpy(topk_inds, idsrc, K*sizeof(int), cudaMemcpyDeviceToDevice) );    
+}
+
+__global__ 
+void sortAndTopK_kernel(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs,const int size, const int K){
+    int i = blockDim.x*blockIdx.x + threadIdx.x;
+    thrust::sort_by_key(thrust::device, src_begin + i * size, src_begin + i * size + size, idsrc + i * size, thrust::greater<float>());
+    thrust::copy_n(thrust::device, src_begin + i * size, K, topk_scores + i * K);
+    thrust::copy_n(thrust::device, idsrc + i * size, K, topk_inds + i * K );
+}
+
+void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes){
+    int blocks = n_classes;
+    int threads = 1;
+    sortAndTopK_kernel<<<blocks, threads, 0>>>(src_begin, idsrc, topk_scores, topk_inds, topk_ys, topk_xs, size, K);   
+}
+
+void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys){    
+    thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), clses, thrust::divides<int>());
+    thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), ids_begin, thrust::modulus<int>());
+    thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(size), ys, thrust::divides<int>());
+    thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(size), xs, thrust::modulus<int>());
+}
+
+void topKxyAddOffset(int * ids_begin, const int K, const int size, 
+                     int *intxs_begin, int *intys_begin, float *xs_begin, 
+                     float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out){
+    thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out);
+    thrust::transform(thrust::device, intxs_begin, intxs_begin + K, src_out, xs_begin, thrust::plus<float>());
+    thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus<int>());
+    thrust::gather(thrust::device, ids_out, ids_out+K, src_begin, src_out);
+    thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus<float>());
+}
+
+void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin, 
+            dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1,
+            float *src_out, int *ids_out){ 
+    thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out);
+    thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides<float>());
+    // x0
+    thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx0, thrust::minus<float>());
+    // x1
+    thrust::transform(thrust::device, xs_begin, xs_begin + K, src_out, bbx1, thrust::plus<float>());
+    thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(size), ids_out, thrust::plus<int>());
+    thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out);
+    thrust::transform(thrust::device, src_out, src_out + K, thrust::make_constant_iterator(2), src_out, thrust::divides<float>());
+    // y0
+    thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby0, thrust::minus<float>());
+    // y1
+    thrust::transform(thrust::device, ys_begin, ys_begin + K, src_out, bby1, thrust::plus<float>());
+}
+

src/kernels/resize.cu

@@ -0,0 +1,46 @@
+#include "kernels.h"
+#include <stdio.h>
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#define MAX(a,b) (((a)>(b))?(a):(b))
+
+__global__ void resize_kernel(  int i_N,float *x, int i_w, int i_h, int i_c,  
+                                int o_w, int o_h, int o_c, int batch, float *out)
+{
+    int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if(i >= i_N) return;
+
+    int out_index = i;
+    int out_w = i%o_w;
+    i = i/o_w;
+    int out_h = i%o_h;
+    i = i/o_h;
+    int out_c = i%o_c;
+    i = i/o_c;
+
+    //copying last column/last row as padding
+    int in_index = ((i*i_c + MIN(out_c,i_c-1))*i_h +  MIN(out_h,i_h-1))*i_w + MIN(out_w, i_w-1);
+    out[out_index] = x[in_index];
+}
+
+
+void resizeForward(  dnnType* srcData, dnnType* dstData, int n, int i_c, int i_h, int i_w,
+     int o_c, int o_h, int o_w, cudaStream_t stream )
+{
+    int i_size = n*i_c*i_h*i_w;
+    int o_size = n*o_c*o_h*o_w;
+
+    int blocks = (o_size+255)/256;
+    int threads = 256;
+
+    if(i_c == o_c && i_h == o_h && i_w == o_w )
+    {
+        checkCuda(cudaMemcpy(dstData, srcData, i_size*sizeof(dnnType), cudaMemcpyDeviceToDevice));
+    }
+    else
+    {
+        checkCuda(cudaMemset(dstData, 0, o_size*sizeof(dnnType)));
+        resize_kernel<<<blocks, threads, 0, stream>>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData);
+        // printDeviceVector(i_size, srcData);
+        // printDeviceVector(o_size, dstData);
+    }
+}

src/kernels/scaladd.cu

@@ -0,0 +1,16 @@
+#include "kernels.h"
+#include <math.h>
+
+__global__ void scal_add_kernel(dnnType* dstData, int size, float alpha, float beta, int inc)
+{
+    int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    if (i < size) dstData[i*inc] = dstData[i*inc] * alpha + beta;
+}
+
+void scalAdd(dnnType* dstData, int size, float alpha, float beta, int inc, cudaStream_t stream)
+{
+    int blocks = (size+255)/256;
+    int threads = 256;
+    
+    scal_add_kernel<<<blocks, threads, 0, stream>>>(dstData, size, alpha, beta, inc);
+}

src/utils.cpp

@@ -20,6 +20,20 @@ bool fileExist(const char *fname) {
     return true;
 }
 
+void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url){
+    if(!fileExist(input_bin.c_str())){
+        std::string mkdir_cmd = "mkdir " + test_folder; 
+        std::string wget_cmd = "wget " + weights_url + " -O " + test_folder + "/weights.zip"; 
+        std::string unzip_cmd = "unzip " + test_folder + "/weights.zip -d" + test_folder;
+        std::string rm_cmd = "rm " + test_folder + "/weights.zip";
+        int err = 0;
+        err = system(mkdir_cmd.c_str());
+        err = system(wget_cmd.c_str());
+        err = system(unzip_cmd.c_str());
+        err = system(rm_cmd.c_str());
+    }
+}
+
 
 void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek)
 {
@@ -48,13 +62,14 @@ void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** dat
     checkCuda( cudaMemcpy(*data_d, *data_h, size_b, cudaMemcpyHostToDevice) );
 }
 
-void printDeviceVector(int size, dnnType* vec_d, bool device)
-{
+
+void printDeviceVector(int size, dnnType* vec_d, bool device){
     dnnType *vec;
     if(device) {
         vec = new dnnType[size];
-        cudaDeviceSynchronize();
-        cudaMemcpy(vec, vec_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost);
+        checkCuda(cudaDeviceSynchronize());
+        checkCuda(cudaMemcpy(vec, vec_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost));
+        checkCuda(cudaDeviceSynchronize());
     } else {
         vec = vec_d;
     }
@@ -76,15 +91,15 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int
     if(device) {
         data_h = new dnnType[size];
         correct_h = new dnnType[size];
-        cudaDeviceSynchronize();
-        cudaMemcpy(data_h, data_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost);
-        cudaMemcpy(correct_h, correct_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost);
-    
+        checkCuda(cudaDeviceSynchronize());
+        checkCuda(cudaMemcpy(data_h, data_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost));
+        checkCuda(cudaMemcpy(correct_h, correct_d, size*sizeof(dnnType), cudaMemcpyDeviceToHost));
+        checkCuda(cudaDeviceSynchronize());
+
     } else {
         data_h = data_d;
         correct_h = correct_d;
     }
-
     int diffs = 0;
     for(int i=0; i<size; i++) {
         if(data_h[i] != data_h[i] || correct_h[i] != correct_h[i] || //nan control
@@ -112,8 +127,18 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int
     return diffs;
 }
 
-void resize(int size, dnnType **data)
-{
+float getColor(const int c, const int x, const int max){
+    float _colors[6][3] = { {1,0,1}, {0,0,1},{0,1,1},{0,1,0},{1,1,0},{1,0,0} };
+    float ratio = ((float)x/max)*5;
+    int i = floor(ratio);
+    int j = ceil(ratio);
+    ratio -= i;
+    float r = (1-ratio) * _colors[i % 6][c % 3] + ratio*_colors[j % 6][c % 3];
+    return r;
+}
+
+
+void resize(int size, dnnType **data){
     if (*data != NULL)
         checkCuda( cudaFree(*data) );
     checkCuda( cudaMalloc(data, size*sizeof(dnnType)) );
@@ -139,3 +164,67 @@ void matrixMulAdd(  cublasHandle_t handle, dnnType* srcData, dnnType* dstData,
     checkERROR( cublasSaxpy(handle, dim, &alpha, srcData, 1, dstData, 1));
 
 }
+
+
+void getMemUsage(double& vm_usage_kb, double& resident_set_kb){
+   using std::ios_base;
+   using std::ifstream;
+   using std::string;
+
+   vm_usage_kb     = 0.0;
+   resident_set_kb = 0.0;
+
+   ifstream stat_stream("/proc/self/stat",ios_base::in);
+
+   //all the stats
+   string pid, comm, state, ppid, pgrp, session, tty_nr;
+   string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+   string utime, stime, cutime, cstime, priority, nice;
+   string O, itrealvalue, starttime;
+
+   unsigned long vsize;
+   long rss;
+
+   stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
+               >> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
+               >> utime >> stime >> cutime >> cstime >> priority >> nice
+               >> O >> itrealvalue >> starttime >> vsize >> rss;
+
+   stat_stream.close();
+
+   long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
+   vm_usage_kb     = vsize / 1024.0;  
+   resident_set_kb = rss * page_size_kb;
+}
+
+void printCudaMemUsage() {
+    size_t free, total;
+    checkCuda( cudaMemGetInfo(&free, &total) );	 
+    std::cout<<"GPU free memory: "<<double(free)/1e6<<" mb.\n";
+}
+
+void removePathAndExtension(const std::string &full_string, std::string &name){
+    name = full_string;
+    std::string tmp_str = full_string;
+	std::string slash = "/";
+    std::string dot = ".";
+    std::size_t current, previous = 0;
+
+    //remove path /path/to/
+	current = tmp_str.find(slash);
+    if (current != std::string::npos) {
+        while (current != std::string::npos) {
+            name = tmp_str.substr(previous, current - previous);
+            previous = current + 1;
+            current = tmp_str.find(slash, previous);
+        }
+        name = tmp_str.substr(previous, current - previous);
+    }
+    // remove extension
+    current = name.find(dot);
+    previous = 0;
+    if (current != std::string::npos) 
+        name = name.substr(previous, current);
+    
+    // std::cout<<"full string: "<<full_string<<" name: "<<name<<std::endl;
+}

tests/backbones/dla34/dla34.cpp

@@ -0,0 +1,350 @@
+#include <iostream>
+#include "tkdnn.h"
+
+const char *input_bin = "dla34/debug/input.bin";
+const char *conv1_bin = "dla34/layers/features-init_block-conv1-conv.bin";
+const char *conv2_bin = "dla34/layers/features-init_block-conv2-conv.bin";
+const char *conv3_bin = "dla34/layers/features-init_block-conv3-conv.bin";
+// s - stage, t - tree
+const char *s1_t1_conv1_bin = "dla34/layers/features-stage1-tree1-body-conv1-conv.bin";
+const char *s1_t1_conv2_bin = "dla34/layers/features-stage1-tree1-body-conv2-conv.bin";
+const char *s1_t1_project = "dla34/layers/features-stage1-tree1-project_conv-conv.bin";
+const char *s1_t2_conv1_bin = "dla34/layers/features-stage1-tree2-body-conv1-conv.bin";
+const char *s1_t2_conv2_bin = "dla34/layers/features-stage1-tree2-body-conv2-conv.bin";
+const char *s1_root_conv1_bin = "dla34/layers/features-stage1-root-conv-conv.bin";
+const char *s2_t1_t1_conv1_bin = "dla34/layers/features-stage2-tree1-tree1-body-conv1-conv.bin";
+const char *s2_t1_t1_conv2_bin = "dla34/layers/features-stage2-tree1-tree1-body-conv2-conv.bin";
+const char *s2_t1_t1_project = "dla34/layers/features-stage2-tree1-tree1-project_conv-conv.bin";
+const char *s2_t1_t2_conv1_bin = "dla34/layers/features-stage2-tree1-tree2-body-conv1-conv.bin";
+const char *s2_t1_t2_conv2_bin = "dla34/layers/features-stage2-tree1-tree2-body-conv2-conv.bin";
+const char *s2_t1_root_conv1_bin = "dla34/layers/features-stage2-tree1-root-conv-conv.bin";
+const char *s2_t2_t1_conv1_bin = "dla34/layers/features-stage2-tree2-tree1-body-conv1-conv.bin";
+const char *s2_t2_t1_conv2_bin = "dla34/layers/features-stage2-tree2-tree1-body-conv2-conv.bin";
+const char *s2_t2_t2_conv1_bin = "dla34/layers/features-stage2-tree2-tree2-body-conv1-conv.bin";
+const char *s2_t2_t2_conv2_bin = "dla34/layers/features-stage2-tree2-tree2-body-conv2-conv.bin";
+const char *s2_t2_root_conv1_bin = "dla34/layers/features-stage2-tree2-root-conv-conv.bin";
+const char *s3_t1_t1_conv1_bin = "dla34/layers/features-stage3-tree1-tree1-body-conv1-conv.bin";
+const char *s3_t1_t1_conv2_bin = "dla34/layers/features-stage3-tree1-tree1-body-conv2-conv.bin";
+const char *s3_t1_t1_project = "dla34/layers/features-stage3-tree1-tree1-project_conv-conv.bin";
+const char *s3_t1_t2_conv1_bin = "dla34/layers/features-stage3-tree1-tree2-body-conv1-conv.bin";
+const char *s3_t1_t2_conv2_bin = "dla34/layers/features-stage3-tree1-tree2-body-conv2-conv.bin";
+const char *s3_t1_root_conv1_bin = "dla34/layers/features-stage3-tree1-root-conv-conv.bin";
+const char *s3_t2_t1_conv1_bin = "dla34/layers/features-stage3-tree2-tree1-body-conv1-conv.bin";
+const char *s3_t2_t1_conv2_bin = "dla34/layers/features-stage3-tree2-tree1-body-conv2-conv.bin";
+const char *s3_t2_t2_conv1_bin = "dla34/layers/features-stage3-tree2-tree2-body-conv1-conv.bin";
+const char *s3_t2_t2_conv2_bin = "dla34/layers/features-stage3-tree2-tree2-body-conv2-conv.bin";
+const char *s3_t2_root_conv1_bin = "dla34/layers/features-stage3-tree2-root-conv-conv.bin";
+const char *s4_t1_conv1_bin = "dla34/layers/features-stage4-tree1-body-conv1-conv.bin";
+const char *s4_t1_conv2_bin = "dla34/layers/features-stage4-tree1-body-conv2-conv.bin";
+const char *s4_t1_project = "dla34/layers/features-stage4-tree1-project_conv-conv.bin";
+const char *s4_t2_conv1_bin = "dla34/layers/features-stage4-tree2-body-conv1-conv.bin";
+const char *s4_t2_conv2_bin = "dla34/layers/features-stage4-tree2-body-conv2-conv.bin";
+const char *s4_root_conv1_bin = "dla34/layers/features-stage4-root-conv-conv.bin";
+
+//final
+const char *fc_bin = "dla34/layers/output.bin";
+
+const char *output_bin = "dla34/debug/output.bin";  
+
+int main()
+{
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 224, 224, 1);
+    tk::dnn::Network net(dim);
+    tk::dnn::Layer *last1, *last2, *last3, *last4;
+
+
+    tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
+    tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU);
+
+    tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true);
+    tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU);
+
+    tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true);
+    tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
+
+    last1 = &relu3;
+    
+    // level 2
+    // tree 1
+        tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true);
+        tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true);
+        last2 = &s1_t1_conv2;
+
+        // get the basicblock input and apply maxpool conv2d and relu
+        tk::dnn::Layer    *route_s1_t1_layers[1] = { last1 };
+        tk::dnn::Route    route_s1_t1(&net, route_s1_t1_layers, 1);
+        // downsample
+        tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+        // project
+        tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true);
+        
+        tk::dnn::Shortcut s1_t1_s1(&net, last2);
+        tk::dnn::Activation   s1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    last1 = &s1_t1_relu;
+    
+    // tree 2
+        tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true);
+        tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true);
+        
+        tk::dnn::Shortcut s1_t2_s1(&net, last1);
+        tk::dnn::Activation   s1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+        last2 = &s1_t2_relu;
+
+    // root
+        // join last1 and net in single input 128, 56, 56
+        tk::dnn::Layer    *route_s1_root_layers[2] = { last2, last1 };
+        tk::dnn::Route    route_s1_root(&net, route_s1_root_layers, 2);
+        tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true);
+        tk::dnn::Activation   s1_root_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    last1 = &s1_root_relu;
+    // level 3
+    // tree 1
+        // tree 1
+            tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true);
+            tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);      
+        
+            tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true);
+            last2 = &s2_t1_t1_conv2;
+
+            // get the basicblock input and apply maxpool conv2d and relu
+            tk::dnn::Layer    *route_s2_t1_t1_layers[1] = { last1 };
+            tk::dnn::Route    route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1);
+            // downsample
+            tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+            last4 = &s2_t1_t1_maxpool1;
+            // project
+            tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true);
+            
+            tk::dnn::Shortcut s2_t1_t1_s1(&net, last2);
+            tk::dnn::Activation   s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s2_t1_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true);
+            tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut s2_t1_t2_s1(&net, last1);
+            tk::dnn::Activation   s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s2_t1_t2_relu;
+
+        // root
+            // join last1 and net in single input 128, 56, 56
+            tk::dnn::Layer    *route_s2_t1_root_layers[2] = { last2, last1 };
+            tk::dnn::Route    route_s2_t1_root(&net, route_s2_t1_root_layers, 2);
+            tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true);
+            tk::dnn::Activation   s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    last1 = &s2_t1_root_relu;
+    last3 = &s2_t1_root_relu;
+    // tree 2
+        // tree 1
+            tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true);
+            tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true);
+            tk::dnn::Shortcut s2_t2_t1_s1(&net, last1);
+            tk::dnn::Activation   s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s2_t2_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true);
+            tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut s2_t2_t2_s1(&net, last1);
+            tk::dnn::Activation   s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s2_t2_t2_relu;
+
+        // root
+            // join last1 and net in single input 128, 56, 56
+            tk::dnn::Layer    *route_s2_t2_root_layers[4] = { last2, last1, last4, last3};
+            tk::dnn::Route    route_s2_t2_root(&net, route_s2_t2_root_layers, 4);
+            tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true);
+            tk::dnn::Activation   s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    
+    last1 = &s2_t2_root_relu;
+    // level 4
+    // tree 1
+        // tree 1
+            tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true);
+            tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);      
+        
+            tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true);
+            last2 = &s3_t1_t1_conv2;
+
+            // get the basicblock input and apply maxpool conv2d and relu
+            tk::dnn::Layer    *route_s3_t1_t1_layers[1] = { last1 };
+            tk::dnn::Route    route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1);
+            // downsample
+            tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+            last4 = &s3_t1_t1_maxpool1;
+            // project
+            tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true);
+            
+            tk::dnn::Shortcut s3_t1_t1_s1(&net, last2);
+            tk::dnn::Activation   s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s3_t1_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true);
+            tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut s3_t1_t2_s1(&net, last1);
+            tk::dnn::Activation   s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s3_t1_t2_relu;
+
+        // root
+            // join last1 and net in single input 256, 56, 56
+            tk::dnn::Layer    *route_s3_t1_root_layers[2] = { last2, last1 };
+            tk::dnn::Route    route_s3_t1_root(&net, route_s3_t1_root_layers, 2);
+            tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true);
+            tk::dnn::Activation   s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    last1 = &s3_t1_root_relu;
+    last3 = &s3_t1_root_relu;
+    // tree 2
+        // tree 1
+            tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true);
+            tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true);
+            tk::dnn::Shortcut s3_t2_t1_s1(&net, last1);
+            tk::dnn::Activation   s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s3_t2_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true);
+            tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut s3_t2_t2_s1(&net, last1);
+            tk::dnn::Activation   s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s3_t2_t2_relu;
+
+        // root
+            // join last1 and net in single input 256, 56, 56
+            tk::dnn::Layer    *route_s3_t2_root_layers[4] = { last2, last1, last4, last3};
+            tk::dnn::Route    route_s3_t2_root(&net, route_s3_t2_root_layers, 4);
+            tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true);
+            tk::dnn::Activation   s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    last1 = &s3_t2_root_relu;
+    // level 4
+    // tree 1
+        tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true);
+        tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true);
+        last2 = &s4_t1_conv2;
+
+        // get the basicblock input and apply maxpool conv2d and relu
+        tk::dnn::Layer    *route_s4_t1_layers[1] = { last1 };
+        tk::dnn::Route    route_s4_t1(&net, route_s4_t1_layers, 1);
+        // downsample
+        tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+        last4 = &s4_t1_maxpool1;
+        // project
+        tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true);
+        
+        tk::dnn::Shortcut s4_t1_s1(&net, last2);
+        tk::dnn::Activation   s4_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    last1 = &s4_t1_relu;
+    
+    // tree 2
+        tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true);
+        tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true);
+        
+        tk::dnn::Shortcut s4_t2_s1(&net, last1);
+        tk::dnn::Activation   s4_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+        last2 = &s4_t2_relu;
+
+    // root
+        // join last1 and net in single input 128, 56, 56
+        tk::dnn::Layer    *route_s4_root_layers[3] = { last2, last1, last4 };
+        tk::dnn::Route    route_s4_root(&net, route_s4_root_layers, 3);
+        tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true);
+        tk::dnn::Activation   s4_root_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    //final
+    tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
+    tk::dnn::Dense   fc(&net, 1000, fc_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    //printDeviceVector(64, data, true);
+
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34"));
+
+    
+    tk::dnn::dataDim_t out_dim;
+    out_dim = net.layers[net.num_layers-1]->output_dim;
+    dnnType *cudnn_out, *rt_out;
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30);
+    {
+        dim1.print();
+        TKDNN_TSTART
+        net.infer(dim1, data);
+        TKDNN_TSTOP
+        dim1.print();
+    }
+    cudnn_out = net.layers[net.num_layers-1]->dstData;
+
+
+    // printDeviceVector(64, cudnn_out, true);
+
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30);
+    {
+        dim2.print();
+        TKDNN_TSTART
+        netRT.infer(dim2, data);
+        TKDNN_TSTOP
+        dim2.print();
+    }
+    rt_out = (dnnType *)netRT.buffersRT[1];
+
+
+    printCenteredTitle(std::string(" RESNET CHECK RESULTS ").c_str(), '=', 30);
+    dnnType *out, *out_h;
+    int odim = out_dim.tot();
+    readBinaryFile(output_bin, odim, &out_h, &out);
+
+    std::cout<<"CUDNN vs correct"; 
+    int ret_cudnn = checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
+    std::cout<<"TRT   vs correct"; 
+    int ret_tensorrt = checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
+    std::cout<<"CUDNN vs TRT    "; 
+    int ret_cudnn_tensorrt = checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
+
+    return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
+}

tests/backbones/dla34/dla34_weightsexporter.py

@@ -0,0 +1,162 @@
+import torch
+import urllib
+from PIL import Image
+from torchvision import transforms
+import numpy as np
+import struct 
+import os
+
+from pytorchcv.model_provider import get_model as ptcv_get_model
+from torch.autograd import Variable
+
+from torchsummary import summary
+import torch.nn as nn
+
+from torch.jit import trace
+
+def create_folders():
+    if not os.path.exists('debug'):
+        os.makedirs('debug')
+    if not os.path.exists('layers'):
+        os.makedirs('layers')
+
+def bin_write(f, data):
+    data =data.flatten()
+    fmt = 'f'*len(data)
+    bin = struct.pack(fmt, *data)
+    f.write(bin)
+
+def hook(module, input, output):
+    setattr(module, "_value_hook", output)
+
+def load_ex_image(model):
+    # Download an example image from the pytorch website
+    url, filename = (
+        "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg")
+    try:
+        urllib.URLopener().retrieve(url, filename)
+    except:
+        urllib.request.urlretrieve(url, filename)
+    
+    # sample execution (requires torchvision)
+    input_image = Image.open(filename)
+    print("input_image: ",input_image.size)
+    preprocess = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[
+                             0.229, 0.224, 0.225]),
+    ])
+    input_tensor = preprocess(input_image)
+    print("input_tensor: ",input_tensor.shape)
+    # create a mini-batch as expected by the model
+    input_batch = input_tensor.unsqueeze(0)
+
+    # move the input and model to GPU for speed if available
+    if torch.cuda.is_available():
+        input_batch = input_batch.to('cuda')
+        model.to('cuda')
+    
+    return model, input_batch
+
+def exp_input(model, input_batch):
+    # Export the input batch 
+    model(input_batch)
+    i = input_batch.cpu().data.numpy()
+    i = np.array(i, dtype=np.float32)
+    i.tofile("debug/input.bin", format="f")
+    print("input: ", i.shape)
+
+def print_wb_output(model):
+    f = None
+    for n, m in model.named_modules():
+        m.eval()
+        if 'DLAResBlock' in str(m.type):
+            continue
+
+        in_output = m._value_hook
+        o = in_output.data.numpy()
+        o = np.array(o, dtype=np.float32)
+        
+        t = '-'.join(n.split('.'))
+        o.tofile("debug/" + t + ".bin", format="f")
+        print('------- ', n, ' ------') 
+        print("debug  ",o.shape)
+
+        if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)):
+            continue
+        
+        if ' of Conv2d' in str(m.type) or ' of Linear' in str(m.type):
+            file_name = "layers/" + t + ".bin"
+            print("open file: ", file_name)
+            f = open(file_name, mode='wb')
+
+        w = np.array([])
+        b = np.array([])
+        if 'weight' in m._parameters and m._parameters['weight'] is not None:
+            w = m._parameters['weight'].data.numpy()
+            w = np.array(w, dtype=np.float32)
+            print ("    weights shape:", np.shape(w))
+        
+        if 'bias' in m._parameters and m._parameters['bias'] is not None:
+            b = m._parameters['bias'].data.numpy()
+            b = np.array(b, dtype=np.float32)
+            print ("    bias shape:", np.shape(b))
+        
+        if 'BatchNorm2d' in str(m.type):
+            b = m._parameters['bias'].data.numpy()
+            b = np.array(b, dtype=np.float32)
+            s = m._parameters['weight'].data.numpy()
+            s = np.array(s, dtype=np.float32)
+            rm = m.running_mean.data.numpy()
+            rm = np.array(rm, dtype=np.float32)
+            rv = m.running_var.data.numpy()
+            rv = np.array(rv, dtype=np.float32)
+            bin_write(f,b)
+            bin_write(f,s)
+            bin_write(f,rm)
+            bin_write(f,rv)
+            print ("    b shape:", np.shape(b))
+            print ("    s shape:", np.shape(s))
+            print ("    rm shape:", np.shape(rm))
+            print ("    rv shape:", np.shape(rv))
+            
+        else:
+            bin_write(f,w)
+            if b.size > 0 and b is not None:
+                bin_write(f,b)
+                
+        if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type):
+            f.close()
+            print("close file")
+            f = None
+
+if __name__ == '__main__':
+    model = ptcv_get_model("dla34", pretrained=True)
+    model.eval()
+
+    # load an example image and load it on model
+    model, input_batch = load_ex_image(model)
+    model.eval()
+    with torch.no_grad():
+        output = model(input_batch)
+
+    # create folders debug and layers if do not exist
+    create_folders()
+
+    # add output attribute to the layers
+    for n, m in model.named_modules():
+        m.register_forward_hook(hook)
+
+    # export input bin
+    exp_input(model, input_batch)
+
+    print_wb_output(model)
+
+    with open("dla34.txt", 'w') as f:
+        for item in list(model.children()):
+            f.write("%s\n" % item)
+
+    summary(model, (3, 224, 224))
+    # print(trace(model, input_batch))

tests/backbones/dla34/env_dla34.yml

@@ -0,0 +1,60 @@
+name: dla34
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _pytorch_select=0.2=gpu_0
+  - blas=1.0=mkl
+  - ca-certificates=2019.10.16=0
+  - certifi=2019.9.11=py36_0
+  - cffi=1.13.1=py36h2e261b9_0
+  - cudatoolkit=10.0.130=0
+  - cudnn=7.6.0=cuda10.0_0
+  - freetype=2.9.1=h8a8886c_1
+  - intel-openmp=2019.4=243
+  - jpeg=9b=h024ee3a_2
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.2.1=hd88cf55_4
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libpng=1.6.37=hbc83047_0
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libtiff=4.0.10=h2733197_2
+  - mkl=2019.4=243
+  - mkl-service=2.3.0=py36he904b0f_0
+  - mkl_fft=1.0.14=py36ha843d7b_0
+  - mkl_random=1.1.0=py36hd6b4f25_0
+  - ncurses=6.1=he6710b0_1
+  - ninja=1.9.0=py36hfd86e86_0
+  - numpy=1.17.2=py36haad9e8e_0
+  - numpy-base=1.17.2=py36hde5b4d6_0
+  - olefile=0.46=py36_0
+  - openssl=1.1.1d=h7b6447c_3
+  - pillow=6.2.0=py36h34e0f95_0
+  - pip=19.3.1=py36_0
+  - pycparser=2.19=py36_0
+  - python=3.6.9=h265db76_0
+  - readline=7.0=h7b6447c_5
+  - setuptools=41.6.0=py36_0
+  - six=1.12.0=py36_0
+  - sqlite=3.30.1=h7b6447c_0
+  - tk=8.6.8=hbc83047_0
+  - wheel=0.33.6=py36_0
+  - xz=5.2.4=h14c3975_4
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.3.7=h0b5b093_0
+  - pip:
+    - chardet==3.0.4
+    - decorator==4.4.1
+    - idna==2.8
+    - lxml==4.4.2
+    - networkx==2.4
+    - nltk==3.4.5
+    - pytorchcv==0.0.55
+    - requests==2.22.0
+    - summary==0.2.0
+    - torch==1.3.0
+    - torchsummary==1.5.1
+    - torchvision==0.4.1
+    - urllib3==1.25.8
+

tests/backbones/resnet101/env_resnet101.yml

@@ -0,0 +1,56 @@
+name: resnet101
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _pytorch_select=0.2=gpu_0
+  - blas=1.0=mkl
+  - ca-certificates=2019.10.16=0
+  - certifi=2019.9.11=py36_0
+  - cffi=1.13.1=py36h2e261b9_0
+  - cudatoolkit=10.0.130=0
+  - cudnn=7.6.0=cuda10.0_0
+  - freetype=2.9.1=h8a8886c_1
+  - intel-openmp=2019.4=243
+  - jpeg=9b=h024ee3a_2
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.2.1=hd88cf55_4
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libpng=1.6.37=hbc83047_0
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libtiff=4.0.10=h2733197_2
+  - mkl=2019.4=243
+  - mkl-service=2.3.0=py36he904b0f_0
+  - mkl_fft=1.0.14=py36ha843d7b_0
+  - mkl_random=1.1.0=py36hd6b4f25_0
+  - ncurses=6.1=he6710b0_1
+  - ninja=1.9.0=py36hfd86e86_0
+  - numpy=1.17.2=py36haad9e8e_0
+  - numpy-base=1.17.2=py36hde5b4d6_0
+  - olefile=0.46=py36_0
+  - openssl=1.1.1d=h7b6447c_3
+  - pillow=6.2.0=py36h34e0f95_0
+  - pip=19.3.1=py36_0
+  - pycparser=2.19=py36_0
+  - python=3.6.9=h265db76_0
+  - pytorch=1.2.0=cuda100py36h938c94c_0
+  - readline=7.0=h7b6447c_5
+  - setuptools=41.6.0=py36_0
+  - six=1.12.0=py36_0
+  - sqlite=3.30.1=h7b6447c_0
+  - tk=8.6.8=hbc83047_0
+  - wheel=0.33.6=py36_0
+  - xz=5.2.4=h14c3975_4
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.3.7=h0b5b093_0
+  - pip:
+    - chardet==3.0.4
+    - idna==2.8
+    - pytorchcv==0.0.55
+    - requests==2.22.0
+    - torch==1.3.0
+    - torchsummary==1.5.1
+    - torchvision==0.4.1
+    - urllib3==1.25.8
+

tests/backbones/resnet101/resnet101.cpp

@@ -0,0 +1,338 @@
+#include <iostream>
+#include "tkdnn.h"
+
+const char *input_bin = "resnet101/debug/input.bin";
+const char *conv1_bin = "resnet101/layers/conv1.bin";
+
+//layer1
+const char *layer1_bin[]={
+"resnet101/layers/layer1-0-conv1.bin",
+"resnet101/layers/layer1-0-conv2.bin",
+"resnet101/layers/layer1-0-conv3.bin",
+"resnet101/layers/layer1-0-downsample-0.bin",
+
+"resnet101/layers/layer1-1-conv1.bin",
+"resnet101/layers/layer1-1-conv2.bin",
+"resnet101/layers/layer1-1-conv3.bin",
+
+"resnet101/layers/layer1-2-conv1.bin",
+"resnet101/layers/layer1-2-conv2.bin",
+"resnet101/layers/layer1-2-conv3.bin"};
+                        
+
+//layer2
+const char *layer2_bin[]={
+"resnet101/layers/layer2-0-conv1.bin",
+"resnet101/layers/layer2-0-conv2.bin",
+"resnet101/layers/layer2-0-conv3.bin",
+"resnet101/layers/layer2-0-downsample-0.bin",
+
+"resnet101/layers/layer2-1-conv1.bin",
+"resnet101/layers/layer2-1-conv2.bin",
+"resnet101/layers/layer2-1-conv3.bin",
+
+"resnet101/layers/layer2-2-conv1.bin",
+"resnet101/layers/layer2-2-conv2.bin",
+"resnet101/layers/layer2-2-conv3.bin",
+
+"resnet101/layers/layer2-3-conv1.bin",
+"resnet101/layers/layer2-3-conv2.bin",
+"resnet101/layers/layer2-3-conv3.bin"
+};
+//layer3
+const char *layer3_bin[]={
+"resnet101/layers/layer3-0-conv1.bin",
+"resnet101/layers/layer3-0-conv2.bin",
+"resnet101/layers/layer3-0-conv3.bin",
+"resnet101/layers/layer3-0-downsample-0.bin",
+
+"resnet101/layers/layer3-1-conv1.bin",
+"resnet101/layers/layer3-1-conv2.bin",
+"resnet101/layers/layer3-1-conv3.bin",
+
+"resnet101/layers/layer3-2-conv1.bin",
+"resnet101/layers/layer3-2-conv2.bin",
+"resnet101/layers/layer3-2-conv3.bin",
+
+"resnet101/layers/layer3-3-conv1.bin",
+"resnet101/layers/layer3-3-conv2.bin",
+"resnet101/layers/layer3-3-conv3.bin",
+
+"resnet101/layers/layer3-4-conv1.bin",
+"resnet101/layers/layer3-4-conv2.bin",
+"resnet101/layers/layer3-4-conv3.bin",
+
+"resnet101/layers/layer3-5-conv1.bin",
+"resnet101/layers/layer3-5-conv2.bin",
+"resnet101/layers/layer3-5-conv3.bin",
+
+"resnet101/layers/layer3-6-conv1.bin",
+"resnet101/layers/layer3-6-conv2.bin",
+"resnet101/layers/layer3-6-conv3.bin",
+
+"resnet101/layers/layer3-7-conv1.bin",
+"resnet101/layers/layer3-7-conv2.bin",
+"resnet101/layers/layer3-7-conv3.bin",
+
+"resnet101/layers/layer3-8-conv1.bin",
+"resnet101/layers/layer3-8-conv2.bin",
+"resnet101/layers/layer3-8-conv3.bin",
+
+"resnet101/layers/layer3-9-conv1.bin",
+"resnet101/layers/layer3-9-conv2.bin",
+"resnet101/layers/layer3-9-conv3.bin",
+
+"resnet101/layers/layer3-10-conv1.bin",
+"resnet101/layers/layer3-10-conv2.bin",
+"resnet101/layers/layer3-10-conv3.bin",
+
+"resnet101/layers/layer3-11-conv1.bin",
+"resnet101/layers/layer3-11-conv2.bin",
+"resnet101/layers/layer3-11-conv3.bin",
+
+"resnet101/layers/layer3-12-conv1.bin",
+"resnet101/layers/layer3-12-conv2.bin",
+"resnet101/layers/layer3-12-conv3.bin",
+
+"resnet101/layers/layer3-13-conv1.bin",
+"resnet101/layers/layer3-13-conv2.bin",
+"resnet101/layers/layer3-13-conv3.bin",
+
+"resnet101/layers/layer3-14-conv1.bin",
+"resnet101/layers/layer3-14-conv2.bin",
+"resnet101/layers/layer3-14-conv3.bin",
+
+"resnet101/layers/layer3-15-conv1.bin",
+"resnet101/layers/layer3-15-conv2.bin",
+"resnet101/layers/layer3-15-conv3.bin",
+
+"resnet101/layers/layer3-16-conv1.bin",
+"resnet101/layers/layer3-16-conv2.bin",
+"resnet101/layers/layer3-16-conv3.bin",
+
+"resnet101/layers/layer3-17-conv1.bin",
+"resnet101/layers/layer3-17-conv2.bin",
+"resnet101/layers/layer3-17-conv3.bin",
+
+"resnet101/layers/layer3-18-conv1.bin",
+"resnet101/layers/layer3-18-conv2.bin",
+"resnet101/layers/layer3-18-conv3.bin",
+
+"resnet101/layers/layer3-19-conv1.bin",
+"resnet101/layers/layer3-19-conv2.bin",
+"resnet101/layers/layer3-19-conv3.bin",
+
+"resnet101/layers/layer3-20-conv1.bin",
+"resnet101/layers/layer3-20-conv2.bin",
+"resnet101/layers/layer3-20-conv3.bin",
+
+"resnet101/layers/layer3-21-conv1.bin",
+"resnet101/layers/layer3-21-conv2.bin",
+"resnet101/layers/layer3-21-conv3.bin",
+
+"resnet101/layers/layer3-22-conv1.bin",
+"resnet101/layers/layer3-22-conv2.bin",
+"resnet101/layers/layer3-22-conv3.bin"};
+
+
+//layer4
+const char *layer4_bin[]={
+"resnet101/layers/layer4-0-conv1.bin",
+"resnet101/layers/layer4-0-conv2.bin",
+"resnet101/layers/layer4-0-conv3.bin",
+"resnet101/layers/layer4-0-downsample-0.bin",
+
+"resnet101/layers/layer4-1-conv1.bin",
+"resnet101/layers/layer4-1-conv2.bin",
+"resnet101/layers/layer4-1-conv3.bin",
+
+"resnet101/layers/layer4-2-conv1.bin",
+"resnet101/layers/layer4-2-conv2.bin",
+"resnet101/layers/layer4-2-conv3.bin"};
+
+//final
+const char *fc_bin = "resnet101/layers/fc.bin";
+
+const char *output_bin = "resnet101/debug/fc.bin";
+
+int main()
+{
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 224, 224, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d conv1(&net, 64, 7, 7, 2, 2, 3, 3, conv1_bin, true);
+    tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Pooling maxpool4(&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX);
+
+    //layer 1
+    int id_layer1_bin = 0;
+    tk::dnn::Layer *last = &maxpool4;
+    for(int i=0; i<3;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 64, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, layer1_bin[id_layer1_bin++], true);
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true);
+        if(i==0) {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } else {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+
+    // tk::dnn::Activation *last_activation = (tk::dnn::Activation *) net.layers[net.num_layers-1];
+    // layer 2
+    int id_layer2_bin = 0;
+    for(int i=0; i<4;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 128, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2;
+        if(i==0)
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 2, 2, 1, 1, layer2_bin[id_layer2_bin++], true);
+        else
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 1, 1, 1, 1, layer2_bin[id_layer2_bin++], true);
+        
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true);
+        if(i==0) 
+        {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 512, 1, 1, 2, 2, 0, 0, layer2_bin[id_layer2_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } 
+        else 
+        {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+    
+    // layer 3
+    int id_layer3_bin = 0;
+    for(int i=0; i<23;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2;
+        if(i==0)
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 2, 2, 1, 1, layer3_bin[id_layer3_bin++], true);
+        else
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, layer3_bin[id_layer3_bin++], true);
+        
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true);
+        if(i==0) 
+        {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 2, 2, 0, 0, layer3_bin[id_layer3_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } 
+        else 
+        {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+
+    // layer 4
+    int id_layer4_bin = 0;
+    for(int i=0; i<3;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2;
+        if(i==0)
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 2, 2, 1, 1, layer4_bin[id_layer4_bin++], true);
+        else
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 1, 1, 1, 1, layer4_bin[id_layer4_bin++], true);
+        
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true);
+        if(i==0) 
+        {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 2, 2, 0, 0, layer4_bin[id_layer4_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } 
+        else 
+        {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+
+    //final
+    tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
+    tk::dnn::Dense   fc(&net, 1000, fc_bin);
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    //printDeviceVector(64, data, true);
+
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("resnet101"));
+
+    
+    tk::dnn::dataDim_t out_dim;
+    out_dim = net.layers[net.num_layers-1]->output_dim;
+    dnnType *cudnn_out, *rt_out;
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30);
+    {
+        dim1.print();
+        TKDNN_TSTART
+        net.infer(dim1, data);
+        TKDNN_TSTOP
+        dim1.print();
+    }
+    cudnn_out = net.layers[net.num_layers-1]->dstData;
+
+    //printDeviceVector(64, cudnn_out, true);
+
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30);
+    {
+        dim2.print();
+        TKDNN_TSTART
+        netRT.infer(dim2, data);
+        TKDNN_TSTOP
+        dim2.print();
+    }
+    rt_out = (dnnType *)netRT.buffersRT[1];
+
+
+    printCenteredTitle(std::string(" RESNET CHECK RESULTS ").c_str(), '=', 30);
+    dnnType *out, *out_h;
+    int odim = out_dim.tot();
+    readBinaryFile(output_bin, odim, &out_h, &out);
+    
+    std::cout<<"CUDNN vs correct"; 
+    int ret_cudnn = checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
+    std::cout<<"TRT   vs correct"; 
+    int ret_tensorrt = checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
+    std::cout<<"CUDNN vs TRT    "; 
+    int ret_cudnn_tensorrt = checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
+
+    return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
+}

tests/backbones/resnet101/resnet101_weightsexporter.py

@@ -0,0 +1,162 @@
+import torch
+import urllib
+from PIL import Image
+from torchvision import transforms
+import numpy as np
+import struct 
+import os
+
+from pytorchcv.model_provider import get_model as ptcv_get_model
+from torch.autograd import Variable
+
+from torchsummary import summary
+import torch.nn as nn
+
+from torch.jit import trace
+
+def create_folders():
+    if not os.path.exists('debug'):
+        os.makedirs('debug')
+    if not os.path.exists('layers'):
+        os.makedirs('layers')
+
+def bin_write(f, data):
+    data =data.flatten()
+    fmt = 'f'*len(data)
+    bin = struct.pack(fmt, *data)
+    f.write(bin)
+
+def hook(module, input, output):
+    setattr(module, "_value_hook", output)
+
+def load_ex_image(model):
+    # Download an example image from the pytorch website
+    url, filename = (
+        "https://github.com/pytorch/hub/raw/master/dog.jpg", "dog.jpg")
+    try:
+        urllib.URLopener().retrieve(url, filename)
+    except:
+        urllib.request.urlretrieve(url, filename)
+    
+    # sample execution (requires torchvision)
+    input_image = Image.open(filename)
+    print("input_image: ",input_image.size)
+    preprocess = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[
+                             0.229, 0.224, 0.225]),
+    ])
+    input_tensor = preprocess(input_image)
+    print("input_tensor: ",input_tensor.shape)
+    # create a mini-batch as expected by the model
+    input_batch = input_tensor.unsqueeze(0)
+
+    # move the input and model to GPU for speed if available
+    if torch.cuda.is_available():
+        input_batch = input_batch.to('cuda')
+        model.to('cuda')
+    
+    return model, input_batch
+
+def exp_input(model, input_batch):
+    # Export the input batch 
+    model(input_batch)
+    i = input_batch.cpu().data.numpy()
+    i = np.array(i, dtype=np.float32)
+    i.tofile("debug/input.bin", format="f")
+    print("input: ", i.shape)
+
+def print_wb_output(model):
+    f = None
+    for n, m in model.named_modules():
+        in_output = m._value_hook
+        o = in_output.data.numpy()
+        o = np.array(o, dtype=np.float32)
+        t = '-'.join(n.split('.'))
+        o.tofile("debug/" + t + ".bin", format="f")
+        print('------- ', n, ' ------') 
+        print("debug  ",o.shape)
+        
+        if not(' of Conv2d' in str(m.type) or ' of Linear' in str(m.type) or ' of BatchNorm2d' in str(m.type)):
+            continue
+        
+        if ' of Conv2d' in str(m.type) or ' of Linear' in str(m.type):
+            file_name = "layers/" + t + ".bin"
+            print("open file: ", file_name)
+            f = open(file_name, mode='wb')
+
+        w = np.array([])
+        b = np.array([])
+        if 'weight' in m._parameters and m._parameters['weight'] is not None:
+            w = m._parameters['weight'].data.numpy()
+            w = np.array(w, dtype=np.float32)
+            print ("    weights shape:", np.shape(w))
+        
+        if 'bias' in m._parameters and m._parameters['bias'] is not None:
+            b = m._parameters['bias'].data.numpy()
+            b = np.array(b, dtype=np.float32)
+            print ("    bias shape:", np.shape(b))
+        
+        if 'BatchNorm2d' in str(m.type):
+            b = m._parameters['bias'].data.numpy()
+            b = np.array(b, dtype=np.float32)
+            s = m._parameters['weight'].data.numpy()
+            s = np.array(s, dtype=np.float32)
+            rm = m.running_mean.data.numpy()
+            rm = np.array(rm, dtype=np.float32)
+            rv = m.running_var.data.numpy()
+            rv = np.array(rv, dtype=np.float32)
+            bin_write(f,b)
+            bin_write(f,s)
+            bin_write(f,rm)
+            bin_write(f,rv)
+            print ("    b shape:", np.shape(b))
+            print ("    s shape:", np.shape(s))
+            print ("    rm shape:", np.shape(rm))
+            print ("    rv shape:", np.shape(rv))
+
+        else:
+            bin_write(f,w)
+            if b.size > 0:
+                bin_write(f,b)
+
+        if ' of BatchNorm2d' in str(m.type) or ' of Linear' in str(m.type):
+            f.close()
+            print("close file")
+            f = None
+
+
+
+
+
+if __name__ == '__main__':
+
+    model = torch.hub.load('pytorch/vision', 'resnet101', pretrained=True)
+    model.eval()
+
+    # load an example image and load it on model
+    model, input_batch = load_ex_image(model)
+    model.eval()
+    with torch.no_grad():
+        output = model(input_batch)
+
+    # create folders debug and layers if do not exist
+    create_folders()
+
+    # add output attribute to the layers
+    for n, m in model.named_modules():
+        m.register_forward_hook(hook)
+
+    # export input bin
+    exp_input(model, input_batch)
+
+    print_wb_output(model)
+
+    with open("resnet101.txt", 'w') as f:
+        for item in list(model.children()):
+            f.write("%s\n" % item)
+
+    summary(model, (3, 224, 224))
+    # print(trace(model, input_batch))

tests/build_models.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-if [ "$1" == "download" ]; then
-    wget https://github.com/ceccocats/tkDNN/releases/download/testData/tkDNN_testwg.tar.gz --no-check-certificate
-    tar -xf tkDNN_testwg.tar.gz
-    rm tkDNN_testwg.tar.gz
-    exit
-fi
-
-echo "build test Model"
-cd test
-python test_model.py
-cd ..
-cd mnist
-python mnist_model.py
-cd ..
-echo "export weights"
-python weights_exporter.py test/net.h5 --output test/layers
-python caffe_weights_exporter.py mnist/lenet.prototxt mnist/lenet.caffemodel --output mnist/layers

tests/centernet/dla34_cnet/dla34_cnet.cpp

@@ -0,0 +1,532 @@
+#include <iostream>
+#include "tkdnn.h"
+
+const char *input_bin = "dla34_cnet/debug/input.bin";
+const char *conv1_bin = "dla34_cnet/layers/base-base_layer-0.bin";
+const char *conv2_bin = "dla34_cnet/layers/base-level0-0.bin";
+const char *conv3_bin = "dla34_cnet/layers/base-level1-0.bin";
+// s - stage, t - tree
+const char *s1_t1_conv1_bin = "dla34_cnet/layers/base-level2-tree1-conv1.bin";
+const char *s1_t1_conv2_bin = "dla34_cnet/layers/base-level2-tree1-conv2.bin";
+const char *s1_t1_project = "dla34_cnet/layers/base-level2-project-0.bin";
+const char *s1_t2_conv1_bin = "dla34_cnet/layers/base-level2-tree2-conv1.bin";
+const char *s1_t2_conv2_bin = "dla34_cnet/layers/base-level2-tree2-conv2.bin";
+const char *s1_root_conv1_bin = "dla34_cnet/layers/base-level2-root-conv.bin";
+const char *s2_t1_t1_conv1_bin = "dla34_cnet/layers/base-level3-tree1-tree1-conv1.bin";
+const char *s2_t1_t1_conv2_bin = "dla34_cnet/layers/base-level3-tree1-tree1-conv2.bin";
+const char *s2_t1_t1_project = "dla34_cnet/layers/base-level3-tree1-project-0.bin";
+const char *s2_t1_t2_conv1_bin = "dla34_cnet/layers/base-level3-tree1-tree2-conv1.bin";
+const char *s2_t1_t2_conv2_bin = "dla34_cnet/layers/base-level3-tree1-tree2-conv2.bin";
+const char *s2_t1_root_conv1_bin = "dla34_cnet/layers/base-level3-tree1-root-conv.bin";
+const char *s2_t2_t1_conv1_bin = "dla34_cnet/layers/base-level3-tree2-tree1-conv1.bin";
+const char *s2_t2_t1_conv2_bin = "dla34_cnet/layers/base-level3-tree2-tree1-conv2.bin";
+const char *s2_t2_t2_conv1_bin = "dla34_cnet/layers/base-level3-tree2-tree2-conv1.bin";
+const char *s2_t2_t2_conv2_bin = "dla34_cnet/layers/base-level3-tree2-tree2-conv2.bin";
+const char *s2_t2_root_conv1_bin = "dla34_cnet/layers/base-level3-tree2-root-conv.bin";
+const char *s3_t1_t1_conv1_bin = "dla34_cnet/layers/base-level4-tree1-tree1-conv1.bin";
+const char *s3_t1_t1_conv2_bin = "dla34_cnet/layers/base-level4-tree1-tree1-conv2.bin";
+const char *s3_t1_t1_project = "dla34_cnet/layers/base-level4-tree1-project-0.bin";
+const char *s3_t1_t2_conv1_bin = "dla34_cnet/layers/base-level4-tree1-tree2-conv1.bin";
+const char *s3_t1_t2_conv2_bin = "dla34_cnet/layers/base-level4-tree1-tree2-conv2.bin";
+const char *s3_t1_root_conv1_bin = "dla34_cnet/layers/base-level4-tree1-root-conv.bin";
+const char *s3_t2_t1_conv1_bin = "dla34_cnet/layers/base-level4-tree2-tree1-conv1.bin";
+const char *s3_t2_t1_conv2_bin = "dla34_cnet/layers/base-level4-tree2-tree1-conv2.bin";
+const char *s3_t2_t2_conv1_bin = "dla34_cnet/layers/base-level4-tree2-tree2-conv1.bin";
+const char *s3_t2_t2_conv2_bin = "dla34_cnet/layers/base-level4-tree2-tree2-conv2.bin";
+const char *s3_t2_root_conv1_bin = "dla34_cnet/layers/base-level4-tree2-root-conv.bin";
+const char *s4_t1_conv1_bin = "dla34_cnet/layers/base-level5-tree1-conv1.bin";
+const char *s4_t1_conv2_bin = "dla34_cnet/layers/base-level5-tree1-conv2.bin";
+const char *s4_t1_project = "dla34_cnet/layers/base-level5-project-0.bin";
+const char *s4_t2_conv1_bin = "dla34_cnet/layers/base-level5-tree2-conv1.bin";
+const char *s4_t2_conv2_bin = "dla34_cnet/layers/base-level5-tree2-conv2.bin";
+const char *s4_root_conv1_bin = "dla34_cnet/layers/base-level5-root-conv.bin";
+
+//final
+// const char *fc_bin = "dla34_cnet/layers/output.bin";
+
+const char *ida_0_p_1_dcn_bin = "dla34_cnet/layers/dla_up-ida_0-proj_1-conv.bin";
+const char *ida_0_p_1_conv_bin = "dla34_cnet/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin";
+const char *ida_0_up_1_deconv_bin = "dla34_cnet/layers/dla_up-ida_0-up_1.bin";
+const char *ida_0_n_1_dcn_bin = "dla34_cnet/layers/dla_up-ida_0-node_1-conv.bin";
+const char *ida_0_n_1_conv_bin = "dla34_cnet/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin";
+
+const char *ida_1_p_1_dcn_bin = "dla34_cnet/layers/dla_up-ida_1-proj_1-conv.bin";
+const char *ida_1_p_1_conv_bin = "dla34_cnet/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin";
+const char *ida_1_up_1_deconv_bin = "dla34_cnet/layers/dla_up-ida_1-up_1.bin";
+const char *ida_1_n_1_dcn_bin = "dla34_cnet/layers/dla_up-ida_1-node_1-conv.bin";
+const char *ida_1_n_1_conv_bin = "dla34_cnet/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin";
+const char *ida_1_p_2_dcn_bin = "dla34_cnet/layers/dla_up-ida_1-proj_2-conv.bin";
+const char *ida_1_p_2_conv_bin = "dla34_cnet/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin";
+const char *ida_1_up_2_deconv_bin = "dla34_cnet/layers/dla_up-ida_1-up_2.bin";
+const char *ida_1_n_2_dcn_bin = "dla34_cnet/layers/dla_up-ida_1-node_2-conv.bin";
+const char *ida_1_n_2_conv_bin = "dla34_cnet/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin";
+
+const char *ida_2_p_1_dcn_bin = "dla34_cnet/layers/dla_up-ida_2-proj_1-conv.bin";
+const char *ida_2_p_1_conv_bin = "dla34_cnet/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin";
+const char *ida_2_up_1_deconv_bin = "dla34_cnet/layers/dla_up-ida_2-up_1.bin";
+const char *ida_2_n_1_dcn_bin = "dla34_cnet/layers/dla_up-ida_2-node_1-conv.bin";
+const char *ida_2_n_1_conv_bin = "dla34_cnet/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin";
+const char *ida_2_p_2_dcn_bin = "dla34_cnet/layers/dla_up-ida_2-proj_2-conv.bin";
+const char *ida_2_p_2_conv_bin = "dla34_cnet/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin";
+const char *ida_2_up_2_deconv_bin = "dla34_cnet/layers/dla_up-ida_2-up_2.bin";
+const char *ida_2_n_2_dcn_bin = "dla34_cnet/layers/dla_up-ida_2-node_2-conv.bin";
+const char *ida_2_n_2_conv_bin = "dla34_cnet/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin";
+const char *ida_2_p_3_dcn_bin = "dla34_cnet/layers/dla_up-ida_2-proj_3-conv.bin";
+const char *ida_2_p_3_conv_bin = "dla34_cnet/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin";
+const char *ida_2_up_3_deconv_bin = "dla34_cnet/layers/dla_up-ida_2-up_3.bin";
+const char *ida_2_n_3_dcn_bin = "dla34_cnet/layers/dla_up-ida_2-node_3-conv.bin";
+const char *ida_2_n_3_conv_bin = "dla34_cnet/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin";
+
+const char *ida_up_p_1_dcn_bin = "dla34_cnet/layers/ida_up-proj_1-conv.bin";
+const char *ida_up_p_1_conv_bin = "dla34_cnet/layers/ida_up-proj_1-conv-conv_offset_mask.bin";
+const char *ida_up_up_1_deconv_bin = "dla34_cnet/layers/ida_up-up_1.bin";
+const char *ida_up_n_1_dcn_bin = "dla34_cnet/layers/ida_up-node_1-conv.bin";
+const char *ida_up_n_1_conv_bin = "dla34_cnet/layers/ida_up-node_1-conv-conv_offset_mask.bin";
+const char *ida_up_p_2_dcn_bin = "dla34_cnet/layers/ida_up-proj_2-conv.bin";
+const char *ida_up_p_2_conv_bin = "dla34_cnet/layers/ida_up-proj_2-conv-conv_offset_mask.bin";
+const char *ida_up_up_2_deconv_bin = "dla34_cnet/layers/ida_up-up_2.bin";
+const char *ida_up_n_2_dcn_bin = "dla34_cnet/layers/ida_up-node_2-conv.bin";
+const char *ida_up_n_2_conv_bin = "dla34_cnet/layers/ida_up-node_2-conv-conv_offset_mask.bin";
+
+const char *hm_conv1_bin = "dla34_cnet/layers/hm-0.bin";
+const char *hm_conv2_bin = "dla34_cnet/layers/hm-2.bin";
+const char *wh_conv1_bin = "dla34_cnet/layers/wh-0.bin";
+const char *wh_conv2_bin = "dla34_cnet/layers/wh-2.bin";
+const char *reg_conv1_bin = "dla34_cnet/layers/reg-0.bin";
+const char *reg_conv2_bin = "dla34_cnet/layers/reg-2.bin";
+
+const char *output_bin[]={
+"dla34_cnet/debug/hm.bin",
+"dla34_cnet/debug/wh.bin",
+"dla34_cnet/debug/reg.bin"};
+
+int main()
+{
+
+    downloadWeightsifDoNotExist(input_bin, "dla34_cnet", "https://cloud.hipert.unimore.it/s/KRZBbCQsKAtQwpZ/download");
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 512, 512, 1);
+    tk::dnn::Network net(dim);
+    tk::dnn::Layer *last1, *last2, *last3, *last4;
+    tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2;
+
+    tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
+    tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU);
+
+    tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true);
+    tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU);
+    base1 = &relu2;
+
+    tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true);
+    tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
+    base2 = &relu3;
+    
+    // level 2
+    // tree 1
+        tk::dnn::Conv2d     s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true);
+        tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d     s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true);
+        last2 = &s1_t1_conv2;
+
+        // get the basicblock input and apply maxpool conv2d and relu
+        tk::dnn::Layer      *route_s1_t1_layers[1] = { base2 };
+        tk::dnn::Route      route_s1_t1(&net, route_s1_t1_layers, 1);
+        // downsample
+        tk::dnn::Pooling    s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+        // project
+        tk::dnn::Conv2d     s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true);
+        
+        tk::dnn::Shortcut   s1_t1_s1(&net, last2);
+        tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    last1 = &s1_t1_relu;
+    // tree 2
+        tk::dnn::Conv2d     s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true);
+        tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d     s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true);
+        
+        tk::dnn::Shortcut   s1_t2_s1(&net, last1);
+        tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+        last2 = &s1_t2_relu;
+
+    // root
+        // join last1 and net in single input 128, 56, 56
+        tk::dnn::Layer      *route_s1_root_layers[2] = { last2, last1 };
+        tk::dnn::Route      route_s1_root(&net, route_s1_root_layers, 2);
+        tk::dnn::Conv2d     s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true);
+        tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    base3 = &s1_root_relu;
+
+    // level 3
+    // tree 1
+        // tree 1
+            tk::dnn::Conv2d     s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true);
+            tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);      
+        
+            tk::dnn::Conv2d     s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true);
+            last2 = &s2_t1_t1_conv2;
+
+            // get the basicblock input and apply maxpool conv2d and relu
+            tk::dnn::Layer      *route_s2_t1_t1_layers[1] = { base3 };
+            tk::dnn::Route      route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1);
+            // downsample
+            tk::dnn::Pooling    s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+            last4 = &s2_t1_t1_maxpool1;
+            // project
+            tk::dnn::Conv2d     s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true);
+            
+            tk::dnn::Shortcut   s2_t1_t1_s1(&net, last2);
+            tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s2_t1_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d     s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true);
+            tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d     s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut   s2_t1_t2_s1(&net, last1);
+            tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s2_t1_t2_relu;
+
+        // root
+            // join last1 and net in single input 128, 56, 56
+            tk::dnn::Layer      *route_s2_t1_root_layers[2] = { last2, last1 };
+            tk::dnn::Route      route_s2_t1_root(&net, route_s2_t1_root_layers, 2);
+            tk::dnn::Conv2d     s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true);
+            tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    last1 = &s2_t1_root_relu;
+    last3 = &s2_t1_root_relu;
+    // tree 2
+        // tree 1
+            tk::dnn::Conv2d     s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true);
+            tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d     s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true);
+            tk::dnn::Shortcut   s2_t2_t1_s1(&net, last1);
+            tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s2_t2_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d     s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true);
+            tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d     s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut   s2_t2_t2_s1(&net, last1);
+            tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s2_t2_t2_relu;
+
+        // root
+            // join last1 and net in single input 128, 56, 56
+            tk::dnn::Layer      *route_s2_t2_root_layers[4] = { last2, last1, last4, last3};
+            tk::dnn::Route      route_s2_t2_root(&net, route_s2_t2_root_layers, 4);
+            tk::dnn::Conv2d     s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true);
+            tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    base4 = &s2_t2_root_relu;
+
+    // level 4
+    // tree 1
+        // tree 1
+            tk::dnn::Conv2d     s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true);
+            tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);      
+        
+            tk::dnn::Conv2d     s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true);
+            last2 = &s3_t1_t1_conv2;
+
+            // get the basicblock input and apply maxpool conv2d and relu
+            tk::dnn::Layer      *route_s3_t1_t1_layers[1] = { base4 };
+            tk::dnn::Route      route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1);
+            // downsample
+            tk::dnn::Pooling    s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+            last4 = &s3_t1_t1_maxpool1;
+            // project
+            tk::dnn::Conv2d     s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true);
+            
+            tk::dnn::Shortcut   s3_t1_t1_s1(&net, last2);
+            tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s3_t1_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d     s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true);
+            tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d     s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut   s3_t1_t2_s1(&net, last1);
+            tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s3_t1_t2_relu;
+
+        // root
+            // join last1 and net in single input 256, 56, 56
+            tk::dnn::Layer      *route_s3_t1_root_layers[2] = { last2, last1 };
+            tk::dnn::Route      route_s3_t1_root(&net, route_s3_t1_root_layers, 2);
+            tk::dnn::Conv2d     s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true);
+            tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    last1 = &s3_t1_root_relu;
+    last3 = &s3_t1_root_relu;
+    // tree 2
+        // tree 1
+            tk::dnn::Conv2d     s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true);
+            tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d     s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true);
+            tk::dnn::Shortcut   s3_t2_t1_s1(&net, last1);
+            tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+        last1 = &s3_t2_t1_relu;
+        
+        // tree 2
+            tk::dnn::Conv2d     s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true);
+            tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+        
+            tk::dnn::Conv2d     s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true);
+            
+            tk::dnn::Shortcut   s3_t2_t2_s1(&net, last1);
+            tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+            last2 = &s3_t2_t2_relu;
+
+        // root
+            // join last1 and net in single input 256, 56, 56
+            tk::dnn::Layer      *route_s3_t2_root_layers[4] = { last2, last1, last4, last3};
+            tk::dnn::Route      route_s3_t2_root(&net, route_s3_t2_root_layers, 4);
+            tk::dnn::Conv2d     s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true);
+            tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
+    
+    base5 = &s3_t2_root_relu;
+
+    // level 5
+    // tree 1
+        tk::dnn::Conv2d     s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true);
+        tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d     s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true);
+        last2 = &s4_t1_conv2;
+
+        // get the basicblock input and apply maxpool conv2d and relu
+        tk::dnn::Layer      *route_s4_t1_layers[1] = { base5 };
+        tk::dnn::Route      route_s4_t1(&net, route_s4_t1_layers, 1);
+        // downsample
+        tk::dnn::Pooling    s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
+        last4 = &s4_t1_maxpool1;
+        // project
+        tk::dnn::Conv2d     s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true);
+        
+        tk::dnn::Shortcut   s4_t1_s1(&net, last2);
+        tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    last1 = &s4_t1_relu;
+    
+    // tree 2
+        tk::dnn::Conv2d     s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true);
+        tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU);       
+       
+        tk::dnn::Conv2d     s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true);
+        
+        tk::dnn::Shortcut   s4_t2_s1(&net, last1);
+        tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU);
+        last2 = &s4_t2_relu;
+
+    // root
+        // join last1 and net in single input 128, 56, 56
+        tk::dnn::Layer      *route_s4_root_layers[3] = { last2, last1, last4 };
+        tk::dnn::Route      route_s4_root(&net, route_s4_root_layers, 3);
+        tk::dnn::Conv2d     s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true);
+        tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU);
+
+    base6 = &s4_root_relu;
+    
+    //final
+    // tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
+    // tk::dnn::Dense   fc(&net, 1000, fc_bin);
+    
+    //ida 0 
+    tk::dnn::DeformConv2d   ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true);
+    tk::dnn::Activation     ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256);
+    tk::dnn::Shortcut       ida_0_shortcut(&net, base5);    
+    tk::dnn::DeformConv2d   ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true);
+    tk::dnn::Activation     ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    ida1 = &ida_0_n_1_relu;
+
+    //ida1-1
+    tk::dnn::Layer          *route_ida1_layers_1[1] = { base5 };
+    tk::dnn::Route          route_ida1_1(&net, route_ida1_layers_1, 1);
+    
+    tk::dnn::DeformConv2d   ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true);
+    tk::dnn::Activation     ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128);
+    tk::dnn::Shortcut       ida_1_shortcut1(&net, base4);    
+    tk::dnn::DeformConv2d   ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true);
+    tk::dnn::Activation     ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    ida2_1 = &ida_1_n_1_relu;
+
+    //ida1-2
+    tk::dnn::Layer          *route_ida1_layers_2[1] = { ida1 };
+    tk::dnn::Route          route_ida1_2(&net, route_ida1_layers_2, 1);
+
+    tk::dnn::DeformConv2d   ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true);
+    tk::dnn::Activation     ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128);
+    tk::dnn::Shortcut       ida_1_shortcut2(&net, ida2_1);    
+    tk::dnn::DeformConv2d   ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true);
+    tk::dnn::Activation     ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
+    ida2_2 = &ida_1_n_2_relu;
+
+    //ida2-1
+    tk::dnn::Layer          *route_ida2_layers_1[1] = { base4 };
+    tk::dnn::Route          route_ida2_1(&net, route_ida2_layers_1, 1);
+    
+    tk::dnn::DeformConv2d   ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true);
+    tk::dnn::Activation     ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64);
+    tk::dnn::Shortcut       ida_2_shortcut1(&net, base3);    
+    tk::dnn::DeformConv2d   ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true);
+    tk::dnn::Activation     ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    ida3_1 = &ida_2_n_1_relu;
+
+    //ida2-2
+    tk::dnn::Layer          *route_ida2_layers_2[1] = { ida2_1 };
+    tk::dnn::Route          route_ida2_2(&net, route_ida2_layers_2, 1);
+    
+    tk::dnn::DeformConv2d   ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true);
+    tk::dnn::Activation     ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64);
+    tk::dnn::Shortcut       ida_2_shortcut2(&net, ida3_1);    
+    tk::dnn::DeformConv2d   ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true);
+    tk::dnn::Activation     ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
+    ida3_2 = &ida_2_n_2_relu;
+
+    //ida2-3
+    tk::dnn::Layer          *route_ida2_layers_3[1] = { ida2_2 };
+    tk::dnn::Route          route_ida2_3(&net, route_ida2_layers_3, 1);
+    
+    tk::dnn::DeformConv2d   ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true);
+    tk::dnn::Activation     ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64);
+    tk::dnn::Shortcut       ida_2_shortcut3(&net, ida3_2);    
+    tk::dnn::DeformConv2d   ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true);
+    tk::dnn::Activation     ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU);
+    ida3_3 = &ida_2_n_3_relu;
+
+    //idaup-1
+    tk::dnn::Layer          *route_idaup_layers_1[1] = { ida2_2 };
+    tk::dnn::Route          route_idaup_1(&net, route_idaup_layers_1, 1);
+    
+    tk::dnn::DeformConv2d   idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true);
+    tk::dnn::Activation     idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64);
+    tk::dnn::Shortcut       idaup_shortcut1(&net, ida3_3);    
+    tk::dnn::DeformConv2d   idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true);
+    tk::dnn::Activation     idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
+    idaup_1 = &idaup_n_1_relu;
+
+    //idaup-2
+    tk::dnn::Layer          *route_idaup_layers_2[1] = { ida1 };
+    tk::dnn::Route          route_idaup_2(&net, route_idaup_layers_2, 1);
+
+    tk::dnn::DeformConv2d   idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true);
+    tk::dnn::Activation     idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d       idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64);
+    tk::dnn::Shortcut       idaup_shortcut2(&net, idaup_1);    
+    tk::dnn::DeformConv2d   idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true);
+    tk::dnn::Activation     idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
+    idaup_2 = &idaup_n_2_relu;
+
+    tk::dnn::Layer    *route_1_0_layers[1] = { idaup_2 };
+
+    // hm
+    tk::dnn::Conv2d     *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
+    tk::dnn::Activation *hm_relu1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     *hm = new tk::dnn::Conv2d(&net, 80, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false);
+    hm->setFinal();
+    int kernel = 3; 
+    int pad = (kernel - 1)/2;
+    tk::dnn::Activation *hm_sig      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
+    tk::dnn::Pooling  *hmax                 = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
+    hmax->setFinal();
+
+    // // wh
+    tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+    tk::dnn::Conv2d     *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
+    tk::dnn::Activation *wh_relu1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false);        
+    wh->setFinal();
+    
+    // // reg
+    tk::dnn::Route    *route_2_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+    tk::dnn::Conv2d     *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false);
+    tk::dnn::Activation *reg_relu1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false);
+    reg->setFinal();
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    //printDeviceVector(64, data, true);
+
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet"));
+
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30);
+    {
+        dim1.print();
+        TKDNN_TSTART
+        net.infer(dim1, data);
+        TKDNN_TSTOP
+        dim1.print();
+    }
+
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30);
+    {
+        dim2.print();
+        TKDNN_TSTART
+        netRT.infer(dim2, data);
+        TKDNN_TSTOP
+        dim2.print();
+    }
+
+    tk::dnn::Layer *outs[3] = { hm, wh, reg }; 
+    int out_count = 1;
+    int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; 
+    for(int i=0; i<3; i++) {
+        printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
+        
+        outs[i]->output_dim.print();
+        
+        dnnType *out, *out_h;
+        int odim = outs[i]->output_dim.tot();
+        readBinaryFile(output_bin[i], odim, &out_h, &out);
+
+        dnnType *cudnn_out, *rt_out;
+        cudnn_out = outs[i]->dstData;
+        rt_out = (dnnType *)netRT.buffersRT[i+out_count];
+        // there is the maxpool. It isn't an output but it is necessary for the process section
+        if(i==0)
+            out_count ++;
+
+        std::cout<<"CUDNN vs correct"; 
+        ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
+        std::cout<<"TRT   vs correct"; 
+        ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
+        std::cout<<"CUDNN vs TRT    "; 
+        ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
+    }
+    return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
+}

tests/centernet/resnet101_cnet/resnet101_cnet.cpp

@@ -0,0 +1,413 @@
+#include <iostream>
+
+#include "kernels.h"
+#include "Yolo3Detection.h"
+#include "tkdnn.h"
+#include <vector>
+#include <numeric>      // std::iota
+#include <algorithm>    // std::sort
+// #include "utils.h"
+
+const char *input_bin = "resnet101_cnet/debug/input.bin";
+const char *conv1_bin = "resnet101_cnet/layers/conv1.bin";
+
+//layer1
+const char *layer1_bin[]={
+"resnet101_cnet/layers/layer1-0-conv1.bin",
+"resnet101_cnet/layers/layer1-0-conv2.bin",
+"resnet101_cnet/layers/layer1-0-conv3.bin",
+"resnet101_cnet/layers/layer1-0-downsample-0.bin",
+
+"resnet101_cnet/layers/layer1-1-conv1.bin",
+"resnet101_cnet/layers/layer1-1-conv2.bin",
+"resnet101_cnet/layers/layer1-1-conv3.bin",
+
+"resnet101_cnet/layers/layer1-2-conv1.bin",
+"resnet101_cnet/layers/layer1-2-conv2.bin",
+"resnet101_cnet/layers/layer1-2-conv3.bin"};
+                        
+
+//layer2
+const char *layer2_bin[]={
+"resnet101_cnet/layers/layer2-0-conv1.bin",
+"resnet101_cnet/layers/layer2-0-conv2.bin",
+"resnet101_cnet/layers/layer2-0-conv3.bin",
+"resnet101_cnet/layers/layer2-0-downsample-0.bin",
+
+"resnet101_cnet/layers/layer2-1-conv1.bin",
+"resnet101_cnet/layers/layer2-1-conv2.bin",
+"resnet101_cnet/layers/layer2-1-conv3.bin",
+
+"resnet101_cnet/layers/layer2-2-conv1.bin",
+"resnet101_cnet/layers/layer2-2-conv2.bin",
+"resnet101_cnet/layers/layer2-2-conv3.bin",
+
+"resnet101_cnet/layers/layer2-3-conv1.bin",
+"resnet101_cnet/layers/layer2-3-conv2.bin",
+"resnet101_cnet/layers/layer2-3-conv3.bin"
+};
+//layer3
+const char *layer3_bin[]={
+"resnet101_cnet/layers/layer3-0-conv1.bin",
+"resnet101_cnet/layers/layer3-0-conv2.bin",
+"resnet101_cnet/layers/layer3-0-conv3.bin",
+"resnet101_cnet/layers/layer3-0-downsample-0.bin",
+
+"resnet101_cnet/layers/layer3-1-conv1.bin",
+"resnet101_cnet/layers/layer3-1-conv2.bin",
+"resnet101_cnet/layers/layer3-1-conv3.bin",
+
+"resnet101_cnet/layers/layer3-2-conv1.bin",
+"resnet101_cnet/layers/layer3-2-conv2.bin",
+"resnet101_cnet/layers/layer3-2-conv3.bin",
+
+"resnet101_cnet/layers/layer3-3-conv1.bin",
+"resnet101_cnet/layers/layer3-3-conv2.bin",
+"resnet101_cnet/layers/layer3-3-conv3.bin",
+
+"resnet101_cnet/layers/layer3-4-conv1.bin",
+"resnet101_cnet/layers/layer3-4-conv2.bin",
+"resnet101_cnet/layers/layer3-4-conv3.bin",
+
+"resnet101_cnet/layers/layer3-5-conv1.bin",
+"resnet101_cnet/layers/layer3-5-conv2.bin",
+"resnet101_cnet/layers/layer3-5-conv3.bin",
+
+"resnet101_cnet/layers/layer3-6-conv1.bin",
+"resnet101_cnet/layers/layer3-6-conv2.bin",
+"resnet101_cnet/layers/layer3-6-conv3.bin",
+
+"resnet101_cnet/layers/layer3-7-conv1.bin",
+"resnet101_cnet/layers/layer3-7-conv2.bin",
+"resnet101_cnet/layers/layer3-7-conv3.bin",
+
+"resnet101_cnet/layers/layer3-8-conv1.bin",
+"resnet101_cnet/layers/layer3-8-conv2.bin",
+"resnet101_cnet/layers/layer3-8-conv3.bin",
+
+"resnet101_cnet/layers/layer3-9-conv1.bin",
+"resnet101_cnet/layers/layer3-9-conv2.bin",
+"resnet101_cnet/layers/layer3-9-conv3.bin",
+
+"resnet101_cnet/layers/layer3-10-conv1.bin",
+"resnet101_cnet/layers/layer3-10-conv2.bin",
+"resnet101_cnet/layers/layer3-10-conv3.bin",
+
+"resnet101_cnet/layers/layer3-11-conv1.bin",
+"resnet101_cnet/layers/layer3-11-conv2.bin",
+"resnet101_cnet/layers/layer3-11-conv3.bin",
+
+"resnet101_cnet/layers/layer3-12-conv1.bin",
+"resnet101_cnet/layers/layer3-12-conv2.bin",
+"resnet101_cnet/layers/layer3-12-conv3.bin",
+
+"resnet101_cnet/layers/layer3-13-conv1.bin",
+"resnet101_cnet/layers/layer3-13-conv2.bin",
+"resnet101_cnet/layers/layer3-13-conv3.bin",
+
+"resnet101_cnet/layers/layer3-14-conv1.bin",
+"resnet101_cnet/layers/layer3-14-conv2.bin",
+"resnet101_cnet/layers/layer3-14-conv3.bin",
+
+"resnet101_cnet/layers/layer3-15-conv1.bin",
+"resnet101_cnet/layers/layer3-15-conv2.bin",
+"resnet101_cnet/layers/layer3-15-conv3.bin",
+
+"resnet101_cnet/layers/layer3-16-conv1.bin",
+"resnet101_cnet/layers/layer3-16-conv2.bin",
+"resnet101_cnet/layers/layer3-16-conv3.bin",
+
+"resnet101_cnet/layers/layer3-17-conv1.bin",
+"resnet101_cnet/layers/layer3-17-conv2.bin",
+"resnet101_cnet/layers/layer3-17-conv3.bin",
+
+"resnet101_cnet/layers/layer3-18-conv1.bin",
+"resnet101_cnet/layers/layer3-18-conv2.bin",
+"resnet101_cnet/layers/layer3-18-conv3.bin",
+
+"resnet101_cnet/layers/layer3-19-conv1.bin",
+"resnet101_cnet/layers/layer3-19-conv2.bin",
+"resnet101_cnet/layers/layer3-19-conv3.bin",
+
+"resnet101_cnet/layers/layer3-20-conv1.bin",
+"resnet101_cnet/layers/layer3-20-conv2.bin",
+"resnet101_cnet/layers/layer3-20-conv3.bin",
+
+"resnet101_cnet/layers/layer3-21-conv1.bin",
+"resnet101_cnet/layers/layer3-21-conv2.bin",
+"resnet101_cnet/layers/layer3-21-conv3.bin",
+
+"resnet101_cnet/layers/layer3-22-conv1.bin",
+"resnet101_cnet/layers/layer3-22-conv2.bin",
+"resnet101_cnet/layers/layer3-22-conv3.bin"};
+
+
+//layer4
+const char *layer4_bin[]={
+"resnet101_cnet/layers/layer4-0-conv1.bin",
+"resnet101_cnet/layers/layer4-0-conv2.bin",
+"resnet101_cnet/layers/layer4-0-conv3.bin",
+"resnet101_cnet/layers/layer4-0-downsample-0.bin",
+
+"resnet101_cnet/layers/layer4-1-conv1.bin",
+"resnet101_cnet/layers/layer4-1-conv2.bin",
+"resnet101_cnet/layers/layer4-1-conv3.bin",
+
+"resnet101_cnet/layers/layer4-2-conv1.bin",
+"resnet101_cnet/layers/layer4-2-conv2.bin",
+"resnet101_cnet/layers/layer4-2-conv3.bin"};
+
+const char *d_conv1_bin = "resnet101_cnet/layers/deconv_layers-0-conv_offset_mask.bin";
+const char *deform1_bin = "resnet101_cnet/layers/deconv_layers-0.bin";
+const char *deconv1_bin = "resnet101_cnet/layers/deconv_layers-3.bin";
+
+const char *d_conv2_bin = "resnet101_cnet/layers/deconv_layers-6-conv_offset_mask.bin";
+const char *deform2_bin = "resnet101_cnet/layers/deconv_layers-6.bin";
+const char *deconv2_bin = "resnet101_cnet/layers/deconv_layers-9.bin";
+
+const char *d_conv3_bin = "resnet101_cnet/layers/deconv_layers-12-conv_offset_mask.bin";
+const char *deform3_bin = "resnet101_cnet/layers/deconv_layers-12.bin";
+const char *deconv3_bin = "resnet101_cnet/layers/deconv_layers-15.bin";
+
+const char *hm_conv1_bin = "resnet101_cnet/layers/hm-0.bin";
+const char *hm_conv2_bin = "resnet101_cnet/layers/hm-2.bin";
+const char *wh_conv1_bin = "resnet101_cnet/layers/wh-0.bin";
+const char *wh_conv2_bin = "resnet101_cnet/layers/wh-2.bin";
+const char *reg_conv1_bin = "resnet101_cnet/layers/reg-0.bin";
+const char *reg_conv2_bin = "resnet101_cnet/layers/reg-2.bin";
+//final
+const char *fc_bin = "resnet101_cnet/layers/fc.bin";
+
+const char *output_bin[]={
+"resnet101_cnet/debug/hm.bin",
+"resnet101_cnet/debug/wh.bin",
+"resnet101_cnet/debug/reg.bin"};
+
+int main()
+{
+    downloadWeightsifDoNotExist(input_bin, "resnet101_cnet", "https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download");
+
+    // Network layout
+    tk::dnn::dataDim_t dim(1, 3, 512, 512, 1);
+    tk::dnn::Network net(dim);
+
+    tk::dnn::Conv2d conv1(&net, 64, 7, 7, 2, 2, 3, 3, conv1_bin, true);
+    tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
+  
+    tk::dnn::Pooling maxpool4(&net, 3, 3, 2, 2, 1, 1, tk::dnn::POOLING_MAX);
+
+
+    //layer 1
+    int id_layer1_bin = 0;
+    tk::dnn::Layer *last = &maxpool4;
+    for(int i=0; i<3;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 64, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, layer1_bin[id_layer1_bin++], true);
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true);
+        if(i==0) {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer1_bin[id_layer1_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } else {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+
+    // layer 2
+    int id_layer2_bin = 0;
+    for(int i=0; i<4;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 128, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2;
+        if(i==0)
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 2, 2, 1, 1, layer2_bin[id_layer2_bin++], true);
+        else
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 128, 3, 3, 1, 1, 1, 1, layer2_bin[id_layer2_bin++], true);
+        
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer2_bin[id_layer2_bin++], true);
+        if(i==0) 
+        {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 512, 1, 1, 2, 2, 0, 0, layer2_bin[id_layer2_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } 
+        else 
+        {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+   
+    // layer 3
+    int id_layer3_bin = 0;
+    for(int i=0; i<23;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 256, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2;
+        if(i==0)
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 2, 2, 1, 1, layer3_bin[id_layer3_bin++], true);
+        else
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, layer3_bin[id_layer3_bin++], true);
+        
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 1, 1, 0, 0, layer3_bin[id_layer3_bin++], true);
+        if(i==0) 
+        {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 1024, 1, 1, 2, 2, 0, 0, layer3_bin[id_layer3_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } 
+        else 
+        {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+ 
+    // layer 4
+    int id_layer4_bin = 0;
+    for(int i=0; i<3;i++)
+    {
+        tk::dnn::Conv2d     *layer1_0_conv1 = new tk::dnn::Conv2d(&net, 512, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true);
+        tk::dnn::Activation *relu1_0_1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv2;
+        if(i==0)
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 2, 2, 1, 1, layer4_bin[id_layer4_bin++], true);
+        else
+            layer1_0_conv2 = new tk::dnn::Conv2d(&net, 512, 3, 3, 1, 1, 1, 1, layer4_bin[id_layer4_bin++], true);
+        
+        tk::dnn::Activation *relu1_0_2      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        tk::dnn::Conv2d     *layer1_0_conv3 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 1, 1, 0, 0, layer4_bin[id_layer4_bin++], true);
+        if(i==0) 
+        {
+            tk::dnn::Layer    *route_1_0_layers[1] = { last };
+            tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+            tk::dnn::Conv2d   *layer1_0_downsample_0 = new tk::dnn::Conv2d(&net, 2048, 1, 1, 2, 2, 0, 0, layer4_bin[id_layer4_bin++], true);
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, layer1_0_conv3);
+        } 
+        else 
+        {
+            tk::dnn::Shortcut *s1_0                  = new tk::dnn::Shortcut(&net, last);
+        }
+        tk::dnn::Activation   *layer1_0_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+        last = layer1_0_relu;
+    }
+
+    tk::dnn::DeformConv2d     *layer0_deform1 = new tk::dnn::DeformConv2d(&net, 256, 1, 3, 3, 1, 1, 1, 1, deform1_bin, d_conv1_bin, true);
+    tk::dnn::Activation   *layer0_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d          *layer0_deconv1 = new tk::dnn::DeConv2d(&net, 256, 4, 4, 2, 2, 1, 1, deconv1_bin, true);
+    tk::dnn::Activation *layer0_deconv1_relu      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+
+    tk::dnn::DeformConv2d     *layer1_deform1 = new tk::dnn::DeformConv2d(&net, 128, 1, 3, 3, 1, 1, 1, 1, deform2_bin, d_conv2_bin, true);
+    tk::dnn::Activation   *layer1_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d          *layer1_deconv1 = new tk::dnn::DeConv2d(&net, 128, 4, 4, 2, 2, 1, 1, deconv2_bin, true);
+    tk::dnn::Activation *layer1_deconv1_relu      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    
+    tk::dnn::DeformConv2d     *layer2_deform1 = new tk::dnn::DeformConv2d(&net, 64, 1, 3, 3, 1, 1, 1, 1, deform3_bin, d_conv3_bin, true);
+    tk::dnn::Activation   *layer2_deform1_relu = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::DeConv2d          *layer2_deconv1 = new tk::dnn::DeConv2d(&net, 64, 4, 4, 2, 2, 1, 1, deconv3_bin, true);
+    tk::dnn::Activation *layer2_deconv1_relu      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);          
+        
+    tk::dnn::Layer    *route_1_0_layers[1] = { layer2_deconv1_relu };
+    tk::dnn::Conv2d     *hm_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
+    tk::dnn::Activation *hm_relu1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     *hm = new tk::dnn::Conv2d(&net, 80, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false);
+    hm->setFinal();
+    int kernel = 3; 
+    int pad = (kernel - 1)/2;
+    tk::dnn::Activation *hm_sig      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
+    tk::dnn::Pooling  *hmax                 = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
+    hmax->setFinal();
+
+    tk::dnn::Route    *route_1_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+    tk::dnn::Conv2d     *wh_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
+    tk::dnn::Activation *wh_relu1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false);
+    wh->setFinal();        
+    
+    tk::dnn::Route    *route_2_0             = new tk::dnn::Route(&net, route_1_0_layers, 1);
+    tk::dnn::Conv2d     *reg_conv1 = new tk::dnn::Conv2d(&net, 64, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false);
+    tk::dnn::Activation *reg_relu1      = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
+    tk::dnn::Conv2d     *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false);
+    reg->setFinal();
+
+    // Load input
+    dnnType *data;
+    dnnType *input_h;
+    readBinaryFile(input_bin, dim.tot(), &input_h, &data);
+    // printDeviceVector(64, data, true);
+
+    //print network model
+    net.print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("resnet101_cnet"));
+
+    
+    tk::dnn::dataDim_t dim1 = dim; //input dim
+    printCenteredTitle(" CUDNN inference ", '=', 30);
+    {
+        dim1.print();
+        TKDNN_TSTART
+        net.infer(dim1, data);
+        TKDNN_TSTOP
+        dim1.print();
+    }
+
+    // printDeviceVector(64, cudnn_out, true);
+
+    tk::dnn::dataDim_t dim2 = dim;
+    printCenteredTitle(" TENSORRT inference ", '=', 30);
+    {
+        dim2.print();
+        TKDNN_TSTART
+        netRT.infer(dim2, data);
+        TKDNN_TSTOP
+        dim2.print();
+    }
+
+    tk::dnn::Layer *outs[3] = { hm, wh, reg }; 
+    int out_count = 1;
+    int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0; 
+    for(int i=0; i<3; i++) {
+        printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
+        
+        outs[i]->output_dim.print();
+        
+        dnnType *out, *out_h;
+        int odim = outs[i]->output_dim.tot();
+        readBinaryFile(output_bin[i], odim, &out_h, &out);
+        // std::cout<<"OUTPUT BIN:\n";
+        // printDeviceVector(odim, cudnn_out, true);
+        // std::cout<<"FILE BIN:\n";
+        // printDeviceVector(odim, out, true);
+
+        dnnType *cudnn_out, *rt_out;
+        cudnn_out = outs[i]->dstData;
+        rt_out = (dnnType *)netRT.buffersRT[i+out_count];
+        // there is the maxpool. It isn't an output but it is necessary for the process section
+        if(i==0)
+            out_count ++;
+
+        std::cout<<"CUDNN vs correct"; 
+        ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
+        std::cout<<"TRT   vs correct"; 
+        ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
+        std::cout<<"CUDNN vs TRT    "; 
+        ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
+    }
+    return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
+}

tests/darknet/cfg/yolo2tiny.cfg

@@ -1,5 +1,5 @@
 [net]
- Training
+# Training
  batch=64
  subdivisions=8
 # Testing
@@ -90,9 +90,9 @@ stride=1
 pad=1
 activation=leaky
 
-#[maxpool]
-#size=2
-#stride=1
+[maxpool]
+size=2
+stride=1
 
 [convolutional]
 batch_normalize=1

tests/darknet/cfg/yolo3.cfg

@@ -0,0 +1,789 @@
+[net]
+# Testing
+# batch=1
+# subdivisions=1
+# Training
+batch=32
+subdivisions=32
+width=416
+height=416
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+######################
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+[yolo]
+mask = 6,7,8
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=80
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 61
+
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+[yolo]
+mask = 3,4,5
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=80
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 36
+
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+[yolo]
+mask = 0,1,2
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=80
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+

tests/darknet/cfg/yolo3_512.cfg

@@ -0,0 +1,789 @@
+[net]
+# Testing
+# batch=1
+# subdivisions=1
+# Training
+batch=32
+subdivisions=32
+width=512
+height=512
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=1000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+######################
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+[yolo]
+mask = 6,7,8
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=80
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 61
+
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+[yolo]
+mask = 3,4,5
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=80
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 36
+
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+
+[yolo]
+mask = 0,1,2
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=80
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+

tests/darknet/cfg/yolo3tiny.cfg

@@ -1,12 +1,12 @@
 [net]
 # Testing
-#batch=1
-#subdivisions=1
+batch=1
+subdivisions=1
 # Training
- batch=64
- subdivisions=16
-width=224
-height=224
+# batch=64
+# subdivisions=2
+width=416
+height=416
 channels=3
 momentum=0.9
 decay=0.0005
@@ -22,6 +22,18 @@ policy=steps
 steps=400000,450000
 scales=.1,.1
 
+[convolutional]
+batch_normalize=1
+filters=16
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
 [convolutional]
 batch_normalize=1
 filters=32
@@ -54,22 +66,6 @@ stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=64
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=3
-stride=1
-pad=1
-activation=leaky
-
 [maxpool]
 size=2
 stride=2
@@ -82,22 +78,6 @@ stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
 [maxpool]
 size=2
 stride=2
@@ -110,41 +90,9 @@ stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
 [maxpool]
 size=2
-stride=2
+stride=1
 
 [convolutional]
 batch_normalize=1
@@ -154,105 +102,81 @@ stride=1
 pad=1
 activation=leaky
 
+###########
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
 [convolutional]
 batch_normalize=1
 filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
 size=3
 stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=1
-pad=1
-activation=leaky
-
-
-#######
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[route]
-layers=-9
-
-[convolutional]
-batch_normalize=1
-size=1
-stride=1
-pad=1
-filters=64
-activation=leaky
-
-[reorg]
-stride=2
-
-[route]
-layers=-1,-4
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
 [convolutional]
 size=1
 stride=1
 pad=1
-filters=425
+filters=255
 activation=linear
 
 
-[region]
-anchors =  0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828
-bias_match=1
+
+[yolo]
+mask = 3,4,5
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
 classes=80
-coords=4
-num=5
-softmax=1
+num=6
 jitter=.3
-rescore=1
-
-object_scale=5
-noobject_scale=1
-class_scale=1
-coord_scale=1
-
-absolute=1
-thresh = .6
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 8
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+[yolo]
+mask = 0,1,2
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
 random=1

tests/darknet/cfg/yolo3tiny_512.cfg

@@ -3,10 +3,10 @@
 batch=1
 subdivisions=1
 # Training
-#batch=64
-#subdivisions=8
-height=416
-width=736
+# batch=64
+# subdivisions=2
+width=512
+height=512
 channels=3
 momentum=0.9
 decay=0.0005
@@ -17,11 +17,23 @@ hue=.1
 
 learning_rate=0.001
 burn_in=1000
-max_batches = 80200
+max_batches = 500200
 policy=steps
-steps=40000,60000
+steps=400000,450000
 scales=.1,.1
 
+[convolutional]
+batch_normalize=1
+filters=16
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[maxpool]
+size=2
+stride=2
+
 [convolutional]
 batch_normalize=1
 filters=32
@@ -54,22 +66,6 @@ stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=64
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=128
-size=3
-stride=1
-pad=1
-activation=leaky
-
 [maxpool]
 size=2
 stride=2
@@ -82,22 +78,6 @@ stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=128
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=3
-stride=1
-pad=1
-activation=leaky
-
 [maxpool]
 size=2
 stride=2
@@ -110,41 +90,9 @@ stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=256
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=512
-size=3
-stride=1
-pad=1
-activation=leaky
-
 [maxpool]
 size=2
-stride=2
+stride=1
 
 [convolutional]
 batch_normalize=1
@@ -154,106 +102,81 @@ stride=1
 pad=1
 activation=leaky
 
+###########
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
 [convolutional]
 batch_normalize=1
 filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
 size=3
 stride=1
 pad=1
 activation=leaky
 
-[convolutional]
-batch_normalize=1
-filters=512
-size=1
-stride=1
-pad=1
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-filters=1024
-size=3
-stride=1
-pad=1
-activation=leaky
-
-
-#######
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
-[route]
-layers=-9
-
-[convolutional]
-batch_normalize=1
-size=1
-stride=1
-pad=1
-filters=64
-activation=leaky
-
-[reorg]
-stride=2
-
-[route]
-layers=-1,-4
-
-[convolutional]
-batch_normalize=1
-size=3
-stride=1
-pad=1
-filters=1024
-activation=leaky
-
 [convolutional]
 size=1
 stride=1
 pad=1
-filters=75
+filters=255
 activation=linear
 
 
-[region]
-anchors =  0.4043,0.4167, 1.2109,1.1018, 2.7258,2.1215, 4.9477,3.9132, 7.9508,6.6806
-bias_match=1
-classes=10
-coords=4
-num=5
-softmax=1
+
+[yolo]
+mask = 3,4,5
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
 jitter=.3
-rescore=1
+ignore_thresh = .7
+truth_thresh = 1
+random=1
 
-object_scale=5
-noobject_scale=1
-class_scale=1
-coord_scale=1
+[route]
+layers = -4
 
-absolute=1
-thresh = .6
-random=0
-flip=1
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 8
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=255
+activation=linear
+
+[yolo]
+mask = 0,1,2
+anchors = 10,14,  23,27,  37,58,  81,82,  135,169,  344,319
+classes=80
+num=6
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1

tests/darknet/csresnext50-panet-spp.cpp

@@ -0,0 +1,34 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path  = "csresnext50-panet-spp";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer115_out.bin",
+        bin_path + "/debug/layer126_out.bin",
+        bin_path + "/debug/layer137_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = "../tests/darknet/cfg/csresnext50-panet-spp.cfg";
+    std::string name_path = "../tests/darknet/names/coco.names";
+    downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/darknet/csresnext50-panet-spp_berkeley.cpp

@@ -0,0 +1,35 @@
+#include<iostream>
+#include<vector>
+#include "tkdnn.h"
+#include "test.h"
+#include "DarknetParser.h"
+
+int main() {
+    std::string bin_path = "csresnext50-panet-spp_berkeley";
+    std::vector<std::string> input_bins = { 
+        bin_path + "/layers/input.bin"
+    };
+    std::vector<std::string> output_bins = {
+        bin_path + "/debug/layer115_out.bin",
+        bin_path + "/debug/layer126_out.bin",
+        bin_path + "/debug/layer137_out.bin"
+    };
+    std::string wgs_path  = bin_path + "/layers";
+    std::string cfg_path  = "../tests/darknet/cfg/csresnext50-panet-spp_berkeley.cfg";
+    std::string name_path = "../tests/darknet/names/berkeley.names";
+    // FIXME: wrong weights
+    // downloadWeightsifDoNotExist(input_bins[0], bin_path, "https://cloud.hipert.unimore.it/s//download");
+
+    // parse darknet network
+    tk::dnn::Network *net = tk::dnn::darknetParser(cfg_path, wgs_path, name_path);
+    net->print();
+
+    //convert network to tensorRT
+    tk::dnn::NetworkRT *netRT = new tk::dnn::NetworkRT(net, net->getNetworkRTName(bin_path.c_str()));
+    
+    int ret = testInference(input_bins, output_bins, net, netRT);
+    net->releaseLayers();
+    delete net;
+    delete netRT;
+    return ret;
+}

tests/darknet/names/berkeley.names

@@ -0,0 +1,10 @@
+person
+car
+truck
+bus
+motor
+bike
+rider
+traffic light
+traffic sign
+train