mmr/tkDNN
Archived
1
0
Fork 0
Code Issues 46 Pull Requests 12 Releases 4 Activity

Compare commits

merge into: mmr/tkDNN:tensorrt7
Branches Tags
mmr/tkDNN:master mmr/tkDNN:dev_rtls mmr/tkDNN:mbzirc mmr/tkDNN:tensorrt7 mmr/tkDNN:gpu-prepocess mmr/tkDNN:resize mmr/tkDNN:yolo4tiny mmr/tkDNN:cudnn-8.0.1 mmr/tkDNN:class mmr/tkDNN:cudnn5 mmr/tkDNN:cudnn2
mmr/tkDNN:v0.7 mmr/tkDNN:v0.6 mmr/tkDNN:v0.5 mmr/tkDNN:v0.4 mmr/tkDNN:testData
..
pull from: mmr/tkDNN:resize
Branches Tags
mmr/tkDNN:master mmr/tkDNN:dev_rtls mmr/tkDNN:mbzirc mmr/tkDNN:tensorrt7 mmr/tkDNN:gpu-prepocess mmr/tkDNN:resize mmr/tkDNN:yolo4tiny mmr/tkDNN:cudnn-8.0.1 mmr/tkDNN:class mmr/tkDNN:cudnn5 mmr/tkDNN:cudnn2
mmr/tkDNN:v0.7 mmr/tkDNN:v0.6 mmr/tkDNN:v0.5 mmr/tkDNN:v0.4 mmr/tkDNN:testData

1 Commits
tensorrt7 .. resize

Author SHA1 Message Date
Micaela Verucchi 8ff7cad2e1 Modified resize and boxes coordinates to float, to achieve same darknet accuracy 
Signed-off-by: Micaela Verucchi <micaelaverucchi@gmail.com>
 2020-08-05 17:25:33 +02:00
123 changed files with 784 additions and 18085 deletions
Expand all files Collapse all files
.gitignore
+1 -7
View File
@@ -12,11 +12,5 @@ build/
*.hdf5
*.pk
*.table
cmake-build-release/
demo/COCO_val2017
demo/BDD100K_val
/.vs
cmake-build-minsizerel/*
scripts/COCO_val2017/*
scripts/COCO_val2017.zip
scripts/all_labels.txt
demo/BDD100K_val
CMakeLists.txt
+6 -58
View File
@@ -1,15 +1,8 @@
cmake_minimum_required(VERSION 3.15)
cmake_minimum_required(VERSION 3.5)
project (tkDNN)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
if(UNIX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -fPIC -Wno-deprecated-declarations")
endif()
if(WIN32)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_FLAGS "/O2 /FS /EHsc")
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif(WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wno-deprecated-declarations -Wno-unused-variable")
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/tkDNN)
# project specific flags
@@ -17,13 +10,7 @@ if(DEBUG)
add_definitions(-DDEBUG)
endif()
if(TKDNN_PATH)
message("SET TKDNN_PATH:"${TKDNN_PATH})
add_definitions(-DTKDNN_PATH="${TKDNN_PATH}")
else()
add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}")
endif()
add_definitions(-DTKDNN_PATH="${CMAKE_CURRENT_SOURCE_DIR}")
#-------------------------------------------------------------------------------
# CUDA
@@ -41,24 +28,19 @@ include_directories(${CUDNN_INCLUDE_DIR})
file(GLOB tkdnn_CUSRC "src/kernels/*.cu" "src/sorting.cu")
cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${CUDNN_INCLUDE_DIRS})
cuda_add_library(kernels SHARED ${tkdnn_CUSRC})
target_link_libraries(kernels ${CUDA_CUBLAS_LIBRARIES})
#-------------------------------------------------------------------------------
# External Libraries
#-------------------------------------------------------------------------------
find_package(Eigen3 REQUIRED)
message("Eigen DIR: " ${EIGEN3_INCLUDE_DIR})
include_directories(${EIGEN3_INCLUDE_DIR})
find_package(OpenCV REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENCV")
# if(OpenCV_CUDA_VERSION)
# add_compile_definitions(OPENCV_CUDACONTRIB)
# endif()
# gives problems in cross-compiling, probably malformed cmake config
find_package(yaml-cpp REQUIRED)
#find_package(yaml-cpp REQUIRED)
#-------------------------------------------------------------------------------
# Build Libraries
@@ -66,7 +48,7 @@ find_package(yaml-cpp REQUIRED)
file(GLOB tkdnn_SRC "src/*.cpp")
set(tkdnn_LIBS kernels ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDNN_LIBRARIES} ${OpenCV_LIBS} yaml-cpp)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CUDA_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} ${NVINFER_INCLUDES})
add_library(tkDNN SHARED ${tkdnn_SRC})
target_link_libraries(tkDNN ${tkdnn_LIBS})
@@ -95,7 +77,6 @@ foreach(test_SRC ${darknet_SRC})
set(test_NAME test_${test_NAME})
add_executable(${test_NAME} ${test_SRC})
target_link_libraries(${test_NAME} tkDNN)
install(TARGETS ${test_NAME} DESTINATION bin)
endforeach()
# MOBILENET
@@ -122,27 +103,6 @@ target_link_libraries(test_resnet101_cnet tkDNN)
add_executable(test_dla34_cnet tests/centernet/dla34_cnet/dla34_cnet.cpp)
target_link_libraries(test_dla34_cnet tkDNN)
add_executable(test_dla34_cnet3d tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp)
target_link_libraries(test_dla34_cnet3d tkDNN)
# CENTERTRACK
add_executable(test_dla34_ctrack tests/centertrack/dla34_ctrack/dla34_ctrack.cpp)
target_link_libraries(test_dla34_ctrack tkDNN)
# SHELFNET
add_executable(test_shelfnet tests/shelfnet/shelfnet.cpp)
target_link_libraries(test_shelfnet tkDNN)
add_executable(test_shelfnet_berkeley tests/shelfnet/shelfnet_berkeley.cpp)
target_link_libraries(test_shelfnet_berkeley tkDNN)
add_executable(test_shelfnet_mapillary tests/shelfnet/shelfnet_mapillary.cpp)
target_link_libraries(test_shelfnet_mapillary tkDNN)
add_executable(test_shelfnet_coco tests/shelfnet/shelfnet_coco.cpp)
target_link_libraries(test_shelfnet_coco tkDNN)
# DEMOS
add_executable(test_rtinference tests/test_rtinference/rtinference.cpp)
target_link_libraries(test_rtinference tkDNN)
@@ -153,15 +113,6 @@ target_link_libraries(map_demo tkDNN)
add_executable(demo demo/demo/demo.cpp)
target_link_libraries(demo tkDNN)
add_executable(demo3D demo/demo/demo3D.cpp)
target_link_libraries(demo3D tkDNN)
add_executable(demoTracker demo/demo/demoTracker.cpp)
target_link_libraries(demoTracker tkDNN)
add_executable(seg_demo demo/demo/seg_demo.cpp)
target_link_libraries(seg_demo tkDNN)
#-------------------------------------------------------------------------------
# Install
#-------------------------------------------------------------------------------
@@ -172,10 +123,7 @@ target_link_libraries(seg_demo tkDNN)
message("install dir:" ${CMAKE_INSTALL_PREFIX})
install(DIRECTORY include/ DESTINATION include/)
install(TARGETS tkDNN kernels DESTINATION lib)
install(TARGETS test_simple test_mnist test_mnistRT test_rtinference demo map_demo DESTINATION bin)
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cmake/" # source directory
DESTINATION "share/tkDNN/cmake/" # target directory
)
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests/" # source directory
DESTINATION "share/tkDNN/tests" # target directory
)
README.md
+246 -102
View File
@@ -3,105 +3,74 @@ tkDNN is a Deep Neural Network library built with cuDNN and tensorRT primitives,
The main goal of this project is to exploit NVIDIA boards as much as possible to obtain the best inference performance. It does not allow training.
If you use tkDNN in your research, please cite the [following paper](https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9212130&casa_token=sQTJXi7tJNoAAAAA:BguH9xCIY48MxbtDS3LXzIXzO-9sWArm7Hd7y7BwaLmqRuM_Gx8bOYizFPNMNtpo5K0kB-P-). For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ .
If you use tkDNN in your research, please cite one of the following papers. For use in commercial solutions, write at gattifrancesco@hotmail.it and micaela.verucchi@unimore.it or refer to https://hipert.unimore.it/ .
```
@inproceedings{verucchi2020systematic,
title={A Systematic Assessment of Embedded Neural Networks for Object Detection},
author={Verucchi, Micaela and Brilli, Gianluca and Sapienza, Davide and Verasani, Mattia and Arena, Marco and Gatti, Francesco and Capotondi, Alessandro and Cavicchioli, Roberto and Bertogna, Marko and Solieri, Marco},
booktitle={2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)},
volume={1},
pages={937--944},
year={2020},
organization={IEEE}
}
Accepted paper @ IRC 2020, will soon be published.
M. Verucchi, L. Bartoli, F. Bagni, F. Gatti, P. Burgio and M. Bertogna, "Real-Time clustering and LiDAR-camera fusion on embedded platforms for self-driving cars", in proceedings in IEEE Robotic Computing (2020)
Accepted paper @ ETFA 2020, will soon be published.
M. Verucchi, G. Brilli, D. Sapienza, M. Verasani, M. Arena, F. Gatti, A. Capotondi, R. Cavicchioli, M. Bertogna, M. Solieri
"A Systematic Assessment of Embedded Neural Networks for Object Detection", in IEEE International Conference on Emerging Technologies and Factory Automation (2020)
```
### What's new
#### 20 July 2021
- [x] Support to sematic segmentation [README](docs/README_seg.md)
- [x] Support 2D/3D Object Detection and Tracking [README](docs/README_2d3dtracking.md)
#### 24 November 2021
- [x] Support to sematic segmentation on cuda 11
- [x] Support to TensorRT8 (thanks to [Harshvardhan Chandirasekar](https://github.com/perseusdg)).
TensorRT8 (and therefore Jetpack 4.6) is currently supported only on the branch tensorrt8 due to [performance issue with TensorRT8](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/tensorrt-8.html)). We will merge it to the master as soon as those issues are fixed (probably in future minor releases).
## FPS Results
Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimension as the input size, on
## Results
Inference FPS of yolov4 with tkDNN, average of 1200 images with the same dimesion as the input size, on
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
* Xavier NX, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ).
* Tx2, Jetpack 4.2 (CUDA 10.0, CUDNN 7.3.1, tensorrt 5.0.6 );
* Jetson Nano, Jetpack 4.4 (CUDA 10.2, CUDNN 8.0.0, tensorrt 7.1.0 ).
| Platform | Network | FP32, B=1 | FP32, B=4 | FP16, B=1 | FP16, B=4 | INT8, B=1 | INT8, B=4 |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | yolo4 320 | 118.59 | 237.31 | 207.81 | 443.32 | 262.37 | 530.93 |
| RTX 2080Ti | yolo4 416 | 104.81 | 162.86 | 169.06 | 293.78 | 206.93 | 353.26 |
| RTX 2080Ti | yolo4 512 | 92.98 | 132.43 | 140.36 | 215.17 | 165.35 | 254.96 |
| RTX 2080Ti | yolo4 608 | 63.77 | 81.53 | 111.39 | 152.89 | 127.79 | 184.72 |
| AGX Xavier | yolo4 320 | 26.78 | 32.05 | 57.14 | 79.05 | 73.15 | 97.56 |
| AGX Xavier | yolo4 416 | 19.96 | 21.52 | 41.01 | 49.00 | 50.81 | 60.61 |
| AGX Xavier | yolo4 512 | 16.58 | 16.98 | 31.12 | 33.84 | 37.82 | 41.28 |
| AGX Xavier | yolo4 608 | 9.45 | 10.13 | 21.92 | 23.36 | 27.05 | 28.93 |
| Xavier NX | yolo4 320 | 14.56 | 16.25 | 30.14 | 41.15 | 42.13 | 53.42 |
| Xavier NX | yolo4 416 | 10.02 | 10.60 | 22.43 | 25.59 | 29.08 | 32.94 |
| Xavier NX | yolo4 512 | 8.10 | 8.32 | 15.78 | 17.13 | 20.51 | 22.46 |
| Xavier NX | yolo4 608 | 5.26 | 5.18 | 11.54 | 12.06 | 15.09 | 15.82 |
| Tx2 | yolo4 320 | 11.18 | 12.07 | 15.32 | 16.31 | - | - |
| Tx2 | yolo4 416 | 7.30 | 7.58 | 9.45 | 9.90 | - | - |
| Tx2 | yolo4 512 | 5.96 | 5.95 | 7.22 | 7.23 | - | - |
| Tx2 | yolo4 608 | 3.63 | 3.65 | 4.67 | 4.70 | - | - |
| Nano | yolo4 320 | 4.23 | 4.55 | 6.14 | 6.53 | - | - |
| Nano | yolo4 416 | 2.88 | 3.00 | 3.90 | 4.04 | - | - |
| Nano | yolo4 512 | 2.32 | 2.34 | 3.02 | 3.04 | - | - |
| Nano | yolo4 608 | 1.40 | 1.41 | 1.92 | 1.93 | - | - |
## MAP Results
Results for COCO val 2017 (5k images), on RTX 2080Ti, with conf threshold=0.001
| | CodaLab | CodaLab | CodaLab | CodaLab | tkDNN map | tkDNN map |
| -------------------- | :-----------: | :-------: | :-----------: | :---------: | :-----------: | :-------: |
| | **tkDNN** | **tkDNN** | **darknet** | **darknet** | **tkDNN** | **tkDNN** |
| | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 | MAP(0.5:0.95) | AP50 |
| Yolov3 (416x416) | 0.381 | 0.675 | 0.380 | 0.675 | 0.372 | 0.663 |
| yolov4 (416x416) | 0.468 | 0.705 | 0.471 | 0.710 | 0.459 | 0.695 |
| yolov3tiny (416x416) | 0.096 | 0.202 | 0.096 | 0.201 | 0.093 | 0.198 |
| yolov4tiny (416x416) | 0.202 | 0.400 | 0.201 | 0.400 | 0.197 | 0.395 |
| Cnet-dla34 (512x512) | 0.366 | 0.543 | \- | \- | 0.361 | 0.535 |
| mv2SSD (512x512) | 0.226 | 0.381 | \- | \- | 0.223 | 0.378 |
| RTX 2080Ti | yolo4 320 | 118,59 |237,31 | 207,81 | 443,32 | 262,37 | 530,93 |
| RTX 2080Ti | yolo4 416 | 104,81 |162,86 | 169,06 | 293,78 | 206,93 | 353,26 |
| RTX 2080Ti | yolo4 512 | 92,98 |132,43 | 140,36 | 215,17 | 165,35 | 254,96 |
| RTX 2080Ti | yolo4 608 | 63,77 |81,53 | 111,39 | 152,89 | 127,79 | 184,72 |
| AGX Xavier | yolo4 320 | 26,78 |32,05 | 57,14 | 79,05 | 73,15 | 97,56 |
| AGX Xavier | yolo4 416 | 19,96 |21,52 | 41,01 | 49,00 | 50,81 | 60,61 |
| AGX Xavier | yolo4 512 | 16,58 |16,98 | 31,12 | 33,84 | 37,82 | 41,28 |
| AGX Xavier | yolo4 608 | 9,45 |10,13 | 21,92 | 23,36 | 27,05 | 28,93 |
| Tx2 | yolo4 320 | 11,18 | 12,07 | 15,32 | 16,31 | - | - |
| Tx2 | yolo4 416 | 7,30 | 7,58 | 9,45 | 9,90 | - | - |
| Tx2 | yolo4 512 | 5,96 | 5,95 | 7,22 | 7,23 | - | - |
| Tx2 | yolo4 608 | 3,63 | 3,65 | 4,67 | 4,70 | - | - |
| Nano | yolo4 320 | 4,23 | 4,55 | 6,14 | 6,53 | - | - |
| Nano | yolo4 416 | 2,88 | 3,00 | 3,90 | 4,04 | - | - |
| Nano | yolo4 512 | 2,32 | 2,34 | 3,02 | 3,04 | - | - |
| Nano | yolo4 608 | 1,40 | 1,41 | 1,92 | 1,93 | - | - |
## Index
- [tkDNN](#tkdnn)
- [Index](#index)
- [Dependencies](#dependencies)
- [About OpenCV](#about-opencv)
- [How to compile this repo](#how-to-compile-this-repo)
- [Workflow](#workflow)
- [Exporting weights](#exporting-weights)
- [Run the demos](#run-the-demos)
- [tkDNN on Windows 10 (experimental)](#tkdnn-on-windows-10-experimental)
- [How to export weights](#how-to-export-weights)
- [1)Export weights from darknet](#1export-weights-from-darknet)
- [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
- [3)Export weights for CenterNet](#3export-weights-for-centernet)
- [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
- [Run the demo](#run-the-demo)
- [FP16 inference](#fp16-inference)
- [INT8 inference](#int8-inference)
- [mAP demo](#map-demo)
- [Existing tests and supported networks](#existing-tests-and-supported-networks)
- [References](#references)
## Dependencies
This branch works on every NVIDIA GPU that supports the following (latest tested) dependencies:
* CUDA 11.0 (or >= 10) [the segmentation only works with CUDA 10 for now]
* cuDNN 8.0.4 (or >= 7.3)
* TensorRT 7.2.0 (or >=5)
* OpenCV 4.5.2 (or >=4)
* cmake 3.21 (or >= 3.15)
* yaml-cpp 0.5.2
* eigen3 3.3.4
* curl 7.58
This branch works on every NVIDIA GPU that supports the dependencies:
* CUDA 10.0
* CUDNN 7.603
* TENSORRT 6.01
* OPENCV 3.4
* yaml-cpp 0.5.2 (sudo apt install libyaml-cpp-dev)
```
sudo apt install libyaml-cpp-dev curl libeigen3-dev
```
#### About OpenCV
## About OpenCV
To compile and install OpenCV4 with contrib us the script ```install_OpenCV4.sh```. It will download and compile OpenCV in Download folder.
```
bash scripts/install_OpenCV4.sh
@@ -127,23 +96,212 @@ Steps needed to do inference on tkDNN with a custom neural network.
* Create a new test and define the network, layer by layer using the weights extracted and the output to check the results.
* Do inference.
## Exporting weights
## How to export weights
For specific details on how to export weights see [HERE](./docs/exporting_weights.md).
Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
```
test_nn
|---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
|---- debug/ (folder containing a binary file for each layer with the corresponding outputs)
```
Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
## Run the demos
### 1)Export weights from darknet
To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
```
git clone https://git.hipert.unimore.it/fgatti/darknet.git
cd darknet
make
mkdir layers debug
./darknet export <path-to-cfg-file> <path-to-weights> layers
```
N.b. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug.
### 2)Export weights for DLA34 and ResNet101
To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.
Create Anaconda environment and activate it:
```
conda env create -f file_name.yml
source activate env_name
python <script name>
```
### 3)Export weights for CenterNet
To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet.
```
git clone https://github.com/sapienzadavide/CenterNet.git
```
* follow the instruction in the README.md and INSTALL.md
```
python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
```
### 4)Export weights for MobileNetSSD
To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network.
```
git clone https://github.com/mive93/pytorch-ssd
cd pytorch-ssd
conda env create -f env_mobv2ssd.yml
python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
```
## Darknet Parser
tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
```
// example of parsing yolo4
tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
net->print();
```
All models from darknet are now parsed directly from cfg, you still need to export the weights with the descripted tools in the previus section.
<details>
<summary>Supported layers</summary>
convolutional
maxpool
avgpool
shortcut
upsample
route
reorg
region
yolo
</details>
<details>
<summary>Supported activations</summary>
relu
leaky
mish
</details>
## Run the demo
This is an example using yolov4.
To run the an object detection first create the .rt file by running:
```
rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo4 # run the yolo test (is slow)
```
If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
```
cmake .. -DDEBUG=True
make
```
Once you have succesfully created your rt file, run the demo:
```
./demo yolo4_fp32.rt ../demo/yolo_test.mp4 y
```
In general the demo program takes 6 parameters:
```
./demo <network-rt-file> <path-to-video> <kind-of-network> <number-of-classes> <n-batches> <show-flag>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<kind-of-network>``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
* ```<number-of-classes>```is the number of classes the network is trained on
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
N.b. By default it is used FP32 inference
For specific details on how to run:
- 2D object detection demos, details on FP16, INT8 and batching see [HERE](./docs/demo.md).
- segmentation demos see [HERE](./docs/README_seg.md).
- 2D/3D object detection and tracking demos see [HERE](./docs/README_2d3dtracking.md).
- mAP demo to evaluate 2D object detectors see [HERE](./docs/mAP_demo.md).
![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
## tkDNN on Windows 10 (experimental)
### FP16 inference
For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows.md).
To run the an object detection demo with FP16 inference follow these steps (example with yolov3):
```
export TKDNN_MODE=FP16 # set the half floating point optimization
rm yolo3_fp16.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # run the yolo test (is slow)
./demo yolo3_fp16.rt ../demo/yolo_test.mp4 y
```
N.b. Using FP16 inference will lead to some errors in the results (first or second decimal).
### INT8 inference
To run the an object detection demo with INT8 inference three environment variables need to be set:
* ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization
* ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image
* ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label
You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder)
```
bash scripts/download_validation.sh COCO
```
to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD insted of COCO to download BDD validation.
Then a complete example using yolo3 and COCO dataset would be:
```
export TKDNN_MODE=INT8
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # run the yolo test (is slow)
./demo yolo3_int8.rt ../demo/yolo_test.mp4 y
```
N.B.
* Using INT8 inference will lead to some errors in the results.
* The test will be slower: this is due to the INT8 calibration, which may take some time to complete.
* INT8 calibration requires TensorRT version greater than or equal to 6.0
* Only 100 images are used to create the calibration table by default (set in the code).
### BatchSize bigger than 1
```
export TKDNN_BATCHSIZE=2
# build tensorRT files
```
This will create a TensorRT file with the desidered **max** batch size.
The test will still run with a batch of 1, but the created tensorRT can manage the desidered batch size.
### Test batch Inference
This will test the network with random input and check if the output of each batch is the same.
```
./test_rtinference <network-rt-file> <number-of-batches>
# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
# example
export TKDNN_BATCHSIZE=4 # set max batch size
rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # build RT file
./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
```
## mAP demo
To compute mAP, precision, recall and f1score, run the map_demo.
A validation set is needed.
To download COCO_val2017 (80 classes) run (form the root folder):
```
bash scripts/download_validation.sh COCO
```
To download Berkeley_val (10 classes) run (form the root folder):
```
bash scripts/download_validation.sh BDD
```
To compute the map, the following parameters are needed:
```
./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
```
where
* ```<network rt>```: rt file of a chosen network on which compute the mAP.
* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg.
* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
Example:
```
cd build
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
```
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to subit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
## Existing tests and supported networks
@@ -170,18 +328,8 @@ For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows
| resnet101_cnet | Centernet (Resnet101 backend)<sup>4</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/5BTjHMWBcJk8g3i/download) |
| csresnext50-panet-spp | Cross Stage Partial Network <sup>7</sup> | [COCO 2014](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/Kcs4xBozwY4wFx8/download) |
| yolo4 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_320 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 320x320 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_512 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_608 | Yolov4 <sup>8</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 608x608 | [weights](https://cloud.hipert.unimore.it/s/d97CFzYqCPCp5Hg/download) |
| yolo4_berkeley | Yolov4 <sup>8</sup> | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 544x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) |
| yolo4tiny | Yolov4 tiny <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
| yolo4x | Yolov4x-mish <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 640x640 | [weights](https://cloud.hipert.unimore.it/s/5MFjtNtgbDGdJEo/download) |
| yolo4tiny_512 | Yolov4 tiny <sup>9</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
| yolo4x-cps | Scaled Yolov4 <sup>10</sup> | [COCO 2017](http://cocodataset.org/) | 80 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/AfzHE4BfTeEm2gH/download) |
| shelfnet | ShelfNet18_realtime<sup>11</sup> | [Cityscapes](https://www.cityscapes-dataset.com/) | 19 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/mEDZMRJaGCFWSJF/download) |
| shelfnet_berkeley | ShelfNet18_realtime<sup>11</sup> | [DeepDrive](https://bdd-data.berkeley.edu/) | 20 | 1024x1024 | [weights](https://cloud.hipert.unimore.it/s/m92e7QdD9gYMF7f/download) |
| dla34_cnet3d | Centernet3D (DLA34 backend)<sup>4</sup> | [KITTI 2017](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d) | 1 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download) |
| dla34_ctrack | CenterTrack (DLA34 backend)<sup>12</sup> | [NuScenes 3D](https://www.nuscenes.org/) | 7 | 512x512 | [weights](https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download) |
| yolo4_berkeley | Yolov4 <sup>8</sup> | [BDD100K ](https://bair.berkeley.edu/blog/2018/05/30/bdd/) | 10 | 540x320 | [weights](https://cloud.hipert.unimore.it/s/nkWFa5fgb4NTdnB/download) |
| yolo4tiny | Yolov4 tiny | [COCO 2017](http://cocodataset.org/) | 80 | 416x416 | [weights](https://cloud.hipert.unimore.it/s/iRnc4pSqmx78gJs/download) |
## References
@@ -194,7 +342,3 @@ For specific details on how to run tkDNN on Windows 10 see [HERE](./docs/windows
6. He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016.
7. Wang, Chien-Yao, et al. "CSPNet: A New Backbone that can Enhance Learning Capability of CNN." arXiv preprint arXiv:1911.11929 (2019).
8. Bochkovskiy, Alexey, Chien-Yao Wang, and Hong-Yuan Mark Liao. "YOLOv4: Optimal Speed and Accuracy of Object Detection." arXiv preprint arXiv:2004.10934 (2020).
9. Bochkovskiy, Alexey, "Yolo v4, v3 and v2 for Windows and Linux" (https://github.com/AlexeyAB/darknet)
10. Wang, Chien-Yao, Alexey Bochkovskiy, and Hong-Yuan Mark Liao. "Scaled-YOLOv4: Scaling Cross Stage Partial Network." arXiv preprint arXiv:2011.08036 (2020).
11. Zhuang, Juntang, et al. "ShelfNet for fast semantic segmentation." Proceedings of the IEEE International Conference on Computer Vision Workshops. 2019.
12. Zhou, Xingyi, Vladlen Koltun, and Philipp Krähenbühl. "Tracking objects as points." European Conference on Computer Vision. Springer, Cham, 2020.
demo/config.yaml
+1 -1
View File
@@ -3,5 +3,5 @@ map_points : 101 #number of recall points (0 for all, 101 for COCO, 11 Pascal
map_levels : 10 #number of IoU step for the AP
map_step : 0.05 #step of IoU
IoU_thresh : 0.5 #starting IoU threshold
conf_thresh : 0.001 #threshold on the condifence of the bbox
conf_thresh : 0.0 #threshold on the condifence of the bbox
verbose : false #print on screen information
demo/demo/demo.cpp
+32 -49
View File
@@ -1,7 +1,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <unistd.h>
#include <mutex>
#include "CenternetDetection.h"
@@ -9,6 +9,7 @@
#include "Yolo3Detection.h"
bool gRun;
bool SAVE_RESULT = false;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
@@ -17,53 +18,35 @@ void sig_handler(int signo) {
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
// get config file path and read it
#ifdef __linux__
std::string config_file = "../demo/demoConfig.yaml";
#elif _WIN32
std::string config_file = "..\\..\\..\\demo\\demoConfig.yaml";
#endif
std::string net = "yolo3_berkeley.rt";
if(argc > 1)
config_file = argv[1];
YAML::Node conf = YAMLloadConf(config_file);
if(!conf)
FatalError("Problem with config file");
net = argv[1];
std::string input = "../demo/yolo_test.mp4";
if(argc > 2)
input = argv[2];
char ntype = 'y';
if(argc > 3)
ntype = argv[3][0];
int n_classes = 80;
if(argc > 4)
n_classes = atoi(argv[4]);
int n_batch = 1;
if(argc > 5)
n_batch = atoi(argv[5]);
bool show = true;
if(argc > 6)
show = atoi(argv[6]);
// read settings from config file
std::string net = YAMLgetConf<std::string>(conf, "net", "yolo4tiny_fp32.rt");
if(!fileExist(net.c_str()))
FatalError("The given network does not exist. Create the rt first.");
#ifdef __linux__
std::string input = YAMLgetConf<std::string>(conf, "input", "../demo/yolo_test.mp4");
#elif _WIN32
std::string input = YAMLgetConf(conf, "win_input", "..\\..\\..\\demo\\yolo_test.mp4");
#endif
if(!fileExist(input.c_str()))
FatalError("The given input video does not exist.");
char ntype = YAMLgetConf<char>(conf, "ntype", 'y');
int n_classes = YAMLgetConf<int>(conf, "n_classes", 80);
int n_batch = YAMLgetConf<int>(conf, "n_batch", 1);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
float conf_thresh = YAMLgetConf<float>(conf, "conf_thresh", 0.3);
bool show = YAMLgetConf<bool>(conf, "show", true);
bool save = YAMLgetConf<bool>(conf, "save", false);
std::cout <<"Net settings - net: "<< net
<<", ntype: "<< ntype
<<", n_classes: "<< n_classes
<<", n_batch: "<< n_batch
<<", conf_thresh: "<< conf_thresh<<"\n";
std::cout <<"Demo settings - input: "<< input
<<", show: "<< show
<<", save: "<< save<<"\n\n";
// create detection network
if(!show)
SAVE_RESULT = true;
tk::dnn::Yolo3Detection yolo;
tk::dnn::CenternetDetection cnet;
tk::dnn::MobilenetDetection mbnet;
@@ -86,9 +69,10 @@ int main(int argc, char *argv[]) {
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net, n_classes, n_batch, conf_thresh);
detNN->init(net, n_classes, n_batch);
gRun = true;
// open video stream
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
@@ -96,21 +80,19 @@ int main(int argc, char *argv[]) {
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(save) {
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
cv::Mat frame;
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
// start detection loop
gRun = true;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
@@ -138,18 +120,19 @@ int main(int argc, char *argv[]) {
cv::waitKey(1);
}
}
if(n_batch == 1 && save)
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())/n_batch<<" ms\n";
for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
std::cout<<"Avg: "<<mean/n_batch<<" ms\t"<<1000/(mean/n_batch)<<" FPS\n"<<COL_END;
return 0;
}
demo/demo/demo3D.cpp
-159
View File
@@ -1,159 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <mutex>
#include "demo_utils.h"
#include "CenternetDetection3D.h"
bool gRun;
bool SAVE_RESULT = false;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
std::string net = "dla34_cnet3d_fp32.rt";
if(argc > 1)
net = argv[1];
#ifdef __linux__
std::string input = "../demo/yolo_test.mp4";
#elif _WIN32
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
#endif
if(argc > 2)
input = argv[2];
std::string calib_params = "";
if(argc > 3)
calib_params = argv[3];
char ntype = 'c';
if(argc > 4)
ntype = argv[4][0];
int n_classes = 3;
if(argc > 5)
n_classes = atoi(argv[5]);
int n_batch = 1;
if(argc > 6)
n_batch = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
float conf_thresh=0.3;
if(argc > 8)
conf_thresh = atof(argv[8]);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
if(!show)
SAVE_RESULT = true;
tk::dnn::CenternetDetection3D cnet;
tk::dnn::DetectionNN3D *detNN;
switch(ntype)
{
case 'c':
detNN = &cnet;
break;
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
std::vector<cv::Mat> calibs;
if(!calib_params.empty() && calib_params!="NULL") {
std::cout<<"calib_params: "<<calib_params<<std::endl;
cv::Mat calib;
// the calibration matrix must be a 3x3 matrix
readCalibrationMatrix(calib_params, calib);
for(int bi=0; bi< n_batch; ++bi)
calibs.push_back(calib);
}
detNN->init(net, n_classes, n_batch, conf_thresh, calibs);
gRun = true;
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
for(int bi=0; bi< n_batch; ++bi){
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
// this will be resized to the net format
batch_dnn_input.push_back(frame.clone());
}
if(!frame.data)
break;
//inference
detNN->update(batch_dnn_input, n_batch, false, nullptr, false);
detNN->draw(batch_frame);
if(show){
for(int bi=0; bi< n_batch; ++bi){
cv::imshow("detection", batch_frame[bi]);
cv::waitKey(1);
}
}
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime preprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->pre_stats.begin(), detNN->pre_stats.end())<<" ms\n";
for(int i=0; i<detNN->pre_stats.size(); i++) mean += detNN->pre_stats[i]; mean /= detNN->pre_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->stats.begin(), detNN->stats.end())<<" ms\n";
for(int i=0; i<detNN->stats.size(); i++) mean += detNN->stats[i]; mean /= detNN->stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime postprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(detNN->post_stats.begin(), detNN->post_stats.end())<<" ms\n";
for(int i=0; i<detNN->post_stats.size(); i++) mean += detNN->post_stats[i]; mean /= detNN->post_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
return 0;
}
demo/demo/demoTracker.cpp
-160
View File
@@ -1,160 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
//#include <unistd.h>
#include <mutex>
#include "demo_utils.h"
#include "CenterTrack.h"
bool gRun;
bool SAVE_RESULT = false;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
std::string net = "dla34_cnet3d_track_fp32.rt";
if(argc > 1)
net = argv[1];
#ifdef __linux__
std::string input = "../demo/yolo_test.mp4";
#elif _WIN32
std::string input = "..\\..\\..\\demo\\yolo_test.mp4";
#endif
if(argc > 2)
input = argv[2];
std::string calib_params = "";
if(argc > 3)
calib_params = argv[3];
char ntype = 'c';
if(argc > 4)
ntype = argv[4][0];
int n_classes = 3;
if(argc > 5)
n_classes = atoi(argv[5]);
int n_batch = 1;
if(argc > 6)
n_batch = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
float conf_thresh=0.3;
if(argc > 8)
conf_thresh = atof(argv[8]);
bool t3d = true;
if(argc > 9)
t3d = atoi(argv[9]);
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
if(!show)
SAVE_RESULT = true;
tk::dnn::CenterTrack ctrack;
tk::dnn::TrackingNN *trackNN;
switch(ntype)
{
case 'c':
trackNN = &ctrack;
break;
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
std::vector<cv::Mat> calibs;
if(!calib_params.empty() && calib_params!="NULL") {
std::cout<<"calib_params: "<<calib_params<<std::endl;
cv::Mat calib;
// the calibration matrix must be a 3x3 matrix
readCalibrationMatrix(calib_params, calib);
for(int bi=0; bi< n_batch; ++bi)
calibs.push_back(calib);
}
trackNN->init(net, n_classes, n_batch, conf_thresh, t3d, calibs);
gRun = true;
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
int h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
std::vector<cv::Mat> batch_frame;
std::vector<cv::Mat> batch_dnn_input;
while(gRun) {
batch_dnn_input.clear();
batch_frame.clear();
for(int bi=0; bi< n_batch; ++bi){
cap >> frame;
if(!frame.data)
break;
batch_frame.push_back(frame);
// this will be resized to the net format
batch_dnn_input.push_back(frame.clone());
}
if(!frame.data)
break;
//inference
trackNN->update(batch_dnn_input, n_batch, false, nullptr, false);
trackNN->draw(batch_frame);
if(show){
for(int bi=0; bi< n_batch; ++bi){
cv::imshow("detection", batch_frame[bi]);
cv::waitKey(1);
}
}
if(n_batch == 1 && SAVE_RESULT)
resultVideo << frame;
}
std::cout<<"detection end\n";
double mean = 0;
std::cout<<COL_GREENB<<"\n\nTime preprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(trackNN->pre_stats.begin(), trackNN->pre_stats.end())<<" ms\n";
for(int i=0; i<trackNN->pre_stats.size(); i++) mean += trackNN->pre_stats[i]; mean /= trackNN->pre_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime stats:\n";
std::cout<<"Min: "<<*std::min_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(trackNN->stats.begin(), trackNN->stats.end())<<" ms\n";
for(int i=0; i<trackNN->stats.size(); i++) mean += trackNN->stats[i]; mean /= trackNN->stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
mean=0;
std::cout<<COL_GREENB<<"\n\nTime postprocessing stats:\n";
std::cout<<"Min: "<<*std::min_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n";
std::cout<<"Max: "<<*std::max_element(trackNN->post_stats.begin(), trackNN->post_stats.end())<<" ms\n";
for(int i=0; i<trackNN->post_stats.size(); i++) mean += trackNN->post_stats[i]; mean /= trackNN->post_stats.size();
std::cout<<"Avg: "<<mean<<" ms\n"<<COL_END;
return 0;
}
demo/demo/map.cpp
+75 -105
View File
@@ -2,10 +2,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
@@ -34,12 +31,10 @@ int main(int argc, char *argv[])
const char *config_filename = "../demo/config.yaml";
const char * net = "yolo3.rt";
const char * labels_path = "../demo/COCO_val2017/all_labels.txt";
int n_batches = 1;
float confidence_thresh = 0.3;
bool show = false;
bool write_dets = false;
bool write_res_on_file = true;
bool write_coco_json = false;
bool write_coco_json = true;
int n_images = 5000;
bool verbose;
@@ -58,12 +53,6 @@ int main(int argc, char *argv[])
labels_path = argv[3];
if(argc > 4)
config_filename = argv[4];
if(argc > 5)
n_batches = atoi(argv[5]);
if(argc > 6)
confidence_thresh = atof(argv[6]);
std::cout<<"conf t: "<<confidence_thresh<<std::endl;
//check if files needed exist
if(!fileExist(config_filename))
@@ -91,9 +80,9 @@ int main(int argc, char *argv[])
}
if(write_res_on_file){
times.open("times_"+net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh)+".csv");
times.open("times_"+net_name+".csv");
memory.open("memory.csv", std::ios_base::app);
memory<<net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh)<<";";
memory<<net<<";";
}
// instantiate detector
@@ -116,7 +105,7 @@ int main(int argc, char *argv[])
default:
FatalError("Network type not allowed (3rd parameter)\n");
}
detNN->init(net, n_classes, 1, conf_thresh);
detNN->init(net, n_classes);
//read images
std::ifstream all_labels(labels_path);
@@ -129,109 +118,90 @@ int main(int argc, char *argv[])
if(show)
cv::namedWindow("detection", cv::WINDOW_NORMAL);
bool file_ok = false;
int images_done;
for (images_done=0 ; images_done < n_images ;) {
for (images_done=0 ; std::getline(all_labels, l_filename) && images_done < n_images ; ++images_done) {
std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\n"<<COL_END;
int cur_batches = 0;
tk::dnn::Frame f;
f.lFilename = l_filename;
f.iFilename = l_filename;
convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
// read frame
if(!fileExist(f.iFilename.c_str()))
FatalError("Wrong image file path.");
cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
std::vector<cv::Mat> batch_frames;
std::vector<cv::Mat> batch_dnn_input;
std::vector<tk::dnn::Frame> cur_frames;
for(;cur_batches<n_batches && images_done < n_images;cur_batches++, ++images_done){
batch_frames.push_back(frame);
int height = frame.rows;
int width = frame.cols;
std::getline(all_labels, l_filename);
file_ok = all_labels ? true : false ;
if (!file_ok)
break;
tk::dnn::Frame f;
f.lFilename = l_filename;
f.iFilename = l_filename;
convertFilename(f.iFilename, "labels", "images", ".txt", ".jpg");
// read frame
if(!fileExist(f.iFilename.c_str()))
FatalError("Wrong image file path.");
cv::Mat frame = cv::imread(f.iFilename.c_str(), cv::IMREAD_COLOR);
batch_frames.push_back(frame);
f.height = frame.rows;
f.width = frame.cols;
if(!frame.data)
break;
batch_dnn_input.push_back(frame.clone());
// read and save groundtruth labels
if(fileExist(f.lFilename.c_str()))
{
std::ifstream labels(f.lFilename);
for(std::string line; std::getline(labels, line); ){
std::istringstream in(line);
tk::dnn::BoundingBox b;
in >> b.cl >> b.x >> b.y >> b.w >> b.h;
b.prob = 1;
b.truthFlag = 1;
f.gt.push_back(b);
if(show)// draw rectangle for groundtruth
cv::rectangle(batch_frames[cur_batches], cv::Point((b.x-b.w/2)*f.width, (b.y-b.h/2)*f.height), cv::Point((b.x+b.w/2)*f.width,(b.y+b.h/2)*f.height), cv::Scalar(0, 255, 0), 2);
}
}
cur_frames.push_back(f);
}
if (!file_ok)
if(!frame.data)
break;
std::vector<cv::Mat> batch_dnn_input;
batch_dnn_input.push_back(frame.clone());
//inference
detNN->update(batch_dnn_input,cur_batches,write_res_on_file, &times, write_coco_json);
detected_bbox.clear();
detNN->update(batch_dnn_input,1,write_res_on_file, &times, write_coco_json);
detNN->draw(batch_frames);
detected_bbox = detNN->detected;
for(int j=0;j<cur_frames.size(); ++j){
if(write_coco_json)
printJsonCOCOFormat(&coco_json, cur_frames[j].iFilename.c_str(), detNN->batchDetected[j], classes, cur_frames[j].width, cur_frames[j].height);
if(write_coco_json)
printJsonCOCOFormat(&coco_json, f.iFilename.c_str(), detected_bbox, classes, width, height);
std::ofstream myfile;
if(write_dets)
myfile.open ("det/"+cur_frames[j].lFilename.substr(cur_frames[j].lFilename.find("labels/") + 7));
std::ofstream myfile;
if(write_dets)
myfile.open ("det/"+f.lFilename.substr(f.lFilename.find("labels/") + 7));
// save detections labels
for(auto d:detNN->batchDetected[j]){
//convert detected bb in the same format as label
//<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
tk::dnn::BoundingBox b;
b.x = (d.x + d.w/2) / cur_frames[j].width;
b.y = (d.y + d.h/2) / cur_frames[j].height;
b.w = d.w / cur_frames[j].width;
b.h = d.h / cur_frames[j].height;
b.prob = d.prob;
b.cl = d.cl;
cur_frames[j].det.push_back(b);
if(write_dets)
myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n";
if(show)// draw rectangle for detection
cv::rectangle(batch_frames[j], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);
}
// save detections labels
for(auto d:detected_bbox){
//convert detected bb in the same format as label
//<x_center>/<image_width> <y_center>/<image_width> <width>/<image_width> <height>/<image_width>
tk::dnn::BoundingBox b;
b.x = (d.x + d.w/2) / width;
b.y = (d.y + d.h/2) / height;
b.w = d.w / width;
b.h = d.h / height;
b.prob = d.prob;
b.cl = d.cl;
f.det.push_back(b);
if(write_dets)
myfile.close();
images.push_back(cur_frames[j]);
if(show){
cv::imshow("detection", batch_frames[j]);
cv::waitKey(0);
}
myfile << d.cl << " "<< d.prob << " "<< b.x << " "<< b.y << " "<< b.w << " "<< b.h <<"\n";
if(show)// draw rectangle for detection
cv::rectangle(batch_frames[0], cv::Point(d.x, d.y), cv::Point(d.x + d.w, d.y + d.h), cv::Scalar(0, 0, 255), 2);
}
std::cout <<COL_ORANGEB<< "Images done:\t" << images_done<< "\tcur batch:\t"<<cur_batches<< "\n"<<COL_END;
if(write_dets)
myfile.close();
// read and save groundtruth labels
if(fileExist(f.lFilename.c_str()))
{
std::ifstream labels(l_filename);
for(std::string line; std::getline(labels, line); ){
std::istringstream in(line);
tk::dnn::BoundingBox b;
in >> b.cl >> b.x >> b.y >> b.w >> b.h;
b.prob = 1;
b.truthFlag = 1;
f.gt.push_back(b);
if(show)// draw rectangle for groundtruth
cv::rectangle(batch_frames[0], cv::Point((b.x-b.w/2)*width, (b.y-b.h/2)*height), cv::Point((b.x+b.w/2)*width,(b.y+b.h/2)*height), cv::Scalar(0, 255, 0), 2);
}
}
images.push_back(f);
if(show){
cv::imshow("detection", batch_frames[0]);
cv::waitKey(0);
}
getMemUsage(vm, rss);
vm_total += vm;
rss_total += rss;
@@ -248,11 +218,11 @@ int main(int argc, char *argv[])
std::cout << "Avg VM[MB]: " << vm_total/images_done/1024.0 << ";Avg RSS[MB]: " << rss_total/images_done/1024.0 << std::endl;
//compute mAP
double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,confidence_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name+"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh));
double AP = tk::dnn::computeMapNIoULevels(images,classes,IoU_thresh,conf_thresh, map_points, map_step, map_levels, verbose, write_res_on_file, net_name);
std::cout<<"mAP "<<IoU_thresh<<":"<<IoU_thresh+map_step*(map_levels-1)<<" = "<<AP<<std::endl;
//compute average precision, recall and f1score
tk::dnn::computeTPFPFN(images,classes,IoU_thresh,confidence_thresh, verbose, write_res_on_file, net_name +"_"+ std::to_string(n_batches)+"_"+std::to_string(confidence_thresh));
tk::dnn::computeTPFPFN(images,classes,IoU_thresh,conf_thresh, verbose, write_res_on_file, net_name);
if(write_res_on_file){
memory<<vm_total/images_done/1024.0<<";"<<rss_total/images_done/1024.0<<"\n";
demo/demo/seg_demo.cpp
-148
View File
@@ -1,148 +0,0 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#include <unistd.h>
#include <mutex>
#include "SegmentationNN.h"
bool gRun;
bool SAVE_RESULT = true;
void sig_handler(int signo) {
std::cout<<"request gateway stop\n";
gRun = false;
}
void writePred(const std::string& images_names, const std::string& gt_folder, const std::string& out_folder, tk::dnn::SegmentationNN& segNN, int& width, int& height, bool show=false){
std::ifstream all_gt(images_names);
std::string filename;
cv::Mat frame;
for (; std::getline(all_gt, filename); ) {
std::cout<<filename<<std::endl;
frame = cv::imread(gt_folder + filename);
height = frame.rows;
width = frame.cols;
segNN.updateOriginal(frame, false);
if(show)
segNN.draw();
cv::imwrite(out_folder + filename, segNN.segmented[0]);
}
}
int main(int argc, char *argv[]) {
std::cout<<"detection\n";
signal(SIGINT, sig_handler);
std::string net = "shelfnet_fp32.rt";
if(argc > 1)
net = argv[1];
std::string input = "../demo/yolo_test.mp4";
if(argc > 2)
input = argv[2];
int n_batch = 1;
if(argc > 3)
n_batch = atoi(argv[3]);
int n_classes = 19;
if(argc > 4)
n_classes = atoi(argv[4]);
bool resize = false;
if(argc > 5)
resize = atoi(argv[5]);
int baseline_resize = 1024;
if(argc > 6)
baseline_resize = atoi(argv[6]);
bool show = true;
if(argc > 7)
show = atoi(argv[7]);
bool write_pred = false;
if(argc > 8)
write_pred = atoi(argv[8]);
if(resize && (baseline_resize < 0 || baseline_resize > 5000))
FatalError("Problem with baseline resize")
if(n_batch < 1 || n_batch > 64)
FatalError("Batch dim not supported");
//net initialization
tk::dnn::SegmentationNN segNN;
segNN.init(net, n_classes, n_batch);
int height = 0, width = 0;
int basewidth=baseline_resize, hsize;
if(write_pred){
std::string gt_folder = "../demo/CityScapes_val/images/";
std::string images_names = "../demo/CityScapes_val/all_images.txt";
std::string out_folder = "seg/";
writePred(images_names, gt_folder, out_folder, segNN, width, height, show);
}
else{
if(!show)
SAVE_RESULT = true;
gRun = true;
cv::VideoCapture cap(input);
if(!cap.isOpened())
gRun = false;
else
std::cout<<"camera started\n";
cv::VideoWriter resultVideo;
if(SAVE_RESULT) {
int w,h;
if(resize){
w = basewidth;
h = int((float(cap.get(cv::CAP_PROP_FRAME_HEIGHT))*float(basewidth/float(cap.get(cv::CAP_PROP_FRAME_WIDTH)))));
}
else{
w = cap.get(cv::CAP_PROP_FRAME_WIDTH);
h = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
}
resultVideo.open("result.mp4", cv::VideoWriter::fourcc('M','P','4','V'), 30, cv::Size(w, h));
}
cv::Mat frame;
while(gRun) {
cap >> frame;
if(!frame.data)
break;
if(resize){
hsize = int((float(frame.rows)*float(basewidth/float(frame.cols))));
cv::resize(frame, frame, cv::Size(basewidth, hsize));
}
height = frame.rows;
width = frame.cols;
//inference
segNN.updateOriginal(frame, true);
if(show)
segNN.draw();
if(SAVE_RESULT)
resultVideo << segNN.segmented[0];
}
}
std::cout<<"segmentation end\n";
double mean = 0, mean_pre = 0, mean_post = 0;
std::cout<<COL_GREENB<<"\n\nTime stats for size ["<<width<<","<<height<<"] :\n";
for(int i=0; i<segNN.stats.size(); i++) mean += segNN.stats[i]; mean /= segNN.stats.size();
for(int i=0; i<segNN.stats_pre.size(); i++) mean_pre += segNN.stats_pre[i]; mean_pre /= segNN.stats_pre.size();
for(int i=0; i<segNN.stats_post.size(); i++) mean_post += segNN.stats_post[i]; mean_post /= segNN.stats_post.size();
std::cout<<"Avg pre:\t"<<mean_pre<<" ms\t"<<1000/(mean_pre)<<" FPS\n";
std::cout<<"Avg inf:\t"<<mean<<" ms\t"<<1000/(mean)<<" FPS\n";
std::cout<<"Avg post:\t"<<mean_post<<" ms\t"<<1000/(mean_post)<<" FPS\n\n";
std::cout<<"Avg tot:\t"<<(mean_pre + mean_post + mean) <<" ms\t"<<1000/((mean_pre + mean_post + mean))<<" FPS\n"<<COL_END;
return 0;
}
demo/demoConfig.yaml
-14
View File
@@ -1,14 +0,0 @@
# video input
input : "../demo/yolo_test.mp4"
win_input : "..\\..\\..\\demo\\yolo_test.mp4"
# network config
net : "yolo4tiny_fp32.rt"
ntype : 'y'
n_classes : 80
n_batch : 1
conf_thresh : 0.3
# demo config
show : true
save : true
docs/README_2d3dtracking.md
-92
View File
@@ -1,92 +0,0 @@
# 2D/3D Object Detection and Tracking
Currently tkDNN supports only CenterTrack as 3DOD & 2D/3D Tracker network.
## 3D Object Detection
To run the 3D object detection demo follow these steps (example with CenterNet based on DLA34):
```
rm dla34_cnet3d_fp32.rt # be sure to delete(or move) old tensorRT files
./test_dla34_cnet3d # run the yolo test (is slow)
./demo3D dla34_cnet3d_fp32.rt ../demo/yolo_test.mp4 NULL c
```
The demo3D program takes the same parameters of the demo program:
```
./demo3D <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh>
```
where
* ```<calibration-file>``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
![demo](https://user-images.githubusercontent.com/11939259/126784875-c4285497-d369-424f-abda-58274cd747ac.gif)
## Object Detection and Tracking
To run the 3D object detection & tracking demo follow these steps (example with CenterTrack based on DLA34):
```
rm dla34_ctrack_fp32.rt # be sure to delete(or move) old tensorRT files
./test_dla34_ctrack # run the yolo test (is slow)
./demoTracker dla34_ctrack_fp32.rt ../demo/yolo_test.mp4 NULL c
```
The demoTracker program takes the same parameters of the demo program:
```
./demoTracker <network-rt-file> <path-to-video> <calibration-file> <kind-of-network> <number-of-classes> <n-batches> <show-flag> <conf-thresh> <2D/3D-flag>
```
where
* ```<calibration-file>``` is the camera calibration file (opencv format). It is important that the file contains entry "camera_matrix" with sub-entry "rows", "cols", "data". If you do not want to pass the calibration file, pass "NULL" instead.
* ```<2D/3D-flag>``` if set to 0 the demo will be in the 2D mode, while if set to 1 the demo will be in the 3D mode (Default is 1 - 3D mode).
![demo](https://user-images.githubusercontent.com/11939259/126784878-513fa9e8-864a-4c24-b4bd-199737184708.gif)
## FPS Results
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
### 3D OD and Tracking
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | pre | 4.43883 | 225.285 | 4.42951 | 225.759 | 4.44278 | 225.084 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | inf | 9.03454 | 110.686 | 6.02013 | 166.109 | 5.31611 | 188.108 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | post | 0.96631 | 1034.87 | 0.96824 | 1032.80 | 0.95066 | 1051.90 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=1) | tot | 14.4397 | 69.2535 | 11.4179 | 87.5818 | 10.7095 | 93.3750 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | pre | 4.60075 | 217.356 | 4.28658 | 233.286 | 4.29473 | 232.844 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | inf | 8.48365 | 117.874 | 5.25150 | 190.422 | 4.58463 | 218.120 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | post | 0.99484 | 1005.19 | 0.91776 | 1089.61 | 0.89853 | 1112.93 |
| RTX 2080Ti | CenterTrack3D 512x512 (B=4) | tot | 14.0792 | 71.0266 | 10.4558 | 95.6405 | 9.77788 | 102.272 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | pre | 34.9915 | 28.5784 | 33.5976 | 29.7440 | 34.4425 | 29.0339 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | inf | 76.3579 | 13.0962 | 52.4759 | 19.0564 | 51.4610 | 19.4322 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | post | 3.38576 | 295.355 | 3.26010 | 306.739 | 3.19770 | 312.725 |
| AGX Xavier | CenterTrack3D 512x512 (B=1) | tot | 114.735 | 8.71574 | 89.3336 | 11.1940 | 89.1012 | 11.2232 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | pre | 32.8933 | 30.4014 | 32.7950 | 30.4925 | 32.9603 | 30.3396 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | inf | 74.2840 | 13.4618 | 50.3858 | 19.8469 | 49.2030 | 20.3240 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | post | 3.14888 | 317.574 | 3.13615 | 318.862 | 3.02550 | 330.524 |
| AGX Xavier | CenterTrack3D 512x512 (B=4) | tot | 110.326 | 9.06404 | 86.3169 | 11.5852 | 85.1888 | 11.7386 |
### 2D OD and Tracking
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | pre | 4.44386 | 225.030 | 4.43828 | 225.313 | 4.47747 | 223.340 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | inf | 9.08365 | 110.088 | 6.04842 | 165.332 | 5.34787 | 186.990 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | post | 0.98593 | 1014.27 | 0.97745 | 1023.07 | 0.96595 | 1035.25 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=1) | tot | 14.5134 | 68.9018 | 11.4642 | 87.2281 | 10.7913 | 92.6672 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | pre | 4.41188 | 226.661 | 4.50800 | 221.828 | 4.29238 | 232.971 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | inf | 8.29015 | 120.625 | 5.38630 | 185.656 | 4.58500 | 218.103 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | post | 0.96847 | 1032.55 | 0.97997 | 1020.44 | 0.91791 | 1089.43 |
| RTX 2080Ti | CenterTrack2D 512x512 (B=4) | tot | 13.6705 | 73.1502 | 10.8743 | 91.9602 | 9.79528 | 102.090 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | pre | 33.4745 | 29.8735 | 33.4847 | 29.8643 | 33.5022 | 29.8488 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | inf | 76.2077 | 13.1220 | 52.5111 | 19.0436 | 51.6057 | 19.3777 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | post | 3.26055 | 306.697 | 3.26806 | 305.992 | 3.21988 | 310.571 |
| AGX Xavier | CenterTrack2D 512x512 (B=1) | tot | 111.943 | 8.93312 | 89.2639 | 11.2027 | 88.3278 | 11.3215 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | pre | 32.8323 | 30.4579 | 32.8595 | 30.4326 | 32.8195 | 30.4697 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | inf | 74.3075 | 13.4576 | 50.3555 | 19.8588 | 49.1805 | 20.3333 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | post | 3.12360 | 320.143 | 3.13570 | 318.908 | 3.04943 | 327.931 |
| AGX Xavier | CenterTrack2D 512x512 (B=4) | tot | 110.263 | 9.06920 | 86.3507 | 11.5807 | 85.0494 | 11.7579 |
docs/README_seg.md
-68
View File
@@ -1,68 +0,0 @@
# Semantic Segmentation with tkDNN
Currently tkDNN supports only ShelfNet as semantic segmentation network.
## Run the demo
To run the semantic segmentation demo follow these steps (example with shelfnet):
```
rm shelfnet_fp32.rt # be sure to delete(or move) old tensorRT files
export TKDNN_BATCHSIZE=4 # be sure you have batch size > than 1 if you want to run inference on images bigger than 1024
./test_shelfnet # run the yolo test (is slow)
./demo shelfnet_fp32.rt ../demo/yolo_test.mp4 1 19
```
In general the demo program takes the following parameters:
```
./seg_demo <network-rt-file> <path-to-video> <n-batches> <number-of-classes> <resize-flag> <baseline-resize> <show-flag> <write-pred>
```
where
* ```<network-rt-file>``` is the rt file generated by a test
* ```<<path-to-video>``` is the path to a video file or a camera input
* ```<n-batches>``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```<number-of-classes>```is the number of classes the network is trained on
* ```<resize-flag>``` if set to 0 the demo will not resize the input frames, but use it as it is, otherwise it will resize it.
* ```<baseline-resize>``` is ```<resize-flag>``` is set to 1, then the input frames will be proportionally resized using ```<baseline-resize>``` as width baseline.
* ```<show-flag>``` if set to 0 the demo will not show the visualization but save the video into result.mp4 (if n-batches ==1)
* ```<write-pred>``` if set to 0 (default) the demo will run, otherwise the evaluation of a dataset will run and the output of the segmentation will be saved. Attention: this is under development and paths are embedded, so change them in the code in advance.
NB) By default it is used FP32 inference
NB) The batching is not used to work on more streams, rather to work on more tiles of the same image. Shelfnet never resized the input image, therefore for images greater than 1024x1024 tiles of 1024x1024 are given in input to the network in batch.
![demo](https://user-images.githubusercontent.com/11939259/126784236-38d24fc3-02df-4514-81c4-497e87e40b65.gif "Results on yolo_test.mp4")
For other demo videos refer to [this playlist](https://www.youtube.com/playlist?list=PLv0nEQYDD45y5EdSiywwCGPBmJVUzIWwe).
NB) The gif and the videos are obtained with Mapillary Vistas weights, that we cannot publicly share due to its license restrictions. However, you can train Shelfnet using Mapillary and [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of the original repo.
## FPS Results
Inference FPS of shelfnet with tkDNN, average of 1200 images on:
* RTX 2080Ti (CUDA 10.2, TensorRT 7.0.0, Cudnn 7.6.5);
* Xavier AGX, Jetpack 4.3 (CUDA 10.0, CUDNN 7.6.3, tensorrt 6.0.1 );
| Platform | Test | Phase | FP32, ms | FP32, FPS | FP16, ms | FP16, FPS | INT8, ms | INT8, FPS |
| :------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | pre | 6.11863 | 163.435 | 5.81465 | 171.979 | 5.88699 | 169.866 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | inf | 11.5464 | 86.6074 | 7.35396 | 135.981 | 6.37623 | 156.832 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | post | 4.09058 | 244.464 | 3.91961 | 255.128 | 4.07343 | 245.493 |
| RTX 2080Ti | shelfnet 1024x1024 (B=1) | tot | 21.7556 | 45.9652 | 17.0882 | 58.5199 | 16.3366 | 61.2121 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | pre | 25.435 | 39.3158 | 25.2953 | 39.5331 | 25.9303 | 38.565 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | inf | 36.5015 | 27.3961 | 17.0534 | 58.6395 | 15.6061 | 64.0773 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | post | 17.3917 | 57.4985 | 17.1649 | 58.2583 | 17.5539 | 56.9675 |
| RTX 2080Ti | shelfnet 2048x2048 (B=4) | tot | 79.3283 | 12.6058 | 59.5136 | 16.8029 | 59.0903 | 16.9233 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | pre | 8.0174 | 124.729 | 7.5117 | 133.126 | 7.47333 | 133.809 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | inf | 72.4173 | 13.8089 | 37.505 | 26.6631 | 31.3286 | 31.9197 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | post | 8.89958 | 112.365 | 8.83576 | 113.176 | 9.42655 | 106.083 |
| AGX Xavier | shelfnet 1024x1024 (B=1) | tot | 89.3342 | 11.1939 | 53.8525 | 18.5692 | 48.2285 | 20.7346 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | pre | 47.1454 | 21.211 | 21.6475 | 46.1947 | 21.4201 | 46.6851 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | inf | 266.537 | 3.75183 | 128.321 | 7.79293 | 107.621 | 9.29185 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | post | 44.0711 | 22.6906 | 40.1732 | 24.8922 | 39.873 | 25.0796 |
| AGX Xavier | shelfnet 2048x2048 (B=4) | tot | 357.753 | 2.79522 | 190.142 | 5.25922 | 168.914 | 5.92016 |
## Known issues
When creating the rt file all the checks returns errors. It is due to a different resize function and handling of the original ShelfNet outputs.
However, the network is supposed to work.
docs/demo.md
-120
View File
@@ -1,120 +0,0 @@
# 2D Object Detection with tkDNN
## Supported Networks
* Yolo4, Yolo4-csp, Yolo4x, Yolo4_berkeley, Yolo4tiny
* Yolo3, Yolo3_berkeley, Yolo3_coco4, Yolo3_flir, Yolo3_512, Yolo3tiny, Yolo3tiny_512
* Yolo2, Yolo2_voc, Yolo2tiny
* Csresnext50-panet-spp, Csresnext50-panet-spp_berkeley
* Resnet101_cnet, Dla34_cnet
* Mobilenetv2ssd, Mobilenetv2ssd512, Bdd-mobilenetv2ssd
## Index
- [2D Object Detection](#2d-object-detection)
- [FP16 inference](#fp16-inference)
- [INT8 inference](#int8-inference)
- [Batching](#batching)
### 2D Object Detection
This is an example using yolov4.
To run the an object detection first create the .rt file by running:
```
rm yolo4_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo4 # run the yolo test (is slow)
```
If you get problems in the creation, try to check the error activating the debug of TensorRT in this way:
```
cmake .. -DDEBUG=True
make
```
Once you have successfully created your rt file, run the demo:
```
./demo <path-to-config>
```
In general the demo program takes 1 parameter, the ```<path-to-config>``` that is the path to che configuration file. The parameter is optional and its default value is ```"../demo/demoConfig.yaml"```.
The config file is a yaml file with the following attributes:
* ```net``` is the rt file generated by a test
* ```input``` is the path to a video file or a camera input (on Linux)
* ```win_input``` is the path to a video file or a camera input (on Windows)
* ```ntype``` is the type of network. Thee types are currently supported: ```y``` (YOLO family), ```c``` (CenterNet family) and ```m``` (MobileNet-SSD family)
* ```n_classes``` is the number of classes the network is trained on
* ```n_batch``` number of batches to use in inference (N.B. you should first export TKDNN_BATCHSIZE to the required n_batches and create again the rt file for the network).
* ```conf_thresh``` confidence threshold for the detector. Only bounding boxes with threshold greater than conf-thresh will be displayed.
* ```show``` if set to 0 the demo will not show the visualization (if n-batches ==1)
* ```save``` if set to 1 the demo will save the video of the demo into result.mp4 (if n-batches ==1)
N.B. By default it is used FP32 inference
![demo](https://user-images.githubusercontent.com/11562617/72547657-540e7800-388d-11ea-83c6-49dfea2a0607.gif)
### FP16 inference
To run the demo with FP16 inference follow these steps (example with yolov3):
```
export TKDNN_MODE=FP16 # set the half floating point optimization
rm yolo3_fp16.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # run the yolo test (is slow)
# set net: yolo3_fp16.rt in the config-file
./demo
```
N.B. Using FP16 inference will lead to some errors in the results (first or second decimal).
### INT8 inference
To run the demo with INT8 inference three environment variables need to be set:
* ```export TKDNN_MODE=INT8```: set the 8-bit integer optimization
* ```export TKDNN_CALIB_IMG_PATH=/path/to/calibration/image_list.txt``` : image_list.txt has in each line the absolute path to a calibration image
* ```export TKDNN_CALIB_LABEL_PATH=/path/to/calibration/label_list.txt```: label_list.txt has in each line the absolute path to a calibration label
You should provide image_list.txt and label_list.txt, using training images. However, if you want to quickly test the INT8 inference you can run (from this repo root folder)
```
bash scripts/download_validation.sh COCO
```
to automatically download COCO2017 validation (inside demo folder) and create those needed file. Use BDD instead of COCO to download BDD validation.
Then a complete example using yolo3 and COCO dataset would be:
```
export TKDNN_MODE=INT8
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
rm yolo3_int8.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # run the yolo test (is slow)
# set net: yolo3_int8.rt in the config-file
./demo
```
N.B.
* Using INT8 inference will lead to some errors in the results.
* The test will be slower: this is due to the INT8 calibration, which may take some time to complete.
* INT8 calibration requires TensorRT version greater than or equal to 6.0
* Only 100 images are used to create the calibration table by default (set in the code).
### Batching
#### BatchSize bigger than 1
```
export TKDNN_BATCHSIZE=2
# build tensorRT files
```
This will create a TensorRT file with the desired **max** batch size.
The test will still run with a batch of 1, but the created tensorRT can manage the desired batch size.
#### Test batch Inference
This will test the network with random input and check if the output of each batch is the same.
```
./test_rtinference <network-rt-file> <number-of-batches>
# <number-of-batches> should be less or equal to the max batch size of the <network-rt-file>
# example
export TKDNN_BATCHSIZE=4 # set max batch size
rm yolo3_fp32.rt # be sure to delete(or move) old tensorRT files
./test_yolo3 # build RT file
./test_rtinference yolo3_fp32.rt 4 # test with a batch size of 4
```
docs/exporting_weights.md
-115
View File
@@ -1,115 +0,0 @@
# tkDNN export weights
## Index
- [How to export weights](#how-to-export-weights)
- [1)Export weights from darknet](#1export-weights-from-darknet)
- [2)Export weights for DLA34 and ResNet101](#2export-weights-for-dla34-and-resnet101)
- [3)Export weights for CenterNet](#3export-weights-for-centernet)
- [4)Export weights for MobileNetSSD](#4export-weights-for-mobilenetssd)
- [5)Export weights for CenterTrack](#5export-weights-for-centertrack)
- [6)Export weights for ShelfNet](#6export-weights-for-shelfnet)
- [Darknet Parser](#darknet-parser)
## How to export weights
Weights are essential for any network to run inference. For each test a folder organized as follow is needed (in the build folder):
```
test_nn
|---- layers/ (folder containing a binary file for each layer with the corresponding wieghts and bias)
|---- debug/ (folder containing a binary file for each layer with the corresponding outputs)
```
Therefore, once the weights have been exported, the folders layers and debug should be placed in the corresponding test.
### 1)Export weights from darknet
To export weights for NNs that are defined in darknet framework, use [this](https://git.hipert.unimore.it/fgatti/darknet.git) fork of darknet and follow these steps to obtain a correct debug and layers folder, ready for tkDNN.
```
git clone https://git.hipert.unimore.it/fgatti/darknet.git
cd darknet
make
mkdir layers debug
./darknet export <path-to-cfg-file> <path-to-weights> layers
```
N.B. Use compilation with CPU (leave GPU=0 in Makefile) if you also want debug.
### 2)Export weights for DLA34 and ResNet101
To get weights and outputs needed to run the tests dla34 and resnet101 use the Python script and the Anaconda environment included in the repository.
Create Anaconda environment and activate it:
```
conda env create -f file_name.yml
source activate env_name
python <script name>
```
### 3)Export weights for CenterNet
To get the weights needed to run Centernet tests use [this](https://github.com/sapienzadavide/CenterNet.git) fork of the original Centernet.
```
git clone https://github.com/sapienzadavide/CenterNet.git
```
* follow the instruction in the README.md and INSTALL.md
```
python demo.py --input_res 512 --arch resdcn_101 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_resdcn101.pth --exp_wo --exp_wo_dim 512
python demo.py --input_res 512 --arch dla_34 ctdet --demo /path/to/image/or/folder/or/video/or/webcam --load_model ../models/ctdet_coco_dla_2x.pth --exp_wo --exp_wo_dim 512
```
### 4)Export weights for MobileNetSSD
To get the weights needed to run Mobilenet tests use [this](https://github.com/mive93/pytorch-ssd) fork of a Pytorch implementation of SSD network.
```
git clone https://github.com/mive93/pytorch-ssd
cd pytorch-ssd
conda env create -f env_mobv2ssd.yml
python run_ssd_live_demo.py mb2-ssd-lite <pth-model-fil> <labels-file>
```
### 5)Export weights for CenterTrack
To get the weights needed to run CenterTrack tests use [this](https://github.com/sapienzadavide/CenterTrack.git) fork of the original CenterTrack.
```
git clone https://github.com/sapienzadavide/CenterTrack.git
```
* follow the instruction in the README.md and INSTALL.md
```
python demo.py tracking,ddd --load_model ../models/nuScenes_3Dtracking.pth --dataset nuscenes --pre_hm --track_thresh 0.1 --demo /path/to/image/or/folder/or/video/or/webcam --test_focal_length 633 --exp_wo --exp_wo_dim 512 --input_h 512 --input_w 512
```
### 6)Export weights for ShelfNet
To get the weights needed to run Shelfnet tests use [this](https://git.hipert.unimore.it/mverucchi/shelfnet) fork of a Pytorch implementation of Shelfnet network.
```
git clone https://git.hipert.unimore.it/mverucchi/shelfnet
cd shelfnet
cd ShelfNet18_realtime
conda env create --file shelfnet_env.yml
conda activate shelfnet
mkdir layer debug
python export.py
```
## Darknet Parser
tkDNN implement and easy parser for darknet cfg files, a network can be converted with *tk::dnn::darknetParser*:
```
// example of parsing yolo4
tk::dnn::Network *net = tk::dnn::darknetParser("yolov4.cfg", "yolov4/layers", "coco.names");
net->print();
```
All models from darknet are now parsed directly from cfg, you still need to export the weights with the described tools in the previous section.
<details>
<summary>Supported layers</summary>
convolutional
maxpool
avgpool
shortcut
upsample
route
reorg
region
yolo
</details>
<details>
<summary>Supported activations</summary>
relu
leaky
mish
logistic
</details>
docs/mAP_demo.md
-32
View File
@@ -1,32 +0,0 @@
# Run the mAP demo
To compute mAP, precision, recall and f1score to evaluate 2D object detectors, run the map_demo.
A validation set is needed.
To download COCO_val2017 (80 classes) run (form the root folder):
```
bash scripts/download_validation.sh COCO
```
To download Berkeley_val (10 classes) run (form the root folder):
```
bash scripts/download_validation.sh BDD
```
To compute the map, the following parameters are needed:
```
./map_demo <network rt> <network type [y|c|m]> <labels file path> <config file path>
```
where
* ```<network rt>```: rt file of a chosen network on which compute the mAP.
* ```<network type [y|c|m]>```: type of network. Right now only y(yolo), c(centernet) and m(mobilenet) are allowed
* ```<labels file path>```: path to a text file containing all the paths of the ground-truth labels. It is important that all the labels of the ground-truth are in a folder called 'labels'. In the folder containing the folder 'labels' there should be also a folder 'images', containing all the ground-truth images having the same same as the labels. To better understand, if there is a label path/to/labels/000001.txt there should be a corresponding image path/to/images/000001.jpg.
* ```<config file path>```: path to a yaml file with the parameters needed for the mAP computation, similar to demo/config.yaml
Example:
```
cd build
./map_demo dla34_cnet_FP32.rt c ../demo/COCO_val2017/all_labels.txt ../demo/config.yaml
```
This demo also creates a json file named ```net_name_COCO_res.json``` containing all the detections computed. The detections are in COCO format, the correct format to submit the results to [CodaLab COCO detection challenge](https://competitions.codalab.org/competitions/20794#participate).
docs/windows.md
-95
View File
@@ -1,95 +0,0 @@
# tkDNN on Windows
## Index
- [Dependencies-Windows](#dependencies-windows)
- [Compiling tkDNN on Windows](#compiling-tkdnn-on-windows)
- [Run the demo on Windows](#run-the-demo-on-windows)
- [FP16 inference windows](#fp16-inference-windows)
- [INT8 inference windows](#int8-inference-windows)
- [Known issues with tkDNN on Windows](#known-issues-with-tkdnn-on-windows)
### Dependencies-Windows
This branch should work on every NVIDIA GPU supported in windows with the following dependencies:
* WINDOWS 10 1803 or HIGHER
* CUDA 10.0 (Recommended CUDA 11.2 )
* CUDNN 7.6 (Recommended CUDNN 8.1.1 )
* TENSORRT 6.0.1 (Recommended TENSORRT 7.2.3.4 )
* OPENCV 3.4 (Recommended OPENCV 4.2.0 )
* MSVC 16.7
* YAML-CPP
* EIGEN3
* 7ZIP (ADD TO PATH)
* NINJA 1.10
All the above mentioned dependencies except 7ZIP can be installed using Microsoft's [VCPKG](https://github.com/microsoft/vcpkg.git) .
After bootstrapping VCPKG the dependencies can be built and installed using the following command :
```
opencv4(normal) - vcpkg.exe install opencv4[tbb,jpeg,tiff,opengl,openmp,png,ffmpeg,eigen]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
opencv4(cuda) - vcpkg.exe install opencv4[cuda,nonfree,contrib,eigen,tbb,jpeg,tiff,opengl,openmp,png,ffmpeg]:x64-windows yaml-cpp:x64-windows eigen3:x64-windows --x-install-root=C:\opt --x-buildtrees-root=C:\temp_vcpkg_build
```
To build opencv4 with cuda and cudnn version corresponding to your cuda version,vcpkg's cudnn portfile needs to be modified by adding ```$ENV{CUDA_PATH}``` at lines 16 and 17 in the portfile.cmake
After VCPKG finishes building and installing all the packages delete C:\temp_vcpkg_build and add C:\opt\x64-windows\bin and C:\opt\x64-windows\debug\bin to path
### Compiling tkDNN on Windows
tkDNN is built with cmake(3.15+) on windows along with ninja.Msbuild and NMake Makefiles are drastically slower when compiling the library compared to windows
```
git clone https://github.com/ceccocats/tkDNN.git
cd tkdnn-windows
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -G"Ninja" ..
ninja -j4
```
### Run the demo on Windows
This example uses yolo4_tiny.\
To run the object detection file create .rt file bu running:
```
.\test_yolo4tiny.exe
```
Once the rt file has been successfully create,run the demo using the following command:
```
.\demo.exe yolo4tiny_fp32.rt ..\demo\yolo_test.mp4 y
```
For general info on more demo paramters,check Run the demo section on top
To run the test_all_tests.sh on windows,use git bash or msys2
### FP16 inference windows
This is an untested feature on windows.To run the object detection demo with FP16 interference follow the below steps(example with yolo4tiny):
```
set TKDNN_MODE=FP16
del /f yolo4tiny_fp16.rt
.\test_yolo4tiny.exe
.\demo.exe yolo4tiny_fp16.rt ..\demo\yolo_test.mp4
```
### INT8 inference windows
To run object detection demo with INT8 (example with yolo4tiny):
```
set TKDNN_MODE=INT8
set TKDNN_CALIB_LABEL_PATH=..\demo\COCO_val2017\all_labels.txt
set TKDNN_CALIB_IMG_PATH=..\demo\COCO_val2017\all_images.txt
del /f yolo4tiny_int8.rt # be sure to delete(or move) old tensorRT files
.\test_yolo4tiny.exe # run the yolo test (is slow)
.\demo.exe yolo4tiny_int8.rt ..\demo\yolo_test.mp4 y
```
### Known issues with tkDNN on Windows
Mobilenet and Centernet demos work properly only when built with msvc 16.7 in Release Mode,when built in debug mode for the mentioned networks one might encounter opencv assert errors
All Darknet models work properly with demo using MSVC version(16.7-16.9)
It is recommended to use Nvidia Driver(465+),Cuda unknown errors have been observed when using older drivers on pascal(SM 61) devices.
include/tkDNN/CenterTrack.h
-181
View File
@@ -1,181 +0,0 @@
#ifndef CENTERTRACK_H
#define CENTERTRACK_H
#include <opencv2/videoio.hpp>
#include "opencv2/opencv.hpp"
#include "kernels.h"
#include "utils.h"
#include "tkdnn.h"
#include <time.h>
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
#include "TrackingNN.h"
#include "kernelsThrust.h"
namespace tk { namespace dnn {
struct detectionRes
{
float score;
int cl;
cv::Mat ct, tr, bb0, bb1;
float dep;
float dim[3];
float alpha;
float x,y,z;
float rot_y;
detectionRes() : ct(cv::Mat(cv::Size(1,2), CV_32F)),
tr(cv::Mat(cv::Size(1,2), CV_32F)),
bb0(cv::Mat(cv::Size(1,2), CV_32F)),
bb1(cv::Mat(cv::Size(1,2), CV_32F)) { }
~detectionRes() {
ct.release();
tr.release();
bb0.release();
bb1.release();
}
};
struct trackingRes
{
struct detectionRes det_res;
int tracking_id;
int age;
int active;
int color;
};
class CenterTrack : public TrackingNN
{
public:
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
tk::dnn::dataDim_t dim_track;
tk::dnn::dataDim_t dim_dep;
tk::dnn::dataDim_t dim_rot;
tk::dnn::dataDim_t dim_dim;
tk::dnn::dataDim_t dim_amodel_offset;
/* preprocessing */
#ifdef OPENCV_CUDACONTRIB
float *mean_d;
float *stddev_d;
#else
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
dnnType *input;
#endif
float *d_ptrs;
std::vector<cv::Mat> inputCalibs;
std::vector<cv::Size> szOld;
cv::Mat src;
cv::Mat dst;
cv::Mat dst2;
cv::Mat trans, trans2, transOut;
/* pre inf */
bool iter0;
dnnType *input_pre_inf_d;
bool test_pre_inf = true;
dnnType *img_d, *hm_d;
tk::dnn::dataDim_t dim_in0;
tk::dnn::dataDim_t dim_in1;
dnnType *out_d;
/* postprocessing */
int K = 100;
int width = 128;//56; // TODO
// pointer used in the kernels
float *src_out;
int *ids_out;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_;
float *ones;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
float *bbx0, *bby0, *bbx1, *bby1;
float *bbx0_d, *bby0_d, *bbx1_d, *bby1_d;
int *intxs, *intys;
float *track, *dep, *rot, *dim_, *wh, *amodel_offset;
float *track_d, *dep_d, *rot_d, *dim_d, *wh_d, *amodel_offset_d;
float *target_coords;
/* visualization */
cv::Mat r;
std::vector<cv::Mat> calibs;
cv::Mat corners, pts3DHomo;
std::vector<std::vector<int>> faceId;
cv::Scalar trColors[256];
bool mode3D;
//processing
struct threshold op;
float outThresh = 0.1;
float newThresh = 0.3;
// float peakThreshold = 0.2;
// float centerThreshold = 0.3; //default 0.5
//detections
std::vector<struct detectionRes> detRes;
int countDet;
//tracks
std::vector<std::vector<struct trackingRes>> trRes;
std::vector<int> countTr;
std::vector<int> trackId;
bool init_preprocessing();
bool init_pre_inf();
bool init_postprocessing();
bool init_visualization(const int n_classes);
void pre_inf(const int bi);
void _get_additional_inputs();
cv::Mat transform_preds_with_trans(float x1, float x2);
void tracking(const int bi);
public:
tk::dnn::Network *pre_phase_net = nullptr;
CenterTrack() {};
~CenterTrack() {};
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
const float conf_thresh=0.3, const bool mode_3d=true,
const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>());
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
void draw(std::vector<cv::Mat>& frames);
};
} // namespace dnn
} // namespace tk
#endif /*CENTERTRACK_H*/
include/tkDNN/CenternetDetection.h
+1 -1
View File
@@ -73,7 +73,7 @@ public:
CenternetDetection() {};
~CenternetDetection() {};
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3);
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/CenternetDetection3D.h
-101
View File
@@ -1,101 +0,0 @@
#ifndef CENTERNETDETECTION3D_H
#define CENTERNETDETECTION3D_H
#include "kernels.h"
#include <opencv2/videoio.hpp>
#include "opencv2/opencv.hpp"
#include <time.h>
#include <vector>
#include <numeric> // std::iota
#include <algorithm> // std::sort
#include "DetectionNN3D.h"
#include "kernelsThrust.h"
namespace tk { namespace dnn {
class CenternetDetection3D : public DetectionNN3D
{
private:
tk::dnn::dataDim_t dim;
tk::dnn::dataDim_t dim2;
tk::dnn::dataDim_t dim_hm;
tk::dnn::dataDim_t dim_wh;
tk::dnn::dataDim_t dim_reg;
tk::dnn::dataDim_t dim_dep;
tk::dnn::dataDim_t dim_rot;
tk::dnn::dataDim_t dim_dim;
std::vector<cv::Mat> inputCalibs;
float *topk_scores;
int *topk_inds_;
float *topk_ys_;
float *topk_xs_;
int *ids_d, *ids_;
float *ones;
float *scores, *scores_d;
int *clses, *clses_d;
int *topk_inds_d;
float *topk_ys_d;
float *topk_xs_d;
int *inttopk_xs_d, *inttopk_ys_d;
float *xs, *ys;
float *dep, *rot, *dim_, *wh;
float *dep_d, *rot_d, *dim_d, *wh_d;
float *target_coords;
#ifdef OPENCV_CUDACONTRIB
float *mean_d;
float *stddev_d;
#else
cv::Vec<float, 3> mean;
cv::Vec<float, 3> stddev;
dnnType *input;
#endif
cv::Mat r;
float *d_ptrs;
cv::Size sz_old;
cv::Mat src;
cv::Mat dst;
cv::Mat dst2;
cv::Mat trans, trans2;
std::vector<cv::Mat> calibs;
//processing
int K = 100;
int width = 128;//56; // TODO
// pointer used in the kernels
float *srcOut;
int *idsOut;
struct threshold op;
cv::Mat corners, pts3DHomo;
std::vector<std::vector<int>> faceId;
public:
CenternetDetection3D() {};
~CenternetDetection3D() {};
bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1, const float conf_thresh=0.3, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>());
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
void draw(std::vector<cv::Mat>& frames);
};
} // namespace dnn
} // namespace tk
#endif /*CENTERNETDETECTION_H*/
include/tkDNN/DarknetParser.h
-3
View File
@@ -24,10 +24,7 @@ namespace tk { namespace dnn {
int num = 1;
int pad = 0;
int coords = 4;
int nms_kind = 0;
int new_coords= 0;
float scale_xy = 1;
float nms_thresh = 0.45;
std::vector<int> layers;
std::string activation = "linear";
include/tkDNN/DetectionNN.h
+7 -9
View File
@@ -4,10 +4,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
@@ -17,7 +14,7 @@
#include "tkdnn.h"
//#define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
@@ -79,15 +76,15 @@ class DetectionNN {
~DetectionNN(){};
/**
* Method used to initialize the class, allocate memory and compute
* Method used to inialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param tensor_path path to the rt file og the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3) = 0;
virtual bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1) = 0;
/**
* This method performs the whole detection of the NN.
@@ -144,15 +141,16 @@ class DetectionNN {
}
/**
* Method to draw bounding boxes and labels on a frame.
* Method to draw boundixg boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
* @param frames orginal frame to draw bounding box on.
*/
void draw(std::vector<cv::Mat>& frames) {
tk::dnn::box b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
int baseline = 0;
float font_scale = 0.5;
int thickness = 2;
include/tkDNN/DetectionNN3D.h
-161
View File
@@ -1,161 +0,0 @@
#ifndef DETECTIONNN3D_H
#define DETECTIONNN3D_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
#include <opencv2/cudaarithm.hpp>
#endif
namespace tk { namespace dnn {
class DetectionNN3D {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
std::vector<cv::Size> originalSize;
cv::Scalar colors[256];
int nBatches = 1;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
#else
cv::Mat bgr[3];
cv::Mat imagePreproc;
dnnType *input;
#endif
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<tk::dnn::box3D> detected3D; /*bounding boxes in output*/
std::vector<std::vector<tk::dnn::box3D>> batchDetected; /*bounding boxes in output*/
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
std::vector<std::string> classesNames;
DetectionNN3D() {};
~DetectionNN3D(){};
/**
* Method used to initialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference.
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
const float conf_thresh=0.3, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>()) = 0;
/**
* This method performs the whole detection of the NN.
*
* @param frames frames to run detection on.
* @param cur_batches number of batches to use in inference.
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times.
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation.
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false,
std::ofstream *times=nullptr, const bool mAP=false){
if(save_times && times==nullptr)
FatalError("save_times set to true, but no valid ofstream given");
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
pre_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
batchDetected.clear();
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, mAP);
TKDNN_TSTOP
post_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<"\n";
}
}
/**
* Method to draw bounding boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
*/
virtual void draw(std::vector<cv::Mat>& frames){};
};
}}
#endif /* DETECTIONNN3D_H*/
include/tkDNN/ImuOdom.h
+2 -9
View File
@@ -1,14 +1,7 @@
#include <iostream>
#include <signal.h>
#include <stdlib.h> /* srand, rand */
#ifdef __linux__
#include <unistd.h>
#elif _WIN32
#define _USE_MATH_DEFINES
#include <math.h>
#endif
#include <mutex>
#include <Eigen/Dense>
#include "utils.h"
@@ -51,7 +44,7 @@ class ImuOdom {
virtual ~ImuOdom() {}
/**
* Method used for initialize the class
* Method used for inizialize the class
*
* @return Success of the initialization
*/
@@ -148,7 +141,7 @@ class ImuOdom {
//odomPOS = odomPOS + deltaP.cast<double>(); // V2
odomROT = odomROT * q.normalized().toRotationMatrix();
// compute Euler
// compute euler
auto newEULER = odomROT.eulerAngles(0, 1, 2);
for(int i=0; i<3; i++) {
while( fabs(newEULER(i) - odomEULER(i)) > M_PI_2 ) {
include/tkDNN/Int8BatchStream.h
+1 -4
View File
@@ -11,11 +11,8 @@
#include <fstream>
#include <iomanip>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <stdlib.h>
#include <unistd.h>
#endif
#include <mutex>
#include "NvInfer.h"
include/tkDNN/Layer.h
+18 -62
View File
@@ -19,10 +19,8 @@ enum layerType_t {
LAYER_ACTIVATION_CRELU,
LAYER_ACTIVATION_LEAKY,
LAYER_ACTIVATION_MISH,
LAYER_ACTIVATION_LOGISTIC,
LAYER_FLATTEN,
LAYER_RESHAPE,
LAYER_RESIZE,
LAYER_MULADD,
LAYER_POOLING,
LAYER_SOFTMAX,
@@ -56,10 +54,6 @@ public:
int id = 0;
bool final; //if the layer is the final one
uint n_params = 0;
uint feature_map_size = 0;
long unsigned MACC = 0;
std::string getLayerName() {
layerType_t type = getLayerType();
@@ -74,10 +68,8 @@ public:
case LAYER_ACTIVATION_CRELU: return "ActivationCReLU";
case LAYER_ACTIVATION_LEAKY: return "ActivationLeaky";
case LAYER_ACTIVATION_MISH: return "ActivationMish";
case LAYER_ACTIVATION_LOGISTIC: return "ActivationLogistic";
case LAYER_FLATTEN: return "Flatten";
case LAYER_RESHAPE: return "Reshape";
case LAYER_RESIZE: return "Resize";
case LAYER_MULADD: return "MulAdd";
case LAYER_POOLING: return "Pooling";
case LAYER_SOFTMAX: return "Softmax";
@@ -179,7 +171,7 @@ public:
/**
Input layer (it doesn't need weights)
Input layer (it doesnt need weigths)
*/
class Input : public Layer {
@@ -215,26 +207,24 @@ public:
/**
Available activation functions
Avaible activation functions
*/
typedef enum {
ACTIVATION_ELU = 100,
ACTIVATION_LEAKY = 101,
ACTIVATION_MISH = 102,
ACTIVATION_LOGISTIC = 103
ACTIVATION_MISH = 102
} tkdnnActivationMode_t;
/**
Activation layer (it doesn't need weights)
Activation layer (it doesnt need weigths)
*/
class Activation : public Layer {
public:
int act_mode;
float ceiling;
float slope;
Activation(Network *net, int act_mode, const float ceiling=0.0, const float slope=0.1);
Activation(Network *net, int act_mode, const float ceiling=0.0);
virtual ~Activation();
virtual layerType_t getLayerType() {
if(act_mode == CUDNN_ACTIVATION_CLIPPED_RELU)
@@ -243,8 +233,6 @@ public:
return LAYER_ACTIVATION_LEAKY;
else if (act_mode == ACTIVATION_MISH)
return LAYER_ACTIVATION_MISH;
else if (act_mode == ACTIVATION_LOGISTIC)
return LAYER_ACTIVATION_LOGISTIC;
else
return LAYER_ACTIVATION;
};
@@ -330,9 +318,9 @@ public:
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
const bool bidirectional = true; /**> is the net bidir */
bool returnSeq = false; /**> if false return only the result of last timestamp */
bool returnSeq = false; /**> if false return only the result of last timestep */
int stateSize = 0; /**> number of hidden states */
int seqLen = 0; /**> number of timestamp */
int seqLen = 0; /**> number of timesteps */
int numLayers = 1; /**> number of internal layers */
protected:
@@ -379,7 +367,7 @@ public:
/**
Deformable Convolutional 2d layer
Deformable Convolutionl 2d layer
*/
class DeformConv2d : public LayerWgs {
@@ -439,23 +427,6 @@ public:
};
enum ResizeMode_t { NEAREST= 0,
LINEAR= 1};
/**
Resize layer
*/
class Resize : public Layer {
public:
Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed=false, ResizeMode_t mode=NEAREST);
virtual ~Resize();
virtual layerType_t getLayerType() { return LAYER_RESIZE; };
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
ResizeMode_t mode;
};
/**
MulAdd layer
@@ -478,7 +449,7 @@ protected:
/**
Available pooling functions (padding on tkDNN is not supported)
Avaible pooling functions (padding on tkDNN is not supported)
*/
typedef enum {
POOLING_MAX = 0,
@@ -489,7 +460,7 @@ typedef enum {
/**
Pooling layer
currently supported only 2d pooing (also on 3d input)
currenty supported only 2d pooing (also on 3d input)
*/
class Pooling : public Layer {
@@ -555,7 +526,7 @@ public:
/**
Reorg layer
Maintains same dimension but change C*H*W distribution
Mantain same dimension but change C*H*W distribution
*/
class Reorg : public Layer {
@@ -576,7 +547,7 @@ public:
class Shortcut : public Layer {
public:
Shortcut(Network *net, Layer *backLayer, bool mul=false);
Shortcut(Network *net, Layer *backLayer);
virtual ~Shortcut();
virtual layerType_t getLayerType() { return LAYER_SHORTCUT; };
@@ -584,12 +555,11 @@ public:
public:
Layer *backLayer;
bool mul = false;
};
/**
Upsample layer
Maintains same dimension but change C*H*W distribution
Mantain same dimension but change C*H*W distribution
*/
class Upsample : public Layer {
@@ -620,16 +590,6 @@ struct sortable_bbox {
int cl;
float **probs;
};
struct box3D {
int cl;
std::vector<float> corners;
float prob;
void print()
{
std::cout<<"\tcl: "<<cl<<"\tprob: "<<prob<<"\tshape corners: "<<corners.size()<<std::endl;
}
};
/**
Yolo3 layer
@@ -650,28 +610,24 @@ public:
int sort_class;
};
enum nmsKind_t {GREEDY_NMS=0, DIOU_NMS=1};
Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1, double nms_thresh=0.45, nmsKind_t nsm_kind=GREEDY_NMS, int new_coords=0);
Yolo(Network *net, int classes, int num, std::string fname_weights,int n_masks=3, float scale_xy=1);
virtual ~Yolo();
virtual layerType_t getLayerType() { return LAYER_YOLO; };
int classes, num, n_masks, new_coords;
int classes, num, n_masks;
dnnType *mask_h, *mask_d; //anchors
dnnType *bias_h, *bias_d; //anchors
float scaleXY;
double nms_thresh;
nmsKind_t nsm_kind;
std::vector<std::string> classesNames;
virtual dnnType* infer(dataDim_t &dim, dnnType* srcData);
int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int new_coords=0);
int computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh);
dnnType *predictions;
static const int MAX_DETECTIONS = 8192*2;
static const int MAX_DETECTIONS = 8192;
static Yolo::detection *allocateDetections(int nboxes, int classes);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes, double nms_thresh=0.45, nmsKind_t nsm_kind=GREEDY_NMS);
static void mergeDetections(Yolo::detection *dets, int ndets, int classes);
};
/**
include/tkDNN/MobilenetDetection.h
+1 -1
View File
@@ -65,7 +65,7 @@ public:
MobilenetDetection() {};
~MobilenetDetection() {};
bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1, const float conf_thresh=0.3);
bool init(const std::string& tensor_path, const int n_classes, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/Network.h
+4 -5
View File
@@ -7,12 +7,12 @@
namespace tk { namespace dnn {
/**
Data representation between layers
Data rapresentation beetween layers
n = batch size
c = channels
h = height (lines)
h = heigth (lines)
w = width (rows)
l = length (3rd dimension)
l = lenght (3rd dimension)
*/
struct dataDim_t {
@@ -43,14 +43,13 @@ public:
void releaseLayers();
/**
Do inference for every added layer
Do inferece for every added layer
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
bool addLayer(Layer *l);
void print();
const char *getNetworkRTName(const char *network_name);
void adjustFeatureMapSizeWithShortcuts();
cudnnDataType_t dataType;
cudnnTensorFormat_t tensorFormat;
include/tkDNN/NetworkRT.h
+1 -8
View File
@@ -6,7 +6,6 @@
#include "Network.h"
#include "Layer.h"
#include "NvInfer.h"
#include <memory>
namespace tk { namespace dnn {
@@ -25,7 +24,6 @@ template<typename T> T readBUF(const char*& buffer)
using namespace nvinfer1;
#include "pluginsRT/ActivationLeakyRT.h"
#include "pluginsRT/ActivationLogisticRT.h"
#include "pluginsRT/ActivationReLUCeilingRT.h"
#include "pluginsRT/ActivationMishRT.h"
#include "pluginsRT/ReorgRT.h"
@@ -61,7 +59,6 @@ public:
#if NV_TENSORRT_MAJOR >= 6
nvinfer1::IBuilderConfig *configRT;
#endif
nvinfer1::ICudaEngine *engineRT;
nvinfer1::IExecutionContext *contextRT;
@@ -94,7 +91,7 @@ public:
}
/**
Do inference
Do inferece
*/
dnnType* infer(dataDim_t &dim, dnnType* data);
void enqueue(int batchSize = 1);
@@ -108,7 +105,6 @@ public:
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Route *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Flatten *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reshape *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Resize *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Reorg *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Region *l);
nvinfer1::ILayer* convert_layer(nvinfer1::ITensor *input, Shortcut *l);
@@ -118,9 +114,6 @@ public:
bool serialize(const char *filename);
bool deserialize(const char *filename);
};
}}
include/tkDNN/NetworkViz.h
+2 -2
View File
@@ -5,8 +5,8 @@
namespace tk { namespace dnn {
cv::Mat vizFloat2colorMap(cv::Mat map, double min=0, double max=0, int classes=19);
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min=0, double max=0, int classes=19);
cv::Mat vizFloat2colorMap(cv::Mat map);
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim);
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim = 1000);
}}
include/tkDNN/SegmentationNN.h
-404
View File
@@ -1,404 +0,0 @@
#ifndef SEGMENTATIONNN_H
#define SEGMENTATIONNN_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#include <unistd.h>
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/core/hal/interface.h>
#include "tkdnn.h"
#include "NetworkViz.h"
#include "kernelsThrust.h"
namespace tk { namespace dnn {
class SegmentationNN {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
int nBatches = 1;
std::vector<cv::Size> originalSize;
cv::Mat bgr[3];
dnnType *input;
dnnType *input_d;
float* confidences_h;
float * tmpInputData_d;
float *tmpOutData_d;
float *tmpOutData_h;
float *mean_d, *stddev_d;
cublasHandle_t cublasHandle;
void computeBorders(const int or_width, const int or_height, int& top, int& bottom, int& left, int&right){
top = 0;
bottom = 0;
left = 0;
right = 0;
if(or_height != or_width){
if(or_height < or_width){
top = (or_width - or_height)/2;
bottom = or_width - top - or_height;
}
else{
left = (or_height - or_width)/2;
right = or_height - left - or_width;
}
}
}
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
void preprocess(cv::Mat &frame, const int bi=0) {
originalSize[bi] = frame.size();
frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0);
int H = frame.rows;
int W = frame.cols;
cv::Mat frame_cropped;
int top, bottom, left, right;
computeBorders(W, H, top, bottom, left, right);
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) );
tk::dnn::dataDim_t idim = netRT->input_dim;
resize(frame_cropped, frame_cropped, cv::Size(idim.w, idim.h));
cv::split(frame_cropped, bgr);
for (int i = 0; i < idim.c; i++){
int idx = i * frame_cropped.rows * frame_cropped.cols;
int ch = idim.c-1 -i;
memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, frame_cropped.rows * frame_cropped.cols * sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d);
}
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
*/
void postprocess(const int bi=0, bool appy_colormap = true) {
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
dataDim_t odim = netRT->output_dim;
matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h);
maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w);
checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost));
dataDim_t vdim = odim;
vdim.c = 1;
cv::Mat colored;
if(appy_colormap)
colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes);
else{
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
colored_fp32.convertTo(colored, CV_8UC1);
}
int max_dim = (originalSize[bi].width > originalSize[bi].height) ? originalSize[bi].width : originalSize[bi].height;
resize(colored, colored, cv::Size(max_dim, max_dim));
int top, bottom, left, right;
computeBorders(originalSize[bi].width, originalSize[bi].height, top, bottom, left, right);
cv::Rect roi(left,top,originalSize[bi].width, originalSize[bi].height);
cv::Mat or_size (colored, roi);
segmented[bi] = or_size;
};
public:
int classes = 0;
std::vector<double> stats; /*keeps track of inference times (ms)*/
std::vector<double> stats_pre;
std::vector<double> stats_post;
std::vector<std::string> classesNames;
std::vector<cv::Mat> segmented;
SegmentationNN() {
checkERROR( cublasCreate(&cublasHandle) );
};
~SegmentationNN(){
checkERROR( cublasDestroy(cublasHandle) );
};
/**
* Method used to inialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file og the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference
* @return true if everything is correct, false otherwise.
*/
bool init(const std::string& tensor_path, const int n_classes=19, const int n_batches=1){
std::cout<<(tensor_path).c_str()<<"\n";
if(!fileExist(tensor_path.c_str()))
FatalError("This file do not exists" + tensor_path );
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
classes = n_classes;
nBatches = n_batches;
checkCuda(cudaMallocHost(&input, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
checkCuda(cudaMalloc(&input_d, sizeof(dnnType) * netRT->input_dim.tot() * nBatches));
dataDim_t odim = netRT->output_dim;
checkCuda(cudaMallocHost(&confidences_h, sizeof(float) * odim.tot()));
checkCuda(cudaMalloc(&tmpInputData_d, sizeof(float) * odim.tot()));
checkCuda(cudaMalloc(&tmpOutData_d, sizeof(float) * odim.w*odim.h));
checkCuda(cudaMallocHost(&tmpOutData_h, sizeof(float) * odim.w*odim.h));
segmented.resize(nBatches);
originalSize.resize(nBatches);
std::vector<float> mean = {0.485, 0.456, 0.406};
std::vector<float> stddev = {0.229, 0.224, 0.225};
checkCuda(cudaMalloc(&mean_d, sizeof(float) * mean.size()));
checkCuda(cudaMalloc(&stddev_d, sizeof(float) * stddev.size()));
checkCuda(cudaMemcpyAsync(mean_d, mean.data(), mean.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
checkCuda(cudaMemcpyAsync(stddev_d, stddev.data(), stddev.size() * sizeof(float), cudaMemcpyHostToDevice, netRT->stream));
return true;
}
/**
* This method performs the whole detection of the NN.
*
* @param frames frames to run detection on.
* @param cur_batches number of batches to use in inference
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool apply_colormap=true){
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
stats_pre.push_back(t_ns);
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
}
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, apply_colormap);
TKDNN_TSTOP
stats_post.push_back(t_ns);
}
}
void updateOriginal(cv::Mat frame, bool apply_colormap=true){
std::vector<cv::Mat> splitted_frames;
int H, W, net_H, net_W;
int top = 0, bottom = 0, left = 0, right = 0;
std::vector<std::pair<int,int>> pos;
{
TKDNN_TSTART
cv::Size original_size = frame.size();
frame.convertTo(frame, CV_32FC3, 1 / 255.0, 0);
H = frame.rows;
W = frame.cols;
net_H = netRT->input_dim.h;
net_W = netRT->input_dim.w;
cv::Mat frame_cropped;
if( H <= net_H && W <= net_W ){ // smaller size wrt network
top = (net_H - H)/2;
bottom = net_H - H - top ;
left = (net_W - W)/2;
right = net_W - W - left ;
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0) );
splitted_frames.push_back(frame_cropped);
}
else{ //bigger size wrt network
if(H < net_H || W < net_W){
if(H < net_H){
top = (net_H - H)/2;
bottom = net_H - H - top ;
}
else{
left = (net_W - W)/2;
right = net_W - W - left ;
}
cv::copyMakeBorder(frame, frame_cropped, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0,0,0));
}
for(int x=0; x+net_W<=W ;){
for(int y=0; y+net_H <=H ; ){
cv::Rect roi(x, y, net_W, net_H);
cv::Mat image_roi = frame(roi);
splitted_frames.push_back(image_roi);
pos.push_back(std::make_pair(x,y));
y += net_H;
if(y == H)
break;
if(y + net_H > H) y = H - net_H;
}
x += net_W;
if(x == W)
break;
if(x + net_W > W) x = W - net_W;
}
}
tk::dnn::dataDim_t idim = netRT->input_dim;
if(splitted_frames.size()> nBatches)
FatalError(std::to_string(splitted_frames.size()) + " min batches required");
for(int bi=0; bi<splitted_frames.size();++bi){
cv::split(splitted_frames[bi], bgr);
for (int i = 0; i < idim.c; i++){
int idx = i * splitted_frames[bi].rows * splitted_frames[bi].cols;
int ch = idim.c-1 -i;
memcpy((void *)&input[idx + idim.tot()*bi], (void *)bgr[ch].data, splitted_frames[bi].rows * splitted_frames[bi].cols * sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ idim.tot()*bi, input + idim.tot()*bi, idim.tot() * sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
normalize(input_d + idim.tot()*bi, idim.c, idim.h, idim.w, mean_d, stddev_d);
}
TKDNN_TSTOP
stats_pre.push_back(t_ns);
}
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = splitted_frames.size();
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
}
dataDim_t odim = netRT->output_dim;
std::vector<cv::Mat> out_img;
{
TKDNN_TSTART
for(int bi=0; bi<splitted_frames.size();++bi){
dnnType *rt_out = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
matrixTranspose(cublasHandle, rt_out, tmpInputData_d, odim.c, odim.w*odim.h);
maxElem(tmpInputData_d, tmpOutData_d, odim.c, odim.h, odim.w);
checkCuda(cudaMemcpy(tmpOutData_h, tmpOutData_d, odim.w*odim.h * sizeof(float), cudaMemcpyDeviceToHost));
dataDim_t vdim = odim;
vdim.c = 1;
cv::Mat colored;
if(apply_colormap)
colored = vizData2Mat(tmpOutData_h, vdim, netRT->input_dim.h, netRT->input_dim.w, 0, classes, classes);
else{
cv::Mat colored_fp32 (cv::Size(odim.w, odim.h),CV_32FC1, tmpOutData_h);
colored_fp32.convertTo(colored, CV_8UC1);
}
out_img.push_back(colored);
}
cv::Mat seg(frame.size(), out_img[0].type());
if(out_img.size() == 1)
{
cv::Rect roi(left, top, W, H);
seg = out_img[0](roi);
}
else{
int bi=0;
if(top == 0 && left == 0){
for(int i=0; i<out_img.size(); ++i){
cv::Mat roi_collage = seg(cv::Rect( pos[i].first ,pos[i].second,out_img[i].cols,out_img[i].rows));
out_img[i].copyTo(roi_collage);
}
}
else{
FatalError("Not handled case")
}
}
segmented[0] = seg;
TKDNN_TSTOP
stats_post.push_back(t_ns);
}
}
/**
* Method to draw boundixg boxes and labels on a frame.
*/
cv::Mat draw(const int cur_batches=1) {
for(int i=0; i<cur_batches; ++i){
cv::imshow("segmented", segmented[i]);
cv::resizeWindow("segmented", cv::Size(512,288));
cv::waitKey(1);
}
return segmented[0];
}
};
}}
#endif /* SEGMENTATIONNN_H*/
include/tkDNN/TrackingNN.h
-158
View File
@@ -1,158 +0,0 @@
#ifndef TRACKINGNN_H
#define TRACKINGNN_H
#include <iostream>
#include <signal.h>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <mutex>
#include "utils.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "tkdnn.h"
// #define OPENCV_CUDACONTRIB //if OPENCV has been compiled with CUDA and contrib.
#ifdef OPENCV_CUDACONTRIB
#include <opencv2/cudawarping.hpp>
#include <opencv2/cudaarithm.hpp>
#endif
namespace tk { namespace dnn {
class TrackingNN {
protected:
tk::dnn::NetworkRT *netRT = nullptr;
dnnType *input_d;
std::vector<cv::Size> originalSize;
cv::Scalar colors[256];
int nBatches = 1;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat bgr[3];
cv::cuda::GpuMat imagePreproc;
#else
cv::Mat bgr[3];
cv::Mat imagePreproc;
dnnType *input;
#endif
/**
* This method preprocess the image, before feeding it to the NN.
*
* @param frame original frame to adapt for inference.
* @param bi batch index
*/
virtual void preprocess(cv::Mat &frame, const int bi=0) = 0;
/**
* This method postprocess the output of the NN to obtain the correct
* boundig boxes.
*
* @param bi batch index
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation
*/
virtual void postprocess(const int bi=0,const bool mAP=false) = 0;
public:
int classes = 0;
float confThreshold = 0.3; /*threshold on the confidence of the boxes*/
std::vector<double> pre_stats, stats, post_stats, visual_stats; /*keeps track of inference times (ms)*/
std::vector<std::string> classesNames;
TrackingNN() {};
~TrackingNN(){};
/**
* Method used to initialize the class, allocate memory and compute
* needed data.
*
* @param tensor_path path to the rt file of the NN.
* @param n_classes number of classes for the given dataset.
* @param n_batches maximum number of batches to use in inference.
* @return true if everything is correct, false otherwise.
*/
virtual bool init(const std::string& tensor_path, const int n_classes=3, const int n_batches=1,
const float conf_thresh=0.3, const bool mode_3d=true, const std::vector<cv::Mat>& k_calibs=std::vector<cv::Mat>()) = 0;
/**
* This method performs the whole detection and tracking of the NN.
*
* @param frames frames to run detection and trcking on.
* @param cur_batches number of batches to use in inference.
* @param save_times if set to true, preprocess, inference and postprocess times
* are saved on a csv file, otherwise not.
* @param times pointer to the output stream where to write times.
* @param mAP set to true only if all the probabilities for a bounding
* box are needed, as in some cases for the mAP calculation.
*/
void update(std::vector<cv::Mat>& frames, const int cur_batches=1, bool save_times=false,
std::ofstream *times=nullptr, const bool mAP=false){
if(save_times && times==nullptr)
FatalError("save_times set to true, but no valid ofstream given");
if(cur_batches > nBatches)
FatalError("A batch size greater than nBatches cannot be used");
originalSize.clear();
if(TKDNN_VERBOSE) printCenteredTitle(" TENSORRT detection ", '=', 30);
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi){
if(!frames[bi].data)
FatalError("No image data feed to detection");
originalSize.push_back(frames[bi].size());
preprocess(frames[bi], bi);
}
TKDNN_TSTOP
pre_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
//do inference
tk::dnn::dataDim_t dim = netRT->input_dim;
dim.n = cur_batches;
{
if(TKDNN_VERBOSE) dim.print();
TKDNN_TSTART
netRT->infer(dim, input_d);
TKDNN_TSTOP
if(TKDNN_VERBOSE) dim.print();
stats.push_back(t_ns);
if(save_times) *times<<t_ns<<";";
}
{
TKDNN_TSTART
for(int bi=0; bi<cur_batches;++bi)
postprocess(bi, mAP);
TKDNN_TSTOP
post_stats.push_back(t_ns);
if(save_times) *times<<t_ns<<"\n";
}
}
/**
* Method to draw bounding boxes and labels on a frame.
*
* @param frames original frame to draw bounding box on.
*/
virtual void draw(std::vector<cv::Mat>& frames){};
};
}}
#endif /* TRACKINGNN_H*/
include/tkDNN/Yolo3Detection.h
+3 -2
View File
@@ -13,6 +13,7 @@ private:
int num = 0;
int nMasks = 0;
int nDets = 0;
bool letterbox = false;
tk::dnn::Yolo::detection *dets = nullptr;
tk::dnn::Yolo* yolo[3];
@@ -21,10 +22,10 @@ private:
cv::Mat bgr_h;
public:
Yolo3Detection() {};
Yolo3Detection(const bool letter_box=false) :letterbox(letter_box){}
~Yolo3Detection() {};
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1, const float conf_thresh=0.3);
bool init(const std::string& tensor_path, const int n_classes=80, const int n_batches=1);
void preprocess(cv::Mat &frame, const int bi=0);
void postprocess(const int bi=0,const bool mAP=false);
};
include/tkDNN/demo_utils.h
-22
View File
@@ -1,22 +0,0 @@
#ifndef DEMO_UTILS_H
#define DEMO_UTILS_H
#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <stdlib.h>
#ifdef __linux__
#include <unistd.h>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <yaml-cpp/yaml.h>
#endif
void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat);
#endif //DEMO_UTILS_H
include/tkDNN/evaluation.h
+4 -6
View File
@@ -18,8 +18,6 @@ struct Frame
std::string iFilename;
std::vector<BoundingBox> gt;
std::vector<BoundingBox> det;
int width;
int height;
void print() const;
};
@@ -75,12 +73,12 @@ double computeMap( std::vector<Frame> &images,const int classes,
* all the recall levels are evaluated, otherwise only
* map_point recall levels are used. For COCO evaluation
* 101 points are used.
* @param map_step step used to increment IoU threshold
* @param map_step step used to increment IoU theshold
* @param map_levels number of IoU step to perform
* @param verbose is set to true, prints on screen additional info
* @param write_on_file if set to true, the results produced by this function
* are written on file
* @param net name of the considered neural network
* @param net name of the considerd neural network
*
* @return mAP IoU_tresh:IoU_tresh+map_step*map_levels (e.g. mAP 0.5:0.95 when
* map_step=0.05 and map_levels=10)
@@ -91,7 +89,7 @@ double computeMapNIoULevels(std::vector<Frame> &images,const int classes,
const int map_levels=10, const bool verbose=false,
const bool write_on_file = false, std::string net = "");
/**
* This method computes the number of True Positive (TP), False Positive (FP),
* This method computes the numper of True Positive (TP), False Positive (FP),
* False Negative (FN), precision, recall and f1-score.
* Those values are computer over all the detections, over all the classes.
*
@@ -103,7 +101,7 @@ double computeMapNIoULevels(std::vector<Frame> &images,const int classes,
* @param verbose is set to true, prints on screen additional info
* @param write_on_file if set to true, the results produced by this function
* are written on file
* @param net name of the considered neural network
* @param net name of the considerd neural network
*/
void computeTPFPFN( std::vector<Frame> &images,const int classes,
const float IoU_thresh=0.5, const float conf_thresh=0.3,
include/tkDNN/kernels.h
+2 -2
View File
@@ -4,7 +4,7 @@
#include "utils.h"
void activationELUForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, float slope, cudaStream_t stream = cudaStream_t(0));
void activationLEAKYForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
void activationReLUCeilingForward(dnnType *srcData, dnnType *dstData, int size, const float ceiling, cudaStream_t stream = cudaStream_t(0));
void activationLOGISTICForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
void activationSIGMOIDForward(dnnType *srcData, dnnType *dstData, int size, cudaStream_t stream = cudaStream_t(0));
@@ -24,7 +24,7 @@ void softmaxForward(float *input, int n, int batch, int batch_offset,
int groups, int group_offset, int stride, float temp, float *output, cudaStream_t stream = cudaStream_t(0));
void shortcutForward(dnnType *srcData, dnnType *dstData, int n1, int c1, int h1, int w1, int s1,
int n2, int c2, int h2, int w2, int s2, bool mul,
int n2, int c2, int h2, int w2, int s2,
cudaStream_t stream = cudaStream_t(0));
void upsampleForward(dnnType *srcData, dnnType *dstData,
include/tkDNN/kernelsThrust.h
-7
View File
@@ -2,7 +2,6 @@
#define KERNELSTHRUST_H
#include <thrust/extrema.h>
#include <thrust/sort.h>
#include <thrust/execution_policy.h>
#include <thrust/functional.h>
@@ -10,8 +9,6 @@
#include <thrust/iterator/constant_iterator.h>
#include <thrust/gather.h>
#include <thrust/copy.h>
#include <thrust/device_ptr.h>
#include "tkdnn.h"
@@ -32,15 +29,11 @@ void topk(dnnType *src_begin, int *idsrc, int K, float *topk_scores,
int *topk_inds, float *topk_ys, float *topk_xs);
// void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int *topk_inds, float *topk_ys, float *topk_xs, const int size, const int K, const int n_classes);
void normalize(float *bgr, const int ch, const int h, const int w, const float *mean, const float *stddev);
void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end);
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op);
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys);
void topKxyAddOffset(int * ids_begin, const int K, const int size, int *intxs_begin, int *intys_begin,
float *xs_begin, float *ys_begin, dnnType *src_begin, float *src_out, int *ids_out);
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin,
dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1, float *src_out, int *ids_out);
void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out);
void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w);
#endif //KERNELSTHRUST_H
include/tkDNN/pluginsRT/ActivationLeakyRT.h
+6 -7
View File
@@ -4,8 +4,9 @@
class ActivationLeakyRT : public IPlugin {
public:
ActivationLeakyRT(float s) {
slope = s;
ActivationLeakyRT() {
}
~ActivationLeakyRT(){
@@ -41,21 +42,19 @@ public:
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
activationLEAKYForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, slope, stream);
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
return 0;
}
virtual size_t getSerializationSize() override {
return 1*sizeof(int) + 1*sizeof(float);
return 1*sizeof(int);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
assert(buf == a + getSerializationSize());
}
int size;
float slope;
};
include/tkDNN/pluginsRT/ActivationLogisticRT.h
-60
View File
@@ -1,60 +0,0 @@
#include<cassert>
#include "../kernels.h"
class ActivationLogisticRT : public IPlugin {
public:
ActivationLogisticRT() {
}
~ActivationLogisticRT(){
}
int getNbOutputs() const override {
return 1;
}
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override {
return inputs[0];
}
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override {
size = 1;
for(int i=0; i<outputDims[0].nbDims; i++)
size *= outputDims[0].d[i];
}
int initialize() override {
return 0;
}
virtual void terminate() override {
}
virtual size_t getWorkspaceSize(int maxBatchSize) const override {
return 0;
}
virtual int enqueue(int batchSize, const void*const * inputs, void** outputs, void* workspace, cudaStream_t stream) override {
activationLOGISTICForward((dnnType*)reinterpret_cast<const dnnType*>(inputs[0]),
reinterpret_cast<dnnType*>(outputs[0]), batchSize*size, stream);
return 0;
}
virtual size_t getSerializationSize() override {
return 1*sizeof(int);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
}
int size;
};
include/tkDNN/pluginsRT/ActivationMishRT.h
+1 -2
View File
@@ -52,9 +52,8 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
assert(buf == a + getSerializationSize());
}
int size;
include/tkDNN/pluginsRT/ActivationReLUCeilingRT.h
+1 -2
View File
@@ -51,10 +51,9 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, ceiling);
tk::dnn::writeBUF(buf, size);
assert(buf = a + getSerializationSize());
}
include/tkDNN/pluginsRT/ActivationSigmoidRT.h
+1 -2
View File
@@ -52,9 +52,8 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, size);
assert(buf == a + getSerializationSize());
}
int size;
include/tkDNN/pluginsRT/DeformableConvRT.h
+3 -4
View File
@@ -30,7 +30,7 @@ public:
height_ones = (i_h + 2 * ph - (1 * (kh - 1) + 1)) / sh + 1;
width_ones = (i_w + 2 * pw - (1 * (kw - 1) + 1)) / sw + 1;
dim_ones = i_c * kh * kw * 1 * height_ones * width_ones;
std::cout<<i_c * o_c * kh * kw * 1<<"\n";
checkCuda( cudaMalloc(&data_d, i_c * o_c * kh * kw * 1 * sizeof(dnnType)));
checkCuda( cudaMalloc(&bias2_d, o_c*sizeof(dnnType)));
checkCuda( cudaMalloc(&ones_d1, height_ones * width_ones * sizeof(dnnType)));
@@ -89,7 +89,7 @@ public:
for(int b=0; b<batchSize; b++) {
checkCuda(cudaMemcpy(offset, output_conv + b * 3 * chunk_dim, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda(cudaMemcpy(mask, output_conv + b * 3 * chunk_dim + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// kernel sigmoid
// kernel sigmoide
activationSIGMOIDForward(mask, mask, chunk_dim);
// deformable convolution
dcnV2CudaForward(stat, handle,
@@ -116,7 +116,7 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, chunk_dim);
tk::dnn::writeBUF(buf, kh);
tk::dnn::writeBUF(buf, kw);
@@ -163,7 +163,6 @@ public:
for(int i=0; i<dim_ones; i++)
tk::dnn::writeBUF(buf, aus[i]);
free(aus);
assert(buf == a + getSerializationSize());
}
cublasStatus_t stat;
include/tkDNN/pluginsRT/FlattenConcatRT.h
+1 -2
View File
@@ -65,13 +65,12 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a = buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
tk::dnn::writeBUF(buf, rows);
tk::dnn::writeBUF(buf, cols);
assert(buf == a + getSerializationSize());
}
int c, h, w;
include/tkDNN/pluginsRT/MaxPoolingFixedSizeRT.h
+1 -2
View File
@@ -55,7 +55,7 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, this->c);
tk::dnn::writeBUF(buf, this->h);
@@ -65,7 +65,6 @@ public:
tk::dnn::writeBUF(buf, this->stride_W);
tk::dnn::writeBUF(buf, this->winSize);
tk::dnn::writeBUF(buf, this->padding);
assert(buf == a + getSerializationSize());
}
int n, c, h, w;
include/tkDNN/pluginsRT/RegionRT.h
+1 -2
View File
@@ -73,14 +73,13 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, classes);
tk::dnn::writeBUF(buf, coords);
tk::dnn::writeBUF(buf, num);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
assert(buf == a + getSerializationSize());
}
int c, h, w;
include/tkDNN/pluginsRT/ReorgRT.h
+1 -2
View File
@@ -52,12 +52,11 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, stride);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
assert(buf == a + getSerializationSize());
}
int c, h, w, stride;
include/tkDNN/pluginsRT/ReshapeRT.h
+1 -2
View File
@@ -50,12 +50,11 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a = buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, n);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
assert(buf == a + getSerializationSize());
}
int n, c, h, w;
include/tkDNN/pluginsRT/ResizeLayerRT.h
+1 -2
View File
@@ -52,7 +52,7 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, o_c);
tk::dnn::writeBUF(buf, o_h);
@@ -61,7 +61,6 @@ public:
tk::dnn::writeBUF(buf, i_c);
tk::dnn::writeBUF(buf, i_h);
tk::dnn::writeBUF(buf, i_w);
assert(buf == a + getSerializationSize());
}
int i_c, i_h, i_w, o_c, o_h, o_w;
include/tkDNN/pluginsRT/RouteRT.h
+1 -2
View File
@@ -75,7 +75,7 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, groups);
tk::dnn::writeBUF(buf, group_id);
tk::dnn::writeBUF(buf, in);
@@ -85,7 +85,6 @@ public:
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
assert(buf == a + getSerializationSize());
}
static const int MAX_INPUTS = 4;
include/tkDNN/pluginsRT/ShortcutRT.h
+5 -8
View File
@@ -4,11 +4,10 @@
class ShortcutRT : public IPlugin {
public:
ShortcutRT(tk::dnn::dataDim_t bdim, bool mul) {
ShortcutRT(tk::dnn::dataDim_t bdim) {
this->bc = bdim.c;
this->bh = bdim.h;
this->bw = bdim.w;
this->mul = mul;
}
~ShortcutRT(){
@@ -48,30 +47,28 @@ public:
dnnType *dstData = reinterpret_cast<dnnType*>(outputs[0]);
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
shortcutForward(srcDataBack, dstData, batchSize, c, h, w, 1, batchSize, bc, bh, bw, 1, mul, stream);
for(int b=0; b < batchSize; ++b)
shortcutForward(srcDataBack + b*bc*bh*bw, dstData + b*c*h*w, 1, c, h, w, 1, 1, bc, bh, bw, 1, stream);
return 0;
}
virtual size_t getSerializationSize() override {
return 6*sizeof(int) + sizeof(bool);
return 6*sizeof(int);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, bc);
tk::dnn::writeBUF(buf, bh);
tk::dnn::writeBUF(buf, bw);
tk::dnn::writeBUF(buf, mul);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
assert(buf == a + getSerializationSize());
}
int c, h, w;
int bc, bh, bw;
bool mul;
};
include/tkDNN/pluginsRT/UpsampleRT.h
+1 -2
View File
@@ -54,12 +54,11 @@ public:
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, stride);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
assert(buf == a + getSerializationSize());
}
int c, h, w, stride;
include/tkDNN/pluginsRT/YoloRT.h
+24 -44
View File
@@ -8,15 +8,12 @@ class YoloRT : public IPlugin {
public:
YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1, float nms_thresh=0.45, int nms_kind=0, int new_coords=0) {
YoloRT(int classes, int num, tk::dnn::Yolo *yolo = nullptr, int n_masks=3, float scale_xy=1) {
this->classes = classes;
this->num = num;
this->n_masks = n_masks;
this->scaleXY = scale_xy;
this->nms_thresh = nms_thresh;
this->nms_kind = nms_kind;
this->new_coords = new_coords;
mask = new dnnType[n_masks];
bias = new dnnType[num*n_masks*2];
@@ -64,23 +61,17 @@ public:
checkCuda( cudaMemcpyAsync(dstData, srcData, batchSize*c*h*w*sizeof(dnnType), cudaMemcpyDeviceToDevice, stream));
for (int b = 0; b < batchSize; ++b){
for(int n = 0; n < n_masks; ++n){
int index = entry_index(b, n*w*h, 0);
activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream);
for (int b = 0; b < batchSize; ++b){
for(int n = 0; n < n_masks; ++n){
int index = entry_index(b, n*w*h, 0);
if (new_coords == 1){
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
}
else{
activationLOGISTICForward(srcData + index, dstData + index, 2*w*h, stream); //x,y
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*w*h, 4);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream);
}
}
}
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * w*h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*w*h, 4);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*w*h, stream);
}
}
//std::cout<<"YOLO END\n";
return 0;
@@ -88,29 +79,22 @@ public:
virtual size_t getSerializationSize() override {
return 8*sizeof(int) + 2*sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
return 6*sizeof(int) + sizeof(float)+ n_masks*sizeof(dnnType) + num*n_masks*2*sizeof(dnnType) + YOLORT_CLASSNAME_W*classes*sizeof(char);
}
virtual void serialize(void* buffer) override {
char *buf = reinterpret_cast<char*>(buffer),*a=buf;
tk::dnn::writeBUF(buf, classes); //std::cout << "Classes :" << classes << std::endl;
tk::dnn::writeBUF(buf, num); //std::cout << "Num : " << num << std::endl;
tk::dnn::writeBUF(buf, n_masks); //std::cout << "N_Masks" << n_masks << std::endl;
tk::dnn::writeBUF(buf, scaleXY); //std::cout << "ScaleXY :" << scaleXY << std::endl;
tk::dnn::writeBUF(buf, nms_thresh); //std::cout << "nms_thresh :" << nms_thresh << std::endl;
tk::dnn::writeBUF(buf, nms_kind); //std::cout << "nms_kind : " << nms_kind << std::endl;
tk::dnn::writeBUF(buf, new_coords); //std::cout << "new_coords : " << new_coords << std::endl;
tk::dnn::writeBUF(buf, c); //std::cout << "C : " << c << std::endl;
tk::dnn::writeBUF(buf, h); //std::cout << "H : " << h << std::endl;
tk::dnn::writeBUF(buf, w); //std::cout << "C : " << c << std::endl;
for (int i = 0; i < n_masks; i++)
{
tk::dnn::writeBUF(buf, mask[i]); //std::cout << "mask[i] : " << mask[i] << std::endl;
}
for (int i = 0; i < n_masks * 2 * num; i++)
{
tk::dnn::writeBUF(buf, bias[i]); //std::cout << "bias[i] : " << bias[i] << std::endl;
}
char *buf = reinterpret_cast<char*>(buffer);
tk::dnn::writeBUF(buf, classes);
tk::dnn::writeBUF(buf, num);
tk::dnn::writeBUF(buf, n_masks);
tk::dnn::writeBUF(buf, c);
tk::dnn::writeBUF(buf, h);
tk::dnn::writeBUF(buf, w);
tk::dnn::writeBUF(buf, scaleXY);
for(int i=0; i<n_masks; i++)
tk::dnn::writeBUF(buf, mask[i]);
for(int i=0; i<n_masks*2*num; i++)
tk::dnn::writeBUF(buf, bias[i]);
// save classes names
for(int i=0; i<classes; i++) {
@@ -120,15 +104,11 @@ public:
tk::dnn::writeBUF(buf, tmp[j]);
}
}
assert(buf == a + getSerializationSize());
}
int c, h, w;
int classes, num, n_masks;
float scaleXY;
float nms_thresh;
int nms_kind;
int new_coords;
std::vector<std::string> classesNames;
dnnType *mask;
include/tkDNN/test.h
+4 -5
View File
@@ -20,7 +20,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
}
if(output_bins.size() != outputs.size()) {
std::cout<<output_bins.size()<<" "<<outputs.size()<<"\n";
FatalError("outputs size mismatch");
FatalError("outputs size missmatch");
}
// Load input
@@ -29,8 +29,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
readBinaryFile(input_bins[0], net->input_dim.tot(), &input_h, &data);
// outputs
//dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()];
std::vector<dnnType *> cudnn_out,rt_out;
dnnType *cudnn_out[outputs.size()], *rt_out[outputs.size()];
tk::dnn::dataDim_t dim1 = net->input_dim; //input dim
printCenteredTitle(" CUDNN inference ", '=', 30); {
@@ -40,7 +39,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
TKDNN_TSTOP
dim1.print();
}
for(int i=0; i<outputs.size(); i++) cudnn_out.push_back(outputs[i]->dstData);
for(int i=0; i<outputs.size(); i++) cudnn_out[i] = outputs[i]->dstData;
if(netRT != nullptr) {
tk::dnn::dataDim_t dim2 = net->input_dim;
@@ -51,7 +50,7 @@ int testInference(std::vector<std::string> input_bins, std::vector<std::string>
TKDNN_TSTOP
dim2.print();
}
for(int i=0; i<outputs.size(); i++) rt_out.push_back((dnnType*)netRT->buffersRT[i+1]);
for(int i=0; i<outputs.size(); i++) rt_out[i] = (dnnType*)netRT->buffersRT[i+1];
}
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
include/tkDNN/utils.h
+1 -32
View File
@@ -12,16 +12,9 @@
#include <cublas_v2.h>
#include <cudnn.h>
#ifdef __linux__
#include <unistd.h>
#endif
#include <ios>
#include <chrono>
#include <yaml-cpp/yaml.h>
#define dnnType float
@@ -46,7 +39,6 @@
#define TKDNN_VERBOSE 0
// Simple Timer
#ifdef __linux__
#define TKDNN_TSTART timespec start, end; \
clock_gettime(CLOCK_MONOTONIC, &start);
@@ -56,14 +48,6 @@
if(show) std::cout<<col<<"Time:"<<std::setw(16)<<t_ns<<" ms\n"<<COL_END;
#define TKDNN_TSTOP TKDNN_TSTOP_C(COL_CYANB, TKDNN_VERBOSE)
#elif _WIN32
#define TKDNN_TSTART auto start = std::chrono::high_resolution_clock::now();
#define TKDNN_TSTOP auto stop = std::chrono::high_resolution_clock::now(); \
std::chrono::duration<double> duration = stop -start; \
auto time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(duration);\
double t_ns = time_ms.count();
#endif
/********************************************************
* Prints the error message, and exits
@@ -121,7 +105,7 @@ void printCenteredTitle(const char *title, char fill, int dim = 30);
bool fileExist(const char *fname);
void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url);
void readBinaryFile(std::string fname, int size, dnnType** data_h, dnnType** data_d, int seek = 0);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10, bool verbose=true);
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device = true, int limit = 10);
void printDeviceVector(int size, dnnType* vec_d, bool device = true);
float getColor(const int c, const int x, const int max);
void resize(int size, dnnType **data);
@@ -138,19 +122,4 @@ static inline bool isCudaPointer(void *data) {
cudaPointerAttributes attr;
return cudaPointerGetAttributes(&attr, data) == 0;
}
inline YAML::Node YAMLloadConf(const std::string& conf_file) {
std::cerr<<"Loading YAML: "<<conf_file<<"\n";
return YAML::LoadFile(conf_file);
}
template<typename T>
inline T YAMLgetConf(YAML::Node conf, std::string key, T defaultVal) {
T val = defaultVal;
if(conf && conf[key]) {
val = conf[key].as<T>();
}
return val;
}
#endif //UTILS_H
scripts/checkExecTimes.py
-37
View File
@@ -1,37 +0,0 @@
import sys
import pandas as pd
if len(sys.argv) < 3:
print("Error: two csv files are needed, old first new second")
exit(1)
old_perf_file = str(sys.argv[1])
new_perf_file = str(sys.argv[2])
verbose = False
if len(sys.argv) == 4:
verbose = bool(sys.argv[3])
print("Comparing {} vs {}".format(old_perf_file, new_perf_file))
df_old = pd.read_csv (old_perf_file, sep=';', header=None, index_col=0)
df_new = pd.read_csv (new_perf_file, sep=';', header=None, index_col=0)
for index, row in df_new.iterrows():
if index in df_old.index:
if verbose:
print("New: ",row[1], row[2], row[3])
print("Old: ",df_old.loc[index][1], df_old.loc[index][2], df_old.loc[index][3])
print(index, end=': ')
if abs(row[1] - df_old.loc[index][1]) < df_old.loc[index][1]*0.1:
print("similar performance")
elif (row[1] < df_old.loc[index][1]):
print('\x1b[3;30;42m' + 'faster' + '\x1b[0m')
elif (row[1] > df_old.loc[index][1]):
if row[1] > df_old.loc[index][1] + df_old.loc[index][1] * 0.5 :
print('\x1b[3;30;41m' + 'WAY SLOWER' + '\x1b[0m')
else:
print('\x1b[3;30;41m' + 'slower' + '\x1b[0m')
scripts/download_validation.py
-39
View File
@@ -1,39 +0,0 @@
import os
import urllib.request as dowReq
import zipfile
val = input("Enter BDD or COCO :")
if(val == "COCO"):
url = "https://cloud.hipert.unimore.it/s/LNxBDk4wzqXPL8c/download"
lib = "..\demo\COCO_val2017"
lib_zip = "COCO_val2017.zip"
elif(val == "BDD"):
url = "https://cloud.hipert.unimore.it/s/bikqk3FzCq2tg4D/download"
lib = "..\demo\BDD100k_val"
lib_zip = "BDD100k_val.zip"
dowReq.urlretrieve(url,lib_zip)
with zipfile.ZipFile(lib_zip,'r') as zip_ref:
zip_ref.extractall(lib)
labelFolder = lib + "\labels"
imageFolder = lib + "\images"
file1 = open(".\\..\\demo\\all_labels.txt","a")
path1 = os.path.realpath(labelFolder)
for file in os.listdir(labelFolder):
valTemp = path1 + "\\" + file
valTemp = valTemp + '\n'
file1.write(valTemp)
file1.close()
file2 = open(".\\..\\demo\\all_images.txt","a")
path2 = os.path.realpath(imageFolder)
for file in os.listdir(imageFolder):
pathtemp = path2 + "\\" + file
pathtemp = pathtemp + '\n'
file2.write(pathtemp)
file2.close()
print("Completed")
scripts/test_all_tests.sh
+29 -45
View File
@@ -1,6 +1,6 @@
#!/bin/bash
#cd build
cd build
RED='\033[1;31m'
GREEN='\033[1;32m'
@@ -29,28 +29,24 @@ function print_output {
}
out_dir=results
out_file=results.log
rm -rf $out_dir/
mkdir -p $out_dir
rm $out_file
function test_net {
./test_$1 &> $out_dir/$1_${TKDNN_MODE}_build_$out_file
./test_$1 &>> $out_file
print_output $? $1
./test_rtinference $1*.rt 1 &> $out_dir/$1_${TKDNN_MODE}_inference_batch1_$out_file
print_output $? "infer $1"
./test_rtinference $1*.rt $TKDNN_BATCHSIZE &> $out_dir/$1_${TKDNN_MODE}_inference_batch${TKDNN_BATCHSIZE}_$out_file
./test_rtinference $1*.rt $TKDNN_BATCHSIZE &>> $out_file
print_output $? "batched $1"
}
# modes=( 1 ) # only FP32
modes=( 1 2 ) # FP32 and FP16
modes=( 1 ) # only FP32
# modes=( 1 2 ) # FP32 and FP16
# modes=( 1 2 3 ) # FP32, FP16 and INT8
for i in "${modes[@]}"
do
rm -f *rt
rm *rt
if [ $i -eq 1 ]
then
export TKDNN_MODE=FP32
@@ -73,41 +69,29 @@ do
echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
test_net mnist
# ./test_imuodom &>> $out_file
# print_output $? imuodom
./test_imuodom &>> $out_file
print_output $? imuodom
test_net yolo4
# test_net yolo4_320
# test_net yolo4_320_coco2
# test_net yolo4_512
# test_net yolo4_608
# test_net yolo4-csp
# test_net yolo4x
# test_net yolo4_berkeley
# test_net yolo4_berkeley_f1
# test_net yolo4tiny
# test_net yolo4tiny_512
# test_net yolo3
# test_net yolo3_berkeley
# test_net yolo3_coco4
# test_net yolo3_flir
# test_net yolo3_512
# test_net yolo3tiny
# test_net yolo3tiny_512
# test_net yolo2
# test_net yolo2_voc
# test_net yolo2tiny
# test_net csresnext50-panet-spp
# test_net csresnext50-panet-spp_berkeley
# test_net resnet101_cnet
# test_net dla34_cnet
# test_net dla34_cnet3d
# test_net mobilenetv2ssd
# test_net mobilenetv2ssd512
# test_net bdd-mobilenetv2ssd
# test_net dla34_ctrack
# test_net shelfnet
# test_net shelfnet_berkeley
test_net yolo4_berkeley
test_net yolo4tiny
test_net yolo3
test_net yolo3_berkeley
test_net yolo3_coco4
test_net yolo3_flir
test_net yolo3_512
test_net yolo3tiny
test_net yolo3tiny_512
test_net yolo2
test_net yolo2_voc
#test_net yolo2tiny
test_net csresnext50-panet-spp
#test_net csresnext50-panet-spp_berkeley
test_net resnet101_cnet
test_net dla34_cnet
test_net mobilenetv2ssd
test_net mobilenetv2ssd512
test_net bdd-mobilenetv2ssd
done
echo "If errors occured, check logfiles in directory: $out_dir"
echo "If errors occured, check logfile $out_file"
scripts/test_inference.sh
-52
View File
@@ -1,52 +0,0 @@
#!/bin/bash
function test_inference {
./test_$1
./test_rtinference $1_$2.rt 1
./test_rtinference $1_$2.rt 4
}
sudo jeston_clock
# modes=( 1 ) # only FP32
# modes=( 1 2 ) # FP32 and FP16
modes=( 1 2 3 ) # FP32, FP16 and INT8
rm times_rtinference.csv
for i in "${modes[@]}"
do
rm *rt
if [ $i -eq 1 ]
then
export TKDNN_MODE=FP32
mode=fp32
echo -e "${ORANGE}Test FP32${NC}"
fi
if [ $i -eq 2 ]
then
export TKDNN_MODE=FP16
mode=fp16
echo -e "${ORANGE}Test FP16${NC}"
fi
if [ $i -eq 3 ]
then
export TKDNN_MODE=INT8
export TKDNN_CALIB_LABEL_PATH=../demo/COCO_val2017/all_labels.txt
export TKDNN_CALIB_IMG_PATH=../demo/COCO_val2017/all_images.txt
mode=int8
echo -e "${ORANGE}Test INT8${NC}"
fi
export TKDNN_BATCHSIZE=4
echo -e "${ORANGE}Batch $TKDNN_BATCHSIZE ${NC}"
test_inference yolo4_320 $mode
test_inference yolo4 $mode
test_inference yolo4_512 $mode
test_inference yolo4_608 $mode
test_inference yolo4tiny $mode
done
src/Activation.cpp
+5 -9
View File
@@ -5,12 +5,11 @@
namespace tk { namespace dnn {
Activation::Activation(Network *net, int act_mode, const float ceiling, const float slope) :
Activation::Activation(Network *net, int act_mode, const float ceiling) :
Layer(net) {
this->act_mode = act_mode;
this->ceiling = ceiling;
this->slope = slope;
this->act_mode = act_mode;
this->ceiling = ceiling;
checkCuda( cudaMalloc(&dstData, input_dim.tot()*sizeof(dnnType)) );
if(int(act_mode) < 100) {
@@ -47,15 +46,12 @@ Activation::~Activation() {
dnnType* Activation::infer(dataDim_t &dim, dnnType* srcData) {
if(act_mode == ACTIVATION_LEAKY) {
activationLEAKYForward(srcData, dstData, dim.tot(), this->slope);
activationLEAKYForward(srcData, dstData, dim.tot());
}
else if(act_mode == ACTIVATION_MISH) {
activationMishForward(srcData, dstData, dim.tot());
}
else if(act_mode == ACTIVATION_LOGISTIC) {
activationLOGISTICForward(srcData, dstData, dim.tot());
} else {
dnnType alpha = dnnType(1);
dnnType beta = dnnType(0);
src/CenterTrack.cpp
-900
View File
@@ -1,900 +0,0 @@
#include "CenterTrack.h"
namespace tk { namespace dnn {
bool CenterTrack::init(const std::string& tensor_path, const int n_classes, const int n_batches,
const float conf_thresh, const bool mode_3d, const std::vector<cv::Mat>& k_calibs) {
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
dim = netRT->input_dim;
dim.c = 3;
nBatches = n_batches;
confThreshold = conf_thresh;
mode3D = mode_3d;
inputCalibs = k_calibs;
init_preprocessing();
init_pre_inf();
init_postprocessing();
init_visualization(n_classes);
return true;
}
bool CenterTrack::init_preprocessing(){
//image transformation
src = cv::Mat(cv::Size(2,3), CV_32F);
dst = cv::Mat(cv::Size(2,3), CV_32F);
dst2 = cv::Mat(cv::Size(2,3), CV_32F);
trans = cv::Mat(cv::Size(3,2), CV_32F);
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
transOut = cv::Mat(cv::Size(3,2), CV_32F);
dst2.at<float>(0,0) = width * 0.5;
dst2.at<float>(0,1) = width * 0.5;
dst2.at<float>(1,0) = width * 0.5;
dst2.at<float>(1,1) = width * 0.5 + width * -0.5;
dst2.at<float>(2,0) = dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
dst2.at<float>(2,1) = dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
for(int bi=0; bi<nBatches; bi++) {
szOld.push_back(cv::Size(0,0));
}
#ifdef OPENCV_CUDACONTRIB
std::cout<<"OPENCV CPMTROB\n";
checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
float mean[3] = {0.40789655, 0.44719303, 0.47026116};
float stddev[3] = {0.2886383, 0.27408165, 0.27809834};
checkCuda( cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda( cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
std::cout<<"NO OPENCV CPMTROB\n";
checkCuda( cudaMallocHost(&input, sizeof(dnnType)*dim.tot() * nBatches));
mean << 0.40789655, 0.44719303, 0.47026116;
stddev << 0.2886383, 0.27408165, 0.27809834;
#endif
checkCuda( cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
checkCuda( cudaMalloc(&input_pre_inf_d, sizeof(dnnType)*dim.tot()));
checkCuda( cudaMalloc(&d_ptrs, dim.tot() * sizeof(float)) );
return true;
}
bool CenterTrack::init_pre_inf(){
// initial steps: the first part of the network
const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin";
const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin";
const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin";
const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin";
dim_in0 = tk::dnn::dataDim_t(1, 3, 512, 512, 1);
dim_in1 = tk::dnn::dataDim_t(1, 1, 512, 512, 1);
checkCuda( cudaMalloc(&out_d, netRT->input_dim.tot()*sizeof(dnnType)) );
checkCuda( cudaMalloc(&img_d, dim_in0.tot()*sizeof(dnnType)) );
checkCuda( cudaMalloc(&hm_d, dim_in1.tot()*sizeof(dnnType)) );
// init to zeros hm
dnnType *hm_h;
checkCuda( cudaMallocHost(&hm_h, 1 * dim.h * dim.w*sizeof(dnnType)) );
for(int i=0; i<1 * dim.h * dim.w; i++)
hm_h[i] = 0.0f;
checkCuda( cudaMemcpy(hm_d, hm_h, 1 * dim.h * dim.w * sizeof(dnnType), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(hm_h) );
dnnType *i0_h, *i1_h, *i2_h;
// dnnType *i0_d, *i1_d, *i2_d;
// const char *input_bin = "dla34_ctrack/debug/input.bin";
// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin";
// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin";
// readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &img_d);
// readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &hm_d);
// readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &input_pre_inf_d);
pre_phase_net = new tk::dnn::Network(dim_in0);
//pre-img
tk::dnn::Input *in_pre_img = new tk::dnn::Input(pre_phase_net, dim_in0, img_d);
tk::dnn::Conv2d *pre_img_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true);
tk::dnn::Activation *pre_img_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
//pre-hm
tk::dnn::Input *in_pre_hm = new tk::dnn::Input(pre_phase_net, dim_in1, hm_d);
tk::dnn::Conv2d *pre_hm_conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true);
tk::dnn::Activation *pre_hm_relu = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
// image input
tk::dnn::Input *input_image = new tk::dnn::Input(pre_phase_net, dim_in0, input_pre_inf_d);
tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(pre_phase_net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
tk::dnn::Activation *relu1 = new tk::dnn::Activation(pre_phase_net, CUDNN_ACTIVATION_RELU);
tk::dnn::Shortcut *s0_input = new tk::dnn::Shortcut(pre_phase_net, pre_img_relu);
tk::dnn::Shortcut *s1_input = new tk::dnn::Shortcut(pre_phase_net, pre_hm_relu);
// output data
out_d = s1_input->dstData;
//print network model
pre_phase_net->print();
iter0=true; // in the first iteration the last input is equal to the current input.
return true;
}
bool CenterTrack::init_postprocessing(){
srand(0); //seed = 0 for random colors
dim_hm = tk::dnn::dataDim_t(1, 10, 128, 128, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_track = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1);
dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1);
dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
dim_amodel_offset = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
for(int i=0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
ids_[i] = i;
}
checkCuda( cudaMalloc(&ones, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
float *ones_h;
checkCuda( cudaMallocHost(&ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
for(int i=0; i<dim_dep.c * dim_dep.h * dim_dep.w; i++)
ones_h[i] = 1.0f;
checkCuda( cudaMemcpy(ones, ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(ones_h) );
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
checkCuda( cudaMallocHost(&bbx0, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bby0, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bbx1, K * sizeof(float)) );
checkCuda( cudaMallocHost(&bby1, K * sizeof(float)) );
checkCuda( cudaMalloc(&bbx0_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bby0_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bbx1_d, K * sizeof(float)) );
checkCuda( cudaMalloc(&bby1_d, K * sizeof(float)) );
checkCuda( cudaMallocHost(&intxs, K * sizeof(int)) );
checkCuda( cudaMallocHost(&intys, K * sizeof(int)) );
checkCuda( cudaMallocHost(&track, K * dim_track.c * sizeof(float)) );
checkCuda( cudaMallocHost(&dep, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMallocHost(&rot, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMallocHost(&dim_, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMallocHost(&wh, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMallocHost(&amodel_offset, K * dim_amodel_offset.c * sizeof(float)) );
checkCuda( cudaMalloc(&track_d, K * dim_track.c * sizeof(float)) );
checkCuda( cudaMalloc(&dep_d, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMalloc(&rot_d, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMalloc(&dim_d, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMalloc(&wh_d, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMalloc(&amodel_offset_d, K * dim_amodel_offset.c * sizeof(float)) );
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
for(int bi=0; bi<nBatches; bi++) {
cv::Mat calibs_ = cv::Mat::zeros(cv::Size(4,3), CV_32F);
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs_.at<float>(0,0) = 633.0;
calibs_.at<float>(1,1) = 633.0;
calibs_.at<float>(2,2) = 1.0;
}
calibs_.at<float>(2,2) = 1.0;
calibs.push_back(calibs_);
}
// Alloc array used in the kernel
checkCuda( cudaMalloc(&src_out, K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_out, K *sizeof(int)) );
trRes.resize(nBatches);
countTr.resize(nBatches, 0);
trackId.resize(nBatches, 0);
return true;
}
bool CenterTrack::init_visualization(const int n_classes){
classes = n_classes;
// const char *kitti_class_name[] = {
// "person", "car", "bicycle"};
// classesNames = std::vector<std::string>(kitti_class_name, std::end( kitti_class_name));
const char *class_name[] = {"car", "truck", "bus", "trailer", "construction_vehicle", "pedestrian",
"motorcycle", "bicycle", "traffic_cone", "barrier"};
classesNames = std::vector<std::string>(class_name, std::end( class_name));
// const char *coco_class_name[] = {
// "person", "bicycle", "car", "motorcycle", "airplane",
// "bus", "train", "truck", "boat", "traffic light", "fire hydrant",
// "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse",
// "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
// "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis",
// "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
// "skateboard", "surfboard", "tennis racket", "bottle", "wine glass",
// "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich",
// "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
// "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv",
// "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
// "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
// "scissors", "teddy bear", "hair drier", "toothbrush"
// };
// classesNames = std::vector<std::string>(coco_class_name, std::end( coco_class_name));
for(int c=0; c<classes; c++) {
int offset = c*123457 % classes;
float r = getColor(2, offset, classes);
float g = getColor(1, offset, classes);
float b = getColor(0, offset, classes);
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
for(int c=0; c<256; c++) {
int offset = c * 123457 % 256;
float r = getColor(2, offset, 256);
float g = getColor(1, offset, 256);
float b = getColor(0, offset, 256);
trColors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
r = cv::Mat(cv::Size(3,3), CV_32F);
r.at<float>(0,1) = 0.0;
r.at<float>(1,0) = 0.0;
r.at<float>(1,1) = 1.0;
r.at<float>(1,2) = 0.0;
r.at<float>(2,1) = 0.0;
corners = cv::Mat(cv::Size(8,3), CV_32F);
corners.at<float>(1,0) = 0.0;
corners.at<float>(1,1) = 0.0;
corners.at<float>(1,2) = 0.0;
corners.at<float>(1,3) = 0.0;
pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F);
pts3DHomo.at<float>(3,0) = 1.0;
pts3DHomo.at<float>(3,1) = 1.0;
pts3DHomo.at<float>(3,2) = 1.0;
pts3DHomo.at<float>(3,3) = 1.0;
pts3DHomo.at<float>(3,4) = 1.0;
pts3DHomo.at<float>(3,5) = 1.0;
pts3DHomo.at<float>(3,6) = 1.0;
pts3DHomo.at<float>(3,7) = 1.0;
faceId.push_back({0,1,5,4});
faceId.push_back({1,2,6, 5});
faceId.push_back({3,0,4,7});
faceId.push_back({2,3,7,6});
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
return true;
}
void CenterTrack::_get_additional_inputs(){
//None no additional input
}
void CenterTrack::pre_inf(const int bi){
TKDNN_TSTART
tk::dnn::dataDim_t dim_aus;
pre_phase_net->infer(dim_aus, nullptr);
TKDNN_TSTOP
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, pre_phase_net->layers[pre_phase_net->num_layers-1]->dstData, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
}
void CenterTrack::preprocess(cv::Mat &frame, const int bi){
cv::Size sz = originalSize[bi];
// float scale = 1.0;
float new_height = dim.h;//sz.height * scale;
float new_width = dim.w;//sz.width * scale;
if(sz.height != szOld[bi].height && sz.width != szOld[bi].width){
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs[bi].at<float>(0,2) = new_width / 2.0f;
calibs[bi].at<float>(1,2) = new_height /2.0f;
}
else {
calibs[bi].at<float>(0,0) = inputCalibs[bi].at<float>(0,0) * dim.w / sz.width;
calibs[bi].at<float>(0,2) = inputCalibs[bi].at<float>(0,2) * dim.w / sz.width;
calibs[bi].at<float>(1,1) = inputCalibs[bi].at<float>(1,1) * dim.h / sz.height;
calibs[bi].at<float>(1,2) = inputCalibs[bi].at<float>(1,2) * dim.h / sz.height;
}
float c[] = {new_width / 2.0f, new_height /2.0f};
float s[] = {float(dim.w), float(dim.h)};
// float s = new_width >= new_height ? new_width : new_height;
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
//dim.print();
src.at<float>(0,0) = c[0];
src.at<float>(0,1) = c[1];
src.at<float>(1,0) = c[0];
src.at<float>(1,1) = c[1] + s[0] * -0.5;
dst.at<float>(0,0) = dim.w * 0.5;
dst.at<float>(0,1) = dim.h * 0.5;
dst.at<float>(1,0) = dim.w * 0.5;
dst.at<float>(1,1) = dim.h * 0.5 + dim.w * -0.5;
src.at<float>(2,0) = src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1) = src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0) = dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1) = dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
trans = cv::getAffineTransform( src, dst );
trans2 = cv::getAffineTransform( dst2, src );
trans2.convertTo(transOut, CV_32F);
}
szOld[bi] = sz;
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat im_Orig;
cv::cuda::GpuMat imageF1_d, imageF2_d;
im_Orig = cv::cuda::GpuMat(frame);
cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h));
// imageF1_d = im_Orig;
checkCuda( cudaDeviceSynchronize() );
sz = imageF1_d.size();
cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR );
checkCuda( cudaDeviceSynchronize() );
imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0);
checkCuda( cudaDeviceSynchronize() );
dim2 = dim;
cv::cuda::GpuMat bgr[3];
cv::cuda::split(imageF1_d,bgr);//split source
for(int i=0; i<dim.c; i++)
checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
checkCuda( cudaMemcpy(input_pre_inf_d, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda( cudaDeviceSynchronize() );
#else
cv::Mat imageF;
resize(frame, imageF, cv::Size(dim.w, dim.h));
// imageF = frame;
sz = imageF.size();
cv::warpAffine(imageF, imageF, trans, cv::Size(dim.w, dim.h), cv::INTER_LINEAR );
// cv::imshow("warp", imageF);
sz = imageF.size();
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
dim2 = dim;
//split channels
cv::Mat bgr[3];
cv::split(imageF,bgr);//split source
for(int i=0; i<3; i++){
bgr[i] = bgr[i] - mean[i];
bgr[i] = bgr[i] / stddev[i];
}
for(int i=0; i<dim2.c; i++) {
int idx = i * imageF.rows * imageF.cols;
int ch = i;
memcpy((void*)&input[idx], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda( cudaMemcpyAsync(input_pre_inf_d, input, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
checkCuda( cudaDeviceSynchronize() );
#endif
if(iter0) {
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
iter0=false;
}
pre_inf(bi);
checkCuda( cudaMemcpy(img_d, input_pre_inf_d, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
checkCuda( cudaDeviceSynchronize() );
}
cv::Mat CenterTrack::transform_preds_with_trans(float x1, float x2){
cv::Mat target_coords(cv::Size(1,3), CV_32F);
target_coords.at<float>(0,0) = x1;
target_coords.at<float>(0,1) = x2;
target_coords.at<float>(0,2) = 1.0;
return transOut * target_coords;
}
void CenterTrack::tracking(const int bi) {
float item_size[countDet];
int item_cl[countDet];
float dets[2*countDet];
for(int i=0; i<countDet; i++){
item_size[i] = (detRes[i].bb1.at<float>(0,0) - detRes[i].bb0.at<float>(0,0)) *
(detRes[i].bb1.at<float>(0,1) - detRes[i].bb0.at<float>(0,1));
item_cl[i] = detRes[i].cl;
dets[i*2] = detRes[i].ct.at<float>(0,0);
dets[i*2+1] = detRes[i].ct.at<float>(0,1);
}
float track_size[countTr[bi]];
int track_cl[countTr[bi]];
float tracks[2*countTr[bi]];
for(int i=0; i<countTr[bi]; i++){
track_size[i] = (trRes[bi][i].det_res.bb1.at<float>(0,0) - trRes[bi][i].det_res.bb0.at<float>(0,0)) *
(trRes[bi][i].det_res.bb1.at<float>(0,1) - trRes[bi][i].det_res.bb0.at<float>(0,1));
track_cl[i] = trRes[bi][i].det_res.cl;
tracks[i*2] = trRes[bi][i].det_res.ct.at<float>(0,0);
tracks[i*2+1] = trRes[bi][i].det_res.ct.at<float>(0,1);
}
float dist[countTr[bi]*countDet];
bool invalid;
for(int i=0; i<countTr[bi]; i++){
for(int j=0; j<countDet; j++){
dist[j*countTr[bi]+i] = pow((tracks[i*2] - dets[j*2]), 2) +
pow((tracks[i*2+1] - dets[j*2+1]), 2);
invalid = dist[j*countTr[bi]+i] > track_size[i] ||
dist[j*countTr[bi]+i] > item_size[j] ||
item_cl[j] != track_cl[i];
dist[j*countTr[bi]+i] = dist[j*countTr[bi]+i] + invalid * (1 << 18);
}
}
int matched_indices[2*countTr[bi]];
float min_tr;
int min_idtr = -1;
for(int i=0; i<countTr[bi]; i++) {
matched_indices[i*2] = -1;
matched_indices[i*2+1] = -1;
}
for(int i=0; i<countDet; i++){
min_tr=(1 << 18);
for(int j=0; j<countTr[bi]; j++){
if(dist[i*countTr[bi]+j]<min_tr) {
min_tr = dist[i*countTr[bi]+j];
min_idtr = j;
}
}
if(min_tr < (1<<16)) {
for(int j=0; j<countDet; j++)
dist[j*countTr[bi]+min_idtr] = (1 << 18);
matched_indices[2*min_idtr] = min_idtr;
matched_indices[2*min_idtr+1] = i;
}
}
bool unmatched_dets[countDet];
for(int i=0; i<countDet; i++)
unmatched_dets[i] = false;
bool unmatched_tracks[countTr[bi]];
for(int i=0; i<countTr[bi]; i++)
unmatched_tracks[i] = false;
for(int i=0; i<countTr[bi]; i++) {
if(matched_indices[2*i] != -1)
unmatched_tracks[matched_indices[2*i]]=true;
if(matched_indices[2*i+1] != -1)
unmatched_dets[matched_indices[2*i+1]]=true;
}
//match
for(int i=0; i<countTr[bi]; i++) {
if(matched_indices[2*i+1] != -1 && matched_indices[2*i] != -1) { //second condition is optional
int tr_id = matched_indices[2*i];
int d_id = matched_indices[2*i+1];
// trRes[tr_id].det_res = detRes[d_id];
trRes[bi][tr_id].det_res.score = detRes[d_id].score;
trRes[bi][tr_id].det_res.cl = detRes[d_id].cl;
trRes[bi][tr_id].det_res.ct = detRes[d_id].ct;
trRes[bi][tr_id].det_res.tr = detRes[d_id].tr;
trRes[bi][tr_id].det_res.bb0 = detRes[d_id].bb0;
trRes[bi][tr_id].det_res.bb1 = detRes[d_id].bb1;
trRes[bi][tr_id].det_res.dep = detRes[d_id].dep;
trRes[bi][tr_id].det_res.dim[0] = detRes[d_id].dim[0];
trRes[bi][tr_id].det_res.dim[1] = detRes[d_id].dim[1];
trRes[bi][tr_id].det_res.dim[2] = detRes[d_id].dim[2];
trRes[bi][tr_id].det_res.alpha = detRes[d_id].alpha;
trRes[bi][tr_id].det_res.x = detRes[d_id].x;
trRes[bi][tr_id].det_res.y = detRes[d_id].y;
trRes[bi][tr_id].det_res.z = detRes[d_id].z;
trRes[bi][tr_id].det_res.rot_y = detRes[d_id].rot_y;
// trRes[bi][matched_indices[2*i]].tracking_id = ; is the same
// trRes[bi][matched_indices[2*i]].color = ; is the same
trRes[bi][tr_id].age = 1;
trRes[bi][tr_id].active = trRes[bi][tr_id].active+1;
}
}
//delete target umatched track
int new_count_tr = 0;
for(int i=0; i<countTr[bi]; i++) {
if(unmatched_tracks[i])
new_count_tr++;
}
if(new_count_tr == 0 && countTr[bi] != 0) { //reset
trRes[bi].clear();
countTr[bi] = 0;
}
int old_count_tr = countTr[bi];
if(countTr[bi] != 0 && new_count_tr != countTr[bi]) {
std::vector<struct trackingRes> new_tr_res;
int id_new_tr=0;
for(int i=0; i<countTr[bi]; i++) {
if(unmatched_tracks[i]) {
struct trackingRes new_tr_res_;
// new_tr_res_new_det_res.det_res = trRes[i].det_res;
new_tr_res_.det_res.score = trRes[bi][i].det_res.score;
new_tr_res_.det_res.cl = trRes[bi][i].det_res.cl;
new_tr_res_.det_res.ct = trRes[bi][i].det_res.ct;
new_tr_res_.det_res.tr = trRes[bi][i].det_res.tr;
new_tr_res_.det_res.bb0 = trRes[bi][i].det_res.bb0;
new_tr_res_.det_res.bb1 = trRes[bi][i].det_res.bb1;
new_tr_res_.det_res.dep = trRes[bi][i].det_res.dep;
new_tr_res_.det_res.dim[0] = trRes[bi][i].det_res.dim[0];
new_tr_res_.det_res.dim[1] = trRes[bi][i].det_res.dim[1];
new_tr_res_.det_res.dim[2] = trRes[bi][i].det_res.dim[2];
new_tr_res_.det_res.alpha = trRes[bi][i].det_res.alpha;
new_tr_res_.det_res.x = trRes[bi][i].det_res.x;
new_tr_res_.det_res.y = trRes[bi][i].det_res.y;
new_tr_res_.det_res.z = trRes[bi][i].det_res.z;
new_tr_res_.det_res.rot_y = trRes[bi][i].det_res.rot_y;
new_tr_res_.tracking_id = trRes[bi][i].tracking_id;
new_tr_res_.age = trRes[bi][i].age;
new_tr_res_.active = trRes[bi][i].active;
new_tr_res_.color = trRes[bi][i].color;
id_new_tr ++;
new_tr_res.push_back(new_tr_res_);
}
}
if(countTr[bi]) {
trRes[bi].clear();
}
countTr[bi] = new_count_tr;
trRes[bi] = new_tr_res;
}
int count_tr_ = countTr[bi];
for(int i=0; i<countDet; i++) {
if((!unmatched_dets[i]) && detRes[i].score > newThresh) {
count_tr_ ++;
struct trackingRes new_tr_res_;
new_tr_res_.det_res.score = detRes[i].score;
new_tr_res_.det_res.cl = detRes[i].cl;
new_tr_res_.det_res.ct = detRes[i].ct;
new_tr_res_.det_res.tr = detRes[i].tr;
new_tr_res_.det_res.bb0 = detRes[i].bb0;
new_tr_res_.det_res.bb1 = detRes[i].bb1;
new_tr_res_.det_res.dep = detRes[i].dep;
new_tr_res_.det_res.dim[0] = detRes[i].dim[0];
new_tr_res_.det_res.dim[1] = detRes[i].dim[1];
new_tr_res_.det_res.dim[2] = detRes[i].dim[2];
new_tr_res_.det_res.alpha = detRes[i].alpha;
new_tr_res_.det_res.x = detRes[i].x;
new_tr_res_.det_res.y = detRes[i].y;
new_tr_res_.det_res.z = detRes[i].z;
new_tr_res_.det_res.rot_y = detRes[i].rot_y;
new_tr_res_.tracking_id = trackId[bi]++;
new_tr_res_.age = 1;
new_tr_res_.active = 1;
new_tr_res_.color = rand() % 256;
if(trRes.size() <= bi) {
std::vector<struct trackingRes> v_new_tr_res_;
v_new_tr_res_.push_back(new_tr_res_);
trRes.push_back(v_new_tr_res_);
}
else
trRes[bi].push_back(new_tr_res_);
}
}
countTr[bi] = count_tr_;
//reset the tracker id
if(trackId[bi] == 1000)
trackId[bi] = 0;
detRes.clear();
}
void CenterTrack::postprocess(const int bi, const bool mAP) {
dnnType *rt_out[9];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
rt_out[7] = (dnnType *)netRT->buffersRT[8]+ netRT->buffersDIM[8].tot()*bi;
rt_out[8] = (dnnType *)netRT->buffersRT[9]+ netRT->buffersDIM[9].tot()*bi;
// ------------------------------------ process --------------------------------------------
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
checkCuda( cudaDeviceSynchronize() );
// output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
activationSIGMOIDForward(rt_out[5], rt_out[5], dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
transformDep(ones, ones + dim_dep.tot(), rt_out[5], rt_out[5] + dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
// nms
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
// ----------- nms end
// ----------- topk
if(K > dim_hm.h * dim_hm.w){
printf ("Error topk (K is too large)\n");
return;
}
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
checkCuda( cudaDeviceSynchronize() );
topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(intxs, inttopk_xs_d, K * sizeof(int), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(intys, inttopk_ys_d, K * sizeof(int), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
// ----------- topk end
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], src_out, ids_out);
checkCuda( cudaDeviceSynchronize() );
bboxes(topk_inds_d, K, dim_wh.h*dim_wh.w, topk_xs_d, topk_ys_d, rt_out[2], bbx0_d, bbx1_d, bby0_d, bby1_d, src_out, ids_out);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(bbx0, bbx0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bby0, bby0_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bbx1, bbx1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(bby1, bby1_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
//regression heads
// ['tracking', 'dep', 'rot', 'dim', 'amodel_offset',
// 'nuscenes_att', 'velocity']
getRecordsFromTopKId(topk_inds_d, K, dim_track.c, dim_track.h * dim_track.w, rt_out[4], track_d, ids_out);
checkCuda( cudaMemcpy(track, track_d, K * dim_track.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[5], dep_d, ids_out);
checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[6], rot_d, ids_out);
checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[7], dim_d, ids_out);
checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_amodel_offset.c, dim_amodel_offset.h * dim_amodel_offset.w, rt_out[8], amodel_offset_d, ids_out);
checkCuda( cudaMemcpy(amodel_offset, amodel_offset_d, K * dim_amodel_offset.c * sizeof(float), cudaMemcpyDeviceToHost) );
// ---------------------------------- post-process -----------------------------------------
countDet = 0;
detRes.clear();
for(int i=0; i<K; i++){
if(scores[i] < outThresh)
break;
countDet ++;
struct detectionRes new_det_res;
new_det_res.score = scores[i];
new_det_res.cl = clses[i]+1;
// ret_s=scores[i];
// ret_c=clses[i]+1;
new_det_res.ct = transform_preds_with_trans(intxs[i], intys[i]);
new_det_res.tr = transform_preds_with_trans(intxs[i] + track[i], intys[i] + track[i+K]);
new_det_res.tr = new_det_res.tr -new_det_res.ct;
new_det_res.bb0 = transform_preds_with_trans(bbx0[i], bby0[i]);
new_det_res.bb1 = transform_preds_with_trans(bbx1[i], bby1[i]);
new_det_res.ct = transform_preds_with_trans(((bbx0[i]+bbx1[i])/2 + amodel_offset[i]),
((bby0[i]+bby1[i])/2 + amodel_offset[i+K]));
new_det_res.dep = dep[i];
new_det_res.dim[0] = dim_[i];
new_det_res.dim[1] = dim_[i+K];
new_det_res.dim[2] = dim_[i+2*K];
// unproject_2d_to_3d
new_det_res.z = dep[i] - calibs[bi].at<float>(2,3);
new_det_res.x = ((float)new_det_res.ct.at<float>(0,0) * dep[i] - calibs[bi].at<float>(0,3) -
calibs[bi].at<float>(0,2) * new_det_res.z) / calibs[bi].at<float>(0,0);
new_det_res.y = ((float)new_det_res.ct.at<float>(0,1) * dep[i] - calibs[bi].at<float>(1,3) -
calibs[bi].at<float>(1,2) * new_det_res.z) / calibs[bi].at<float>(1,1) + (dim_[i] / 2);
// alpha2rot_y
// idx = rot[:, 1] > rot[:, 5]
// alpha1 = np.arctan2(rot[:, 2], rot[:, 3]) + (-0.5 * np.pi)
// alpha2 = np.arctan2(rot[:, 6], rot[:, 7]) + ( 0.5 * np.pi)
// return alpha1 * idx + alpha2 * (1 - idx)
if(rot[1*K + i] > rot[5*K + i])
new_det_res.alpha = std::atan2(rot[2*K + i], rot[3*K + i]) -0.5 * M_PI;
else
new_det_res.alpha = std::atan2(rot[6*K + i], rot[7*K + i]) +0.5 * M_PI;
new_det_res.rot_y = (new_det_res.alpha + std::atan2((float)new_det_res.ct.at<float>(0,0) - calibs[bi].at<float>(0,2), calibs[bi].at<float>(0,0)));
new_det_res.ct = new_det_res.ct + new_det_res.tr; //dest
detRes.push_back(new_det_res);
}
// track step
tracking(bi);
}
void CenterTrack::draw(std::vector<cv::Mat>& frames) {
struct trackingRes t;
float sc;
int id;
std::string txt;
int baseline = 0;
float font_scale = 0.8;
int thickness = 2;
for(int bi=0; bi<frames.size(); ++bi) {
float scale_x = float(originalSize[bi].width)/dim.w;
float scale_y = float(originalSize[bi].height)/dim.h;
resize(frames[bi], frames[bi], originalSize[bi]);
// draw dets
for(int i=0; trRes.size() != 0 && i<trRes[bi].size(); i++) {
t = trRes[bi][i];
id = t.tracking_id;
txt = classesNames[t.det_res.cl-1]+'-'+std::to_string(id); //forse ha bisogno di cl-1
cv::Size text_size = getTextSize(txt, cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
if(t.det_res.score > confThreshold){// && t.active!=0) {
if(!mode3D) {
cv::rectangle(frames[bi],
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y),
cv::Point(t.det_res.bb1.at<float>(0,0) * scale_x, t.det_res.bb1.at<float>(0,1) * scale_y),
trColors[t.color], thickness);
cv::rectangle(frames[bi],
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y - text_size.height - thickness),
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x + text_size.width, t.det_res.bb0.at<float>(0,1) * scale_y),
trColors[t.color], -1);
cv::putText(frames[bi], txt,
cv::Point(t.det_res.bb0.at<float>(0,0) * scale_x, t.det_res.bb0.at<float>(0,1) * scale_y - thickness -1),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
cv::arrowedLine(frames[bi],
cv::Point((int)t.det_res.ct.at<float>(0,0) * scale_x, (int)t.det_res.ct.at<float>(0,1) * scale_y),
cv::Point((int)(t.det_res.ct.at<float>(0,0) * scale_x + t.det_res.tr.at<float>(0,0) * scale_x),
(int)(t.det_res.ct.at<float>(0,1) * scale_y + t.det_res.tr.at<float>(0,1) * scale_y)),
cv::Scalar(255, 0, 255), 2);
}
//3d
if(mode3D && t.det_res.z > 1){
r.at<float>(0,0) = std::cos(t.det_res.rot_y);
r.at<float>(0,2) = std::sin(t.det_res.rot_y);
r.at<float>(2,0) = -std::sin(t.det_res.rot_y);
r.at<float>(2,2) = std::cos(t.det_res.rot_y);
corners.at<float>(0,0) = t.det_res.dim[2]/2;
corners.at<float>(0,1) = t.det_res.dim[2]/2;
corners.at<float>(0,2) = -t.det_res.dim[2]/2;
corners.at<float>(0,3) = -t.det_res.dim[2]/2;
corners.at<float>(0,4) = t.det_res.dim[2]/2;
corners.at<float>(0,5) = t.det_res.dim[2]/2;
corners.at<float>(0,6) = -t.det_res.dim[2]/2;
corners.at<float>(0,7) = -t.det_res.dim[2]/2;
corners.at<float>(1,4) = -t.det_res.dim[0];
corners.at<float>(1,5) = -t.det_res.dim[0];
corners.at<float>(1,6) = -t.det_res.dim[0];
corners.at<float>(1,7) = -t.det_res.dim[0];
corners.at<float>(2,0) = t.det_res.dim[1]/2;
corners.at<float>(2,1) = -t.det_res.dim[1]/2;
corners.at<float>(2,2) = -t.det_res.dim[1]/2;
corners.at<float>(2,3) = t.det_res.dim[1]/2;
corners.at<float>(2,4) = t.det_res.dim[1]/2;
corners.at<float>(2,5) = -t.det_res.dim[1]/2;
corners.at<float>(2,6) = -t.det_res.dim[1]/2;
corners.at<float>(2,7) = t.det_res.dim[1]/2;
cv::Mat aus = r * corners;
for(int k=0; k<8; k++) {
aus.at<float>(0,k) += t.det_res.x;
aus.at<float>(1,k) += t.det_res.y;
aus.at<float>(2,k) += t.det_res.z;
}
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
for(int k1=0; k1<3; k1++) {
for(int k2=0; k2<8; k2++)
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
}
aus.release();
aus = calibs[bi] * pts3DHomo;
std::vector<float> res_corners;
for(int k=0; k<8; k++) {
res_corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
res_corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
}
aus.release();
for(int ind_f=3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frames[bi],
cv::Point((int)res_corners.at(faceId.at(ind_f).at(j) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y),
cv::Point((int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y),
trColors[t.color], 2);
if(ind_f == 0 && j==3) {
cv::line(frames[bi],
cv::Point((int)res_corners.at(faceId.at(ind_f).at(0) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(0) * 2 + 1) * scale_y),
cv::Point((int)res_corners.at(faceId.at(ind_f).at(2) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), trColors[t.color], 2);
cv::line(frames[bi],
cv::Point((int)res_corners.at(faceId.at(ind_f).at(1) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(1) * 2 + 1) * scale_y),
cv::Point((int)res_corners.at(faceId.at(ind_f).at(3) * 2) * scale_x,
(int)res_corners.at(faceId.at(ind_f).at(3) * 2 + 1) * scale_y), trColors[t.color], 2);
}
}
}
float bb0=(1 << 10), bb1=0, bb2=(1 << 10), bb3=0;
for(int k=0; k<8; k++) {
if(res_corners[2*k] < bb0)
bb0 = res_corners[2*k];
if(res_corners[2*k] > bb1)
bb1 = res_corners[2*k];
if(res_corners[2*k+1] < bb2)
bb2 = res_corners[2*k+1];
if(res_corners[2*k+1] > bb3)
bb3 = res_corners[2*k+1];
}
// if(not no_bbox):
// cv::rectangle(frame,
// cv::Point(bb0, bb2),
// cv::Point(bb1, bb3),
// trColors[t.color], thickness);
cv::rectangle(frames[bi],
cv::Point(bb0 * scale_x, bb2 * scale_y - text_size.height - thickness),
cv::Point(bb0 * scale_x + text_size.width, bb2 * scale_y),
trColors[t.color], -1);
cv::putText(frames[bi], txt,
cv::Point(bb0 * scale_x, bb2 * scale_y - thickness -1),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), 1);
cv::arrowedLine(frames[bi],
cv::Point((int)((bb0 + bb1)/2) * scale_x, (int)((bb2 + bb3)/2) * scale_y),
cv::Point((int)((bb0 + bb1)/2 + t.det_res.tr.at<float>(0,0)) * scale_x,
(int)((bb2 + bb3)/2 + t.det_res.tr.at<float>(0,1)) * scale_y),
cv::Scalar(255, 0, 255), 2);
}
}
}
}
}
}}
src/CenternetDetection.cpp
+5 -7
View File
@@ -3,12 +3,11 @@
namespace tk { namespace dnn {
bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){
bool CenternetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
dim = netRT->input_dim;
@@ -119,7 +118,6 @@ bool CenternetDetection::init(const std::string& tensor_path, const int n_classe
dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
return true;
}
@@ -373,10 +371,10 @@ void CenternetDetection::postprocess(const int bi, const bool mAP){
// std::cout<<"th: "<<scores[j]<<" - cl: "<<clses[j]<<" i: "<<i<<std::endl;
//add coco bbox
//det[0:4], i, det[4]
float x0 = target_coords[j*4];
float y0 = target_coords[j*4+1];
float x1 = target_coords[j*4+2];
float y1 = target_coords[j*4+3];
int x0 = target_coords[j*4];
int y0 = target_coords[j*4+1];
int x1 = target_coords[j*4+2];
int y1 = target_coords[j*4+3];
int obj_class = clses[j];
float prob = scores[j];
// std::cout<<"("<<x0<<", "<<y0<<"),("<<x1<<", "<<y1<<")"<<std::endl;
src/CenternetDetection3D.cpp
-541
View File
@@ -1,541 +0,0 @@
#include "CenternetDetection3D.h"
namespace tk { namespace dnn {
bool CenternetDetection3D::init(const std::string& tensor_path, const int n_classes, const int n_batches,
const float conf_thresh, const std::vector<cv::Mat>& k_calibs) {
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
inputCalibs = k_calibs;
dim = netRT->input_dim;
const char *kitti_class_name[] = {
"person", "car", "bicycle"};
classesNames = std::vector<std::string>(kitti_class_name, std::end( kitti_class_name));
for(int c=0; c<classes; c++) {
int offset = c*123457 % classes;
float r = getColor(2, offset, classes);
float g = getColor(1, offset, classes);
float b = getColor(0, offset, classes);
colors[c] = cv::Scalar(int(255.0*b), int(255.0*g), int(255.0*r));
}
src = cv::Mat(cv::Size(2,3), CV_32F);
dst = cv::Mat(cv::Size(2,3), CV_32F);
dst2 = cv::Mat(cv::Size(2,3), CV_32F);
trans = cv::Mat(cv::Size(3,2), CV_32F);
trans2 = cv::Mat(cv::Size(3,2), CV_32F);
checkCuda(cudaMalloc(&input_d, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
dim_hm = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
dim_wh = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_reg = tk::dnn::dataDim_t(1, 2, 128, 128, 1);
dim_dep = tk::dnn::dataDim_t(1, 1, 128, 128, 1);
dim_rot = tk::dnn::dataDim_t(1, 8, 128, 128, 1);
dim_dim = tk::dnn::dataDim_t(1, 3, 128, 128, 1);
checkCuda( cudaMalloc(&topk_scores, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_inds_, dim_hm.c * K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_, dim_hm.c * K *sizeof(float)) );
checkCuda( cudaMalloc(&ids_d, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
checkCuda( cudaMallocHost(&ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int)) );
for(int i =0; i<dim_hm.c * dim_hm.h * dim_hm.w; i++){
ids_[i] = i;
}
checkCuda( cudaMalloc(&ones, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
float *ones_h;
checkCuda( cudaMallocHost(&ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float)) );
for(int i=0; i<dim_dep.c * dim_dep.h * dim_dep.w; i++)
ones_h[i]=1.0f;
checkCuda( cudaMemcpy(ones, ones_h, dim_dep.c * dim_dep.h * dim_dep.w * sizeof(float), cudaMemcpyHostToDevice) );
checkCuda( cudaFreeHost(ones_h) );
checkCuda( cudaMallocHost(&scores, K *sizeof(float)) );
checkCuda( cudaMalloc(&scores_d, K *sizeof(float)) );
checkCuda( cudaMallocHost(&clses, K *sizeof(int)) );
checkCuda( cudaMalloc(&clses_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_inds_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&topk_ys_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&topk_xs_d, K *sizeof(float)) );
checkCuda( cudaMalloc(&inttopk_ys_d, K *sizeof(int)) );
checkCuda( cudaMalloc(&inttopk_xs_d, K *sizeof(int)) );
checkCuda( cudaMallocHost(&xs, K * sizeof(float)) );
checkCuda( cudaMallocHost(&ys, K * sizeof(float)) );
checkCuda( cudaMallocHost(&dep, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMallocHost(&rot, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMallocHost(&dim_, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMallocHost(&wh, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMalloc(&dep_d, K * dim_dep.c * sizeof(float)) );
checkCuda( cudaMalloc(&rot_d, K * dim_rot.c * sizeof(float)) );
checkCuda( cudaMalloc(&dim_d, K * dim_dim.c * sizeof(float)) );
checkCuda( cudaMalloc(&wh_d, K * dim_wh.c * sizeof(float)) );
checkCuda( cudaMallocHost(&target_coords, 4 * K *sizeof(float)) );
#ifdef OPENCV_CUDACONTRIB
checkCuda( cudaMalloc(&mean_d, 3 * sizeof(float)) );
checkCuda( cudaMalloc(&stddev_d, 3 * sizeof(float)) );
float mean[3] = {0.485, 0.456, 0.406};
float stddev[3] = {0.229, 0.224, 0.225};
checkCuda(cudaMemcpy(mean_d, mean, 3*sizeof(float), cudaMemcpyHostToDevice));
checkCuda(cudaMemcpy(stddev_d, stddev, 3*sizeof(float), cudaMemcpyHostToDevice));
#else
checkCuda(cudaMallocHost(&input, sizeof(dnnType)*netRT->input_dim.tot() * nBatches));
mean << 0.485, 0.456, 0.406;
stddev << 0.229, 0.224, 0.225;
#endif
for(int bi=0; bi<nBatches; bi++) {
cv::Mat calibs_ = cv::Mat::zeros(cv::Size(4,3), CV_32F);
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs_.at<float>(0,0) = 707.0493;
calibs_.at<float>(0,2) = 604.0814;
calibs_.at<float>(1,1) = 707.0493;
calibs_.at<float>(1,2) = 180.5066;
calibs_.at<float>(0,3) = 45.75831;
calibs_.at<float>(1,3) = -0.3454157;
calibs_.at<float>(2,2) = 1.0;
calibs_.at<float>(2,3) = 0.004981016;
}
else {
calibs_.at<float>(0,0) = inputCalibs[bi].at<float>(0,0);// * (1440.0/dim.w);// / 1440;
calibs_.at<float>(0,2) = inputCalibs[bi].at<float>(0,2);// * (1440.0/dim.w);// / 1440;
calibs_.at<float>(1,1) = inputCalibs[bi].at<float>(1,1);// * (1080.0/dim.h);//dim.h / 1080;
calibs_.at<float>(1,2) = inputCalibs[bi].at<float>(1,2);// * (1080.0/dim.h);//dim.h / 1080;
calibs_.at<float>(2,2) = 1.0;
}
// calibs_.at<float>(0,3) = 45.75831;
// calibs_.at<float>(1,3) = -0.3454157;
// calibs_.at<float>(2,2) = 1.0;
// calibs_.at<float>(2,3) = 0.004981016;
calibs.push_back(calibs_);
}
r = cv::Mat(cv::Size(3,3), CV_32F);
r.at<float>(0,1) = 0.0;
r.at<float>(1,0) = 0.0;
r.at<float>(1,1) = 1.0;
r.at<float>(1,2) = 0.0;
r.at<float>(2,1) = 0.0;
corners = cv::Mat(cv::Size(8,3), CV_32F);
corners.at<float>(1,0) = 0.0;
corners.at<float>(1,1) = 0.0;
corners.at<float>(1,2) = 0.0;
corners.at<float>(1,3) = 0.0;
pts3DHomo = cv::Mat(cv::Size(8,4), CV_32F);
pts3DHomo.at<float>(3,0) = 1.0;
pts3DHomo.at<float>(3,1) = 1.0;
pts3DHomo.at<float>(3,2) = 1.0;
pts3DHomo.at<float>(3,3) = 1.0;
pts3DHomo.at<float>(3,4) = 1.0;
pts3DHomo.at<float>(3,5) = 1.0;
pts3DHomo.at<float>(3,6) = 1.0;
pts3DHomo.at<float>(3,7) = 1.0;
checkCuda( cudaMalloc(&d_ptrs, dim.c * dim.h*dim.w * sizeof(float)) );
// Alloc array used in the kernel
checkCuda( cudaMalloc(&srcOut, K *sizeof(float)) );
checkCuda( cudaMalloc(&idsOut, K *sizeof(int)) );
dst2.at<float>(0,0)=width * 0.5;
dst2.at<float>(0,1)=width * 0.5;
dst2.at<float>(1,0)=width * 0.5;
dst2.at<float>(1,1)=width * 0.5 + width * -0.5;
dst2.at<float>(2,0)=dst2.at<float>(1,0) + (-dst2.at<float>(0,1)+dst2.at<float>(1,1) );
dst2.at<float>(2,1)=dst2.at<float>(1,1) + (dst2.at<float>(0,0)-dst2.at<float>(1,0) );
faceId.push_back({0,1,5,4});
faceId.push_back({1,2,6, 5});
faceId.push_back({2,3,7,6});
faceId.push_back({3,0,4,7});
// ([[0,1,5,4], [1,2,6, 5], [2,3,7,6], [3,0,4,7]]);
return true;
}
void CenternetDetection3D::preprocess(cv::Mat &frame, const int bi){
cv::Size sz = originalSize[bi];
float new_height = dim.h;//sz.height * scale;
float new_width = dim.w;//sz.width * scale;
if(sz.height != sz_old.height && sz.width != sz_old.width){
if(inputCalibs.size() == 0 || inputCalibs[bi].empty()) {
calibs[bi].at<float>(0,2) = new_width / 2.0f;
calibs[bi].at<float>(1,2) = new_height /2.0f;
}
else {
calibs[bi].at<float>(0,0) = inputCalibs[bi].at<float>(0,0) * 2.0 * dim.w / sz.width;
calibs[bi].at<float>(0,2) = inputCalibs[bi].at<float>(0,2) * dim.w / sz.width ;
calibs[bi].at<float>(1,1) = inputCalibs[bi].at<float>(1,1) * 2.0 * dim.h / sz.height;
calibs[bi].at<float>(1,2) = inputCalibs[bi].at<float>(1,2) * dim.h / sz.height;
}
float c[] = {new_width / 2.0f, new_height /2.0f};
float s[] = {new_width, new_height};
// ----------- get_affine_transform
// rot_rad = pi * 0 / 100 --> 0
src.at<float>(0,0)=c[0];
src.at<float>(0,1)=c[1];
src.at<float>(1,0)=c[0];
src.at<float>(1,1)=c[1] + s[0] * -0.5;
dst.at<float>(0,0)=netRT->input_dim.w * 0.5;
dst.at<float>(0,1)=netRT->input_dim.h * 0.5;
dst.at<float>(1,0)=netRT->input_dim.w * 0.5;
dst.at<float>(1,1)=netRT->input_dim.h * 0.5 + netRT->input_dim.w * -0.5;
src.at<float>(2,0)=src.at<float>(1,0) + (-src.at<float>(0,1)+src.at<float>(1,1) );
src.at<float>(2,1)=src.at<float>(1,1) + (src.at<float>(0,0)-src.at<float>(1,0) );
dst.at<float>(2,0)=dst.at<float>(1,0) + (-dst.at<float>(0,1)+dst.at<float>(1,1) );
dst.at<float>(2,1)=dst.at<float>(1,1) + (dst.at<float>(0,0)-dst.at<float>(1,0) );
trans = cv::getAffineTransform( src, dst );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME gett affine trans: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
trans2 = cv::getAffineTransform( dst2, src );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME getAffineTrans 2: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
}
sz_old = sz;
#ifdef OPENCV_CUDACONTRIB
// std::cout<<"OPENCV CPMTROB\n";
cv::cuda::GpuMat im_Orig;
cv::cuda::GpuMat imageF1_d, imageF2_d;
im_Orig = cv::cuda::GpuMat(frame);
cv::cuda::resize (im_Orig, imageF1_d, cv::Size(dim.w, dim.h));//cv::Size(new_width, new_height));
// imageF1_d = im_Orig;
checkCuda( cudaDeviceSynchronize() );
sz = imageF1_d.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
cv::cuda::warpAffine(imageF1_d, imageF2_d, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
checkCuda( cudaDeviceSynchronize() );
imageF2_d.convertTo(imageF1_d, CV_32FC3, 1/255.0);
checkCuda( cudaDeviceSynchronize() );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME convert: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
dim2 = dim;
cv::cuda::GpuMat bgr[3];
cv::cuda::split(imageF1_d,bgr);//split source
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME split: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
for(int i=0; i<dim.c; i++)
checkCuda( cudaMemcpy(d_ptrs + i*dim.h * dim.w, (float*)bgr[i].data, dim.h * dim.w * sizeof(float), cudaMemcpyDeviceToDevice) );
normalize(d_ptrs, dim.c, dim.h, dim.w, mean_d, stddev_d);
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME normalize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
checkCuda(cudaMemcpy(input_d+ netRT->input_dim.tot()*bi, d_ptrs, dim2.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME Memcpy to input_d: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
#else
// std::cout<<"NO OPENCV CPMTROB\n";
cv::Mat imageF;
resize(frame, imageF, cv::Size(dim.w, dim.h));//cv::Size(new_width, new_height));
// imageF = frame;
sz = imageF.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME resize: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
cv::Mat trans = cv::getAffineTransform( src, dst );
cv::warpAffine(imageF, imageF, trans, cv::Size(netRT->input_dim.w, netRT->input_dim.h), cv::INTER_LINEAR );
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME warpAffine: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
sz = imageF.size();
// std::cout<<"size: "<<sz.height<<" "<<sz.width<<" - "<<std::endl;
imageF.convertTo(imageF, CV_32FC3, 1/255.0);
// end_t = std::chrono::steady_clock::now();
// std::cout << " TIME convertto: " << std::chrono::duration_cast<std::chrono:: microseconds>(end_t - step_t).count() << " us" << std::endl;
// step_t = end_t;
dim2 = dim;
//split channels
cv::Mat bgr[3];
cv::split(imageF,bgr);//split source
for(int i=0; i<3; i++){
bgr[i] = bgr[i] - mean[i];
bgr[i] = bgr[i] / stddev[i];
}
//write channels
for(int i=0; i<dim2.c; i++) {
int idx = i*imageF.rows*imageF.cols;
int ch = dim2.c-3 +i;
// std::cout<<"i: "<<i<<", idx: "<<idx<<", ch: "<<ch<<std::endl;
memcpy((void*)&input[idx+ netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imageF.rows*imageF.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d+ netRT->input_dim.tot()*bi, input+ netRT->input_dim.tot()*bi, dim2.tot()*sizeof(dnnType), cudaMemcpyHostToDevice));
#endif
}
void CenternetDetection3D::postprocess(const int bi, const bool mAP) {
dnnType *rt_out[7];
rt_out[0] = (dnnType *)netRT->buffersRT[1]+ netRT->buffersDIM[1].tot()*bi;
rt_out[1] = (dnnType *)netRT->buffersRT[2]+ netRT->buffersDIM[2].tot()*bi;
rt_out[2] = (dnnType *)netRT->buffersRT[3]+ netRT->buffersDIM[3].tot()*bi;
rt_out[3] = (dnnType *)netRT->buffersRT[4]+ netRT->buffersDIM[4].tot()*bi;
rt_out[4] = (dnnType *)netRT->buffersRT[5]+ netRT->buffersDIM[5].tot()*bi;
rt_out[5] = (dnnType *)netRT->buffersRT[6]+ netRT->buffersDIM[6].tot()*bi;
rt_out[6] = (dnnType *)netRT->buffersRT[7]+ netRT->buffersDIM[7].tot()*bi;
// ------------------------------------ process --------------------------------------------
activationSIGMOIDForward(rt_out[0], rt_out[0], dim_hm.tot());
checkCuda( cudaDeviceSynchronize() );
// output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
activationSIGMOIDForward(rt_out[4], rt_out[4], dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
transformDep(ones, ones + dim_dep.tot(), rt_out[4], rt_out[4] + dim_dep.tot());
checkCuda( cudaDeviceSynchronize() );
subtractWithThreshold(rt_out[0], rt_out[0] + dim_hm.tot(), rt_out[1], rt_out[0], op);
// ----------- nms end
// ----------- topk
if(K > dim_hm.h * dim_hm.w){
printf ("Error topk (K is too large)\n");
return;
}
checkCuda( cudaMemcpy(ids_d, ids_, dim_hm.c * dim_hm.h * dim_hm.w*sizeof(int), cudaMemcpyHostToDevice) );
sort(rt_out[0],rt_out[0]+dim_hm.tot(),ids_d);
checkCuda( cudaDeviceSynchronize() );
topk(rt_out[0], ids_d, K, scores_d, topk_inds_d, topk_ys_d, topk_xs_d);
checkCuda( cudaDeviceSynchronize() );
checkCuda( cudaMemcpy(scores, scores_d, K *sizeof(float), cudaMemcpyDeviceToHost) );
topKxyclasses(topk_inds_d, topk_inds_d+K, K, width, dim_hm.w*dim_hm.h, clses_d, inttopk_xs_d, inttopk_ys_d);
checkCuda( cudaMemcpy(topk_xs_d, (float *)inttopk_xs_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(topk_ys_d, (float *)inttopk_ys_d, K*sizeof(float), cudaMemcpyDeviceToDevice) );
checkCuda( cudaMemcpy(clses, clses_d, K*sizeof(int), cudaMemcpyDeviceToHost) );
// ----------- topk end
topKxyAddOffset(topk_inds_d, K, dim_reg.h*dim_reg.w, inttopk_xs_d, inttopk_ys_d, topk_xs_d, topk_ys_d, rt_out[3], srcOut, idsOut);
// checkCuda( cudaDeviceSynchronize() );
getRecordsFromTopKId(topk_inds_d, K, dim_dep.c, dim_dep.h * dim_dep.w, rt_out[4], dep_d, idsOut);
checkCuda( cudaMemcpy(dep, dep_d, K * dim_dep.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_rot.c, dim_rot.h * dim_rot.w, rt_out[5], rot_d, idsOut);
checkCuda( cudaMemcpy(rot, rot_d, K * dim_rot.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_dim.c, dim_dim.h * dim_dim.w, rt_out[6], dim_d, idsOut);
checkCuda( cudaMemcpy(dim_, dim_d, K * dim_dim.c * sizeof(float), cudaMemcpyDeviceToHost) );
getRecordsFromTopKId(topk_inds_d, K, dim_wh.c, dim_wh.h * dim_wh.w, rt_out[2], wh_d, idsOut);
checkCuda( cudaMemcpy(wh, wh_d, K * dim_wh.c * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(xs, topk_xs_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
checkCuda( cudaMemcpy(ys, topk_ys_d, K * sizeof(float), cudaMemcpyDeviceToHost) );
// ---------------------------------- post-process -----------------------------------------
// ddd_post_process_2d
cv::Mat new_pt1(cv::Size(1,2), CV_32F);
cv::Mat new_pt2(cv::Size(1,2), CV_32F);
for(int i = 0; i<K; i++){
new_pt1.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*xs[i] +
static_cast<float>(trans2.at<double>(0,1))*ys[i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt1.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*xs[i] +
static_cast<float>(trans2.at<double>(1,1))*ys[i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
new_pt2.at<float>(0,0)=static_cast<float>(trans2.at<double>(0,0))*wh[i] +
static_cast<float>(trans2.at<double>(0,1))*wh[K+i] +
static_cast<float>(trans2.at<double>(0,2))*1.0;
new_pt2.at<float>(0,1)=static_cast<float>(trans2.at<double>(1,0))*wh[i] +
static_cast<float>(trans2.at<double>(1,1))*wh[K+i] +
static_cast<float>(trans2.at<double>(1,2))*1.0;
target_coords[i*4] = new_pt1.at<float>(0,0);
target_coords[i*4+1] = new_pt1.at<float>(0,1);
target_coords[i*4+2] = new_pt2.at<float>(0,0);
target_coords[i*4+3] = new_pt2.at<float>(0,1);
}
float alpha;
float x, y, z, rot_y;
detected3D.clear();
for(int i = 0; i<classes; i++){
for(int j=0; j<K; j++){
if(clses[j] == i){
//get alpha
if(rot[1*K + j] > rot[5*K + j])
alpha = std::atan2(rot[2*K + j], rot[3*K + j]) -0.5 * M_PI;
else
alpha = std::atan2(rot[6*K + j], rot[7*K + j]) +0.5 * M_PI;
// unproject_2d_to_3d
z = dep[j] - calibs[bi].at<float>(2,3);// z = depth - P[2, 3]
x = (target_coords[j*4] * dep[j] - calibs[bi].at<float>(0,3) - calibs[bi].at<float>(0,2) * z) / calibs[bi].at<float>(0,0);
y = (target_coords[j*4+1] * dep[j] - calibs[bi].at<float>(1,3) - calibs[bi].at<float>(1,2) * z) / calibs[bi].at<float>(1,1) + (dim_[j] / 2);
// alpha2rot_y
rot_y = (alpha + std::atan2(target_coords[j*4] - calibs[bi].at<float>(0,2), calibs[bi].at<float>(0,0)));
if(rot_y>M_PI)
rot_y -= 2*M_PI;
if(rot_y<M_PI)
rot_y += 2*M_PI;
if(scores[j] > confThreshold) {
if(z>0) {
// compute_box_3d
r.at<float>(0,0) = std::cos(rot_y);
r.at<float>(0,2) = std::sin(rot_y);
r.at<float>(2,0) = -std::sin(rot_y);
r.at<float>(2,2) = std::cos(rot_y);
corners.at<float>(0,0) = dim_[2*K+j]/2;
corners.at<float>(0,1) = dim_[2*K+j]/2;
corners.at<float>(0,2) = -dim_[2*K+j]/2;
corners.at<float>(0,3) = -dim_[2*K+j]/2;
corners.at<float>(0,4) = dim_[2*K+j]/2;
corners.at<float>(0,5) = dim_[2*K+j]/2;
corners.at<float>(0,6) = -dim_[2*K+j]/2;
corners.at<float>(0,7) = -dim_[2*K+j]/2;
corners.at<float>(1,4) = -dim_[j];
corners.at<float>(1,5) = -dim_[j];
corners.at<float>(1,6) = -dim_[j];
corners.at<float>(1,7) = -dim_[j];
corners.at<float>(2,0) = dim_[K+j]/2;
corners.at<float>(2,1) = -dim_[K+j]/2;
corners.at<float>(2,2) = -dim_[K+j]/2;
corners.at<float>(2,3) = dim_[K+j]/2;
corners.at<float>(2,4) = dim_[K+j]/2;
corners.at<float>(2,5) = -dim_[K+j]/2;
corners.at<float>(2,6) = -dim_[K+j]/2;
corners.at<float>(2,7) = dim_[K+j]/2;
cv::Mat aus = r * corners;
for(int k=0; k<8; k++) {
aus.at<float>(0,k) += x;
aus.at<float>(1,k) += y;
aus.at<float>(2,k) += z;
}
// corners.copyTo(pts3DHomo(cv::Rect(0, 0, 8, 3)));
for(int k1=0; k1<3; k1++) {
for(int k2=0; k2<8; k2++)
pts3DHomo.at<float>(k1,k2) = aus.at<float>(k1,k2);
}
aus.release();
aus = calibs[bi] * pts3DHomo;
tk::dnn::box3D res;
for(int k=0; k<8; k++) {
res.corners.push_back(aus.at<float>(0,k) / aus.at<float>(2,k));
res.corners.push_back(aus.at<float>(1,k) / aus.at<float>(2,k));
}
res.cl = i;
res.prob = scores[j];
//res.print();
detected3D.push_back(res);
}
}
}
}
}
batchDetected.push_back(detected3D);
}
void CenternetDetection3D::draw(std::vector<cv::Mat>& frames) {
tk::dnn::box3D b;
int x0, w, x1, y0, h, y1;
int objClass;
std::string det_class;
int baseline = 0;
float font_scale = 0.5;
int thickness = 2;
for(int bi=0; bi<frames.size(); ++bi){
float scale_x = float(originalSize[bi].width)/dim.w;
float scale_y = float(originalSize[bi].height)/dim.h;
resize(frames[bi], frames[bi], originalSize[bi]);
// draw dets
for(int i=0; i<batchDetected[bi].size(); i++) {
b = batchDetected[bi][i];
for(int ind_f = 3; ind_f>=0; ind_f--) {
for(int j=0; j<4; j++) {
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(j) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at(j) * 2 + 1) * scale_y),
cv::Point(b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at((j+1)%4) * 2 + 1) * scale_y),
colors[b.cl], 2);
if(ind_f == 0) {
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(0) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at(0) * 2 + 1)* scale_y),
cv::Point(b.corners.at(faceId.at(ind_f).at(2) * 2) * scale_x,
b.corners.at(faceId.at(ind_f).at(2) * 2 + 1) * scale_y), colors[b.cl], 2);
cv::line(frames[bi], cv::Point(b.corners.at(faceId.at(ind_f).at(1) * 2)* scale_x,
b.corners.at(faceId.at(ind_f).at(1) * 2 + 1)* scale_y),
cv::Point(b.corners.at(faceId.at(ind_f).at(3) * 2)* scale_x,
b.corners.at(faceId.at(ind_f).at(3) * 2 + 1)* scale_y), colors[b.cl], 2);
}
}
}
// draw label
cv::Size text_size = getTextSize(classesNames[b.cl], cv::FONT_HERSHEY_SIMPLEX, font_scale, thickness, &baseline);
cv::rectangle(frames[bi], cv::Point(b.corners.at(faceId.at(0).at(0) * 2)* scale_x,
b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y),
cv::Point((b.corners.at(faceId.at(0).at(0) * 2)* scale_x + text_size.width - 2),
(b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y - text_size.height - 2)), colors[b.cl], -1);
cv::putText(frames[bi], classesNames[b.cl], cv::Point(b.corners.at(faceId.at(0).at(0) * 2)* scale_x,
(b.corners.at(faceId.at(0).at(0) * 2 + 1)* scale_y - (baseline / 2))),
cv::FONT_HERSHEY_SIMPLEX, font_scale, cv::Scalar(255, 255, 255), thickness);
}
}
}
}}
src/Conv2d.cpp
-5
View File
@@ -166,11 +166,6 @@ Conv2d::Conv2d( Network *net, int out_ch, int kernelH, int kernelW,
}
initCUDNN(deConv);
if(this->groups != 1)
MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
else
MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
// allocate warkspace
if (ws_sizeInBytes!=0) {
checkCuda( cudaMalloc(&workSpace, ws_sizeInBytes) );
src/DarknetParser.cpp
+3 -14
View File
@@ -37,10 +37,7 @@ namespace tk { namespace dnn {
std::string name,value;
if(!divideNameAndValue(line, name, value))
return false;
if(name.find("new_coords") != std::string::npos)
fields.new_coords = std::stoi(value);
else if(name.find("width") != std::string::npos)
if(name.find("width") != std::string::npos)
fields.width = std::stoi(value);
else if(name.find("height") != std::string::npos)
fields.height = std::stoi(value);
@@ -82,13 +79,6 @@ namespace tk { namespace dnn {
fields.group_id = std::stoi(value);
else if(name.find("scale_x_y") != std::string::npos)
fields.scale_xy = std::stof(value);
else if(name.find("beta_nms") != std::string::npos)
fields.nms_thresh = std::stof(value);
else if(name.find("nms_kind") != std::string::npos){
if(value == "greedynms") fields.nms_kind = 0;
else if(value == "diounms") fields.nms_kind = 1;
else std::cout<<"Not supported nms_kind "<<value<<", setting to greedynms"<<std::endl;
}
else if(name.find("from") != std::string::npos)
fields.layers.push_back(std::stof(value));
else if(name.find("mask") != std::string::npos){
@@ -171,7 +161,7 @@ namespace tk { namespace dnn {
} else if(f.type == "yolo") {
std::string wgs = wgs_path + "/g" + std::to_string(netLayers.size()) + ".bin";
//printf("%d %d %s %d %f\n", f.classes, f.num/f.n_mask, wgs.c_str(), f.n_mask, f.scale_xy);
tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy, f.nms_thresh, (tk::dnn::Yolo::nmsKind_t) f.nms_kind, f.new_coords);
tk::dnn::Yolo *l = new tk::dnn::Yolo(net, f.classes, f.num/f.n_mask, wgs, f.n_mask, f.scale_xy);
if(names.size() != f.classes)
FatalError("Mismatch between number of classes and names");
l->classesNames = names;
@@ -187,7 +177,6 @@ namespace tk { namespace dnn {
if(f.activation == "relu") act = tkdnnActivationMode_t(CUDNN_ACTIVATION_RELU);
else if(f.activation == "leaky") act = tk::dnn::ACTIVATION_LEAKY;
else if(f.activation == "mish") act = tk::dnn::ACTIVATION_MISH;
else if(f.activation == "logistic") act = tk::dnn::ACTIVATION_LOGISTIC;
else { FatalError("activation not supported: " + f.activation); }
netLayers[netLayers.size()-1] = new tk::dnn::Activation(net, act);
};
@@ -212,7 +201,7 @@ namespace tk { namespace dnn {
tk::dnn::Network *net = nullptr;
// layers without activations to retrieve correct id number
// layers without activations to retrive correct id number
std::vector<tk::dnn::Layer*> netLayers;
std::ifstream if_cfg(cfg_file);
src/DeformConv2d.cpp
+1 -7
View File
@@ -73,12 +73,6 @@ DeformConv2d::DeformConv2d( Network *net, int out_ch, int deformable_group, int
output_dim.c = out_ch;
initCUDNN();
if(this->deformableGroup != 1)
MACC = kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
else
MACC = input_dim.c*kernelH*kernelW*output_dim.c*output_dim.w*output_dim.h;
//allocate data for infer result
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
}
@@ -101,7 +95,7 @@ dnnType* DeformConv2d::infer(dataDim_t &dim, dnnType* srcData) {
// split conv2d outputs into offset and mask
checkCuda(cudaMemcpy(offset, output_conv, 2*chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
checkCuda(cudaMemcpy(mask, output_conv + 2*chunk_dim, chunk_dim*sizeof(dnnType), cudaMemcpyDeviceToDevice));
// kernel sigmoid
// kernel sigmoide
activationSIGMOIDForward(mask, mask, chunk_dim);
// deformable convolution
src/Dense.cpp
+1 -1
View File
@@ -37,7 +37,7 @@ dnnType* Dense::infer(dataDim_t &dim, dnnType* srcData) {
// place bias into dstData
checkCuda( cudaMemcpy(dstData, bias_d, dim_y*sizeof(dnnType), cudaMemcpyDeviceToDevice) );
//do matrix multiplication
//do matrix moltiplication
checkERROR( cublasSgemv(net->cublasHandle, CUBLAS_OP_T,
dim_x, dim_y,
&alpha,
src/LSTM.cpp
+9 -14
View File
@@ -87,22 +87,17 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig
checkCUDNN(cudnnCreateRNNDescriptor(&rnnDesc));
#if CUDNN_MAJOR > 7
checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle,rnnDesc, stateSize, numLayers, dropoutDesc,
cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT,
//(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL),
cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL,
cudnnRNNMode_t::CUDNN_LSTM,
cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD,
net->dataType));
checkCUDNN(cudnnSetRNNDescriptor_v6(net->cudnnHandle,
#else
checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle,rnnDesc, stateSize, numLayers, dropoutDesc,
checkCUDNN(cudnnSetRNNDescriptor(net->cudnnHandle,
#endif
rnnDesc, stateSize, numLayers, dropoutDesc,
cudnnRNNInputMode_t::CUDNN_LINEAR_INPUT,
//(bidirectional ? cudnnDirectionMode_t::CUDNN_BIDIRECTIONAL : cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL),
cudnnDirectionMode_t::CUDNN_UNIDIRECTIONAL,
cudnnRNNMode_t::CUDNN_LSTM,
cudnnRNNAlgo_t::CUDNN_RNN_ALGO_STANDARD,
net->dataType));
#endif
// Get temp space sizes
@@ -138,7 +133,7 @@ LSTM::LSTM( Network *net, int hiddensize, bool returnSeq, std::string fname_weig
output_dim = input_dim;
output_dim.c = stateSize*(bidirectional ? 2 : 1);
// if retunseq is disabled only the last timestamp is returned
// if retunseq is disabled only the last timestep is returned
if(!returnSeq) {
output_dim.h = 1;
output_dim.w = 1;
@@ -259,7 +254,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
rnnDesc,
seqLen, // number of time steps (nT)
x_desc_vec_.data(), // input array of desc (nT*nC_in)
srcF, // input pointer
srcF, // input pointer
hx_desc_, // initial hidden state desc
hx_ptr, // initial hidden state pointer
cx_desc_, // initial cell state desc
@@ -286,7 +281,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
rnnDesc,
seqLen, // number of time steps (nT)
x_desc_vec_.data(), // input array of desc (nT*nC_in)
srcB, // input pointer
srcB, // input pointer
hx_desc_, // initial hidden state desc
hx_ptr, // initial hidden state pointer
cx_desc_, // initial cell state desc
@@ -294,7 +289,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
w_desc_, // weights desc
wb_ptr, // weights pointer
y_desc_vec_.data(), // output desc (nT*nC_out)
dstB_NR, // output pointer
dstB_NR, // output pointer
hy_desc_, // final hidden state desc
hy_ptr, // final hidden state pointer
cy_desc_, // final cell state desc
@@ -312,7 +307,7 @@ dnnType* LSTM::infer(dataDim_t &dim, dnnType* srcData) {
one_output_dim.c*sizeof(dnnType), cudaMemcpyDeviceToDevice));
}
// if retunseq is disabled only the last timestamp is returned
// if retunseq is disabled only the last timestep is returned
if(returnSeq) {
// forward transpose
matrixTranspose(net->cublasHandle, dstF, dstData,
src/Layer.cpp
-2
View File
@@ -18,8 +18,6 @@ Layer::Layer(Network *net) {
if(!net->addLayer(this))
FatalError("Net reached max number of layers");
}
feature_map_size = input_dim.tot() + output_dim.tot();
}
Layer::~Layer() {
src/LayerWgs.cpp
+2 -6
View File
@@ -19,8 +19,6 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
int seek = 0;
readBinaryFile(weights_path.c_str(), inputs*outputs*kh*kw*kl, &data_h, &data_d, seek);
seek += inputs*outputs*kh*kw*kl;
n_params = seek;
this->additional_bias = additional_bias;
if(additional_bias) {
readBinaryFile(weights_path.c_str(), outputs, &bias2_h, &bias2_d, seek);
@@ -28,17 +26,15 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
}
readBinaryFile(weights_path.c_str(), outputs, &bias_h, &bias_d, seek);
seek += outputs;
this->batchnorm = batchnorm;
if(batchnorm) {
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &scales_h, &scales_d, seek);
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &mean_h, &mean_d, seek);
seek += outputs;
readBinaryFile(weights_path.c_str(), outputs, &variance_h, &variance_d, seek);
seek += outputs;
float eps = TKDNN_BN_MIN_EPSILON;
@@ -109,7 +105,7 @@ LayerWgs::LayerWgs(Network *net, int inputs, int outputs,
float2half(tmp_d, variance16_d, b_size);
cudaMemcpy(variance16_h, variance16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
//convert scales
//conver scales
float2half(scales_d, scales16_d, b_size);
cudaMemcpy(scales16_h, scales16_d, b_size*sizeof(__half), cudaMemcpyDeviceToHost);
src/MobilenetDetection.cpp
+3 -4
View File
@@ -126,13 +126,12 @@ float MobilenetDetection::iou(const tk::dnn::box &a, const tk::dnn::box &b){
return iou;
}
bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh){
bool MobilenetDetection::init(const std::string& tensor_path, const int n_classes, const int n_batches){
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str());
imageSize = netRT->input_dim.h;
classes = n_classes;
nBatches = n_batches;
confThreshold = conf_thresh;
SSDSpec specs[N_SSDSPEC];
@@ -198,7 +197,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe
"bottle" , "wine glass" , "cup" , "fork" , "knife" , "spoon" , "bowl" , "banana" ,
"apple" , "sandwich" , "orange" , "broccoli" , "carrot" , "hot dog" , "pizza" ,
"donut" , "cake" , "chair" , "sofa" , "pottedplant" , "bed" , "diningtable" ,
"toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" ,
"toilet" , "tvmonitor" , "laptop" , "mouse" , "remote" , "keyboard" ,
"cell phone" , "microwave" , "oven" , "toaster" , "sink" , "refrigerator" ,
"book" , "clock" , "vase" , "scissors" , "teddy bear" , "hair drier" , "toothbrush"};
classesNames = std::vector<std::string>(classes_names_, std::end(classes_names_));
@@ -207,7 +206,7 @@ bool MobilenetDetection::init(const std::string& tensor_path, const int n_classe
else{
FatalError("Number of classes not supported for mobilenet");
}
return true;
return 1;
}
void MobilenetDetection::preprocess(cv::Mat &frame, const int bi){
src/MulAdd.cpp
+1 -1
View File
@@ -12,7 +12,7 @@ MulAdd::MulAdd(Network *net, dnnType mul, dnnType add) : Layer(net) {
int size = input_dim.tot();
// create a vector with all value set to add
// create a vector with all value setted to add
dnnType *add_vector_h = new dnnType[size];
for(int i=0; i<size; i++)
add_vector_h[i] = add;
src/Network.cpp
-36
View File
@@ -96,28 +96,6 @@ dataDim_t Network::getOutputDim() {
return layers[num_layers-1]->output_dim;
}
void Network::adjustFeatureMapSizeWithShortcuts(){
layerType_t layer_type;
int shortcutted_idx;
for(int i=0; i<num_layers; i++) {
layer_type = layers[i]->getLayerType();
if(layer_type == LAYER_SHORTCUT){
shortcutted_idx = -1;
for(int j=0; j<num_layers; j++) {
if(static_cast<tk::dnn::Shortcut*>(layers[i])->backLayer == layers[j]){
shortcutted_idx = j;
break;
}
}
if(shortcutted_idx == -1)
FatalError("Problem when computing featuer_map_size with shortcuts");
for(int j=shortcutted_idx+1; j<i; ++j)
layers[j]->feature_map_size += layers[shortcutted_idx]->output_dim.tot();
}
}
}
void Network::print() {
printCenteredTitle(" NETWORK MODEL ", '=', 60);
@@ -128,21 +106,10 @@ void Network::print() {
std::cout.width(16); std::cout<<std::left<<"output (H*W,CH)";
std::cout<<"\n";
adjustFeatureMapSizeWithShortcuts();
long long unsigned int tot_params = 0;
long long unsigned int max_feature_map_size = 0;
long long unsigned int tot_MACC = 0;
for(int i=0; i<num_layers; i++) {
dataDim_t in = layers[i]->input_dim;
dataDim_t out = layers[i]->output_dim;
tot_params += layers[i]->n_params;
tot_MACC += layers[i]->MACC;
if(layers[i]->feature_map_size> max_feature_map_size)
max_feature_map_size = layers[i]->feature_map_size;
std::cout.width(3); std::cout<<std::right<<i;
std::cout<<" ";
std::cout.width(16); std::cout<<std::left<<layers[i]->getLayerName();
@@ -161,9 +128,6 @@ void Network::print() {
}
printCenteredTitle("", '=', 60);
std::cout<<"\n";
std::cout<<"N params: "<<tot_params<<std::endl;
std::cout<<"Max feature map size: "<<max_feature_map_size<<std::endl;
std::cout<<"N MACC: "<<tot_MACC<<std::endl<<std::endl;
printCudaMemUsage();
}
const char *Network::getNetworkRTName(const char *network_name){
src/NetworkRT.cpp
+41 -114
View File
@@ -140,7 +140,6 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
engineRT = builderRT->buildEngineWithConfig(*networkRT, *configRT);
#else
engineRT = builderRT->buildCudaEngine(*networkRT);
//engineRT = std::shared_ptr<nvinfer1::ICudaEngine>(builderRT->buildCudaEngine(*networkRT));
#endif
if(engineRT == nullptr)
FatalError("cloud not build cuda engine")
@@ -164,7 +163,7 @@ NetworkRT::NetworkRT(Network *net, const char *name) {
// note that indices are guaranteed to be less than IEngine::getNbBindings()
buf_input_idx = engineRT->getBindingIndex("data");
buf_output_idx = engineRT->getBindingIndex("out");
std::cout<<"input index = "<<buf_input_idx<<" -> output index = "<<buf_output_idx<<"\n";
std::cout<<"input idex = "<<buf_input_idx<<" -> output index = "<<buf_output_idx<<"\n";
Dims iDim = engineRT->getBindingDimensions(buf_input_idx);
@@ -227,7 +226,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
return convert_layer(input, (Conv2d*) l);
if(type == LAYER_POOLING)
return convert_layer(input, (Pooling*) l);
if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH || type == LAYER_ACTIVATION_LOGISTIC)
if(type == LAYER_ACTIVATION || type == LAYER_ACTIVATION_CRELU || type == LAYER_ACTIVATION_LEAKY || type == LAYER_ACTIVATION_MISH)
return convert_layer(input, (Activation*) l);
if(type == LAYER_SOFTMAX)
return convert_layer(input, (Softmax*) l);
@@ -237,8 +236,6 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Layer *l) {
return convert_layer(input, (Flatten*) l);
if(type == LAYER_RESHAPE)
return convert_layer(input, (Reshape*) l);
if(type == LAYER_RESIZE)
return convert_layer(input, (Resize*) l);
if(type == LAYER_REORG)
return convert_layer(input, (Reorg*) l);
if(type == LAYER_REGION)
@@ -392,13 +389,13 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
#if NV_TENSORRT_MAJOR < 6
// plugin version
IPlugin *plugin = new ActivationLeakyRT(l->slope);
IPlugin *plugin = new ActivationLeakyRT();
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
checkNULL(lRT);
return lRT;
#else
IActivationLayer *lRT = networkRT->addActivation(*input, ActivationType::kLEAKY_RELU);
lRT->setAlpha(l->slope);
lRT->setAlpha(0.1);
checkNULL(lRT);
return lRT;
#endif
@@ -424,12 +421,6 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Activation *l) {
checkNULL(lRT);
return lRT;
}
else if(l->act_mode == ACTIVATION_LOGISTIC) {
IPlugin *plugin = new ActivationLogisticRT();
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
checkNULL(lRT);
return lRT;
}
else {
FatalError("this Activation mode is not yet implemented");
return NULL;
@@ -481,23 +472,13 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Flatten *l) {
ILayer* NetworkRT::convert_layer(ITensor *input, Reshape *l) {
// std::cout<<"convert Reshape\n";
l->output_dim.print();
IPlugin *plugin = new ReshapeRT(l->output_dim);
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
checkNULL(lRT);
return lRT;
}
ILayer* NetworkRT::convert_layer(ITensor *input, Resize *l) {
// std::cout<<"convert Resize\n";
IResizeLayer *lRT = networkRT->addResize(*input); //default is kNEAREST
checkNULL(lRT);
Dims d{};
lRT->setResizeMode(ResizeMode(l->mode));
lRT->setOutputDimensions(DimsCHW{l->output_dim.c, l->output_dim.h, l->output_dim.w});
return lRT;
}
ILayer* NetworkRT::convert_layer(ITensor *input, Reorg *l) {
//std::cout<<"convert Reorg\n";
@@ -525,7 +506,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
ITensor *back_tens = tensors[l->backLayer];
if(l->backLayer->output_dim.c == l->output_dim.c && !l->mul)
if(l->backLayer->output_dim.c == l->output_dim.c)
{
IElementWiseLayer *lRT = networkRT->addElementWise(*input, *back_tens, ElementWiseOperation::kSUM);
checkNULL(lRT);
@@ -534,7 +515,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Shortcut *l) {
else
{
// plugin version
IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim, l->mul);
IPlugin *plugin = new ShortcutRT(l->backLayer->output_dim);
ITensor **inputs = new ITensor*[2];
inputs[0] = input;
inputs[1] = back_tens;
@@ -548,7 +529,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, Yolo *l) {
//std::cout<<"convert Yolo\n";
//std::cout<<"New plugin YOLO\n";
IPlugin *plugin = new YoloRT(l->classes, l->num, l, l->n_masks, l->scaleXY, l->nms_thresh, l->nsm_kind, l->new_coords);
IPlugin *plugin = new YoloRT(l->classes, l->num, l, l->n_masks, l->scaleXY);
IPluginLayer *lRT = networkRT->addPlugin(&input, 1, *plugin);
checkNULL(lRT);
return lRT;
@@ -580,7 +561,7 @@ ILayer* NetworkRT::convert_layer(ITensor *input, DeformConv2d *l) {
IPluginLayer *lRT = networkRT->addPlugin(inputs, 2, *plugin);
checkNULL(lRT);
lRT->setName( ("Deformable" + std::to_string(l->id)).c_str() );
delete[](inputs);
delete(inputs);
// batchnorm
void *bias_b, *power_b, *mean_b, *variance_b, *scales_b;
if(dtRT == DataType::kHALF) {
@@ -657,61 +638,43 @@ bool NetworkRT::deserialize(const char *filename) {
IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, size_t serialLength) {
const char * buf = reinterpret_cast<const char*>(serialData),*bufCheck = buf;
const char * buf = reinterpret_cast<const char*>(serialData);
std::string name(layerName);
//std::cout<<name<<std::endl;
if(name.find("ActivationLeaky") == 0) {
ActivationLeakyRT *a = new ActivationLeakyRT(readBUF<float>(buf));
ActivationLeakyRT *a = new ActivationLeakyRT();
a->size = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return a;
}
if(name.find("ActivationMish") == 0) {
ActivationMishRT *a = new ActivationMishRT();
a->size = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return a;
}
if(name.find("ActivationLogistic") == 0) {
ActivationLogisticRT *a = new ActivationLogisticRT();
a->size = readBUF<int>(buf);
return a;
}
if(name.find("ActivationLogistic") == 0) {
ActivationLogisticRT *a = new ActivationLogisticRT();
a->size = readBUF<int>(buf);
return a;
}
if(name.find("ActivationCReLU") == 0) {
float activationReluTemp = readBUF<float>(buf);
ActivationReLUCeiling* a = new ActivationReLUCeiling(activationReluTemp);
ActivationReLUCeiling *a = new ActivationReLUCeiling(readBUF<float>(buf));
a->size = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return a;
}
if(name.find("Region") == 0) {
int classesTemp = readBUF<int>(buf);
int coordsTemp = readBUF<int>(buf);
int numTemp = readBUF<int>(buf);
RegionRT* r = new RegionRT(classesTemp, coordsTemp, numTemp);
RegionRT *r = new RegionRT(readBUF<int>(buf), //classes
readBUF<int>(buf), //coords
readBUF<int>(buf)); //num
r->c = readBUF<int>(buf);
r->h = readBUF<int>(buf);
r->w = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return r;
}
if(name.find("Reorg") == 0) {
int strideTemp = readBUF<int>(buf);
ReorgRT *r = new ReorgRT(strideTemp);
ReorgRT *r = new ReorgRT(readBUF<int>(buf)); //stride
r->c = readBUF<int>(buf);
r->h = readBUF<int>(buf);
r->w = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return r;
}
@@ -722,39 +685,32 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
bdim.w = readBUF<int>(buf);
bdim.l = 1;
ShortcutRT *r = new ShortcutRT(bdim, readBUF<bool>(buf));
ShortcutRT *r = new ShortcutRT(bdim);
r->c = readBUF<int>(buf);
r->h = readBUF<int>(buf);
r->w = readBUF<int>(buf);
return r;
assert(buf == bufCheck + serialLength);
}
if(name.find("Pooling") == 0) {
int cTemp = readBUF<int>(buf);
int hTemp = readBUF<int>(buf);
int wTemp = readBUF<int>(buf);
int nTemp = readBUF<int>(buf);
int strideHTemp = readBUF<int>(buf);
int strideWTemp = readBUF<int>(buf);
int winSizeTemp = readBUF<int>(buf);
int paddingTemp = readBUF<int>(buf);
MaxPoolFixedSizeRT* r = new MaxPoolFixedSizeRT(cTemp, hTemp, wTemp, nTemp, strideHTemp, strideWTemp, winSizeTemp, paddingTemp);
assert(buf == bufCheck + serialLength);
MaxPoolFixedSizeRT *r = new MaxPoolFixedSizeRT( readBUF<int>(buf), //c
readBUF<int>(buf), //h
readBUF<int>(buf), //w
readBUF<int>(buf), //n
readBUF<int>(buf), //strideH
readBUF<int>(buf), //strideW
readBUF<int>(buf), //winSize
readBUF<int>(buf)); //padding
return r;
}
if(name.find("Resize") == 0) {
int o_cTemp = readBUF<int>(buf);
int o_hTemp = readBUF<int>(buf);
int o_wTemp = readBUF<int>(buf);
ResizeLayerRT* r = new ResizeLayerRT(o_cTemp, o_hTemp, o_wTemp);
ResizeLayerRT *r = new ResizeLayerRT(readBUF<int>(buf), //o_c
readBUF<int>(buf), //o_h
readBUF<int>(buf)); //o_w
r->i_c = readBUF<int>(buf);
r->i_h = readBUF<int>(buf);
r->i_w = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return r;
}
@@ -765,7 +721,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
r->w = readBUF<int>(buf);
r->rows = readBUF<int>(buf);
r->cols = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return r;
}
@@ -777,28 +732,19 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
new_dim.h = readBUF<int>(buf);
new_dim.w = readBUF<int>(buf);
ReshapeRT *r = new ReshapeRT(new_dim);
assert(buf == bufCheck + serialLength);
return r;
}
if(name.find("Yolo") == 0) {
int classes_temp = readBUF<int>(buf);
int num_temp = readBUF<int>(buf);
int n_masks_temp = readBUF<int>(buf);
float scale_xy_temp = readBUF<float>(buf);
float nms_thresh_temp = readBUF<float>(buf);
int nms_kind_temp = readBUF<int>(buf);
int new_coords_temp = readBUF<int>(buf);
YoloRT *r = new YoloRT(classes_temp,num_temp,nullptr,n_masks_temp,scale_xy_temp,nms_thresh_temp,nms_kind_temp,new_coords_temp);
YoloRT *r = new YoloRT(readBUF<int>(buf), //classes
readBUF<int>(buf), //num
nullptr,
readBUF<int>(buf)); //n_masks
r->c = readBUF<int>(buf);
r->h = readBUF<int>(buf);
r->w = readBUF<int>(buf);
r->scaleXY = readBUF<float>(buf);
for(int i=0; i<r->n_masks; i++)
r->mask[i] = readBUF<dnnType>(buf);
for(int i=0; i<r->n_masks*2*r->num; i++)
@@ -812,54 +758,36 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
tmp[j] = readBUF<char>(buf);
r->classesNames[i] = std::string(tmp);
}
assert(buf == bufCheck + serialLength);
yolos[n_yolos++] = r;
return r;
}
if(name.find("Upsample") == 0) {
int strideTemp = readBUF<int>(buf);
UpsampleRT* r = new UpsampleRT(strideTemp);
UpsampleRT *r = new UpsampleRT(readBUF<int>(buf)); //stride
r->c = readBUF<int>(buf);
r->h = readBUF<int>(buf);
r->w = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return r;
}
if(name.find("Route") == 0) {
int groupsTemp = readBUF<int>(buf);
int group_idTemp = readBUF<int>(buf);
RouteRT* r = new RouteRT(groupsTemp, group_idTemp);
RouteRT *r = new RouteRT(readBUF<int>(buf),readBUF<int>(buf));
r->in = readBUF<int>(buf);
for(int i=0; i<RouteRT::MAX_INPUTS; i++)
r->c_in[i] = readBUF<int>(buf);
r->c = readBUF<int>(buf);
r->h = readBUF<int>(buf);
r->w = readBUF<int>(buf);
assert(buf == bufCheck + serialLength);
return r;
}
if(name.find("Deformable") == 0) {
int chuck_dimTemp = readBUF<int>(buf);
int khTemp = readBUF<int>(buf);
int kwTemp = readBUF<int>(buf);
int shTemp = readBUF<int>(buf);
int swTemp = readBUF<int>(buf);
int phTemp = readBUF<int>(buf);
int pwTemp = readBUF<int>(buf);
int deformableGroupTemp = readBUF<int>(buf);
int i_nTemp = readBUF<int>(buf);
int i_cTemp = readBUF<int>(buf);
int i_hTemp = readBUF<int>(buf);
int i_wTemp = readBUF<int>(buf);
int o_nTemp = readBUF<int>(buf);
int o_cTemp = readBUF<int>(buf);
int o_hTemp = readBUF<int>(buf);
int o_wTemp = readBUF<int>(buf);
DeformableConvRT* r = new DeformableConvRT(chuck_dimTemp, khTemp, kwTemp, shTemp, swTemp, phTemp, pwTemp, deformableGroupTemp, i_nTemp, i_cTemp, i_hTemp, i_wTemp, o_nTemp, o_cTemp, o_hTemp, o_wTemp, nullptr);
DeformableConvRT *r = new DeformableConvRT(readBUF<int>(buf), readBUF<int>(buf), readBUF<int>(buf),
readBUF<int>(buf), readBUF<int>(buf), readBUF<int>(buf),
readBUF<int>(buf), readBUF<int>(buf),
readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),
readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),readBUF<int>(buf),
nullptr);
dnnType *aus = new dnnType[r->chunk_dim*2];
for(int i=0; i<r->chunk_dim*2; i++)
aus[i] = readBUF<dnnType>(buf);
@@ -890,7 +818,6 @@ IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialDa
aus[i] = readBUF<dnnType>(buf);
checkCuda( cudaMemcpy(r->ones_d2, aus, sizeof(dnnType)*r->dim_ones, cudaMemcpyHostToDevice) );
free(aus);
assert(buf == bufCheck + serialLength);
return r;
}
src/NetworkViz.cpp
+15 -380
View File
@@ -6,389 +6,23 @@
namespace tk { namespace dnn {
cv::Mat mapillary_15_map(cv::Mat adjMap){
// cv::imshow("test", adjMap);
// cv::waitKey(0);
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
//animal
M3.at<uchar>(0)=165;
M2.at<uchar>(0)=42;
M1.at<uchar>(0)=45;
//curb
M3.at<uchar>(1)=196;
M2.at<uchar>(1)=196;
M1.at<uchar>(1)=196;
//barrier
M3.at<uchar>(2)=90;
M2.at<uchar>(2)=120;
M1.at<uchar>(2)=150;
//road
M3.at<uchar>(3)=128;
M2.at<uchar>(3)=64;
M1.at<uchar>(3)=128;
//building
M3.at<uchar>(4)=70;
M2.at<uchar>(4)=70;
M1.at<uchar>(4)=70;
//person
M3.at<uchar>(5)=220;
M2.at<uchar>(5)=20;
M1.at<uchar>(5)=60;
//roadmark
M3.at<uchar>(6)=255;
M2.at<uchar>(6)=255;
M1.at<uchar>(6)=255;
//nature
M3.at<uchar>(7)=107;
M2.at<uchar>(7)=142;
M1.at<uchar>(7)=35;
//sky
M3.at<uchar>(8)=70;
M2.at<uchar>(8)=130;
M1.at<uchar>(8)=180;
//billboard
M3.at<uchar>(9)=220;
M2.at<uchar>(9)=220;
M1.at<uchar>(9)=220;
//pole
M3.at<uchar>(10)=153;
M2.at<uchar>(10)=153;
M1.at<uchar>(10)=153;
//traffic sign
M3.at<uchar>(11)=128;
M2.at<uchar>(11)=128;
M1.at<uchar>(11)=128;
//bike
M3.at<uchar>(12)=119;
M2.at<uchar>(12)=11;
M1.at<uchar>(12)=32;
//vehicle
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
//void
for(int i=14;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat berkeley_20_map(cv::Mat adjMap){
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
//road
M3.at<uchar>(0)=128;
M2.at<uchar>(0)=64;
M1.at<uchar>(0)=128;
//sidewalk
M3.at<uchar>(1)=244;
M2.at<uchar>(1)=35;
M1.at<uchar>(1)=232;
//building
M3.at<uchar>(2)=70;
M2.at<uchar>(2)=70;
M1.at<uchar>(2)=70;
//wall
M3.at<uchar>(3)=102;
M2.at<uchar>(3)=102;
M1.at<uchar>(3)=156;
//fence
M3.at<uchar>(4)=90;
M2.at<uchar>(4)=120;
M1.at<uchar>(4)=150;
//pole
M3.at<uchar>(5)=153;
M2.at<uchar>(5)=153;
M1.at<uchar>(5)=153;
//traffic light
M3.at<uchar>(6)=250;
M2.at<uchar>(6)=170;
M1.at<uchar>(6)=30;
//traffic sign
M3.at<uchar>(7)=128;
M2.at<uchar>(7)=128;
M1.at<uchar>(7)=128;
//nature
M3.at<uchar>(8)=107;
M2.at<uchar>(8)=142;
M1.at<uchar>(8)=35;
//ground
M3.at<uchar>(9)=0;
M2.at<uchar>(9)=192;
M1.at<uchar>(9)=0;
//sky
M3.at<uchar>(10)=70;
M2.at<uchar>(10)=130;
M1.at<uchar>(10)=180;
//person
M3.at<uchar>(11)=220;
M2.at<uchar>(11)=20;
M1.at<uchar>(11)=60;
//rider
M3.at<uchar>(12)=255;
M2.at<uchar>(12)=0;
M1.at<uchar>(12)=100;
//car
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
//truck
M3.at<uchar>(14)=0;
M2.at<uchar>(14)=0;
M1.at<uchar>(14)=70;
//bus
M3.at<uchar>(15)=0;
M2.at<uchar>(15)=60;
M1.at<uchar>(15)=100;
//train
M3.at<uchar>(16)=0;
M2.at<uchar>(16)=0;
M1.at<uchar>(16)=192;
//motorbike
M3.at<uchar>(17)=0;
M2.at<uchar>(17)=0;
M1.at<uchar>(17)=230;
//bike
M3.at<uchar>(18)=119;
M2.at<uchar>(18)=11;
M1.at<uchar>(18)=32;
//void
for(int i=19;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat cityscapes_19_map(cv::Mat adjMap){
cv::Mat M1(1, 256, CV_8UC1), M2(1, 256, CV_8UC1), M3(1, 256, CV_8UC1);
//road
M3.at<uchar>(0)=128;
M2.at<uchar>(0)=64;
M1.at<uchar>(0)=128;
//sidewalk
M3.at<uchar>(1)=244;
M2.at<uchar>(1)=35;
M1.at<uchar>(1)=232;
//building
M3.at<uchar>(2)=70;
M2.at<uchar>(2)=70;
M1.at<uchar>(2)=70;
//wall
M3.at<uchar>(3)=102;
M2.at<uchar>(3)=102;
M1.at<uchar>(3)=156;
//fence
M3.at<uchar>(4)=190;
M2.at<uchar>(4)=153;
M1.at<uchar>(4)=153;
//pole
M3.at<uchar>(5)=153;
M2.at<uchar>(5)=153;
M1.at<uchar>(5)=153;
//traffic light
M3.at<uchar>(6)=250;
M2.at<uchar>(6)=170;
M1.at<uchar>(6)=30;
//traffic sign
M3.at<uchar>(7)=220;
M2.at<uchar>(7)=220;
M1.at<uchar>(7)=0;
//vegetation
M3.at<uchar>(8)=107;
M2.at<uchar>(8)=142;
M1.at<uchar>(8)=35;
//terrain
M3.at<uchar>(9)=152;
M2.at<uchar>(9)=251;
M1.at<uchar>(9)=152;
//sky
M3.at<uchar>(10)=70;
M2.at<uchar>(10)=130;
M1.at<uchar>(10)=180;
//person
M3.at<uchar>(11)=220;
M2.at<uchar>(11)=20;
M1.at<uchar>(11)=60;
//rider
M3.at<uchar>(12)=255;
M2.at<uchar>(12)=0;
M1.at<uchar>(12)=0;
//car
M3.at<uchar>(13)=0;
M2.at<uchar>(13)=0;
M1.at<uchar>(13)=142;
//truck
M3.at<uchar>(14)=0;
M2.at<uchar>(14)=0;
M1.at<uchar>(14)=70;
//bus
M3.at<uchar>(15)=0;
M2.at<uchar>(15)=60;
M1.at<uchar>(15)=100;
//train
M3.at<uchar>(16)=0;
M2.at<uchar>(16)=80;
M1.at<uchar>(16)=100;
//motorcycle
M3.at<uchar>(17)=0;
M2.at<uchar>(17)=0;
M1.at<uchar>(17)=230;
//bicycle
M3.at<uchar>(18)=119;
M2.at<uchar>(18)=11;
M1.at<uchar>(18)=32;
//void
for(int i=19;i<256;i++)
{
M1.at<uchar>(i)=0;
M2.at<uchar>(i)=0;
M3.at<uchar>(i)=0;
}
cv::Mat r1,r2,r3;
cv::LUT(adjMap,M1,r1);
cv::LUT(adjMap,M2,r2);
cv::LUT(adjMap,M3,r3);
std::vector<cv::Mat> planes;
planes.push_back(r1);
planes.push_back(r2);
planes.push_back(r3);
cv::Mat dst;
cv::merge(planes,dst);
return dst;
}
cv::Mat vizFloat2colorMap(cv::Mat map,double min, double max, int classes) {
if(min == 0 && max == 0)
cv::minMaxIdx(map, &min, &max);
cv::Mat vizFloat2colorMap(cv::Mat map) {
double min;
double max;
cv::minMaxIdx(map, &min, &max);
cv::Mat adjMap;
// expand your range to 0..255. Similar to histEq();
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
//return adjMap;
cv::Mat falseColorsMap;
switch (classes)
{
case 15:
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = mapillary_15_map(adjMap);
break;
case 20:
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = berkeley_20_map(adjMap);
break;
case 19:
map.convertTo(adjMap,CV_8UC1);
falseColorsMap = cityscapes_19_map(adjMap);
break;
default:
// expand your range to 0..255. Similar to histEq();
map.convertTo(adjMap,CV_8UC1, 255 / (max-min), -min);
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_JET);
}
applyColorMap(adjMap, falseColorsMap, cv::COLORMAP_HOT);
return falseColorsMap;
}
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int img_w, double min, double max, int classes) {
cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int imgdim) {
dnnType *data = nullptr;
// copy to CPU
@@ -404,13 +38,14 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int i
cv::Mat grid = cv::Mat(gridSize, CV_8UC3, cv::Scalar(0));
for(int i=0; i<dim.c;i++) {
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i), min, max, classes);
cv::Mat raw = vizFloat2colorMap(cv::Mat(cv::Size(dim.w, dim.h),CV_32FC1, data + dim.w*dim.h*i));
int r = i / gridDim;
int c = i - r * gridDim;
raw.copyTo(grid.rowRange(r*dim.h, r*dim.h + dim.h).colRange(c*dim.w, c*dim.w + dim.w));
}
cv::Size vdim(img_w, img_h);
float ar = float(dim.w)/dim.h;
cv::Size vdim(ar*imgdim, imgdim);
cv::Mat viz;
cv::resize(grid, viz, vdim, 0, 0, 0);
@@ -424,7 +59,7 @@ cv::Mat vizData2Mat(dnnType *dataInput, tk::dnn::dataDim_t dim, int img_h, int i
cv::Mat vizLayer2Mat(tk::dnn::Network *net, int layer, int imgdim) {
if(layer >= net->num_layers)
FatalError("Could not viz layer\n");
return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim, imgdim);
return vizData2Mat(net->layers[layer]->dstData, net->layers[layer]->output_dim, imgdim);
//cv::imwrite("viz/layer" + std::to_string(layer) + ".png", viz);
//cv::imshow("layer", viz);
src/Region.cpp
+1 -1
View File
@@ -63,7 +63,7 @@ dnnType* Region::infer(dataDim_t &dim, dnnType* srcData) {
}
/* Interpret class */
/* Intepret class */
RegionInterpret::RegionInterpret(dataDim_t input_dim, dataDim_t output_dim,
int classes, int coords, int num, float thresh, std::string fname_weights) {
src/Reshape.cpp
-5
View File
@@ -15,11 +15,6 @@ Reshape::Reshape(Network *net, dataDim_t new_dim) : Layer(net) {
output_dim.w = new_dim.w;
output_dim.l = new_dim.l;
output_dim = new_dim;
if(input_dim.tot() != output_dim.tot())
FatalError("Reshape dimension mismatch");
}
Reshape::~Reshape() {
src/Resize.cpp
-39
View File
@@ -1,39 +0,0 @@
#include <iostream>
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
Resize::Resize(Network *net, int scale_c, int scale_h, int scale_w, bool fixed, ResizeMode_t mode) : Layer(net) {
this->mode = mode;
if(fixed){
output_dim.c = scale_c;
output_dim.h = scale_h;
output_dim.w = scale_w;
}
else{
output_dim.c *= scale_c;
output_dim.h *= scale_h;
output_dim.w *= scale_w;
}
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
}
Resize::~Resize() {
checkCuda( cudaFree(dstData) );
}
dnnType* Resize::infer(dataDim_t &dim, dnnType* srcData) {
resizeForward(srcData, dstData, dim.n, dim.c, dim.h, dim.w,
output_dim.c, output_dim.h, output_dim.w);
dim = output_dim;
return dstData;
}
}}
src/Shortcut.cpp
+5 -6
View File
@@ -5,16 +5,15 @@
namespace tk { namespace dnn {
Shortcut::Shortcut(Network *net, Layer *backLayer, bool mul) : Layer(net) {
Shortcut::Shortcut(Network *net, Layer *backLayer) : Layer(net) {
this->backLayer = backLayer;
this->mul = mul;
checkCuda( cudaMalloc(&dstData, output_dim.tot()*sizeof(dnnType)) );
if( ( backLayer->output_dim.c != input_dim.c && mul ) ||
(( backLayer->output_dim.w != input_dim.w || backLayer->output_dim.h != input_dim.h ) && !mul ) )
if( /*backLayer->output_dim.c != input_dim.c ||*/
backLayer->output_dim.w != input_dim.w ||
backLayer->output_dim.h != input_dim.h )
FatalError("Shortcut dim missmatch");
}
Shortcut::~Shortcut() {
@@ -27,7 +26,7 @@ dnnType* Shortcut::infer(dataDim_t &dim, dnnType* srcData) {
dataDim_t bdim = this->backLayer->output_dim;
checkCuda(cudaMemcpy(dstData, srcData, dim.tot()*sizeof(dnnType), cudaMemcpyDeviceToDevice));
shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1, mul);
shortcutForward(this->backLayer->dstData, dstData, dim.n, dim.c, dim.h, dim.w, 1, bdim.n, bdim.c, bdim.h, bdim.w, 1);
//update data dimensions
dim = output_dim;
src/Yolo.cpp
+18 -61
View File
@@ -9,10 +9,9 @@
#include "Layer.h"
#include "kernels.h"
namespace tk { namespace dnn {
Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy, double nms_thresh, nmsKind_t nsm_kind, int new_coords) :
Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_masks, float scale_xy) :
Layer(net) {
this->final = true;
@@ -20,9 +19,6 @@ Yolo::Yolo(Network *net, int classes, int num, std::string fname_weights, int n_
this->num = num;
this->n_masks = n_masks;
this->scaleXY = scale_xy;
this->nms_thresh = nms_thresh;
this->nsm_kind = nsm_kind;
this->new_coords = new_coords;
// load anchors
if(fname_weights != "") {
@@ -63,21 +59,12 @@ int entry_index(int batch, int location, int entry,
entry*input_dim.w*input_dim.h + loc;
}
Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride, int new_coords) {
Yolo::box get_yolo_box(float *x, float *biases, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride) {
Yolo::box b;
if(new_coords == 0){
b.x = (i + x[index + 0*stride]) / lw;
b.y = (j + x[index + 1*stride]) / lh;
b.w = exp(x[index + 2*stride]) * biases[2*n] / w;
b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
}
else{
b.x = (i + x[index + 0 * stride] ) / lw;
b.y = (j + x[index + 1 * stride] ) / lh;
b.w = x[index + 2 * stride] * x[index + 2 * stride] * 4 * biases[2 * n] / w;
b.h = x[index + 3 * stride] * x[index + 3 * stride] * 4 * biases[2 * n + 1] / h;
}
b.x = (i + x[index + 0*stride]) / lw;
b.y = (j + x[index + 1*stride]) / lh;
b.w = exp(x[index + 2*stride]) * biases[2*n] / w;
b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
return b;
}
@@ -88,16 +75,12 @@ dnnType* Yolo::infer(dataDim_t &dim, dnnType* srcData) {
for (int b = 0; b < dim.n; ++b){
for(int n = 0; n < n_masks; ++n){
int index = entry_index(b, n*dim.w*dim.h, 0, classes, input_dim, output_dim);
if (new_coords == 1){
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
}
else{
activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
activationLOGISTICForward(srcData + index, dstData + index, 2*dim.w*dim.h);
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h);
}
if (this->scaleXY != 1) scalAdd(dstData + index, 2 * dim.w*dim.h, this->scaleXY, -0.5*(this->scaleXY - 1), 1);
index = entry_index(b, n*dim.w*dim.h, 4, classes, input_dim, output_dim);
activationLOGISTICForward(srcData + index, dstData + index, (1+classes)*dim.w*dim.h);
}
}
@@ -133,7 +116,7 @@ void correct_yolo_boxes(Yolo::detection *dets, int n, int w, int h, int netw, in
}
}
int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh, int new_coords) {
int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int neth, float thresh) {
if(predictions == nullptr)
predictions = new dnnType[output_dim.tot()];
@@ -157,7 +140,7 @@ int Yolo::computeDetections(Yolo::detection *dets, int &ndets, int netw, int net
if(objectness <= thresh) continue;
int box_index = entry_index(0, n*lw*lh + i, 0, classes, input_dim, output_dim);
dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh, new_coords);
dets[count].bbox = get_yolo_box(predictions, bias_h, mask_h[n], box_index, col, row, lw, lh, netw, neth, lw*lh);
dets[count].objectness = objectness;
dets[count].classes = classes;
for(j = 0; j < classes; ++j){
@@ -210,32 +193,6 @@ float yolo_box_iou(Yolo::box a, Yolo::box b)
return yolo_box_intersection(a, b)/yolo_box_union(a, b);
}
void box_c(const Yolo::box a, const Yolo::box b, float& top, float& bot, float& left, float& right) {
top = (std::min)(a.y - a.h / 2, b.y - b.h / 2);
bot = (std::max)(a.y + a.h / 2, b.y + b.h / 2);
left = (std::min)(a.x - a.w / 2, b.x - b.w / 2);
right = (std::max)(a.x + a.w / 2, b.x + b.w / 2);
}
// https://github.com/Zzh-tju/DIoU-darknet
// https://arxiv.org/abs/1911.08287
float yolo_box_diou(const Yolo::box a, const Yolo::box b, const float nms_thresh=0.6)
{
float top, bot, left, right;
box_c(a, b, top, bot, left, right);
float w = right - left;
float h = bot - top;
float c = w * w + h * h;
float iou = yolo_box_iou(a, b);
if (c == 0)
return iou;
float d = (a.x - b.x) * (a.x - b.x) + (a.y - b.y) * (a.y - b.y);
float u = pow(d / c, nms_thresh);
float diou_term = u;
return iou - diou_term;
}
int yolo_nms_comparator(const void *pa, const void *pb)
{
Yolo::detection a = *(Yolo::detection *)pa;
@@ -262,7 +219,8 @@ Yolo::detection *Yolo::allocateDetections(int nboxes, int classes) {
return dets;
}
void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double nms_thresh, nmsKind_t nsm_kind) {
void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes) {
double nms_thresh = 0.45;
int total = ndets;
int i, j, k;
@@ -278,7 +236,6 @@ void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double
}
total = k+1;
float thresh = 0.45f;
for(k = 0; k < classes; ++k){
for(i = 0; i < total; ++i){
dets[i].sort_class = k;
@@ -289,13 +246,13 @@ void Yolo::mergeDetections(Yolo::detection *dets, int ndets, int classes, double
box a = dets[i].bbox;
for(j = i+1; j < total; ++j){
box b = dets[j].bbox;
if (nsm_kind == GREEDY_NMS && yolo_box_iou(a, b) > thresh)
dets[j].prob[k] = 0;
else if (nsm_kind == DIOU_NMS && yolo_box_diou(a, b, nms_thresh) > thresh)
if (yolo_box_iou(a, b) > nms_thresh){
dets[j].prob[k] = 0;
}
}
}
}
}
}}
src/Yolo3Detection.cpp
+112 -41
View File
@@ -3,14 +3,13 @@
namespace tk { namespace dnn {
bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches, const float conf_thresh) {
bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, const int n_batches) {
//convert network to tensorRT
std::cout<<(tensor_path).c_str()<<"\n";
netRT = new tk::dnn::NetworkRT(NULL, (tensor_path).c_str() );
nBatches = n_batches;
confThreshold = conf_thresh;
tk::dnn::dataDim_t idim = netRT->input_dim;
idim.n = nBatches;
@@ -32,9 +31,6 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
memcpy(yolo[i]->bias_h, yRT->bias, sizeof(dnnType)*num*nMasks*2);
yolo[i]->input_dim = yolo[i]->output_dim = tk::dnn::dataDim_t(1, yRT->c, yRT->h, yRT->w);
yolo[i]->classesNames = yRT->classesNames;
yolo[i]->nms_thresh = yRT->nms_thresh;
yolo[i]->nsm_kind = (tk::dnn::Yolo::nmsKind_t) yRT->nms_kind;
yolo[i]->new_coords = yRT->new_coords;
}
dets = tk::dnn::Yolo::allocateDetections(tk::dnn::Yolo::MAX_DETECTIONS, classes);
@@ -56,28 +52,80 @@ bool Yolo3Detection::init(const std::string& tensor_path, const int n_classes, c
return true;
}
void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
#ifdef OPENCV_CUDACONTRIB
cv::cuda::GpuMat orig_img, img_resized;
orig_img = cv::cuda::GpuMat(frame);
cv::cuda::resize(orig_img, img_resized, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
img_resized.convertTo(imagePreproc, CV_32FC3, 1/255.0);
//split channels
cv::cuda::split(imagePreproc,bgr);//split source
//write channels
for(int i=0; i<netRT->input_dim.c; i++) {
int size = imagePreproc.rows * imagePreproc.cols;
int ch = netRT->input_dim.c-1 -i;
bgr[ch].download(bgr_h); //TODO: don't copy back on CPU
checkCuda( cudaMemcpy(input_d + i*size + netRT->input_dim.tot()*bi, (float*)bgr_h.data, size*sizeof(dnnType), cudaMemcpyHostToDevice));
cv::Mat resize_image(cv::Mat im, int w, int h)
{
cv::Mat resized = cv::Mat(cv::Size(w,h), CV_32FC3, cv::Scalar(0) );
cv::Mat part = cv::Mat(cv::Size(w,im.rows), CV_32FC3, cv::Scalar(0) );
int r, c, k;
float w_scale = (float)(im.cols - 1) / (w - 1);
float h_scale = (float)(im.rows - 1) / (h - 1);
for(k = 0; k < im.channels(); ++k){
for(r = 0; r < im.rows; ++r){
for(c = 0; c < w; ++c){
float val = 0;
if(c == w-1 || im.cols == 1){
val = im.at<cv::Vec3f>(r, im.cols-1)[k];
} else {
float sx = c*w_scale;
int ix = (int) sx;
float dx = sx - ix;
val = (1 - dx) * im.at<cv::Vec3f>(r, ix)[k] + dx * im.at<cv::Vec3f>(r,ix+1)[k];
}
part.at<cv::Vec3f>(r,c)[k] = val;
}
}
}
#else
cv::resize(frame, frame, cv::Size(netRT->input_dim.w, netRT->input_dim.h));
for(k = 0; k < im.channels(); ++k){
for(r = 0; r < h; ++r){
float sy = r*h_scale;
int iy = (int) sy;
float dy = sy - iy;
for(c = 0; c < w; ++c){
float val = (1-dy) * part.at<cv::Vec3f>(iy, c)[k];
resized.at<cv::Vec3f>(r, c)[k] = val;
}
if(r == h-1 || im.rows == 1) continue;
for(c = 0; c < w; ++c){
float val = dy * part.at<cv::Vec3f>(iy+1, c)[k];
resized.at<cv::Vec3f>(r,c)[k] += val;
}
}
}
return resized;
}
void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
frame.convertTo(imagePreproc, CV_32FC3, 1/255.0);
if(letterbox){
int im_w = frame.cols;
int im_h = frame.rows;
int net_w = netRT->input_dim.w;
int net_h = netRT->input_dim.h;
if(net_w == net_h && letterbox){
float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
int new_h = im_h/ratio;
int new_w = im_w/ratio;
imagePreproc = resize_image(imagePreproc, new_w, new_h);
cv::Mat borders;
int top = (net_h - new_h)/2;
int bottom = (net_h - new_h) - top;
int left = (net_w - new_w)/2;
int right = (net_w - new_w) - left;
cv::copyMakeBorder(imagePreproc,imagePreproc, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0.5,0.5,0.5));
}
else
FatalError("letterbox not spported with h!=w");
}
else
imagePreproc = resize_image(imagePreproc, netRT->input_dim.w, netRT->input_dim.h);
//split channels
cv::split(imagePreproc,bgr);//split source
@@ -88,16 +136,14 @@ void Yolo3Detection::preprocess(cv::Mat &frame, const int bi){
memcpy((void*)&input[idx + netRT->input_dim.tot()*bi], (void*)bgr[ch].data, imagePreproc.rows*imagePreproc.cols*sizeof(dnnType));
}
checkCuda(cudaMemcpyAsync(input_d + netRT->input_dim.tot()*bi, input + netRT->input_dim.tot()*bi, netRT->input_dim.tot()*sizeof(dnnType), cudaMemcpyHostToDevice, netRT->stream));
#endif
}
void Yolo3Detection::postprocess(const int bi, const bool mAP){
//get yolo outputs
std::vector<float *> rt_out;
//dnnType *rt_out[netRT->pluginFactory->n_yolos];
for(int i=0; i<netRT->pluginFactory->n_yolos; i++)
rt_out.push_back((dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi);
dnnType *rt_out[netRT->pluginFactory->n_yolos];
for(int i=0; i<netRT->pluginFactory->n_yolos; i++)
rt_out[i] = (dnnType*)netRT->buffersRT[i+1] + netRT->buffersDIM[i+1].tot()*bi;
float x_ratio = float(originalSize[bi].width) / float(netRT->input_dim.w);
float y_ratio = float(originalSize[bi].height) / float(netRT->input_dim.h);
@@ -106,18 +152,49 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
nDets = 0;
for(int i=0; i<netRT->pluginFactory->n_yolos; i++) {
yolo[i]->dstData = rt_out[i];
yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold, yolo[i]->new_coords);
yolo[i]->computeDetections(dets, nDets, netRT->input_dim.w, netRT->input_dim.h, confThreshold);
}
tk::dnn::Yolo::mergeDetections(dets, nDets, classes, yolo[0]->nms_thresh, yolo[0]->nsm_kind);
tk::dnn::Yolo::mergeDetections(dets, nDets, classes);
int im_w = originalSize[bi].width;
int im_h = originalSize[bi].height;
int net_w = netRT->input_dim.w;
int net_h = netRT->input_dim.h;
int new_h, new_w;
int top = 0, left = 0;
if(letterbox){
float ratio = ( im_w > im_h ) ? float(im_w)/float(net_w) : float(im_h)/float(net_h);
x_ratio = ratio;
y_ratio = ratio;
std::cout<<ratio<<std::endl;
int new_h = im_h/ratio;
int new_w = im_w/ratio;
top = (net_h - new_h)/2;
left = (net_w - new_w)/2;
}
else{
new_h = net_h;
new_w = net_w;
}
float deltaw = net_w - new_w;
float deltah = net_h - new_h;
float ratiow = (float)new_w / net_w;
float ratioh = (float)new_h / net_h;
// fill detected
detected.clear();
for(int j=0; j<nDets; j++) {
tk::dnn::Yolo::box b = dets[j].bbox;
float x0 = (b.x-b.w/2.);
float x1 = (b.x+b.w/2.);
float y0 = (b.y-b.h/2.);
float y1 = (b.y+b.h/2.);
float x0 = (b.x - left - b.w/2.);
float x1 = (b.x - left + b.w/2.);
float y0 = (b.y - top - b.h/2.);
float y1 = (b.y - top + b.h/2.);
// convert to image coords
x0 = x_ratio*x0;
@@ -138,15 +215,9 @@ void Yolo3Detection::postprocess(const int bi, const bool mAP){
res.w = x1 - x0;
res.h = y1 - y0;
// FIXME: this shuld be useless
// if(mAP)
// for(int c=0; c<classes; c++)
// res.probs.push_back(dets[j].prob[c]);
detected.push_back(res);
}
}
}
batchDetected.push_back(detected);
}
src/demo_utils.cpp
-19
View File
@@ -1,19 +0,0 @@
#include "demo_utils.h"
void readCalibrationMatrix(const std::string& path, cv::Mat& calib_mat){
YAML::Node config = YAML::LoadFile(path);
//read camera matrix
int rows = config["camera_matrix"]["rows"].as<int>();
int cols = config["camera_matrix"]["cols"].as<int>();
cv::Mat calib = cv::Mat(cv::Size(rows, cols), CV_32F);
float *vals = (float *)calib.data;
for(int i=0; i < config["camera_matrix"]["data"].size(); ++i )
vals[i] = config["camera_matrix"]["data"][i].as<float>();
calib_mat = calib;
}
src/evaluation.cpp
+7 -5
View File
@@ -63,7 +63,7 @@ double computeMap( std::vector<Frame> &images,const int classes,
int gt_checked = 0;
// for each detection compute IoU with groundtruth and match detetcion and
// for each detection comput IoU with groundtruth and match detetcion and
// groundtruth with IoU greater than IoU_thresh
for(auto &img:images){
for(size_t i=0; i<img.det.size(); i++){
@@ -153,7 +153,7 @@ double computeMap( std::vector<Frame> &images,const int classes,
}
}
//compute average precision for each class. Two methods are available,
//compute average precision for each class. Two methods are avaible,
//based on map_points required
double mean_average_precision = 0;
double last_recall, last_precision, delta_recall;
@@ -287,7 +287,7 @@ void computeTPFPFN( std::vector<Frame> &images,const int classes,
}
}
//count all TP, FP, FN and compute precision, recall and f1-score
//count all TP, FP, FN and compute precsion, recall and f1-score
double avg_precision = 0, avg_recall = 0, f1_score = 0;
int TP = 0, FP = 0, FN = 0;
for(size_t i=0; i<classes; i++){
@@ -342,14 +342,16 @@ void printJsonCOCOFormat(std::ofstream *out_file, const std::string image_path,
//min threshold confidence is set in DetectionNN.h
if (bbox[i].probs[j] > 0) {
*out_file << "{\"image_id\":" << image_id <<
*out_file << std::fixed << std::setprecision(6) <<
"{\"image_id\":" << image_id <<
", \"category_id\":" << coco_ids[j] <<
", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh <<
"], \"score\":" << bbox[i].probs[j] << "},\n";
}
}
else
*out_file << "{\"image_id\":" << image_id <<
*out_file << std::fixed << std::setprecision(6) <<
"{\"image_id\":" << image_id <<
", \"category_id\":" << coco_ids[bbox[i].cl] <<
", \"bbox\":[" << bx << ", " << by << ", " << bw << ", " << bh <<
"], \"score\":" << bbox[i].prob << "},\n";
src/kernels/activation_leaky.cu
+4 -4
View File
@@ -1,7 +1,7 @@
#include "kernels.h"
__global__
void activation_leaky(dnnType *input, dnnType *output, int size, float slope) {
void activation_leaky(dnnType *input, dnnType *output, int size) {
int i = blockDim.x*blockIdx.x + threadIdx.x;
@@ -9,7 +9,7 @@ void activation_leaky(dnnType *input, dnnType *output, int size, float slope) {
if (input[i]>0)
output[i] = input[i];
else
output[i] = slope*input[i];
output[i] = 0.1f*input[i];
}
}
@@ -17,12 +17,12 @@ void activation_leaky(dnnType *input, dnnType *output, int size, float slope) {
/**
ELU activation function
*/
void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, float slope, cudaStream_t stream)
void activationLEAKYForward(dnnType* srcData, dnnType* dstData, int size, cudaStream_t stream)
{
int blocks = (size+255)/256;
int threads = 256;
activation_leaky<<<blocks, threads, 0, stream>>>(srcData, dstData, size, slope);
activation_leaky<<<blocks, threads, 0, stream>>>(srcData, dstData, size);
}
src/kernels/deformable_conv.cu
+1 -1
View File
@@ -18,7 +18,7 @@ inline int GET_BLOCKS(const int N)
}
__device__ __host__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width,
__device__ float dmcn_im2col_bilinear(const float *bottom_data, const int data_width,
const int height, const int width, float h, float w) {
int h_low = floor(h);
int w_low = floor(w);
src/kernels/postprocessing.cu
-38
View File
@@ -1,11 +1,5 @@
#include "kernelsThrust.h"
void transformDep(float *src_begin, float *src_end, float *dst_begin, float *dst_end) {
int e = exp(-6);
thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(e), dst_begin, thrust::plus<float>());
thrust::transform(thrust::device, src_begin, src_end, dst_begin, dst_begin, thrust::divides<float>());
thrust::transform(thrust::device, dst_begin, dst_end, thrust::make_constant_iterator(-1.0), dst_begin, thrust::plus<float>());
}
void subtractWithThreshold(dnnType *src_begin, dnnType *src_end, dnnType *src2_begin, dnnType *src_out, struct threshold op){
thrust::transform(thrust::device, src_begin, src_end, src2_begin, src_out, op);
@@ -40,30 +34,6 @@ void sortAndTopKonDevice(dnnType *src_begin, int *idsrc, float *topk_scores, int
sortAndTopK_kernel<<<blocks, threads, 0>>>(src_begin, idsrc, topk_scores, topk_inds, topk_ys, topk_xs, size, K);
}
__global__
void maxElem_kernel(float *src_begin, float *dst_begin, const int n_classes, const int size){
int i = blockDim.x*blockIdx.x + threadIdx.x;
if (i > size)
return;
float max = 0;
int max_idx = 0;
for( int j = i*n_classes; j < i*n_classes + n_classes; ++j ){
if( src_begin[j] > max ){
max = src_begin[j];
max_idx = j;
}
}
dst_begin[i] = max_idx - i*n_classes;
}
void maxElem(dnnType *src_begin, dnnType *dst_begin, const int c, const int h, const int w){
int blocks = (h*w)/32+1;
int threads = 32;
maxElem_kernel<<<blocks, threads, 0>>>(src_begin, dst_begin, c, h*w);
}
void topKxyclasses(int *ids_begin, int *ids_end, const int K, const int size, const int wh, int *clses, int *xs, int *ys){
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), clses, thrust::divides<int>());
thrust::transform(thrust::device, ids_begin, ids_end, thrust::make_constant_iterator(wh), ids_begin, thrust::modulus<int>());
@@ -81,14 +51,6 @@ void topKxyAddOffset(int * ids_begin, const int K, const int size,
thrust::transform(thrust::device, intys_begin, intys_begin + K, src_out, ys_begin, thrust::plus<float>());
}
void getRecordsFromTopKId(int * ids_begin, const int K, const int ch, const int size, dnnType *src_begin, float *src_out, int *ids_out) {
for(int i=0; i<ch; i++) {
// thrust::gather(thrust::device, ids_begin, ids_begin + K, src_begin, src_out);
thrust::transform(thrust::device, ids_begin, ids_begin + K, thrust::make_constant_iterator(i*size), ids_out, thrust::plus<int>());
thrust::gather(thrust::device, ids_out, ids_out + K, src_begin, src_out+i*K);
}
}
void bboxes(int * ids_begin, const int K, const int size, float *xs_begin, float *ys_begin,
dnnType *src_begin, float *bbx0, float *bbx1, float *bby0, float *bby1,
float *src_out, int *ids_out){
src/kernels/resize.cu
+27 -14
View File
@@ -1,33 +1,46 @@
#include "kernels.h"
#include <stdio.h>
#define MIN(a,b) (((a)<(b))?(a):(b))
#define MAX(a,b) (((a)>(b))?(a):(b))
__global__ void resize_kernel( int size,float *x, int i_w, int i_h, int i_c,
__global__ void resize_kernel( int i_N,float *x, int i_w, int i_h, int i_c,
int o_w, int o_h, int o_c, int batch, float *out)
{
int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
if(id >= size) return;
int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
if(i >= i_N) return;
int i = id % o_w;
id /= o_w;
int j = id % o_h;
id /= o_h;
int k = id % o_c;
id /= o_c;
int b = id % batch;
int out_index = i;
int out_w = i%o_w;
i = i/o_w;
int out_h = i%o_h;
i = i/o_h;
int out_c = i%o_c;
i = i/o_c;
int out_index = i + o_w*(j + o_h*(k + o_c*b));
int add_index = i/(o_w/i_w) + i_w*(j/(o_h/i_h) + i_h*(k + i_c*b));
out[out_index] = x[add_index];
//copying last column/last row as padding
int in_index = ((i*i_c + MIN(out_c,i_c-1))*i_h + MIN(out_h,i_h-1))*i_w + MIN(out_w, i_w-1);
out[out_index] = x[in_index];
}
void resizeForward( dnnType* srcData, dnnType* dstData, int n, int i_c, int i_h, int i_w,
int o_c, int o_h, int o_w, cudaStream_t stream )
{
int i_size = n*i_c*i_h*i_w;
int o_size = n*o_c*o_h*o_w;
int blocks = (o_size+255)/256;
int threads = 256;
resize_kernel<<<blocks, threads, 0, stream>>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData);
if(i_c == o_c && i_h == o_h && i_w == o_w )
{
checkCuda(cudaMemcpy(dstData, srcData, i_size*sizeof(dnnType), cudaMemcpyDeviceToDevice));
}
else
{
checkCuda(cudaMemset(dstData, 0, o_size*sizeof(dnnType)));
resize_kernel<<<blocks, threads, 0, stream>>>(o_size, srcData, i_w, i_h, i_c, o_w, o_h, o_c, n, dstData);
// printDeviceVector(i_size, srcData);
// printDeviceVector(o_size, dstData);
}
}
src/kernels/shortcut.cu
+15 -48
View File
@@ -21,60 +21,27 @@ __global__ void shortcut_kernel(int size, int minw, int minh, int minc, int stri
//out[out_index] += add[add_index];
}
__global__ void shortcut_mul_kernel(int size, int minw, int minh, int minc, int sample, int batch,
int w1, int h1, int c1, dnnType *mul,
int w2, int h2, int c2, float s1, float s2, dnnType *out)
{
int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
if (id >= size) return;
int i = id % minw;
id /= minw;
int j = id % minh;
id /= minh;
int k = id % minc;
id /= minc;
int b = id % batch;
int out_index = i*sample + w1*(j*sample + h1*(k + c1*b));
out[out_index] = out[out_index] * mul[k + c2*b];
}
void shortcutForward(dnnType* srcData, dnnType* dstData, int n1, int c1, int h1, int w1, int s1,
int n2, int c2, int h2, int w2, int s2,
bool mul, cudaStream_t stream)
cudaStream_t stream)
{
assert(n1 == n2);
int batch = n1;
if(!mul){
int minw = (w1 < w2) ? w1 : w2;
int minh = (h1 < h2) ? h1 : h2;
int minc = (c1 < c2) ? c1 : c2;
int stride = w1/w2;
int sample = w2/w1;
assert(stride == h1/h2);
assert(sample == h2/h1);
if(stride < 1) stride = 1;
if(sample < 1) sample = 1;
int minw = (w1 < w2) ? w1 : w2;
int minh = (h1 < h2) ? h1 : h2;
int minc = (c1 < c2) ? c1 : c2;
int size = batch * minw * minh * minc;
int blocks = (size+255)/256;
int threads = 256;
shortcut_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, stride, sample, batch,
w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
}
else{
int minw = w1;
int minh = h1;
int minc = c1;
int sample = 1;
int stride = w1/w2;
int sample = w2/w1;
assert(stride == h1/h2);
assert(sample == h2/h1);
if(stride < 1) stride = 1;
if(sample < 1) sample = 1;
int size = batch * minw * minh * minc;
int blocks = (size+255)/256;
int threads = 256;
shortcut_mul_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, sample, batch,
w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
}
int size = batch * minw * minh * minc;
int blocks = (size+255)/256;
int threads = 256;
shortcut_kernel<<<blocks, threads, 0, stream>>>(size, minw, minh, minc, stride, sample, batch,
w1, h1, c1, srcData, w2, h2, c2, s1, s2, dstData);
}
src/utils.cpp
+10 -26
View File
@@ -23,23 +23,14 @@ bool fileExist(const char *fname) {
void downloadWeightsifDoNotExist(const std::string& input_bin, const std::string& test_folder, const std::string& weights_url){
if(!fileExist(input_bin.c_str())){
std::string mkdir_cmd = "mkdir " + test_folder;
std::string wget_cmd = "curl " + weights_url + " --output " + test_folder + "/weights.zip";
#ifdef __linux__
std::string wget_cmd = "wget " + weights_url + " -O " + test_folder + "/weights.zip";
std::string unzip_cmd = "unzip " + test_folder + "/weights.zip -d" + test_folder;
std::string rm_cmd = "rm " + test_folder + "/weights.zip";
#elif _WIN32
std::string unzip_cmd = "7z x " + test_folder + "/weights.zip -o" + test_folder;
#endif
int err = 0;
err = system(mkdir_cmd.c_str());
err = system(wget_cmd.c_str());
err = system(unzip_cmd.c_str());
#ifdef __linux__
err = system(rm_cmd.c_str());
#endif
}
}
@@ -92,7 +83,7 @@ void printDeviceVector(int size, dnnType* vec_d, bool device){
delete [] vec;
}
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit, bool verbose) {
int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int limit) {
dnnType *data_h, *correct_h;
const float eps = 0.02f;
@@ -111,7 +102,6 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int
}
int diffs = 0;
for(int i=0; i<size; i++) {
// data_h[i] = data_h[i]*1e-2;
if(data_h[i] != data_h[i] || correct_h[i] != correct_h[i] || //nan control
fabs(data_h[i] - correct_h[i]) > eps) {
diffs += 1;
@@ -127,15 +117,13 @@ int checkResult(int size, dnnType *data_d, dnnType *correct_d, bool device, int
delete [] correct_h;
}
if(verbose){
std::cout<<" | ";
if(diffs == 0)
std::cout<<COL_GREENB<<"OK";
else
std::cout<<COL_REDB<<"Wrongs: "<<diffs;
std::cout<<" | ";
if(diffs == 0)
std::cout<<COL_GREENB<<"OK";
else
std::cout<<COL_REDB<<"Wrongs: "<<diffs;
std::cout<<COL_END<<" ~"<<eps<<"\n";
}
std::cout<<COL_END<<" ~"<<eps<<"\n";
return diffs;
}
@@ -203,12 +191,8 @@ void getMemUsage(double& vm_usage_kb, double& resident_set_kb){
>> O >> itrealvalue >> starttime >> vsize >> rss;
stat_stream.close();
#ifdef __linux__
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
#elif _WIN32
long page_size_kb = 4096/1024;
#endif
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
vm_usage_kb = vsize / 1024.0;
resident_set_kb = rss * page_size_kb;
}
@@ -243,4 +227,4 @@ void removePathAndExtension(const std::string &full_string, std::string &name){
name = name.substr(previous, current);
// std::cout<<"full string: "<<full_string<<" name: "<<name<<std::endl;
}
}
tests/centernet/dla34_cnet/dla34_cnet.cpp
-12
View File
@@ -479,18 +479,6 @@ int main()
//print network model
net.print();
// for(int i=0; i<net.num_layers; i++) {
// if(net.layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
// tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net.layers[i];
// c->releaseDevice();
// c->releaseHost(true, false);
// }
// if(net.layers[i]->dstData != nullptr) {
// cudaFree(net.layers[i]->dstData);
// net.layers[i]->dstData = nullptr;
// }
// }
//convert network to tensorRT
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet"));
tests/centernet/dla34_cnet3d/dla34_cnet3d.cpp
-562
View File
@@ -1,562 +0,0 @@
#include <iostream>
#include "tkdnn.h"
const char *input_bin = "dla34_cnet3d/debug/input.bin";
const char *conv1_bin = "dla34_cnet3d/layers/base-base_layer-0.bin";
const char *conv2_bin = "dla34_cnet3d/layers/base-level0-0.bin";
const char *conv3_bin = "dla34_cnet3d/layers/base-level1-0.bin";
// s - stage, t - tree
const char *s1_t1_conv1_bin = "dla34_cnet3d/layers/base-level2-tree1-conv1.bin";
const char *s1_t1_conv2_bin = "dla34_cnet3d/layers/base-level2-tree1-conv2.bin";
const char *s1_t1_project = "dla34_cnet3d/layers/base-level2-project-0.bin";
const char *s1_t2_conv1_bin = "dla34_cnet3d/layers/base-level2-tree2-conv1.bin";
const char *s1_t2_conv2_bin = "dla34_cnet3d/layers/base-level2-tree2-conv2.bin";
const char *s1_root_conv1_bin = "dla34_cnet3d/layers/base-level2-root-conv.bin";
const char *s2_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv1.bin";
const char *s2_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree1-conv2.bin";
const char *s2_t1_t1_project = "dla34_cnet3d/layers/base-level3-tree1-project-0.bin";
const char *s2_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv1.bin";
const char *s2_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree1-tree2-conv2.bin";
const char *s2_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree1-root-conv.bin";
const char *s2_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv1.bin";
const char *s2_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree1-conv2.bin";
const char *s2_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv1.bin";
const char *s2_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level3-tree2-tree2-conv2.bin";
const char *s2_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level3-tree2-root-conv.bin";
const char *s3_t1_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv1.bin";
const char *s3_t1_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree1-conv2.bin";
const char *s3_t1_t1_project = "dla34_cnet3d/layers/base-level4-tree1-project-0.bin";
const char *s3_t1_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv1.bin";
const char *s3_t1_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree1-tree2-conv2.bin";
const char *s3_t1_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree1-root-conv.bin";
const char *s3_t2_t1_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv1.bin";
const char *s3_t2_t1_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree1-conv2.bin";
const char *s3_t2_t2_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv1.bin";
const char *s3_t2_t2_conv2_bin = "dla34_cnet3d/layers/base-level4-tree2-tree2-conv2.bin";
const char *s3_t2_root_conv1_bin = "dla34_cnet3d/layers/base-level4-tree2-root-conv.bin";
const char *s4_t1_conv1_bin = "dla34_cnet3d/layers/base-level5-tree1-conv1.bin";
const char *s4_t1_conv2_bin = "dla34_cnet3d/layers/base-level5-tree1-conv2.bin";
const char *s4_t1_project = "dla34_cnet3d/layers/base-level5-project-0.bin";
const char *s4_t2_conv1_bin = "dla34_cnet3d/layers/base-level5-tree2-conv1.bin";
const char *s4_t2_conv2_bin = "dla34_cnet3d/layers/base-level5-tree2-conv2.bin";
const char *s4_root_conv1_bin = "dla34_cnet3d/layers/base-level5-root-conv.bin";
//final
// const char *fc_bin = "dla34_cnet3d/layers/output.bin";
const char *ida_0_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv.bin";
const char *ida_0_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin";
const char *ida_0_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_0-up_1.bin";
const char *ida_0_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv.bin";
const char *ida_0_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin";
const char *ida_1_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv.bin";
const char *ida_1_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin";
const char *ida_1_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_1.bin";
const char *ida_1_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv.bin";
const char *ida_1_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin";
const char *ida_1_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv.bin";
const char *ida_1_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin";
const char *ida_1_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_1-up_2.bin";
const char *ida_1_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv.bin";
const char *ida_1_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin";
const char *ida_2_p_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv.bin";
const char *ida_2_p_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin";
const char *ida_2_up_1_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_1.bin";
const char *ida_2_n_1_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv.bin";
const char *ida_2_n_1_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin";
const char *ida_2_p_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv.bin";
const char *ida_2_p_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin";
const char *ida_2_up_2_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_2.bin";
const char *ida_2_n_2_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv.bin";
const char *ida_2_n_2_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin";
const char *ida_2_p_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv.bin";
const char *ida_2_p_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin";
const char *ida_2_up_3_deconv_bin = "dla34_cnet3d/layers/dla_up-ida_2-up_3.bin";
const char *ida_2_n_3_dcn_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv.bin";
const char *ida_2_n_3_conv_bin = "dla34_cnet3d/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin";
const char *ida_up_p_1_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv.bin";
const char *ida_up_p_1_conv_bin = "dla34_cnet3d/layers/ida_up-proj_1-conv-conv_offset_mask.bin";
const char *ida_up_up_1_deconv_bin = "dla34_cnet3d/layers/ida_up-up_1.bin";
const char *ida_up_n_1_dcn_bin = "dla34_cnet3d/layers/ida_up-node_1-conv.bin";
const char *ida_up_n_1_conv_bin = "dla34_cnet3d/layers/ida_up-node_1-conv-conv_offset_mask.bin";
const char *ida_up_p_2_dcn_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv.bin";
const char *ida_up_p_2_conv_bin = "dla34_cnet3d/layers/ida_up-proj_2-conv-conv_offset_mask.bin";
const char *ida_up_up_2_deconv_bin = "dla34_cnet3d/layers/ida_up-up_2.bin";
const char *ida_up_n_2_dcn_bin = "dla34_cnet3d/layers/ida_up-node_2-conv.bin";
const char *ida_up_n_2_conv_bin = "dla34_cnet3d/layers/ida_up-node_2-conv-conv_offset_mask.bin";
const char *hm_conv1_bin = "dla34_cnet3d/layers/hm-0.bin";
const char *hm_conv2_bin = "dla34_cnet3d/layers/hm-2.bin";
const char *wh_conv1_bin = "dla34_cnet3d/layers/wh-0.bin";
const char *wh_conv2_bin = "dla34_cnet3d/layers/wh-2.bin";
const char *reg_conv1_bin = "dla34_cnet3d/layers/reg-0.bin";
const char *reg_conv2_bin = "dla34_cnet3d/layers/reg-2.bin";
const char *dep_conv1_bin = "dla34_cnet3d/layers/dep-0.bin";
const char *dep_conv2_bin = "dla34_cnet3d/layers/dep-2.bin";
const char *rot_conv1_bin = "dla34_cnet3d/layers/rot-0.bin";
const char *rot_conv2_bin = "dla34_cnet3d/layers/rot-2.bin";
const char *dim_conv1_bin = "dla34_cnet3d/layers/dim-0.bin";
const char *dim_conv2_bin = "dla34_cnet3d/layers/dim-2.bin";
const char *output_bin[]={
"dla34_cnet3d/debug/hm.bin",
"dla34_cnet3d/debug/wh.bin",
"dla34_cnet3d/debug/reg.bin",
"dla34_cnet3d/debug/dep.bin",
"dla34_cnet3d/debug/rot.bin",
"dla34_cnet3d/debug/dim.bin"};
int main()
{
downloadWeightsifDoNotExist(input_bin, "dla34_cnet3d", "https://cloud.hipert.unimore.it/s/2MDyWGzQsTKMjmR/download");
// Network layout
tk::dnn::dataDim_t dim(1, 3, 512, 512, 1);
tk::dnn::Network net(dim);
tk::dnn::Layer *last1, *last2, *last3, *last4;
tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2;
tk::dnn::Conv2d conv1(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true);
tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU);
base1 = &relu2;
tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true);
tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
base2 = &relu3;
// level 2
// tree 1
tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true);
tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true);
last2 = &s1_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s1_t1_layers[1] = { base2 };
tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1);
// downsample
tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
// project
tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true);
tk::dnn::Shortcut s1_t1_s1(&net, last2);
tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s1_t1_relu;
// tree 2
tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true);
tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true);
tk::dnn::Shortcut s1_t2_s1(&net, last1);
tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s1_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 };
tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2);
tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true);
tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU);
base3 = &s1_root_relu;
// level 3
// tree 1
// tree 1
tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true);
tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true);
last2 = &s2_t1_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 };
tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1);
// downsample
tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
last4 = &s2_t1_t1_maxpool1;
// project
tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true);
tk::dnn::Shortcut s2_t1_t1_s1(&net, last2);
tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s2_t1_t1_relu;
// tree 2
tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true);
tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true);
tk::dnn::Shortcut s2_t1_t2_s1(&net, last1);
tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s2_t1_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 };
tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2);
tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true);
tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s2_t1_root_relu;
last3 = &s2_t1_root_relu;
// tree 2
// tree 1
tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true);
tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true);
tk::dnn::Shortcut s2_t2_t1_s1(&net, last1);
tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s2_t2_t1_relu;
// tree 2
tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true);
tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true);
tk::dnn::Shortcut s2_t2_t2_s1(&net, last1);
tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s2_t2_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3};
tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4);
tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true);
tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
base4 = &s2_t2_root_relu;
// level 4
// tree 1
// tree 1
tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true);
tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true);
last2 = &s3_t1_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 };
tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1);
// downsample
tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
last4 = &s3_t1_t1_maxpool1;
// project
tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true);
tk::dnn::Shortcut s3_t1_t1_s1(&net, last2);
tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s3_t1_t1_relu;
// tree 2
tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true);
tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true);
tk::dnn::Shortcut s3_t1_t2_s1(&net, last1);
tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s3_t1_t2_relu;
// root
// join last1 and net in single input 256, 56, 56
tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 };
tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2);
tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true);
tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s3_t1_root_relu;
last3 = &s3_t1_root_relu;
// tree 2
// tree 1
tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true);
tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true);
tk::dnn::Shortcut s3_t2_t1_s1(&net, last1);
tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s3_t2_t1_relu;
// tree 2
tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true);
tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true);
tk::dnn::Shortcut s3_t2_t2_s1(&net, last1);
tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s3_t2_t2_relu;
// root
// join last1 and net in single input 256, 56, 56
tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3};
tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4);
tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true);
tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
base5 = &s3_t2_root_relu;
// level 5
// tree 1
tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true);
tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true);
last2 = &s4_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s4_t1_layers[1] = { base5 };
tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1);
// downsample
tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
last4 = &s4_t1_maxpool1;
// project
tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true);
tk::dnn::Shortcut s4_t1_s1(&net, last2);
tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s4_t1_relu;
// tree 2
tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true);
tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true);
tk::dnn::Shortcut s4_t2_s1(&net, last1);
tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s4_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 };
tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3);
tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true);
tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU);
base6 = &s4_root_relu;
//final
// tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
// tk::dnn::Dense fc(&net, 1000, fc_bin);
//ida 0
tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true);
tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256);
tk::dnn::Shortcut ida_0_shortcut(&net, base5);
tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true);
tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
ida1 = &ida_0_n_1_relu;
//ida1-1
tk::dnn::Layer *route_ida1_layers_1[1] = { base5 };
tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1);
tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true);
tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128);
tk::dnn::Shortcut ida_1_shortcut1(&net, base4);
tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true);
tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
ida2_1 = &ida_1_n_1_relu;
//ida1-2
tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 };
tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1);
tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true);
tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128);
tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1);
tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true);
tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
ida2_2 = &ida_1_n_2_relu;
//ida2-1
tk::dnn::Layer *route_ida2_layers_1[1] = { base4 };
tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1);
tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true);
tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64);
tk::dnn::Shortcut ida_2_shortcut1(&net, base3);
tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true);
tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
ida3_1 = &ida_2_n_1_relu;
//ida2-2
tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 };
tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1);
tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true);
tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64);
tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1);
tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true);
tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
ida3_2 = &ida_2_n_2_relu;
//ida2-3
tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 };
tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1);
tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true);
tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64);
tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2);
tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true);
tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU);
ida3_3 = &ida_2_n_3_relu;
//idaup-1
tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 };
tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1);
tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true);
tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64);
tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3);
tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true);
tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
idaup_1 = &idaup_n_1_relu;
//idaup-2
tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 };
tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1);
tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true);
tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64);
tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1);
tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true);
tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
idaup_2 = &idaup_n_2_relu;
tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 };
// hm
tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false);
hm->setFinal();
int kernel = 3;
int pad = (kernel - 1)/2;
tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
hmax->setFinal();
// wh
tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false);
wh->setFinal();
// reg
tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false);
tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false);
reg->setFinal();
// dep
tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false);
tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false);
dep->setFinal();
// rot
tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false);
tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false);
rot->setFinal();
// dim
tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false);
tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false);
dim_->setFinal();
// Load input
dnnType *data;
dnnType *input_h;
readBinaryFile(input_bin, dim.tot(), &input_h, &data);
//printDeviceVector(64, data, true);
//print network model
net.print();
//convert network to tensorRT
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_cnet3d"));
tk::dnn::dataDim_t dim1 = dim; //input dim
printCenteredTitle(" CUDNN inference ", '=', 30);
{
dim1.print();
TKDNN_TSTART
net.infer(dim1, data);
TKDNN_TSTOP
dim1.print();
}
tk::dnn::dataDim_t dim2 = dim;
printCenteredTitle(" TENSORRT inference ", '=', 30);
{
dim2.print();
TKDNN_TSTART
netRT.infer(dim2, data);
TKDNN_TSTOP
dim2.print();
}
tk::dnn::Layer *outs[6] = { hm, wh, reg, dep, rot, dim_ };
int out_count = 1;
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
for(int i=0; i<6; i++) {
printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
outs[i]->output_dim.print();
dnnType *out, *out_h;
int odim = outs[i]->output_dim.tot();
readBinaryFile(output_bin[i], odim, &out_h, &out);
dnnType *cudnn_out, *rt_out;
cudnn_out = outs[i]->dstData;
rt_out = (dnnType *)netRT.buffersRT[i+out_count];
// there is the maxpool. It isn't an output but it is necessary for the process section
if(i==0)
out_count ++;
std::cout<<"CUDNN vs correct";
ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
std::cout<<"TRT vs correct";
ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
std::cout<<"CUDNN vs TRT ";
ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
}
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
}
tests/centernet/resnet101_cnet/resnet101_cnet.cpp
-12
View File
@@ -353,18 +353,6 @@ int main()
//print network model
net.print();
// for(int i=0; i<net.num_layers; i++) {
// if(net.layers[i]->getLayerType() == tk::dnn::LAYER_CONV2D) {
// tk::dnn::Conv2d *c = (tk::dnn::Conv2d*) net.layers[i];
// c->releaseDevice();
// c->releaseHost(true, false);
// }
// if(net.layers[i]->dstData != nullptr) {
// cudaFree(net.layers[i]->dstData);
// net.layers[i]->dstData = nullptr;
// }
// }
//convert network to tensorRT
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("resnet101_cnet"));
tests/centertrack/dla34_ctrack/dla34_ctrack.cpp
-632
View File
@@ -1,632 +0,0 @@
#include <iostream>
#include "tkdnn.h"
const char *input_bin = "dla34_ctrack/debug/input_base-level0-0.bin";
// const char *input_bin = "dla34_ctrack/debug/input.bin";
// const char *pre_img_bin = "dla34_ctrack/debug/pre_imgages.bin";
// const char *pre_hm_bin = "dla34_ctrack/debug/pre_hms.bin";
// //pre
// const char *pre_img_conv1_bin = "dla34_ctrack/layers/base-pre_img_layer-0.bin";
// const char *pre_hm_conv1_bin = "dla34_ctrack/layers/base-pre_hm_layer-0.bin";
// const char *conv1_bin = "dla34_ctrack/layers/base-base_layer-0.bin";
const char *conv2_bin = "dla34_ctrack/layers/base-level0-0.bin";
const char *conv3_bin = "dla34_ctrack/layers/base-level1-0.bin";
// s - stage, t - tree
const char *s1_t1_conv1_bin = "dla34_ctrack/layers/base-level2-tree1-conv1.bin";
const char *s1_t1_conv2_bin = "dla34_ctrack/layers/base-level2-tree1-conv2.bin";
const char *s1_t1_project = "dla34_ctrack/layers/base-level2-project-0.bin";
const char *s1_t2_conv1_bin = "dla34_ctrack/layers/base-level2-tree2-conv1.bin";
const char *s1_t2_conv2_bin = "dla34_ctrack/layers/base-level2-tree2-conv2.bin";
const char *s1_root_conv1_bin = "dla34_ctrack/layers/base-level2-root-conv.bin";
const char *s2_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv1.bin";
const char *s2_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree1-conv2.bin";
const char *s2_t1_t1_project = "dla34_ctrack/layers/base-level3-tree1-project-0.bin";
const char *s2_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv1.bin";
const char *s2_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree1-tree2-conv2.bin";
const char *s2_t1_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree1-root-conv.bin";
const char *s2_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv1.bin";
const char *s2_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree1-conv2.bin";
const char *s2_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv1.bin";
const char *s2_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level3-tree2-tree2-conv2.bin";
const char *s2_t2_root_conv1_bin = "dla34_ctrack/layers/base-level3-tree2-root-conv.bin";
const char *s3_t1_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv1.bin";
const char *s3_t1_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree1-conv2.bin";
const char *s3_t1_t1_project = "dla34_ctrack/layers/base-level4-tree1-project-0.bin";
const char *s3_t1_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv1.bin";
const char *s3_t1_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree1-tree2-conv2.bin";
const char *s3_t1_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree1-root-conv.bin";
const char *s3_t2_t1_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv1.bin";
const char *s3_t2_t1_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree1-conv2.bin";
const char *s3_t2_t2_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv1.bin";
const char *s3_t2_t2_conv2_bin = "dla34_ctrack/layers/base-level4-tree2-tree2-conv2.bin";
const char *s3_t2_root_conv1_bin = "dla34_ctrack/layers/base-level4-tree2-root-conv.bin";
const char *s4_t1_conv1_bin = "dla34_ctrack/layers/base-level5-tree1-conv1.bin";
const char *s4_t1_conv2_bin = "dla34_ctrack/layers/base-level5-tree1-conv2.bin";
const char *s4_t1_project = "dla34_ctrack/layers/base-level5-project-0.bin";
const char *s4_t2_conv1_bin = "dla34_ctrack/layers/base-level5-tree2-conv1.bin";
const char *s4_t2_conv2_bin = "dla34_ctrack/layers/base-level5-tree2-conv2.bin";
const char *s4_root_conv1_bin = "dla34_ctrack/layers/base-level5-root-conv.bin";
//final
// const char *fc_bin = "dla34_ctrack/layers/output.bin";
const char *ida_0_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv.bin";
const char *ida_0_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-proj_1-conv-conv_offset_mask.bin";
const char *ida_0_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_0-up_1.bin";
const char *ida_0_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv.bin";
const char *ida_0_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_0-node_1-conv-conv_offset_mask.bin";
const char *ida_1_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv.bin";
const char *ida_1_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_1-conv-conv_offset_mask.bin";
const char *ida_1_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_1.bin";
const char *ida_1_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv.bin";
const char *ida_1_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_1-conv-conv_offset_mask.bin";
const char *ida_1_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv.bin";
const char *ida_1_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-proj_2-conv-conv_offset_mask.bin";
const char *ida_1_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_1-up_2.bin";
const char *ida_1_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv.bin";
const char *ida_1_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_1-node_2-conv-conv_offset_mask.bin";
const char *ida_2_p_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv.bin";
const char *ida_2_p_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_1-conv-conv_offset_mask.bin";
const char *ida_2_up_1_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_1.bin";
const char *ida_2_n_1_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv.bin";
const char *ida_2_n_1_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_1-conv-conv_offset_mask.bin";
const char *ida_2_p_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv.bin";
const char *ida_2_p_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_2-conv-conv_offset_mask.bin";
const char *ida_2_up_2_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_2.bin";
const char *ida_2_n_2_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv.bin";
const char *ida_2_n_2_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_2-conv-conv_offset_mask.bin";
const char *ida_2_p_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv.bin";
const char *ida_2_p_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-proj_3-conv-conv_offset_mask.bin";
const char *ida_2_up_3_deconv_bin = "dla34_ctrack/layers/dla_up-ida_2-up_3.bin";
const char *ida_2_n_3_dcn_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv.bin";
const char *ida_2_n_3_conv_bin = "dla34_ctrack/layers/dla_up-ida_2-node_3-conv-conv_offset_mask.bin";
const char *ida_up_p_1_dcn_bin = "dla34_ctrack/layers/ida_up-proj_1-conv.bin";
const char *ida_up_p_1_conv_bin = "dla34_ctrack/layers/ida_up-proj_1-conv-conv_offset_mask.bin";
const char *ida_up_up_1_deconv_bin = "dla34_ctrack/layers/ida_up-up_1.bin";
const char *ida_up_n_1_dcn_bin = "dla34_ctrack/layers/ida_up-node_1-conv.bin";
const char *ida_up_n_1_conv_bin = "dla34_ctrack/layers/ida_up-node_1-conv-conv_offset_mask.bin";
const char *ida_up_p_2_dcn_bin = "dla34_ctrack/layers/ida_up-proj_2-conv.bin";
const char *ida_up_p_2_conv_bin = "dla34_ctrack/layers/ida_up-proj_2-conv-conv_offset_mask.bin";
const char *ida_up_up_2_deconv_bin = "dla34_ctrack/layers/ida_up-up_2.bin";
const char *ida_up_n_2_dcn_bin = "dla34_ctrack/layers/ida_up-node_2-conv.bin";
const char *ida_up_n_2_conv_bin = "dla34_ctrack/layers/ida_up-node_2-conv-conv_offset_mask.bin";
const char *hm_conv1_bin = "dla34_ctrack/layers/hm-0.bin";
const char *hm_conv2_bin = "dla34_ctrack/layers/hm-2.bin";
const char *wh_conv1_bin = "dla34_ctrack/layers/wh-0.bin";
const char *wh_conv2_bin = "dla34_ctrack/layers/wh-2.bin";
const char *reg_conv1_bin = "dla34_ctrack/layers/reg-0.bin";
const char *reg_conv2_bin = "dla34_ctrack/layers/reg-2.bin";
const char *track_conv1_bin = "dla34_ctrack/layers/tracking-0.bin";
const char *track_conv2_bin = "dla34_ctrack/layers/tracking-2.bin";
const char *dep_conv1_bin = "dla34_ctrack/layers/dep-0.bin";
const char *dep_conv2_bin = "dla34_ctrack/layers/dep-2.bin";
const char *rot_conv1_bin = "dla34_ctrack/layers/rot-0.bin";
const char *rot_conv2_bin = "dla34_ctrack/layers/rot-2.bin";
const char *dim_conv1_bin = "dla34_ctrack/layers/dim-0.bin";
const char *dim_conv2_bin = "dla34_ctrack/layers/dim-2.bin";
const char *a_off_conv1_bin = "dla34_ctrack/layers/amodel_offset-0.bin";
const char *a_off_conv2_bin = "dla34_ctrack/layers/amodel_offset-2.bin";
const char *output_bin[]={
"dla34_ctrack/debug/hm.bin",
"dla34_ctrack/debug/wh.bin",
"dla34_ctrack/debug/reg.bin",
"dla34_ctrack/debug/tracking.bin",
"dla34_ctrack/debug/dep.bin",
"dla34_ctrack/debug/rot.bin",
"dla34_ctrack/debug/dim.bin",
"dla34_ctrack/debug/amodel_offset.bin"};
// const char *output_bin = "dla34_ctrack/debug/base-level0-2.bin";
int main()
{
downloadWeightsifDoNotExist("dla34_ctrack/debug/input.bin", "dla34_ctrack", "https://cloud.hipert.unimore.it/s/rjNfgGL9FtAXLHp/download");
// Network layout
// tk::dnn::dataDim_t dim_in0(1, 3, 512, 512, 1);
// tk::dnn::dataDim_t dim_in1(1, 1, 512, 512, 1);
tk::dnn::dataDim_t dim_in0(1, 16, 512, 512, 1);
// dnnType *i0_d, *i1_d, *i2_d;
// dnnType *i0_h, *i1_h, *i2_h;
// checkCuda( cudaMalloc(&i0_d, dim_in0.tot()*sizeof(dnnType)) );
// checkCuda( cudaMalloc(&i1_d, dim_in1.tot()*sizeof(dnnType)) );
// checkCuda( cudaMalloc(&i2_d, dim_in0.tot()*sizeof(dnnType)) );
tk::dnn::Network net(dim_in0);
tk::dnn::Layer *last1, *last2, *last3, *last4;
tk::dnn::Layer *base1, *base2, *base3, *base4, *base5, *base6, *ida1, *ida2_1, *ida2_2, *ida3_1, *ida3_2, *ida3_3, *idaup_1, *idaup_2;
// tk::dnn::Layer *pre_img, *pre_hm;
// //pre-img
// readBinaryFile(pre_img_bin, dim_in0.tot(), &i0_h, &i0_d);
// tk::dnn::Input *in_pre_img = new tk::dnn::Input(&net, dim_in0, i0_d);
// tk::dnn::Conv2d pre_img_conv1(&net, 16, 7, 7, 1, 1, 3, 3, pre_img_conv1_bin, true);
// tk::dnn::Activation pre_img_relu(&net, CUDNN_ACTIVATION_RELU);
// pre_img = &pre_img_relu;
// //pre-hm
// readBinaryFile(pre_hm_bin, dim_in1.tot(), &i1_h, &i1_d);
// tk::dnn::Input *in_pre_hm = new tk::dnn::Input(&net, dim_in1, i1_d);
// tk::dnn::Conv2d pre_hm_conv1(&net, 16, 7, 7, 1, 1, 3, 3, pre_hm_conv1_bin, true);
// tk::dnn::Activation pre_hm_relu(&net, CUDNN_ACTIVATION_RELU);
// pre_hm = &pre_hm_relu;
// // image input
// readBinaryFile(input_bin, dim_in0.tot(), &i2_h, &i2_d);
// tk::dnn::Input *input_image = new tk::dnn::Input(&net, dim_in0, i2_d);
// tk::dnn::Conv2d *conv1 = new tk::dnn::Conv2d(&net, 16, 7, 7, 1, 1, 3, 3, conv1_bin, true);
// tk::dnn::Activation relu1(&net, CUDNN_ACTIVATION_RELU);
// tk::dnn::Shortcut s0_input(&net, pre_img);
// tk::dnn::Shortcut s1_input(&net, pre_hm);
tk::dnn::Conv2d conv2(&net, 16, 3, 3, 1, 1, 1, 1, conv2_bin, true);
tk::dnn::Activation relu2(&net, CUDNN_ACTIVATION_RELU);
base1 = &relu2;
tk::dnn::Conv2d conv3(&net, 32, 3, 3, 2, 2, 1, 1, conv3_bin, true);
tk::dnn::Activation relu3(&net, CUDNN_ACTIVATION_RELU);
base2 = &relu3;
// level 2
// tree 1
tk::dnn::Conv2d s1_t1_conv1(&net, 64, 3, 3, 2, 2, 1, 1, s1_t1_conv1_bin, true);
tk::dnn::Activation s1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s1_t1_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t1_conv2_bin, true);
last2 = &s1_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s1_t1_layers[1] = { base2 };
tk::dnn::Route route_s1_t1(&net, route_s1_t1_layers, 1);
// downsample
tk::dnn::Pooling s1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
// project
tk::dnn::Conv2d s1_t1_residual1_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_t1_project, true);
tk::dnn::Shortcut s1_t1_s1(&net, last2);
tk::dnn::Activation s1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s1_t1_relu;
// tree 2
tk::dnn::Conv2d s1_t2_conv1(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv1_bin, true);
tk::dnn::Activation s1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s1_t2_conv2(&net, 64, 3, 3, 1, 1, 1, 1, s1_t2_conv2_bin, true);
tk::dnn::Shortcut s1_t2_s1(&net, last1);
tk::dnn::Activation s1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s1_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s1_root_layers[2] = { last2, last1 };
tk::dnn::Route route_s1_root(&net, route_s1_root_layers, 2);
tk::dnn::Conv2d s1_root_conv1(&net, 64, 1, 1, 1, 1, 0, 0, s1_root_conv1_bin, true);
tk::dnn::Activation s1_root_relu(&net, CUDNN_ACTIVATION_RELU);
base3 = &s1_root_relu;
// level 3
// tree 1
// tree 1
tk::dnn::Conv2d s2_t1_t1_conv1(&net, 128, 3, 3, 2, 2, 1, 1, s2_t1_t1_conv1_bin, true);
tk::dnn::Activation s2_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t1_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t1_conv2_bin, true);
last2 = &s2_t1_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s2_t1_t1_layers[1] = { base3 };
tk::dnn::Route route_s2_t1_t1(&net, route_s2_t1_t1_layers, 1);
// downsample
tk::dnn::Pooling s2_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
last4 = &s2_t1_t1_maxpool1;
// project
tk::dnn::Conv2d s2_t1_t1_residual1_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_t1_project, true);
tk::dnn::Shortcut s2_t1_t1_s1(&net, last2);
tk::dnn::Activation s2_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s2_t1_t1_relu;
// tree 2
tk::dnn::Conv2d s2_t1_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv1_bin, true);
tk::dnn::Activation s2_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t1_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t1_t2_conv2_bin, true);
tk::dnn::Shortcut s2_t1_t2_s1(&net, last1);
tk::dnn::Activation s2_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s2_t1_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s2_t1_root_layers[2] = { last2, last1 };
tk::dnn::Route route_s2_t1_root(&net, route_s2_t1_root_layers, 2);
tk::dnn::Conv2d s2_t1_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t1_root_conv1_bin, true);
tk::dnn::Activation s2_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s2_t1_root_relu;
last3 = &s2_t1_root_relu;
// tree 2
// tree 1
tk::dnn::Conv2d s2_t2_t1_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv1_bin, true);
tk::dnn::Activation s2_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t2_t1_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t1_conv2_bin, true);
tk::dnn::Shortcut s2_t2_t1_s1(&net, last1);
tk::dnn::Activation s2_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s2_t2_t1_relu;
// tree 2
tk::dnn::Conv2d s2_t2_t2_conv1(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv1_bin, true);
tk::dnn::Activation s2_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s2_t2_t2_conv2(&net, 128, 3, 3, 1, 1, 1, 1, s2_t2_t2_conv2_bin, true);
tk::dnn::Shortcut s2_t2_t2_s1(&net, last1);
tk::dnn::Activation s2_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s2_t2_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s2_t2_root_layers[4] = { last2, last1, last4, last3};
tk::dnn::Route route_s2_t2_root(&net, route_s2_t2_root_layers, 4);
tk::dnn::Conv2d s2_t2_root_conv1(&net, 128, 1, 1, 1, 1, 0, 0, s2_t2_root_conv1_bin, true);
tk::dnn::Activation s2_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
base4 = &s2_t2_root_relu;
// level 4
// tree 1
// tree 1
tk::dnn::Conv2d s3_t1_t1_conv1(&net, 256, 3, 3, 2, 2, 1, 1, s3_t1_t1_conv1_bin, true);
tk::dnn::Activation s3_t1_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t1_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t1_conv2_bin, true);
last2 = &s3_t1_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s3_t1_t1_layers[1] = { base4 };
tk::dnn::Route route_s3_t1_t1(&net, route_s3_t1_t1_layers, 1);
// downsample
tk::dnn::Pooling s3_t1_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
last4 = &s3_t1_t1_maxpool1;
// project
tk::dnn::Conv2d s3_t1_t1_residual1_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_t1_project, true);
tk::dnn::Shortcut s3_t1_t1_s1(&net, last2);
tk::dnn::Activation s3_t1_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s3_t1_t1_relu;
// tree 2
tk::dnn::Conv2d s3_t1_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv1_bin, true);
tk::dnn::Activation s3_t1_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t1_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t1_t2_conv2_bin, true);
tk::dnn::Shortcut s3_t1_t2_s1(&net, last1);
tk::dnn::Activation s3_t1_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s3_t1_t2_relu;
// root
// join last1 and net in single input 256, 56, 56
tk::dnn::Layer *route_s3_t1_root_layers[2] = { last2, last1 };
tk::dnn::Route route_s3_t1_root(&net, route_s3_t1_root_layers, 2);
tk::dnn::Conv2d s3_t1_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t1_root_conv1_bin, true);
tk::dnn::Activation s3_t1_root_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s3_t1_root_relu;
last3 = &s3_t1_root_relu;
// tree 2
// tree 1
tk::dnn::Conv2d s3_t2_t1_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv1_bin, true);
tk::dnn::Activation s3_t2_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t2_t1_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t1_conv2_bin, true);
tk::dnn::Shortcut s3_t2_t1_s1(&net, last1);
tk::dnn::Activation s3_t2_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s3_t2_t1_relu;
// tree 2
tk::dnn::Conv2d s3_t2_t2_conv1(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv1_bin, true);
tk::dnn::Activation s3_t2_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s3_t2_t2_conv2(&net, 256, 3, 3, 1, 1, 1, 1, s3_t2_t2_conv2_bin, true);
tk::dnn::Shortcut s3_t2_t2_s1(&net, last1);
tk::dnn::Activation s3_t2_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s3_t2_t2_relu;
// root
// join last1 and net in single input 256, 56, 56
tk::dnn::Layer *route_s3_t2_root_layers[4] = { last2, last1, last4, last3};
tk::dnn::Route route_s3_t2_root(&net, route_s3_t2_root_layers, 4);
tk::dnn::Conv2d s3_t2_root_conv1(&net, 256, 1, 1, 1, 1, 0, 0, s3_t2_root_conv1_bin, true);
tk::dnn::Activation s3_t2_root_relu(&net, CUDNN_ACTIVATION_RELU);
base5 = &s3_t2_root_relu;
// level 5
// tree 1
tk::dnn::Conv2d s4_t1_conv1(&net, 512, 3, 3, 2, 2, 1, 1, s4_t1_conv1_bin, true);
tk::dnn::Activation s4_t1_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s4_t1_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t1_conv2_bin, true);
last2 = &s4_t1_conv2;
// get the basicblock input and apply maxpool conv2d and relu
tk::dnn::Layer *route_s4_t1_layers[1] = { base5 };
tk::dnn::Route route_s4_t1(&net, route_s4_t1_layers, 1);
// downsample
tk::dnn::Pooling s4_t1_maxpool1(&net, 2, 2, 2, 2, 0, 0, tk::dnn::POOLING_MAX);
last4 = &s4_t1_maxpool1;
// project
tk::dnn::Conv2d s4_t1_residual1_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_t1_project, true);
tk::dnn::Shortcut s4_t1_s1(&net, last2);
tk::dnn::Activation s4_t1_relu(&net, CUDNN_ACTIVATION_RELU);
last1 = &s4_t1_relu;
// tree 2
tk::dnn::Conv2d s4_t2_conv1(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv1_bin, true);
tk::dnn::Activation s4_t2_relu1(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d s4_t2_conv2(&net, 512, 3, 3, 1, 1, 1, 1, s4_t2_conv2_bin, true);
tk::dnn::Shortcut s4_t2_s1(&net, last1);
tk::dnn::Activation s4_t2_relu(&net, CUDNN_ACTIVATION_RELU);
last2 = &s4_t2_relu;
// root
// join last1 and net in single input 128, 56, 56
tk::dnn::Layer *route_s4_root_layers[3] = { last2, last1, last4 };
tk::dnn::Route route_s4_root(&net, route_s4_root_layers, 3);
tk::dnn::Conv2d s4_root_conv1(&net, 512, 1, 1, 1, 1, 0, 0, s4_root_conv1_bin, true);
tk::dnn::Activation s4_root_relu(&net, CUDNN_ACTIVATION_RELU);
base6 = &s4_root_relu;
//final
// tk::dnn::Pooling avgpool(&net, 7, 7, 7, 7, 0, 0, tk::dnn::POOLING_AVERAGE);
// tk::dnn::Dense fc(&net, 1000, fc_bin);
//ida 0
tk::dnn::Layer *route_ida0[1] = { base6 };
tk::dnn::Route route_ida0_0(&net, route_ida0, 1);
tk::dnn::DeformConv2d ida_0_p_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_p_1_dcn_bin, ida_0_p_1_conv_bin, true);
tk::dnn::Activation ida_0_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_0_up_1_deconv(&net, 256, 4, 4, 2, 2, 1, 1, ida_0_up_1_deconv_bin, false, 256);
tk::dnn::Shortcut ida_0_shortcut(&net, base5);
tk::dnn::DeformConv2d ida_0_n_1_dcn(&net, 256, 1, 3, 3, 1, 1, 1, 1, ida_0_n_1_dcn_bin, ida_0_n_1_conv_bin, true);
tk::dnn::Activation ida_0_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
ida1 = &ida_0_n_1_relu;
//ida1-1
tk::dnn::Layer *route_ida1_layers_1[1] = { base5 };
tk::dnn::Route route_ida1_1(&net, route_ida1_layers_1, 1);
tk::dnn::DeformConv2d ida_1_p_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_1_dcn_bin, ida_1_p_1_conv_bin, true);
tk::dnn::Activation ida_1_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_1_up_1_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_1_deconv_bin, false, 128);
tk::dnn::Shortcut ida_1_shortcut1(&net, base4);
tk::dnn::DeformConv2d ida_1_n_1_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_1_dcn_bin, ida_1_n_1_conv_bin, true);
tk::dnn::Activation ida_1_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
ida2_1 = &ida_1_n_1_relu;
//ida1-2
tk::dnn::Layer *route_ida1_layers_2[1] = { ida1 };
tk::dnn::Route route_ida1_2(&net, route_ida1_layers_2, 1);
tk::dnn::DeformConv2d ida_1_p_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_p_2_dcn_bin, ida_1_p_2_conv_bin, true);
tk::dnn::Activation ida_1_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_1_up_2_deconv(&net, 128, 4, 4, 2, 2, 1, 1, ida_1_up_2_deconv_bin, false, 128);
tk::dnn::Shortcut ida_1_shortcut2(&net, ida2_1);
tk::dnn::DeformConv2d ida_1_n_2_dcn(&net, 128, 1, 3, 3, 1, 1, 1, 1, ida_1_n_2_dcn_bin, ida_1_n_2_conv_bin, true);
tk::dnn::Activation ida_1_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
ida2_2 = &ida_1_n_2_relu;
//ida2-1
tk::dnn::Layer *route_ida2_layers_1[1] = { base4 };
tk::dnn::Route route_ida2_1(&net, route_ida2_layers_1, 1);
tk::dnn::DeformConv2d ida_2_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_1_dcn_bin, ida_2_p_1_conv_bin, true);
tk::dnn::Activation ida_2_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_2_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_1_deconv_bin, false, 64);
tk::dnn::Shortcut ida_2_shortcut1(&net, base3);
tk::dnn::DeformConv2d ida_2_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_1_dcn_bin, ida_2_n_1_conv_bin, true);
tk::dnn::Activation ida_2_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
ida3_1 = &ida_2_n_1_relu;
//ida2-2
tk::dnn::Layer *route_ida2_layers_2[1] = { ida2_1 };
tk::dnn::Route route_ida2_2(&net, route_ida2_layers_2, 1);
tk::dnn::DeformConv2d ida_2_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_2_dcn_bin, ida_2_p_2_conv_bin, true);
tk::dnn::Activation ida_2_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_2_up_2_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_2_deconv_bin, false, 64);
tk::dnn::Shortcut ida_2_shortcut2(&net, ida3_1);
tk::dnn::DeformConv2d ida_2_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_2_dcn_bin, ida_2_n_2_conv_bin, true);
tk::dnn::Activation ida_2_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
ida3_2 = &ida_2_n_2_relu;
//ida2-3
tk::dnn::Layer *route_ida2_layers_3[1] = { ida2_2 };
tk::dnn::Route route_ida2_3(&net, route_ida2_layers_3, 1);
tk::dnn::DeformConv2d ida_2_p_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_p_3_dcn_bin, ida_2_p_3_conv_bin, true);
tk::dnn::Activation ida_2_p_3_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d ida_2_up_3_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_2_up_3_deconv_bin, false, 64);
tk::dnn::Shortcut ida_2_shortcut3(&net, ida3_2);
tk::dnn::DeformConv2d ida_2_n_3_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_2_n_3_dcn_bin, ida_2_n_3_conv_bin, true);
tk::dnn::Activation ida_2_n_3_relu(&net, CUDNN_ACTIVATION_RELU);
ida3_3 = &ida_2_n_3_relu;
//idaup-1
tk::dnn::Layer *route_idaup_layers_1[1] = { ida2_2 };
tk::dnn::Route route_idaup_1(&net, route_idaup_layers_1, 1);
tk::dnn::DeformConv2d idaup_p_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_1_dcn_bin, ida_up_p_1_conv_bin, true);
tk::dnn::Activation idaup_p_1_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d idaup_up_1_deconv(&net, 64, 4, 4, 2, 2, 1, 1, ida_up_up_1_deconv_bin, false, 64);
tk::dnn::Shortcut idaup_shortcut1(&net, ida3_3);
tk::dnn::DeformConv2d idaup_n_1_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_1_dcn_bin, ida_up_n_1_conv_bin, true);
tk::dnn::Activation idaup_n_1_relu(&net, CUDNN_ACTIVATION_RELU);
idaup_1 = &idaup_n_1_relu;
//idaup-2
tk::dnn::Layer *route_idaup_layers_2[1] = { ida1 };
tk::dnn::Route route_idaup_2(&net, route_idaup_layers_2, 1);
tk::dnn::DeformConv2d idaup_p_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_p_2_dcn_bin, ida_up_p_2_conv_bin, true);
tk::dnn::Activation idaup_p_2_relu(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::DeConv2d idaup_up_2_deconv(&net, 64, 8, 8, 4, 4, 2, 2, ida_up_up_2_deconv_bin, false, 64);
tk::dnn::Shortcut idaup_shortcut2(&net, idaup_1);
tk::dnn::DeformConv2d idaup_n_2_dcn(&net, 64, 1, 3, 3, 1, 1, 1, 1, ida_up_n_2_dcn_bin, ida_up_n_2_conv_bin, true);
tk::dnn::Activation idaup_n_2_relu(&net, CUDNN_ACTIVATION_RELU);
idaup_2 = &idaup_n_2_relu;
tk::dnn::Layer *route_1_0_layers[1] = { idaup_2 };
// hm
tk::dnn::Conv2d *hm_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, hm_conv1_bin, false);
tk::dnn::Activation *hm_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *hm = new tk::dnn::Conv2d(&net, 10, 1, 1, 1, 1, 0, 0, hm_conv2_bin, false);
hm->setFinal();
int kernel = 3;
int pad = (kernel - 1)/2;
tk::dnn::Activation *hm_sig = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_SIGMOID);
tk::dnn::Pooling *hmax = new tk::dnn::Pooling(&net, kernel, kernel, 1, 1, pad, pad, tk::dnn::POOLING_MAX);
hmax->setFinal();
// wh
tk::dnn::Route *route_1_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *wh_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, wh_conv1_bin, false);
tk::dnn::Activation *wh_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *wh = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, wh_conv2_bin, false);
wh->setFinal();
// reg
tk::dnn::Route *route_2_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *reg_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, reg_conv1_bin, false);
tk::dnn::Activation *reg_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *reg = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, reg_conv2_bin, false);
reg->setFinal();
// tracking
tk::dnn::Route *route_3_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *track_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, track_conv1_bin, false);
tk::dnn::Activation *track_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *track = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, track_conv2_bin, false);
track->setFinal();
// dep
tk::dnn::Route *route_4_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *dep_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dep_conv1_bin, false);
tk::dnn::Activation *dep_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *dep = new tk::dnn::Conv2d(&net, 1, 1, 1, 1, 1, 0, 0, dep_conv2_bin, false);
dep->setFinal();
// rot
tk::dnn::Route *route_5_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *rot_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, rot_conv1_bin, false);
tk::dnn::Activation *rot_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *rot = new tk::dnn::Conv2d(&net, 8, 1, 1, 1, 1, 0, 0, rot_conv2_bin, false);
rot->setFinal();
// dim
tk::dnn::Route *route_6_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *dim_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, dim_conv1_bin, false);
tk::dnn::Activation *dim_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *dim_ = new tk::dnn::Conv2d(&net, 3, 1, 1, 1, 1, 0, 0, dim_conv2_bin, false);
dim_->setFinal();
// amodel_offset
tk::dnn::Route *route_7_0 = new tk::dnn::Route(&net, route_1_0_layers, 1);
tk::dnn::Conv2d *a_off_conv1 = new tk::dnn::Conv2d(&net, 256, 3, 3, 1, 1, 1, 1, a_off_conv1_bin, false);
tk::dnn::Activation *a_off_relu1 = new tk::dnn::Activation(&net, CUDNN_ACTIVATION_RELU);
tk::dnn::Conv2d *a_off = new tk::dnn::Conv2d(&net, 2, 1, 1, 1, 1, 0, 0, a_off_conv2_bin, false);
a_off->setFinal();
// Load input
dnnType *data;
dnnType *input_h;
readBinaryFile(input_bin, dim_in0.tot(), &input_h, &data);
//printDeviceVector(64, data, true);
//print network model
net.print();
//convert network to tensorRT
tk::dnn::NetworkRT netRT(&net, net.getNetworkRTName("dla34_ctrack"));
tk::dnn::dataDim_t dim1 = dim_in0; //input dim
printCenteredTitle(" CUDNN inference ", '=', 30);
{
dim1.print();
TKDNN_TSTART
// tk::dnn::dataDim_t dim_aus;
// net.infer(dim_aus, nullptr);
net.infer(dim1, data);
TKDNN_TSTOP
dim1.print();
}
tk::dnn::dataDim_t dim2 = dim_in0;
printCenteredTitle(" TENSORRT inference ", '=', 30);
{
dim2.print();
TKDNN_TSTART
netRT.infer(dim2, data);
TKDNN_TSTOP
dim2.print();
}
// dnnType *out, *out_h;
// int odim = net.layers[net.num_layers-1]->output_dim.tot();
// readBinaryFile(output_bin, odim, &out_h, &out);
// dnnType *cudnn_out;
// cudnn_out = net.layers[net.num_layers-1]->dstData;
// std::cout<<"CUDNN vs correct";
// checkResult(odim, cudnn_out, out);
tk::dnn::Layer *outs[8] = { hm, wh, reg, track, dep, rot, dim_, a_off};
int out_count = 1;
int ret_cudnn = 0, ret_tensorrt = 0, ret_cudnn_tensorrt = 0;
for(int i=0; i<8; i++) {
printCenteredTitle((std::string(" RESNET CHECK RESULTS ") + std::to_string(i) + " ").c_str(), '=', 30);
outs[i]->output_dim.print();
dnnType *out, *out_h;
int odim = outs[i]->output_dim.tot();
readBinaryFile(output_bin[i], odim, &out_h, &out);
dnnType *cudnn_out, *rt_out;
cudnn_out = outs[i]->dstData;
rt_out = (dnnType *)netRT.buffersRT[i+out_count];
// there is the maxpool. It isn't an output but it is necessary for the process section
if(i==0)
out_count ++;
std::cout<<"CUDNN vs correct";
ret_cudnn |= checkResult(odim, cudnn_out, out) == 0 ? 0: ERROR_CUDNN;
std::cout<<"TRT vs correct";
ret_tensorrt |= checkResult(odim, rt_out, out) == 0 ? 0 : ERROR_TENSORRT;
std::cout<<"CUDNN vs TRT ";
ret_cudnn_tensorrt |= checkResult(odim, cudnn_out, rt_out) == 0 ? 0 : ERROR_CUDNNvsTENSORRT;
}
return ret_cudnn | ret_tensorrt | ret_cudnn_tensorrt;
}
tests/darknet/cfg/yolo4-csp.cfg
-1279
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4-csp_crowd.cfg
-1279
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4_320.cfg
-1156
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4_320_coco2.cfg
-1158
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4_512.cfg
-1156
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4_608.cfg
-1156
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4_mmr.cfg
-1158
View File
File diff suppressed because it is too large Load Diff
tests/darknet/cfg/yolo4tiny_512.cfg
-281
View File
@@ -1,281 +0,0 @@
[net]
# Testing
#batch=1
#subdivisions=1
# Training
batch=64
subdivisions=1
width=512
height=512
channels=3
momentum=0.9
decay=0.0005
angle=0
saturation = 1.5
exposure = 1.5
hue=.1
learning_rate=0.00261
burn_in=1000
max_batches = 500200
policy=steps
steps=400000,450000
scales=.1,.1
[convolutional]
batch_normalize=1
filters=32
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=3
stride=1
pad=1
activation=leaky
[route]
layers=-1
groups=2
group_id=1
[convolutional]
batch_normalize=1
filters=32
size=3
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=32
size=3
stride=1
pad=1
activation=leaky
[route]
layers = -1,-2
[convolutional]
batch_normalize=1
filters=64
size=1
stride=1
pad=1
activation=leaky
[route]
layers = -6,-1
[maxpool]
size=2
stride=2
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[route]
layers=-1
groups=2
group_id=1
[convolutional]
batch_normalize=1
filters=64
size=3
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=3
stride=1
pad=1
activation=leaky
[route]
layers = -1,-2
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[route]
layers = -6,-1
[maxpool]
size=2
stride=2
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[route]
layers=-1
groups=2
group_id=1
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[route]
layers = -1,-2
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[route]
layers = -6,-1
[maxpool]
size=2
stride=2
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
##################################
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=255
activation=linear
[yolo]
mask = 3,4,5
anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319
classes=80
num=6
jitter=.3
scale_x_y = 1.05
cls_normalizer=1.0
iou_normalizer=0.07
iou_loss=ciou
ignore_thresh = .7
truth_thresh = 1
random=0
resize=1.5
nms_kind=greedynms
beta_nms=0.6
[route]
layers = -4
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[upsample]
stride=2
[route]
layers = -1, 23
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=255
activation=linear
[yolo]
mask = 1,2,3
anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319
classes=80
num=6
jitter=.3
scale_x_y = 1.05
cls_normalizer=1.0
iou_normalizer=0.07
iou_loss=ciou
ignore_thresh = .7
truth_thresh = 1
random=0
resize=1.5
nms_kind=greedynms
beta_nms=0.6

Some files were not shown because too many files have changed in this diff Show More